edenapi: Implement uploading changesets in hg cloud upload command

Summary:
Implement using of uploading changesets in `hg cloud upload` command.

This is the last part for `hg cloud upload` - uploading changesets via Edenapi
test
```

```
# machine #2
liubovd {emoji:1f352}  ~/fbsource
 [15] → hg pull -r 0b6075b4bda143d5212c1525323fb285d96a1afb
pulling from mononoke://mononoke.c2p.facebook.net/fbsource
connected to twshared27150.03.cln3.facebook.com session RaIPDgvF6l8rmXkA
abort: 0b6075b4bda143d5212c1525323fb285d96a1afb not found!
```

```
# machine #1
devvm1006.cln0 {emoji:1f440}   ~/fbsource/fbcode/eden/scm
 [6] →  EDENSCM_LOG="edenapi::client=info" ./hg cloud upload
Jul 11 13:26:26.322  INFO edenapi::client: Requesting lookup for 1 item(s)
commitcloud: head '0b6075b4bda1' hasn't been uploaded yet
Jul 11 13:26:26.472  INFO edenapi::client: Requesting lookup for 6 item(s)
commitcloud: queue 1 commit for upload
Jul 11 13:26:26.648  INFO edenapi::client: Requesting lookup for 1 item(s)
commitcloud: queue 0 files for upload
Jul 11 13:26:26.698  INFO edenapi::client: Requesting lookup for 4 item(s)
commitcloud: queue 4 trees for upload
Jul 11 13:26:27.393  INFO edenapi::client: Requesting trees upload for 4 item(s)
commitcloud: uploaded 4 trees
commitcloud: uploading commit '0b6075b4bda143d5212c1525323fb285d96a1afb'...
Jul 11 13:26:28.426  INFO edenapi::client: Requesting changesets upload for 1 item(s)
commitcloud: uploaded 1 commit
```

```
# machine #2
liubovd {emoji:1f352}  ~/fbsource
 [16] → hg pull -r 0b6075b4bda143d5212c1525323fb285d96a1afb
pulling from mononoke://mononoke.c2p.facebook.net/fbsource
connected to twshared16001.08.cln2.facebook.com session QCpy1x9yrflRF6xF
searching for changes
adding commits
adding manifests
adding file changes
added 895 commits with 0 changes to 0 files
(running background incremental repack)
prefetching trees for 4 commits

liubovd {emoji:1f352}  ~/fbsource
 [17] → hg up 0b6075b4bda143d5212c1525323fb285d96a1afb
warning: watchman has recently started (pid 93231) - operation will be slower than usual
connected to twshared32054.08.cln2.facebook.com session Hw91G8kRYzt4c5BV
1 files updated, 0 files merged, 0 files removed, 0 files unresolved

liubovd {emoji:1f352}  ~/fbsource
 [18] → hg diff -c .
connected to twshared0965.07.cln2.facebook.com session rrYSvRM6pnBYZ2Fn
 diff --git a/fbcode/eden/scm/test b/fbcode/eden/scm/test
new file mode 100644
 --- /dev/null
+++ b/fbcode/eden/scm/test
@@ -0,0 +1,1 @@
+test
```

Initial perf wins:

Having a large stack of 6 commits (total 24 files changed), tested *adding a single line to a file at the top commit*. We can see at least 2X win but it should be more because I have tested with a local instance of edenapi service that runs on my devserver.

```
╷
╷ @  5582fc8ee  6 minutes ago  liubovd
╷ │  test
╷ │
╷ o  d55f9bb65  86 minutes ago  liubovd  D29644738
╷ │  [hg] edenapi: Implement using of uploading changesets in `hg cloud upload` command
╷ │
╷ o  561149783  Friday at 15:10  liubovd  D29644797
╷ │  [hg] edenapi: Add request handler for uploading hg changesets
╷ │
╷ o  c3dda964a  Friday at 15:10  liubovd  D29644800
╷ │  [edenapi_service] Add new /:repo/upload/changesets endpoint
╷ │
╷ o  28ce2fa0c  Friday at 15:10  liubovd  D29644799
╷ │  [hg] edenapi/edenapi_service: Add new API for uploading Hg Changesets
╷ │
╷ o  13325b361  Yesterday at 15:23  liubovd  D29644798
╭─╯  [edenapi_service] Implement uploading of hg changesets
```

```
# adding new line to a file test in the test commit, and then run:
devvm1006.cln0 {emoji:1f440}   ~/fbsource/fbcode/eden/scm
 [8] → time hg cloud upload
commitcloud: head '4e4f947d73e6' hasn't been uploaded yet
commitcloud: queue 1 commit for upload
commitcloud: queue 0 files for upload
commitcloud: queue 4 trees for upload
commitcloud: uploaded 4 trees
commitcloud: uploading commit '4e4f947d73e676b63df7c90c4e707d38e6d0a93b'...
commitcloud: uploaded 1 commit

real	0m3.778s
user	0m0.017s
sys	0m0.027s
```

```
# adding another new line to a file test in the test commit, and then run:
devvm1006.cln0 {emoji:1f440}   ~/fbsource/fbcode/eden/scm
 [11] → time hg cloud backup
connected to twshared30574.02.cln2.facebook.com session uvOvhxtBfeM7pMgl
backing up stack rooted at 13325b3612d2
commitcloud: backed up 1 commit

real	0m7.507s
user	0m0.013s
sys	0m0.030s
```

Test force mode of the new command that reupload everything:
```
devvm1006.cln0 {emoji:1f440}   ~/fbsource/fbcode/eden/scm
 [13] → time hg cloud upload --force
commitcloud: head '5582fc8ee382' hasn't been uploaded yet
commitcloud: queue 6 commits for upload
commitcloud: queue 24 files for upload
commitcloud: uploaded 24 files
commitcloud: queue 61 trees for upload
commitcloud: uploaded 61 trees
commitcloud: uploading commit '13325b3612d20c176923d1aab8a28383cea2ba9a'...
commitcloud: uploading commit '28ce2fa0c6a02de57cdc732db742fd5c8f2611ad'...
commitcloud: uploading commit 'c3dda964a71b65f01fc4ccadc9429ee887ea982c'...
commitcloud: uploading commit '561149783e2fb5916378fe27757dcc2077049f8c'...
commitcloud: uploading commit 'd55f9bb65a0829b1731baa686cb8a6e0c5500cc2'...
commitcloud: uploading commit '5582fc8ee382c4c367a057db2a1781377bf55ba4'...
commitcloud: uploaded 6 commits

real	0m7.830s
user	0m0.011s
sys	0m0.032s
```

We can see the time is similar to the current `hg cloud backup` command.

Reviewed By: markbt

Differential Revision: D29644738

fbshipit-source-id: cbbfcb2e8018f83f323f447848b3b6045baf47c5

eden/mononoke/tests/integration/test-commitcloud-upload.t

@@ -120,11 +120,11 @@ This test also checks file content deduplication. We upload 1 file content and 1
     INFO edenapi::client: Requesting trees upload for 2 item(s)
   commitcloud: uploaded 2 trees
   commitcloud: uploading commit '536d3fb3929eab4b01e63ab7fc9b25a5c8a08bc9'...
+    INFO edenapi::client: Requesting changesets upload for 1 item(s)
+  commitcloud: uploaded 1 changeset
 
 
 Make another commit in the first client and upload it
-(It will try to upload the first commit again because uploading of changesets haven't been implemented yet
-but it shouldn't try to reupload any trees or filenodes for the first commit)
 The files of the second commit are identical to the files of the first commit, so we don't expect any new content uploads
   $ hgedenapi prev -q
   [8b2dca] base_commit
@@ -133,16 +133,19 @@ The files of the second commit are identical to the files of the first commit, s
   $ hgedenapi commit -m "New files Dir2"
   $ EDENSCM_LOG="edenapi::client=info" hgedenapi cloud upload
     INFO edenapi::client: Requesting lookup for 2 item(s)
-  commitcloud: head '536d3fb3929e' hasn't been uploaded yet
   commitcloud: head '65289540f44d' hasn't been uploaded yet
-    INFO edenapi::client: Requesting lookup for 2 item(s)
-  commitcloud: queue 2 commits for upload
-    INFO edenapi::client: Requesting lookup for 200 item(s)
+    INFO edenapi::client: Requesting lookup for 1 item(s)
+  commitcloud: queue 1 commit for upload
+    INFO edenapi::client: Requesting lookup for 100 item(s)
   commitcloud: queue 0 files for upload
-    INFO edenapi::client: Requesting lookup for 4 item(s)
+    INFO edenapi::client: Requesting lookup for 2 item(s)
   commitcloud: queue 1 tree for upload
     INFO edenapi::client: Requesting trees upload for 1 item(s)
   commitcloud: uploaded 1 tree
-  commitcloud: uploading commit '536d3fb3929eab4b01e63ab7fc9b25a5c8a08bc9'...
   commitcloud: uploading commit '65289540f44d80cecffca8a3fd655c0ca6243cd9'...
-
+    INFO edenapi::client: Requesting changesets upload for 1 item(s)
+  commitcloud: uploaded 1 changeset
+
+  $ EDENSCM_LOG="edenapi::client=info" hgedenapi cloud upload
+    INFO edenapi::client: Requesting lookup for 2 item(s)
+  commitcloud: nothing to upload

eden/scm/edenscm/hgext/commitcloud/upload.py

@@ -92,6 +92,28 @@ def uploadtrees(repo, trees):
         raise error.Abort(e)
 
 
+def uploadchangesets(repo, changesets):
+    """Upload changesets"""
+    if not changesets:
+        return
+    try:
+        stream, _stats = repo.edenapi.uploadchangesets(
+            ccutil.getreponame(repo), changesets
+        )
+        foundindices = {item[INDEX_KEY] for item in stream if item[TOKEN_KEY]}
+        repo.ui.status(
+            _n(
+                "uploaded %d changeset\n",
+                "uploaded %d changesets\n",
+                len(foundindices),
+            )
+            % len(foundindices),
+            component="commitcloud",
+        )
+    except (error.RustError, error.HttpError) as e:
+        raise error.Abort(e)
+
+
 def getblobs(repo, nodes):
     """Get changed files"""
     toupload = set()
@@ -232,9 +254,41 @@ def upload(repo, revs, force=False):
     # Upload missing trees
     uploadtrees(repo, uploadtreesqueue)
 
-    # TODO (liubovd): finally: implement upload of hg changesets
+    # Uploading changesets
+    changesets = []
     for node in uploadcommitqueue.iterrev():
         ui.status(
-            _("uploading commit '%s'...\n") % nodemod.hex(node),
-            component="commitcloud",
+            _("uploading commit '%s'...\n") % nodemod.hex(node), component="commitcloud"
         )
+        ctx = repo[node]
+        extras = [
+            {"key": key.encode(), "value": value.encode()}
+            for key, value in ctx.extra().items()
+            if key != "branch"
+        ]
+        (time, timezone) = ctx.date()
+        p1 = ctx.p1().node()
+        p2 = ctx.p2().node()
+        if p1 != nodemod.nullid and p2 != nodemod.nullid:
+            parents = (p1, p2)
+        elif p1 != nodemod.nullid:
+            parents = p1
+        else:
+            parents = None
+        changesets.append(
+            (
+                node,
+                {
+                    "parents": parents,
+                    "manifestid": ctx.manifestnode(),
+                    "user": ctx.user().encode(),
+                    "time": int(time),
+                    "tz": timezone,
+                    "extras": extras,
+                    "files": ctx.files(),
+                    "message": ctx.description().encode(),
+                },
+            )
+        )
+
+    uploadchangesets(repo, changesets)

eden/scm/edenscmnative/bindings/modules/pyedenapi/src/client.rs

@@ -19,8 +19,9 @@ use edenapi_types::CommitGraphEntry;
 use edenapi_types::CommitKnownResponse;
 use edenapi_types::TreeEntry;
 use edenapi_types::{
-    CommitHashToLocationResponse, CommitLocationToHashResponse, CommitRevlogData, LookupResponse,
-    UploadHgFilenodeResponse, UploadTreeResponse,
+    CommitHashToLocationResponse, CommitLocationToHashResponse, CommitRevlogData,
+    HgChangesetContent, LookupResponse, UploadHgChangesetsResponse, UploadHgFilenodeResponse,
+    UploadTreeResponse,
 };
 use progress::{NullProgressFactory, ProgressFactory};
 use pyconfigparser::config;
@@ -299,6 +300,20 @@ py_class!(pub class client |py| {
         let progress = self.progress(py).clone();
         self.inner(py).clone().uploadtrees_py(py, repo, items, callback, progress)
     }
+
+    /// Upload changesets
+    def uploadchangesets(
+        &self,
+        repo: String,
+        changesets: Vec<(
+            PyBytes,                   /* hgid (node_id) */
+            Serde<HgChangesetContent>  /* changeset content */
+        )>,
+        callback: Option<PyObject> = None
+    ) -> PyResult<(TStream<anyhow::Result<Serde<UploadHgChangesetsResponse>>>, PyFuture)> {
+        let progress = self.progress(py).clone();
+        self.inner(py).clone().uploadchangesets_py(py, repo, changesets, callback, progress)
+    }
 });
 
 impl ExtractInnerRef for client {

eden/scm/edenscmnative/bindings/modules/pyedenapi/src/pyext.rs

@@ -20,12 +20,12 @@ use cpython_ext::PyCell;
 use cpython_ext::{PyPathBuf, ResultPyErrExt};
 use dag_types::{Location, VertexName};
 use edenapi::{EdenApi, EdenApiBlocking, EdenApiError, Fetch, Stats};
-use edenapi_types::CommitGraphEntry;
-use edenapi_types::CommitKnownResponse;
 use edenapi_types::{
-    AnyFileContentId, AnyId, CommitHashToLocationResponse, CommitLocationToHashRequest,
-    CommitLocationToHashResponse, CommitRevlogData, EdenApiServerError, FileEntry, HgFilenodeData,
-    HistoryEntry, LookupResponse, TreeEntry, UploadHgFilenodeResponse, UploadTreeResponse,
+    AnyFileContentId, AnyId, CommitGraphEntry, CommitHashToLocationResponse, CommitKnownResponse,
+    CommitLocationToHashRequest, CommitLocationToHashResponse, CommitRevlogData,
+    EdenApiServerError, FileEntry, HgChangesetContent, HgFilenodeData, HistoryEntry,
+    LookupResponse, TreeEntry, UploadHgChangeset, UploadHgChangesetsResponse,
+    UploadHgFilenodeResponse, UploadTreeResponse,
 };
 use progress::{ProgressBar, ProgressFactory, Unit};
 use pyrevisionstore::as_legacystore;
@@ -650,6 +650,44 @@ pub trait EdenApiPyExt: EdenApi {
         let stats_py = PyFuture::new(py, stats.map_ok(PyStats))?;
         Ok((responses_py.into(), stats_py))
     }
+
+    /// Upload changesets
+    fn uploadchangesets_py(
+        self: Arc<Self>,
+        py: Python,
+        repo: String,
+        changesets: Vec<(
+            PyBytes,                   /* hgid (node_id) */
+            Serde<HgChangesetContent>, /* changeset content */
+        )>,
+        callback: Option<PyObject>,
+        _progress: Arc<dyn ProgressFactory>,
+    ) -> PyResult<(
+        TStream<anyhow::Result<Serde<UploadHgChangesetsResponse>>>,
+        PyFuture,
+    )> {
+        let callback = callback.map(wrap_callback);
+        let changesets = changesets
+            .into_iter()
+            .map(|(node_id, content)| UploadHgChangeset {
+                node_id: to_hgid(py, &node_id),
+                changeset_content: content.0,
+            })
+            .collect();
+        let (responses, stats) = py
+            .allow_threads(|| {
+                block_unless_interrupted(async move {
+                    let response = self.upload_changesets(repo, changesets, callback).await?;
+                    Ok::<_, EdenApiError>((response.entries, response.stats))
+                })
+            })
+            .map_pyerr(py)?
+            .map_pyerr(py)?;
+
+        let responses_py = responses.map_ok(Serde).map_err(Into::into);
+        let stats_py = PyFuture::new(py, stats.map_ok(PyStats))?;
+        Ok((responses_py.into(), stats_py))
+    }
 }
 
 impl<T: EdenApi + ?Sized> EdenApiPyExt for T {}