modern sync: use upload_identical_changeset instead of upload_hg_chan…

…geset

Summary: Lots of type conversions, usual piping

Reviewed By: markbt

Differential Revision: D67379173

fbshipit-source-id: 7d7a2b19c8a1cb42bed1deedfec24449331c1598

eden/mononoke/edenapi_service/src/handlers/commit.rs

@@ -566,7 +566,16 @@ impl SaplingRemoteApiHandler for UploadBonsaiChangesetHandler {
             .await
             .map_err(HttpError::e429)?;
 
-        let cs_id = upload_bonsai_changeset(cs.clone(), repo, bubble_id).await?;
+        let parents = stream::iter(cs.hg_parents)
+            .then(|hgid| async move {
+                repo.get_bonsai_from_hg(hgid.into())
+                    .await?
+                    .ok_or_else(|| anyhow!("Parent HgId {} is invalid", hgid))
+            })
+            .try_collect()
+            .await?;
+
+        let cs_id = upload_bonsai_changeset(cs.clone(), repo, bubble_id, parents).await?;
 
         Ok(stream::once(async move {
             Ok(UploadTokensResponse {
@@ -584,16 +593,8 @@ async fn upload_bonsai_changeset(
     cs: BonsaiChangesetContent,
     repo: &HgRepoContext<Repo>,
     bubble_id: Option<BubbleId>,
+    parents: Vec<ChangesetId>,
 ) -> anyhow::Result<ChangesetId> {
-    let parents = stream::iter(cs.hg_parents)
-        .then(|hgid| async move {
-            repo.get_bonsai_from_hg(hgid.into())
-                .await?
-                .ok_or_else(|| anyhow!("Parent HgId {} is invalid", hgid))
-        })
-        .try_collect()
-        .await?;
-
     let (_hg_extra, cs_ctx) = repo
         .repo_ctx()
         .create_changeset(
@@ -1163,7 +1164,14 @@ impl SaplingRemoteApiHandler for UploadIdenticalChangesetsHandler {
         let mut changesets_data = Vec::new();
         for changeset in changesets {
             let bs_c = BonsaiChangesetContent::from(changeset.clone());
-            let bcs_id = upload_bonsai_changeset(bs_c, &repo, None).await?;
+
+            let parents = changeset
+                .bonsai_parents
+                .to_vec()
+                .into_iter()
+                .map(|p| p.into())
+                .collect();
+            let bcs_id = upload_bonsai_changeset(bs_c, &repo, None, parents).await?;
             let blobstore = repo.repo_ctx().repo_blobstore();
             let bcs = bcs_id
                 .load(&ctx, &blobstore)

eden/mononoke/modern_sync/Cargo.toml

@@ -41,5 +41,6 @@ sapling-edenapi = { version = "0.1.0", path = "../../scm/lib/edenapi" }
 sapling-edenapi_types = { version = "0.1.0", path = "../../scm/lib/edenapi/types" }
 sharding_ext = { version = "0.1.0", path = "../cmdlib/sharding_ext" }
 slog = { version = "2.7", features = ["max_level_trace", "nested-values"] }
+sorted_vector_map = { version = "0.2.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 stats = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 url = "2.5.2"

eden/mononoke/modern_sync/src/sender.rs

@@ -12,6 +12,7 @@ use mercurial_types::blobs::HgBlobChangeset;
 use mercurial_types::HgChangesetId;
 use mercurial_types::HgFileNodeId;
 use mercurial_types::HgManifestId;
+use mononoke_types::BonsaiChangeset;
 use mononoke_types::FileContents;
 pub mod dummy;
 pub mod edenapi;
@@ -40,6 +41,11 @@ pub trait ModernSyncSender {
 
     async fn upload_hg_changeset(&self, hg_css: Vec<HgBlobChangeset>) -> Result<()>;
 
+    async fn upload_identical_changeset(
+        &self,
+        css: Vec<(HgBlobChangeset, BonsaiChangeset)>,
+    ) -> Result<()>;
+
     async fn set_bookmark(
         &self,
         bookmark: String,

eden/mononoke/modern_sync/src/sender/dummy.rs

@@ -12,6 +12,7 @@ use mercurial_types::blobs::HgBlobChangeset;
 use mercurial_types::HgChangesetId;
 use mercurial_types::HgFileNodeId;
 use mercurial_types::HgManifestId;
+use mononoke_types::BonsaiChangeset;
 use mononoke_types::FileContents;
 use slog::info;
 use slog::Logger;
@@ -88,4 +89,19 @@ impl ModernSyncSender for DummySender {
         );
         Ok(())
     }
+
+    async fn upload_identical_changeset(
+        &self,
+        css: Vec<(HgBlobChangeset, BonsaiChangeset)>,
+    ) -> Result<()> {
+        for (hg_cs, bs_cs) in css {
+            info!(
+                &self.logger,
+                "Uploading hg changeset with hgid {} and bsid{}",
+                hg_cs.get_changeset_id(),
+                bs_cs.get_changeset_id()
+            );
+        }
+        Ok(())
+    }
 }

eden/mononoke/modern_sync/src/sender/edenapi.rs

@@ -7,6 +7,7 @@
 
 use std::collections::HashMap;
 use std::collections::HashSet;
+use std::iter::Iterator;
 
 use anyhow::Result;
 use async_trait::async_trait;
@@ -29,6 +30,7 @@ use mercurial_types::HgChangesetId;
 use mercurial_types::HgFileNodeId;
 use mercurial_types::HgManifestId;
 use mononoke_app::args::TLSArgs;
+use mononoke_types::BonsaiChangeset;
 use mononoke_types::FileContents;
 use repo_blobstore::RepoBlobstore;
 use slog::info;
@@ -229,4 +231,23 @@ impl ModernSyncSender for EdenapiSender {
         info!(&self.logger, "Move bookmark response {:?}", res);
         Ok(())
     }
+
+    async fn upload_identical_changeset(
+        &self,
+        css: Vec<(HgBlobChangeset, BonsaiChangeset)>,
+    ) -> Result<()> {
+        let entries = stream::iter(css)
+            .map(util::to_identical_changeset)
+            .try_collect::<Vec<_>>()
+            .await?;
+
+        let res = self.client.upload_identical_changesets(entries).await?;
+        info!(
+            &self.logger,
+            "Upload hg changeset response: {:?}",
+            res.entries.try_collect::<Vec<_>>().await?
+        );
+
+        Ok(())
+    }
 }

eden/mononoke/modern_sync/src/sender/edenapi/util.rs

@@ -5,16 +5,24 @@
  * GNU General Public License version 2.
  */
 
+use anyhow::ensure;
+use anyhow::Error;
 use anyhow::Result;
 use blobstore::Loadable;
 use bytes::Bytes;
 use bytes::BytesMut;
 use context::CoreContext;
+use edenapi_service::utils::to_hg_path;
+use edenapi_types::commit::BonsaiExtra;
+use edenapi_types::commit::BonsaiParents;
+use edenapi_types::commit::HgInfo;
 use edenapi_types::AnyFileContentId;
 use edenapi_types::AnyId;
+use edenapi_types::BonsaiFileChange;
 use edenapi_types::Extra;
 use edenapi_types::HgChangesetContent;
 use edenapi_types::HgFilenodeData;
+use edenapi_types::IdenticalChangesetContent;
 use edenapi_types::Parents;
 use edenapi_types::RepoPathBuf;
 use edenapi_types::UploadHgChangeset;
@@ -24,7 +32,13 @@ use mercurial_types::blobs::HgBlobChangeset;
 use mercurial_types::fetch_manifest_envelope;
 use mercurial_types::HgFileNodeId;
 use mercurial_types::HgManifestId;
+use mononoke_types::BonsaiChangeset;
+use mononoke_types::BonsaiChangesetMut;
+use mononoke_types::ChangesetId;
+use mononoke_types::FileChange;
+use mononoke_types::NonRootMPath;
 use repo_blobstore::RepoBlobstore;
+use sorted_vector_map::SortedVectorMap;
 
 pub async fn from_tree_to_entry(
     id: HgManifestId,
@@ -111,3 +125,117 @@ pub fn concatenate_bytes(vec_of_bytes: Vec<Bytes>) -> Bytes {
     }
     bytes_mut.freeze()
 }
+
+pub fn to_identical_changeset(
+    css: (HgBlobChangeset, BonsaiChangeset),
+) -> Result<IdenticalChangesetContent> {
+    let (hg_cs, bcs) = css;
+    let BonsaiChangesetMut {
+        parents,
+        author,
+        author_date,
+        committer: _,
+        committer_date: _,
+        message,
+        hg_extra: _,
+        git_extra_headers,
+        file_changes,
+        is_snapshot,
+        git_tree_hash,
+        git_annotated_tag,
+    } = bcs.clone().into_mut();
+
+    let hg_info = HgInfo {
+        node_id: hg_cs.get_changeset_id().into_nodehash().into(),
+        manifestid: hg_cs.manifestid().into_nodehash().into(),
+        extras: hg_cs
+            .extra()
+            .iter()
+            .map(|(key, value)| Extra {
+                key: key.to_vec(),
+                value: value.to_vec(),
+            })
+            .collect(),
+    };
+
+    let bonsai_parents = BonsaiParents::from_iter(parents.iter().map(|p| (*p).into()));
+
+    // Ensure items are indeed equivalent between bonsai and hg changeset
+    ensure!(author.as_bytes() == hg_cs.user(), "Author mismatch");
+    ensure!(author_date == *hg_cs.time(), "Time mismatch");
+    ensure!(message.as_bytes() == hg_cs.message(), "Message mismatch");
+    ensure!(git_tree_hash.is_none(), "Unexpected git tree hash found");
+    ensure!(
+        git_annotated_tag.is_none(),
+        "Unexpected git annotated tag found"
+    );
+    ensure!(
+        git_extra_headers.is_none(),
+        "Unexpected git extra headers found"
+    );
+
+    Ok(IdenticalChangesetContent {
+        bcs_id: bcs.get_changeset_id().into(),
+        hg_parents: hg_cs.parents().into(),
+        bonsai_parents,
+        author: author.to_string(),
+        time: author_date.timestamp_secs(),
+        tz: author_date.tz_offset_secs(),
+        extras: bcs
+            .hg_extra()
+            .map(|(key, value)| BonsaiExtra {
+                key: key.to_string(),
+                value: value.to_vec(),
+            })
+            .collect(),
+        file_changes: to_file_change(&file_changes, bcs.parents())?,
+        message: message.to_string(),
+        is_snapshot,
+        hg_info,
+    })
+}
+
+fn to_file_change(
+    map: &SortedVectorMap<NonRootMPath, FileChange>,
+    mut parents: impl Iterator<Item = ChangesetId>,
+) -> Result<Vec<(RepoPathBuf, BonsaiFileChange)>> {
+    let res = map
+        .into_iter()
+        .map(|(path, fc)| {
+            let path = RepoPathBuf::from_string(path.to_string())?;
+            let fc = match fc {
+                FileChange::Deletion => BonsaiFileChange::Deletion,
+                FileChange::UntrackedDeletion => BonsaiFileChange::UntrackedDeletion,
+                FileChange::Change(tc) => BonsaiFileChange::Change {
+                    upload_token: UploadToken::new_fake_token(
+                        AnyId::AnyFileContentId(AnyFileContentId::ContentId(
+                            tc.content_id().into(),
+                        )),
+                        None,
+                    ),
+                    file_type: tc.file_type().try_into()?,
+                    copy_info: match tc.copy_from() {
+                        Some((path, cs_id)) => {
+                            let index = parents.position(|parent| parent == *cs_id).ok_or(
+                                anyhow::anyhow!("Error: Copy from info doesn't match parents"),
+                            )?;
+                            Some((to_hg_path(path)?, index))
+                        }
+                        None => None,
+                    },
+                },
+                FileChange::UntrackedChange(uc) => BonsaiFileChange::UntrackedChange {
+                    upload_token: UploadToken::new_fake_token(
+                        AnyId::AnyFileContentId(AnyFileContentId::ContentId(
+                            uc.content_id().into(),
+                        )),
+                        None,
+                    ),
+                    file_type: uc.file_type().try_into()?,
+                },
+            };
+            Ok((path, fc))
+        })
+        .collect::<Result<Vec<(RepoPathBuf, BonsaiFileChange)>, Error>>()?;
+    Ok(res)
+}

eden/mononoke/modern_sync/src/sync.rs

@@ -290,7 +290,7 @@ pub async fn process_one_changeset(
     mf_ids.push(hg_mf_id);
     sender.upload_trees(mf_ids).await?;
     sender.upload_filenodes(file_ids).await?;
-    sender.upload_hg_changeset(vec![hg_cs]).await?;
+    sender.upload_identical_changeset(vec![(hg_cs, bs)]).await?;
 
     if log_completion {
         STATS::synced_commits.add_value(1, (repo.repo_identity().name().to_string(),));

eden/mononoke/mononoke_types/src/bonsai_changeset.rs

@@ -362,6 +362,11 @@ impl BonsaiChangeset {
         self.inner.git_tree_hash.as_ref()
     }
 
+    /// Get the git annotated tag corresponding to the current changeset
+    pub fn git_annotated_tag(&self) -> Option<&BonsaiAnnotatedTag> {
+        self.inner.git_annotated_tag.as_ref()
+    }
+
     /// Get the changeset ID of this changeset.
     pub fn get_changeset_id(&self) -> ChangesetId {
         self.id