resolve part of #2300 improve bit export performance by pushing new t…

…ags only

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [unreleased]
 
+- [#2300](https://github.com/teambit/bit/issues/2300) improve `bit export` performance by pushing new tags only
+
 ## [[14.7.3] - 2020-02-02](https://github.com/teambit/bit/releases/tag/v14.7.3)
 
 ### New

src/scope/component-ops/export-scope-components.ts

@@ -100,9 +100,21 @@ export async function exportMany({
     const componentsAndObjects = [];
     const processComponentObjects = async (componentObject: ComponentObjects) => {
       const componentAndObject = componentObject.toObjects(scope.objects);
+      const localVersions = componentAndObject.component.getLocalVersions();
       componentAndObject.component.clearStateData();
-      await convertToCorrectScope(scope, componentAndObject, remoteNameStr, includeDependencies, bitIds, codemod);
-      await changePartialNamesToFullNamesInDists(scope, componentAndObject.component, componentAndObject.objects);
+      const didConvertScope = await convertToCorrectScope(
+        scope,
+        componentAndObject,
+        remoteNameStr,
+        includeDependencies,
+        bitIds,
+        codemod
+      );
+      const didChangeDists = await changePartialNamesToFullNamesInDists(
+        scope,
+        componentAndObject.component,
+        componentAndObject.objects
+      );
       const remoteObj = { url: remote.host, name: remote.name, date: Date.now().toString() };
       componentAndObject.component.addScopeListItem(remoteObj);
 
@@ -116,9 +128,13 @@ export async function exportMany({
         // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
         componentsAndObjects.push(componentAndObjectCloned);
       }
-      const componentBuffer = await componentAndObject.component.compress();
-      const objectsBuffer = await Promise.all(componentAndObject.objects.map(obj => obj.compress()));
-      return new ComponentObjects(componentBuffer, objectsBuffer);
+
+      if (didConvertScope || didChangeDists) {
+        const componentBuffer = await componentAndObject.component.compress();
+        const objectsBuffer = await Promise.all(componentAndObject.objects.map(obj => obj.compress()));
+        return new ComponentObjects(componentBuffer, objectsBuffer);
+      }
+      return componentAndObject.component.collectObjects(scope.objects, localVersions);
     };
     // don't use Promise.all, otherwise, it'll throw "JavaScript heap out of memory" on a large set of data
     const manyObjects: ComponentObjects[] = await pMapSeries(componentObjects, processComponentObjects);
@@ -287,16 +303,19 @@ async function convertToCorrectScope(
   fork: boolean,
   exportingIds: BitIds,
   codemod: boolean
-): Promise<void> {
+): Promise<boolean> {
   // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   const versionsObjects: Version[] = componentsObjects.objects.filter(object => object instanceof Version);
-  await Promise.all(
+  const hasVersionsChanged = await Promise.all(
     versionsObjects.map(async (objectVersion: Version) => {
       const hashBefore = objectVersion.hash().toString();
-      if (codemod) await _replaceSrcOfVersionIfNeeded(objectVersion);
-      changeDependencyScope(objectVersion);
+      const didCodeMod = codemod ? await _replaceSrcOfVersionIfNeeded(objectVersion) : false;
+      const didDependencyChange = changeDependencyScope(objectVersion);
       const hashAfter = objectVersion.hash().toString();
       if (hashBefore !== hashAfter) {
+        if (!didCodeMod && !didDependencyChange) {
+          throw new Error('hash should not be changed if there was not any dependency scope changes nor codemod');
+        }
         logger.debugAndAddBreadCrumb(
           'scope._convertToCorrectScope',
           `switching {id} version hash from ${hashBefore} to ${hashAfter}`,
@@ -309,18 +328,28 @@ async function convertToCorrectScope(
           }
         });
       }
+      return didCodeMod || didDependencyChange;
     })
   );
   componentsObjects.component.scope = remoteScope;
 
-  function changeDependencyScope(version: Version): void {
+  // return true if one of the component has changed
+  return hasVersionsChanged.some(x => x);
+
+  function changeDependencyScope(version: Version): boolean {
+    let hasChanged = false;
     version.getAllDependencies().forEach(dependency => {
-      dependency.id = getIdWithUpdatedScope(dependency.id);
+      const updatedScope = getIdWithUpdatedScope(dependency.id);
+      if (!updatedScope.isEqual(dependency.id)) {
+        hasChanged = true;
+        dependency.id = updatedScope;
+      }
     });
     version.flattenedDependencies = getBitIdsWithUpdatedScope(version.flattenedDependencies);
     version.flattenedDevDependencies = getBitIdsWithUpdatedScope(version.flattenedDevDependencies);
     version.flattenedCompilerDependencies = getBitIdsWithUpdatedScope(version.flattenedCompilerDependencies);
     version.flattenedTesterDependencies = getBitIdsWithUpdatedScope(version.flattenedTesterDependencies);
+    return hasChanged;
   }
 
   function getIdWithUpdatedScope(dependencyId: BitId): BitId {
@@ -342,18 +371,21 @@ async function convertToCorrectScope(
     const updatedIds = bitIds.map(id => getIdWithUpdatedScope(id));
     return BitIds.fromArray(updatedIds);
   }
-  async function _replaceSrcOfVersionIfNeeded(version: Version) {
+  async function _replaceSrcOfVersionIfNeeded(version: Version): Promise<boolean> {
+    let hasVersionChanged = false;
     const files = [...version.files, ...(version.dists || [])];
     await Promise.all(
       files.map(async file => {
         const newFileObject = await _createNewFileIfNeeded(version, file);
         if (newFileObject) {
           file.file = newFileObject.hash();
           componentsObjects.objects.push(newFileObject);
+          hasVersionChanged = true;
         }
         return null;
       })
     );
+    return hasVersionChanged;
   }
   async function _createNewFileIfNeeded(
     version: Version,
@@ -395,24 +427,29 @@ async function changePartialNamesToFullNamesInDists(
   scope: Scope,
   component: ModelComponent,
   objects: BitObject[]
-): Promise<void> {
+): Promise<boolean> {
   // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   const versions: Version[] = objects.filter(object => object instanceof Version);
-  await Promise.all(versions.map(version => _replaceDistsOfVersionIfNeeded(version)));
+  const haveVersionsChanged = await Promise.all(versions.map(version => _replaceDistsOfVersionIfNeeded(version)));
+
+  return haveVersionsChanged.some(x => x);
 
-  async function _replaceDistsOfVersionIfNeeded(version: Version) {
+  async function _replaceDistsOfVersionIfNeeded(version: Version): Promise<boolean> {
     const dists = version.dists;
-    if (!dists) return;
-    await Promise.all(
+    if (!dists) return false;
+    const hasDistsChanged = await Promise.all(
       dists.map(async dist => {
         const newDistObject = await _createNewDistIfNeeded(version, dist);
         if (newDistObject) {
           dist.file = newDistObject.hash();
           objects.push(newDistObject);
+          return true;
         }
-        return null;
+        return false;
       })
     );
+    // return true if one of the dists has changed
+    return hasDistsChanged.some(x => x);
   }
 
   async function _createNewDistIfNeeded(

src/scope/models/model-component.ts

@@ -292,8 +292,31 @@ export default class Component extends BitObject {
     return versionRef.loadSync(repository, throws);
   }
 
-  collectObjects(repo: Repository): Promise<ComponentObjects> {
-    return Promise.all([this.asRaw(repo), this.collectRaw(repo)])
+  async collectRaw(repo: Repository, versions?: string[]): Promise<Buffer[]> {
+    if (!versions) return super.collectRaw(repo);
+
+    const collectRefs = async (): Promise<Ref[]> => {
+      const refsCollection: Ref[] = [];
+
+      async function addRefs(object: BitObject) {
+        const refs = object.refs();
+        const objs = await Promise.all(refs.map(ref => ref.load(repo, true)));
+        refsCollection.push(...refs);
+        await Promise.all(objs.map(obj => addRefs(obj)));
+      }
+
+      const versionsRefs = versions.map(version => this.versions[version]);
+      refsCollection.push(...versionsRefs);
+      const versionsObjects = await Promise.all(versions.map(version => this.versions[version].load(repo)));
+      await Promise.all(versionsObjects.map(versionObject => addRefs(versionObject)));
+      return refsCollection;
+    };
+    const refs = await collectRefs();
+    return Promise.all(refs.map(ref => ref.loadRaw(repo)));
+  }
+
+  collectObjects(repo: Repository, versions?: string[]): Promise<ComponentObjects> {
+    return Promise.all([this.asRaw(repo), this.collectRaw(repo, versions)])
       .then(([rawComponent, objects]) => new ComponentObjects(rawComponent, objects))
       .catch(err => {
         if (err.code === 'ENOENT') {