Skip to content

Commit

Permalink
Improve bit export performance by pushing new tags only (#2304)
Browse files Browse the repository at this point in the history 
* resolve part of #2300 improve bit export performance by pushing new tags only

* add a new flag "--all-version" to preserve the old behavior of exporting all component versions
  • Loading branch information
@davidfirst
davidfirst authored Feb 6, 2020
1 parent 54c8f6d commit 530d4da
Show file tree
Hide file tree
Showing 7 changed files with 117 additions and 35 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).

## [unreleased]

- [#2300](https://github.com/teambit/bit/issues/2300) improve `bit export` performance by pushing new tags only

## [[14.7.3] - 2020-02-02](https://github.com/teambit/bit/releases/tag/v14.7.3)

### New
Expand Down
20 changes: 11 additions & 9 deletions e2e/commands/export.e2e.1.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -600,6 +600,7 @@ describe('bit export command', function() {
describe('some components were exported to one scope and other to another scope', () => {
let localScopeBefore;
let remoteScopeBefore;
let anotherRemoteScopeBefore;
let anotherRemote;
let anotherRemotePath;
before(() => {
Expand All @@ -618,6 +619,7 @@ describe('bit export command', function() {
helper.command.tagScope('2.0.0');
localScopeBefore = helper.scopeHelper.cloneLocalScope();
remoteScopeBefore = helper.scopeHelper.cloneRemoteScope();
anotherRemoteScopeBefore = helper.scopeHelper.cloneScope(anotherRemotePath);
});
describe('export with no ids, no remote and no flags', () => {
let output;
Expand All @@ -644,7 +646,7 @@ describe('bit export command', function() {
before(() => {
helper.scopeHelper.getClonedLocalScope(localScopeBefore);
helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
output = helper.command.exportToCurrentScope('foo1 foo2');
});
it('should export successfully all ids, each to its own remote', () => {
Expand All @@ -667,7 +669,7 @@ describe('bit export command', function() {
before(() => {
helper.scopeHelper.getClonedLocalScope(localScopeBefore);
helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
helper.fs.outputFile('foo1.js', "require('./foo2');");
helper.command.tagScope('3.0.0');
helper.scopeHelper.addRemoteScope(anotherRemotePath, helper.scopes.remotePath);
Expand All @@ -686,7 +688,7 @@ describe('bit export command', function() {
before(() => {
helper.scopeHelper.getClonedLocalScope(localScopeBefore);
helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
helper.fs.outputFile('foo1.js', "require('./foo2');");
helper.fs.outputFile('foo2.js', "require('./foo1');");
helper.command.tagScope('3.0.0');
Expand All @@ -704,7 +706,7 @@ describe('bit export command', function() {
before(() => {
helper.scopeHelper.getClonedLocalScope(localScopeBefore);
helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
helper.fs.outputFile('foo1.js', "require('./foo2');");

helper.command.tagScope('3.0.0');
Expand All @@ -727,7 +729,7 @@ describe('bit export command', function() {
before(() => {
helper.scopeHelper.getClonedLocalScope(localScopeBefore);
helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
helper.fs.outputFile('foo3.js', '');
helper.command.addComponent('foo3.js');
helper.command.tagAllComponents();
Expand All @@ -748,7 +750,7 @@ describe('bit export command', function() {
before(() => {
helper.scopeHelper.getClonedLocalScope(localScopeBefore);
helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
helper.fs.outputFile('foo3.js');
helper.command.addComponent('foo3.js');
helper.command.tagAllComponents();
Expand All @@ -769,7 +771,7 @@ describe('bit export command', function() {
before(() => {
helper.scopeHelper.getClonedLocalScope(beforeExportScope);
helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
helper.bitJson.addKeyVal(undefined, 'defaultScope', helper.scopes.remote);
output = helper.command.export();
});
Expand Down Expand Up @@ -827,7 +829,7 @@ describe('bit export command', function() {
helper.scopeHelper.reInitRemoteScope(forkScopePath);
helper.fs.createFile('utils', 'is-string.js', ''); // remove the is-type dependency
helper.command.tagAllComponents();
helper.command.exportAllComponents();
helper.command.export('--all-versions');

helper.command.export(`${forkScope} utils/is-string --include-dependencies`);
const forkScopeList = helper.command.listScopeParsed(forkScope);
Expand All @@ -852,7 +854,7 @@ describe('bit export command', function() {
helper.fs.createFile('utils', 'is-string.js', ''); // remove the is-type dependency
helper.fs.createFile('utils', 'is-type.js', ''); // add another version for is-type
helper.command.tagAllComponents();
helper.command.exportAllComponents();
helper.command.export('--all-versions');

helper.scopeHelper.reInitLocalScope();
helper.scopeHelper.addRemoteScope();
Expand Down
4 changes: 4 additions & 0 deletions src/api/consumer/lib/export.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ export default (async function exportAction(params: {
eject: boolean;
includeDependencies: boolean;
setCurrentScope: boolean;
allVersions: boolean;
includeNonStaged: boolean;
codemod: boolean;
force: boolean;
Expand All @@ -52,6 +53,7 @@ async function exportComponents({
setCurrentScope,
includeNonStaged,
codemod,
allVersions,
force
}: {
ids: string[];
Expand All @@ -60,6 +62,7 @@ async function exportComponents({
setCurrentScope: boolean;
includeNonStaged: boolean;
codemod: boolean;
allVersions: boolean;
force: boolean;
}): Promise<{ updatedIds: BitId[]; nonExistOnBitMap: BitId[]; missingScope: BitId[]; exported: BitId[] }> {
const consumer: Consumer = await loadConsumer();
Expand All @@ -79,6 +82,7 @@ async function exportComponents({
includeDependencies,
changeLocallyAlthoughRemoteIsDifferent: setCurrentScope,
codemod,
allVersions,
idsWithFutureScope
});
const { updatedIds, nonExistOnBitMap } = _updateIdsOnBitMap(consumer.bitMap, updatedLocally);
Expand Down
3 changes: 3 additions & 0 deletions src/cli/commands/public-cmds/export-cmd.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ export default class Export extends Command {
'rewire',
'EXPERIMENTAL. when exporting to a different scope, replace import/require statements in the source code to the new scope'
],
['', 'all-versions', 'export not only staged versions but all of them'],
['f', 'force', 'force changing a component remote without asking for a confirmation']
];
loader = true;
Expand All @@ -49,6 +50,7 @@ export default class Export extends Command {
includeDependencies = false,
setCurrentScope = false,
all = false,
allVersions = false,
force = false,
rewire = false
}: any
Expand All @@ -74,6 +76,7 @@ export default class Export extends Command {
includeDependencies,
setCurrentScope,
includeNonStaged: all,
allVersions,
codemod: rewire,
force
}).then(results => ({
Expand Down
24 changes: 14 additions & 10 deletions src/e2e-helper/e2e-scope-helper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,21 +181,25 @@ export default class ScopeHelper {
}

cloneRemoteScope() {
return this.cloneScope(this.scopes.remotePath);
}

cloneScope(scopePath: string) {
const clonedScope = generateRandomStr();
const clonedScopePath = path.join(this.scopes.e2eDir, clonedScope);
if (this.debugMode) console.log(`cloning a scope from ${this.scopes.remotePath} to ${clonedScopePath}`);
fs.copySync(this.scopes.remotePath, clonedScopePath);
if (this.debugMode) console.log(`cloning a scope from ${scopePath} to ${clonedScopePath}`);
fs.copySync(scopePath, clonedScopePath);
this.clonedScopes.push(clonedScopePath);
return clonedScopePath;
}

getClonedRemoteScope(clonedScopePath: string, deleteCurrentScope = true) {
if (deleteCurrentScope) {
fs.removeSync(this.scopes.remotePath);
} else {
this.getNewBareScope();
}
if (this.debugMode) console.log(`cloning a scope from ${clonedScopePath} to ${this.scopes.remotePath}`);
fs.copySync(clonedScopePath, this.scopes.remotePath);
getClonedScope(clonedScopePath: string, scopePath: string) {
fs.removeSync(scopePath);
if (this.debugMode) console.log(`cloning a scope from ${clonedScopePath} to ${scopePath}`);
fs.copySync(clonedScopePath, scopePath);
}

getClonedRemoteScope(clonedScopePath: string) {
return this.getClonedScope(clonedScopePath, this.scopes.remotePath);
}
}
78 changes: 62 additions & 16 deletions src/scope/component-ops/export-scope-components.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export async function exportMany({
includeDependencies = false, // kind of fork. by default dependencies only cached, with this, their scope-name is changed
changeLocallyAlthoughRemoteIsDifferent = false, // by default only if remote stays the same the component is changed from staged to exported
codemod = false,
allVersions,
idsWithFutureScope
}: {
scope: Scope;
Expand All @@ -68,6 +69,7 @@ export async function exportMany({
includeDependencies: boolean;
changeLocallyAlthoughRemoteIsDifferent: boolean;
codemod: boolean;
allVersions: boolean;
idsWithFutureScope: BitIds;
}): Promise<{ exported: BitIds; updatedLocally: BitIds }> {
logger.debugAndAddBreadCrumb('scope.exportMany', 'ids: {ids}', { ids: ids.toString() });
Expand Down Expand Up @@ -100,9 +102,21 @@ export async function exportMany({
const componentsAndObjects = [];
const processComponentObjects = async (componentObject: ComponentObjects) => {
const componentAndObject = componentObject.toObjects(scope.objects);
const localVersions = componentAndObject.component.getLocalVersions();
componentAndObject.component.clearStateData();
await convertToCorrectScope(scope, componentAndObject, remoteNameStr, includeDependencies, bitIds, codemod);
await changePartialNamesToFullNamesInDists(scope, componentAndObject.component, componentAndObject.objects);
const didConvertScope = await convertToCorrectScope(
scope,
componentAndObject,
remoteNameStr,
includeDependencies,
bitIds,
codemod
);
const didChangeDists = await changePartialNamesToFullNamesInDists(
scope,
componentAndObject.component,
componentAndObject.objects
);
const remoteObj = { url: remote.host, name: remote.name, date: Date.now().toString() };
componentAndObject.component.addScopeListItem(remoteObj);

Expand All @@ -116,8 +130,18 @@ export async function exportMany({
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
componentsAndObjects.push(componentAndObjectCloned);
}

const componentBuffer = await componentAndObject.component.compress();
const objectsBuffer = await Promise.all(componentAndObject.objects.map(obj => obj.compress()));
const getObjectsBuffer = () => {
if (allVersions || includeDependencies || didConvertScope || didChangeDists) {
// only when really needed (e.g. fork or version changes), collect all versions objects
return Promise.all(componentAndObject.objects.map(obj => obj.compress()));
}
// when possible prefer collecting only new/local versions. the server has already
// the rest, so no point of sending them.
return componentAndObject.component.collectVersionsObjects(scope.objects, localVersions);
};
const objectsBuffer = await getObjectsBuffer();
return new ComponentObjects(componentBuffer, objectsBuffer);
};
// don't use Promise.all, otherwise, it'll throw "JavaScript heap out of memory" on a large set of data
Expand Down Expand Up @@ -287,16 +311,19 @@ async function convertToCorrectScope(
fork: boolean,
exportingIds: BitIds,
codemod: boolean
): Promise<void> {
): Promise<boolean> {
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
const versionsObjects: Version[] = componentsObjects.objects.filter(object => object instanceof Version);
await Promise.all(
const haveVersionsChanged = await Promise.all(
versionsObjects.map(async (objectVersion: Version) => {
const hashBefore = objectVersion.hash().toString();
if (codemod) await _replaceSrcOfVersionIfNeeded(objectVersion);
changeDependencyScope(objectVersion);
const didCodeMod = codemod ? await _replaceSrcOfVersionIfNeeded(objectVersion) : false;
const didDependencyChange = changeDependencyScope(objectVersion);
const hashAfter = objectVersion.hash().toString();
if (hashBefore !== hashAfter) {
if (!didCodeMod && !didDependencyChange) {
throw new Error('hash should not be changed if there was not any dependency scope changes nor codemod');
}
logger.debugAndAddBreadCrumb(
'scope._convertToCorrectScope',
`switching {id} version hash from ${hashBefore} to ${hashAfter}`,
Expand All @@ -309,18 +336,29 @@ async function convertToCorrectScope(
}
});
}
return didCodeMod || didDependencyChange;
})
);
const hasComponentChanged = remoteScope !== componentsObjects.component.scope;
componentsObjects.component.scope = remoteScope;

function changeDependencyScope(version: Version): void {
// return true if one of the versions has changed or the component itself
return haveVersionsChanged.some(x => x) || hasComponentChanged;

function changeDependencyScope(version: Version): boolean {
let hasChanged = false;
version.getAllDependencies().forEach(dependency => {
dependency.id = getIdWithUpdatedScope(dependency.id);
const updatedScope = getIdWithUpdatedScope(dependency.id);
if (!updatedScope.isEqual(dependency.id)) {
hasChanged = true;
dependency.id = updatedScope;
}
});
version.flattenedDependencies = getBitIdsWithUpdatedScope(version.flattenedDependencies);
version.flattenedDevDependencies = getBitIdsWithUpdatedScope(version.flattenedDevDependencies);
version.flattenedCompilerDependencies = getBitIdsWithUpdatedScope(version.flattenedCompilerDependencies);
version.flattenedTesterDependencies = getBitIdsWithUpdatedScope(version.flattenedTesterDependencies);
return hasChanged;
}

function getIdWithUpdatedScope(dependencyId: BitId): BitId {
Expand All @@ -342,18 +380,21 @@ async function convertToCorrectScope(
const updatedIds = bitIds.map(id => getIdWithUpdatedScope(id));
return BitIds.fromArray(updatedIds);
}
async function _replaceSrcOfVersionIfNeeded(version: Version) {
async function _replaceSrcOfVersionIfNeeded(version: Version): Promise<boolean> {
let hasVersionChanged = false;
const files = [...version.files, ...(version.dists || [])];
await Promise.all(
files.map(async file => {
const newFileObject = await _createNewFileIfNeeded(version, file);
if (newFileObject) {
file.file = newFileObject.hash();
componentsObjects.objects.push(newFileObject);
hasVersionChanged = true;
}
return null;
})
);
return hasVersionChanged;
}
async function _createNewFileIfNeeded(
version: Version,
Expand Down Expand Up @@ -395,24 +436,29 @@ async function changePartialNamesToFullNamesInDists(
scope: Scope,
component: ModelComponent,
objects: BitObject[]
): Promise<void> {
): Promise<boolean> {
// @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
const versions: Version[] = objects.filter(object => object instanceof Version);
await Promise.all(versions.map(version => _replaceDistsOfVersionIfNeeded(version)));
const haveVersionsChanged = await Promise.all(versions.map(version => _replaceDistsOfVersionIfNeeded(version)));

return haveVersionsChanged.some(x => x);

async function _replaceDistsOfVersionIfNeeded(version: Version) {
async function _replaceDistsOfVersionIfNeeded(version: Version): Promise<boolean> {
const dists = version.dists;
if (!dists) return;
await Promise.all(
if (!dists) return false;
const hasDistsChanged = await Promise.all(
dists.map(async dist => {
const newDistObject = await _createNewDistIfNeeded(version, dist);
if (newDistObject) {
dist.file = newDistObject.hash();
objects.push(newDistObject);
return true;
}
return null;
return false;
})
);
// return true if one of the dists has changed
return hasDistsChanged.some(x => x);
}

async function _createNewDistIfNeeded(
Expand Down
Loading

0 comments on commit 530d4da

Please sign in to comment.