From 72c4b4301e4ce2e579d73aa79b05d67206c73235 Mon Sep 17 00:00:00 2001 From: Konstantin Raev Date: Fri, 13 May 2016 11:54:25 +0100 Subject: [PATCH] Enabled save to mirror in tarball downloader --- package.json | 1 + src/cli/commands/install.js | 14 ++- src/config.js | 24 ++++- src/fetchers/_base.js | 12 +-- src/fetchers/tarball.js | 57 +++++++++++- src/lockfile/index.js | 10 ++- src/package-fetcher.js | 2 +- src/package-reference.js | 3 + src/package-request.js | 14 ++- src/registries/npm.js | 7 +- src/resolvers/exotics/tarball.js | 2 +- test/commands/install.js | 82 ++++++++++++++++-- .../install-from-offline-mirror/.npmrc | 1 + .../install-from-offline-mirror/fbkpm.lock | 4 + .../fake-fbkpm-dependency-2.0.1.tgz | Bin 0 -> 873 bytes .../install-from-offline-mirror/package.json | 5 ++ .../.npmrc | 1 + .../install-with-save-offline-mirror/.npmrc | 1 + test/package-resolver.js | 2 +- 19 files changed, 211 insertions(+), 31 deletions(-) create mode 100644 test/fixtures/install/install-from-offline-mirror/.npmrc create mode 100644 test/fixtures/install/install-from-offline-mirror/fbkpm.lock create mode 100644 test/fixtures/install/install-from-offline-mirror/mirror-for-offline/fake-fbkpm-dependency-2.0.1.tgz create mode 100644 test/fixtures/install/install-from-offline-mirror/package.json create mode 100644 test/fixtures/install/install-with-save-no-offline-mirror/.npmrc create mode 100644 test/fixtures/install/install-with-save-offline-mirror/.npmrc diff --git a/package.json b/package.json index da90f96c6b..d6cbdaf41a 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "cmd-shim": "^2.0.1", "commander": "^2.9.0", "diff": "^2.2.1", + "ini": "^1.3.4", "invariant": "^2.2.0", "is-builtin-module": "^1.0.0", "kreporters": "^1.0.2", diff --git a/src/cli/commands/install.js b/src/cli/commands/install.js index 6b365f7f4f..48569bcd38 100644 --- a/src/cli/commands/install.js +++ b/src/cli/commands/install.js @@ -93,8 +93,6 @@ export class Install { let patterns = []; let deps = []; - let foundConfig = false; - for (let registry of Object.keys(registries)) { let filenames = registries[registry].filenames; @@ -103,7 +101,6 @@ export class Install { if (!(await fs.exists(loc))) continue; this.registries.push(registry); - foundConfig = true; let json = await fs.readJson(loc); Object.assign(this.resolutions, json.resolutions); @@ -159,15 +156,13 @@ export class Install { let totalStepsThis = totalSteps + 2; this.reporter.step(1, totalStepsThis, "Rehydrating dependency graph", emoji.get("fast_forward")); - await this.resolver.init(requests, true); - - patterns = await this.flatten(patterns); + await this.resolver.init(requests); this.reporter.step(2, totalStepsThis, "Fetching packages", emoji.get("package")); await this.resolver.fetcher.init(); return { - patterns, + patterns: await this.flatten(patterns), total: totalStepsThis, step: 2 }; @@ -231,7 +226,7 @@ export class Install { * Save added packages to `package.json` if any of the --save flags were used */ - async savePackages(patterns: Array) { + async savePackages(patterns: Array): Promise { if (!this.args.length) return; let { save, saveDev, saveExact, saveOptional } = this.flags; @@ -417,7 +412,8 @@ export async function run( throw new MessageError("Missing package names for --save flags"); } - let lockfile = await Lockfile.fromDirectory(config.cwd, reporter, isStrictLockfile(flags, args)); + let lockfile = await Lockfile.fromDirectory(config.cwd, reporter, isStrictLockfile(flags, args), + hasSaveFlags(flags)); let install = new Install("install", flags, args, config, reporter, lockfile); return install.init(); } diff --git a/src/config.js b/src/config.js index b2e75216d3..5ee5434a55 100644 --- a/src/config.js +++ b/src/config.js @@ -24,6 +24,7 @@ import map from "./util/map.js"; let invariant = require("invariant"); let userHome = require("user-home"); let path = require("path"); +let url = require("url"); type ConfigOptions = { cwd?: string, @@ -140,6 +141,27 @@ export default class Config { return path.join(this.tempFolder, filename); } + /** + * Remote packages may be cached in a file system to be available for offline installation + * Second time the same package needs to be installed it will be loaded from there + */ + + getOfflineMirrorPath(registry: RegistryNames, tarUrl: ?string): string { + let mirrorPath = this.registries[registry] && this.registries[registry] + .config["kpm-offline-mirror"]; + if (!mirrorPath) { + return ""; + } + if (!tarUrl) { + return mirrorPath; + } + let parsed = url.parse(tarUrl); + if (!parsed || !parsed.pathname) { + return mirrorPath; + } + return path.join(mirrorPath, path.basename(parsed.pathname)); + } + /** * Find temporary folder. */ @@ -160,7 +182,7 @@ export default class Config { return opts.packagesRoot; } - // walk up from current directory looking for fbkpm_modules folders + // walk up from current directory looking for .fbkpm folders let parts = this.cwd.split(path.sep); for (let i = parts.length; i > 0; i--) { let loc = parts.slice(0, i).concat(constants.MODULE_CACHE_DIRECTORY).join(path.sep); diff --git a/src/fetchers/_base.js b/src/fetchers/_base.js index 6f1f14bba3..737d643caf 100644 --- a/src/fetchers/_base.js +++ b/src/fetchers/_base.js @@ -20,17 +20,19 @@ import * as fs from "../util/fs.js"; let path = require("path"); export default class BaseFetcher { - constructor(remote: PackageRemote, config: Config) { - this.reference = remote.reference; - this.registry = remote.registry; - this.hash = remote.hash; - this.config = config; + constructor(remote: PackageRemote, config: Config, saveForOffline: boolean) { + this.reference = remote.reference; + this.registry = remote.registry; + this.hash = remote.hash; + this.config = config; + this.saveForOffline = saveForOffline; } registry: RegistryNames; reference: string; config: Config; hash: ?string; + saveForOffline: boolean; async _fetch(dest: string): Promise { throw new Error("Not implemented"); diff --git a/src/fetchers/tarball.js b/src/fetchers/tarball.js index 4bf9cbe02c..674981264a 100644 --- a/src/fetchers/tarball.js +++ b/src/fetchers/tarball.js @@ -9,24 +9,61 @@ * @flow */ -import { SecurityError } from "../errors.js"; +import { SecurityError, MessageError } from "../errors.js"; import * as crypto from "../util/crypto.js"; import BaseFetcher from "./_base.js"; +import * as fsUtil from "../util/fs.js"; let zlib = require("zlib"); let tar = require("tar"); let url = require("url"); +let through = require("through2"); +let fs = require("fs"); +let path = require("path"); export default class TarballFetcher extends BaseFetcher { + async _fetch(dest: string): Promise { - let { reference: ref, hash } = this; + let { reference: ref, hash, config, saveForOffline, registry } = this; + let parts = url.parse(ref); if (!hash) { - let parts = url.parse(ref); if (parts.protocol === "http:") { throw new SecurityError(`${ref}: Refusing to fetch tarball over plain HTTP without a hash`); } } + if (parts.protocol === null) { + let localTarball = path.resolve( + this.config.getOfflineMirrorPath(registry, null), + ref); + if (!await fsUtil.exists(localTarball)) { + throw new MessageError(`${ref}: Tarball is not in network and can't be located in cache`); + } + return new Promise((resolve, reject) => { + let validateStream = crypto.hashStreamValidation(); + + let extractor = tar.Extract({ path: dest, strip: 1 }) + .on("error", reject) + .on("end", function () { + let expectHash = hash; + let actualHash = validateStream.getHash(); + if (!expectHash || expectHash === actualHash) { + resolve(actualHash); + } else { + reject(new SecurityError( + `Bad hash. Expected ${expectHash} but got ${actualHash}` + )); + } + }); + // flow gets confused with the pipe/on types chain + let cachedStream: Object = fs.createReadStream(localTarball); + cachedStream + .pipe(validateStream) + .pipe(zlib.createUnzip()) + .on("error", reject) + .pipe(extractor); + }); + } return this.config.requestManager.request({ url: ref, @@ -50,6 +87,19 @@ export default class TarballFetcher extends BaseFetcher { } }); + let mirrorPath = config.getOfflineMirrorPath(registry, ref); + let mirrorTarballStream; + if (mirrorPath && saveForOffline) { + mirrorTarballStream = fs.createWriteStream(mirrorPath); + mirrorTarballStream.on("error", reject); + } + let mirrorSaver = through(function (chunk, enc, callback) { + if (mirrorTarballStream) { + mirrorTarballStream.write(chunk, enc); + } + callback(null, chunk); + }); + req .on("redirect", function () { if (hash) return; @@ -64,6 +114,7 @@ export default class TarballFetcher extends BaseFetcher { } }) .pipe(validateStream) + .pipe(mirrorSaver) .pipe(zlib.createUnzip()) .on("error", reject) .pipe(extractor); diff --git a/src/lockfile/index.js b/src/lockfile/index.js index a7f87b09db..90fea9cc93 100644 --- a/src/lockfile/index.js +++ b/src/lockfile/index.js @@ -24,13 +24,18 @@ export { default as parse } from "./parse"; export { default as stringify } from "./stringify"; export default class Lockfile { - constructor(cache: ?Object, strict?: boolean) { + constructor(cache: ?Object, strict?: boolean, save?: boolean) { this.strict = !!strict; + this.save = !!save; this.cache = cache; } + // true if operation is just rehydrating node_modules folder strict: boolean; + // true if lockfile will be persisted + save: boolean; + cache: ?{ [key: string]: string | { name: string, @@ -49,6 +54,7 @@ export default class Lockfile { dir: string, reporter: Reporter, strictIfPresent: boolean, + save: boolean, ): Promise { // read the package.json in this directory let lockfileLoc = path.join(dir, constants.LOCKFILE_FILENAME); @@ -68,7 +74,7 @@ export default class Lockfile { reporter.info("No lockfile found."); } - return new Lockfile(lockfile, strict); + return new Lockfile(lockfile, strict, save); } isStrict(): boolean { diff --git a/src/package-fetcher.js b/src/package-fetcher.js index 6d98011f20..b7f8b890b2 100644 --- a/src/package-fetcher.js +++ b/src/package-fetcher.js @@ -55,7 +55,7 @@ export default class PackageFetcher { await fs.mkdirp(dest); try { - let fetcher = new Fetcher(remote, this.config); + let fetcher = new Fetcher(remote, this.config, ref.saveForOffline); return await fetcher.fetch(dest); } catch (err) { try { diff --git a/src/package-reference.js b/src/package-reference.js index 669242e8ca..1680455422 100644 --- a/src/package-reference.js +++ b/src/package-reference.js @@ -22,6 +22,7 @@ export default class PackageReference { request: PackageRequest, info: Manifest, remote: PackageRemote, + saveForOffline: boolean, ) { this.lockfile = request.rootLockfile; this.requests = [request]; @@ -40,6 +41,7 @@ export default class PackageReference { this.patterns = []; this.optional = null; this.location = null; + this.saveForOffline = !!saveForOffline; } requests: Array; @@ -50,6 +52,7 @@ export default class PackageReference { version: string; uid: string; optional: ?boolean; + saveForOffline: boolean; dependencies: Array; patterns: Array; permissions: { [key: string]: boolean }; diff --git a/src/package-request.js b/src/package-request.js index 0b3f7b23d6..4c76519e13 100644 --- a/src/package-request.js +++ b/src/package-request.js @@ -252,7 +252,7 @@ export default class PackageRequest { invariant(remote, "Missing remote"); // set package reference - let ref = new PackageReference(this, info, remote); + let ref = new PackageReference(this, info, remote, this.resolver.lockfile.save); // in order to support lockfiles inside transitive dependencies we need to block // resolution to fetch the package so we can peek inside of it for a fbkpm.lock @@ -270,9 +270,19 @@ export default class PackageRequest { info.name, () => this.resolver.fetcher.fetch(ref) ); + let offlineMirrorPath = this.config.getOfflineMirrorPath(ref.remote.registry, + ref.remote.reference); + // replace resolved remote URL with local path + if (this.resolver.lockfile.save && offlineMirrorPath) { + if (await fs.exists(offlineMirrorPath)) { + remote.resolved = path.relative( + this.config.getOfflineMirrorPath(ref.remote.registry), + offlineMirrorPath) + `#${ref.remote.hash}`; + } + } + remote.hash = hash; newInfo.reference = ref; newInfo.remote = remote; - remote.hash = hash; info = newInfo; // find and load in fbkpm.lock from this module if it exists diff --git a/src/registries/npm.js b/src/registries/npm.js index 6d5365fb22..a99efc767b 100644 --- a/src/registries/npm.js +++ b/src/registries/npm.js @@ -15,6 +15,7 @@ import Registry from "./_base.js"; let userHome = require("user-home"); let path = require("path"); let _ = require("lodash"); +let ini = require("ini"); function getGlobalPrefix(): string { if (process.env.PREFIX) { @@ -52,9 +53,13 @@ export default class NpmRegistry extends Registry { for (let loc of possibles) { if (await fs.exists(loc)) { - // TODO: merge it in! + _.defaults(this.config, ini.parse(await fs.readFile(loc))); } } + if (this.config["kpm-offline-mirror"]) { + this.config["kpm-offline-mirror"] = path.resolve(this.cwd, this.config["kpm-offline-mirror"]); + await fs.mkdirp(this.config["kpm-offline-mirror"]); + } _.defaults(this.config, { registry: "http://registry.npmjs.org" diff --git a/src/resolvers/exotics/tarball.js b/src/resolvers/exotics/tarball.js index cb1a968502..8798bd554d 100644 --- a/src/resolvers/exotics/tarball.js +++ b/src/resolvers/exotics/tarball.js @@ -58,7 +58,7 @@ export default class TarballResolver extends ExoticResolver { reference: url, registry, hash - }, this.config); + }, this.config, false); // fetch file and get it's hash let fetched: FetchedManifest = await fetcher.fetch(dest); diff --git a/test/commands/install.js b/test/commands/install.js index 0e220e1a8c..675d2d1c74 100644 --- a/test/commands/install.js +++ b/test/commands/install.js @@ -9,10 +9,11 @@ import * as reporters from "kreporters"; import * as constants from "../../src/constants.js"; -import Lockfile from "../../src/lockfile/index.js"; +import { default as Lockfile, parse } from "../../src/lockfile/index.js"; import { Install } from "../../src/cli/commands/install.js"; import Config from "../../src/config.js"; import * as fs from "../../src/util/fs.js"; +import assert from "assert"; let test = require("ava"); let path = require("path"); @@ -20,25 +21,37 @@ let path = require("path"); let fixturesLoc = path.join(__dirname, "..", "fixtures", "install"); async function clean(cwd, removeLock) { - await fs.unlink(path.join(cwd, constants.MODULE_DIRECTORY)); + await fs.unlink(path.join(cwd, constants.MODULE_CACHE_DIRECTORY)); await fs.unlink(path.join(cwd, "node_modules")); if (removeLock) await fs.unlink(path.join(cwd, constants.LOCKFILE_FILENAME)); } -async function run(flags, args, name) { - let lockfile = new Lockfile; +async function createLockfile(dir, strict, save) { + let lockfileLoc = path.join(dir, constants.LOCKFILE_FILENAME); + let lockfile; + + if (await fs.exists(lockfileLoc)) { + let rawLockfile = await fs.readFile(lockfileLoc); + lockfile = parse(rawLockfile); + } + + return new Lockfile(lockfile, strict, save); +} + +async function run(flags, args, name, checkInstalled) { let reporter = new reporters.NoopReporter; let cwd = path.join(fixturesLoc, name); // remove the lockfile if we create one and it didn't exist before let removeLock = !(await fs.exists(path.join(cwd, constants.LOCKFILE_FILENAME))); + let lockfile = await createLockfile(cwd, flags.strict, flags.save); // clean up if we weren't successful last time await clean(cwd); // create directories - await fs.mkdirp(path.join(cwd, constants.MODULE_DIRECTORY)); + await fs.mkdirp(path.join(cwd, constants.MODULE_CACHE_DIRECTORY)); await fs.mkdirp(path.join(cwd, "node_modules")); let config = new Config(reporter, { cwd }); @@ -47,6 +60,10 @@ async function run(flags, args, name) { let install = new Install("install", flags, args, config, reporter, lockfile); await install.init(); + if (checkInstalled) { + await checkInstalled(cwd); + } + // clean up await clean(cwd, removeLock); } @@ -70,3 +87,58 @@ test("install with arg", () => { test("install with arg that has binaries", () => { return run({}, ["react-native-cli"], "install-with-arg-and-bin"); }); + +test("install with --save and offline mirror", () => { + let mirrorPath = "mirror-for-offline"; + return run({save: true}, ["is-array@1.0.1"], "install-with-save-offline-mirror", async (cwd) => { + + let allFiles = await fs.walk(cwd); + + assert(allFiles.findIndex((file) => { + return file.relative === `${mirrorPath}/is-array-1.0.1.tgz`; + }) !== -1); + + let rawLockfile = await fs.readFile(path.join(cwd, constants.LOCKFILE_FILENAME)); + let lockfile = parse(rawLockfile); + assert(lockfile["is-array@1.0.1"]["resolved"] === + "is-array-1.0.1.tgz#e9850cc2cc860c3bc0977e84ccf0dd464584279a"); + + await fs.unlink(path.join(cwd, mirrorPath)); + await fs.unlink(path.join(cwd, "package.json")); + return allFiles; + }); +}); + +test("install with --save and without offline mirror", () => { + let mirrorPath = "mirror-for-offline"; + return run({save: true}, ["is-array@1.0.1"], "install-with-save-no-offline-mirror", async (cwd) => { + + let allFiles = await fs.walk(cwd); + + assert(allFiles.findIndex((file) => { + return file.relative === `${mirrorPath}/is-array-1.0.1.tgz`; + }) === -1); + + let rawLockfile = await fs.readFile(path.join(cwd, constants.LOCKFILE_FILENAME)); + let lockfile = parse(rawLockfile); + assert(lockfile["is-array@1.0.1"]["resolved"] === + "https://registry.npmjs.org/is-array/-/is-array-1.0.1.tgz#e9850cc2cc860c3bc0977e84ccf0dd464584279a"); + + await fs.unlink(path.join(cwd, mirrorPath)); + await fs.unlink(path.join(cwd, "package.json")); + return allFiles; + }); +}); + +test("install from offline mirror", () => { + return run({}, [], "install-from-offline-mirror", async (cwd) => { + + let allFiles = await fs.walk(cwd); + + assert(allFiles.findIndex((file) => { + return file.relative === "node_modules/fake-fbkpm-dependency/package.json"; + }) !== -1); + + return allFiles; + }); +}); diff --git a/test/fixtures/install/install-from-offline-mirror/.npmrc b/test/fixtures/install/install-from-offline-mirror/.npmrc new file mode 100644 index 0000000000..d912bd785c --- /dev/null +++ b/test/fixtures/install/install-from-offline-mirror/.npmrc @@ -0,0 +1 @@ +kpm-offline-mirror=./mirror-for-offline diff --git a/test/fixtures/install/install-from-offline-mirror/fbkpm.lock b/test/fixtures/install/install-from-offline-mirror/fbkpm.lock new file mode 100644 index 0000000000..afcb15583f --- /dev/null +++ b/test/fixtures/install/install-from-offline-mirror/fbkpm.lock @@ -0,0 +1,4 @@ +fake-fbkpm-dependency@2.0.1: + name fake-fbkpm-dependency + version "2.0.1" + resolved fake-fbkpm-dependency-2.0.1.tgz#ef965c2c237a58e3d4c3f09569d303a1dc9c0d72 diff --git a/test/fixtures/install/install-from-offline-mirror/mirror-for-offline/fake-fbkpm-dependency-2.0.1.tgz b/test/fixtures/install/install-from-offline-mirror/mirror-for-offline/fake-fbkpm-dependency-2.0.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..f903c3b4d1c4e99f0d835b05d4833e7b250896b0 GIT binary patch literal 873 zcmV-v1D5~vzAGLl3i!1fxXzCVlOmOe{8LGhGI**7S(sd=;S*`^pW_}dBN9D1s@C3&-iH0 zr=nT?*SS``TM@=M3Pb4qZvXx&nf>9`p8oR$mm(C~{}CF~K@d_L;T|S~a6q89zMb3u zvzB?mjospZtyBFN*sh<6$Pj*f{^Z+NkACvUA}eN$AR2B%6P>2|4V&;Z7bu9faXD4m zi0@=7avw8aWs53ulZP;WDvQRG9Q07ue)CYctJK65UkJOGxj30z&a=gd-LOz&p+;9I zO7`4ik&leb3aJ%)JQJfi$O#lP0aLjW3RXN_3XnN)1&@s3S11VfYOI9GT|V)#Pud%A z40CbyMw_wgg3RkO;mt0>L_TOYY0p6!ud0wePbx=3<%0e2>=mLz$hdu4hYl^GyWHX` z6U_A&(EwgPe){bDUu|Pnj6`u9Q;HEuny^Zxn%l=uVpz&C8-`JkBs7VgWoiQ?i6cUA z94GBJ<7Fc>CLFhqnx>7h746t`0KXr8kK{oiNcOV)>@{!VTF za*AjYvp>1}W1kiKyKqiY1~-;PGH&<+ zt`~AG#vAu-DWgUdOQYcWQtNb6^=YBU%T)McooQ3#wp~r1-@b*(QjOf_3hvng_rXo( zb%i^3Tur!p7mm(!+287^oF4&WuJ8Vcr)imXiZ-VmRXcU+^d9;Pt!KSg02lxOaRb1L literal 0 HcmV?d00001 diff --git a/test/fixtures/install/install-from-offline-mirror/package.json b/test/fixtures/install/install-from-offline-mirror/package.json new file mode 100644 index 0000000000..953c1806f5 --- /dev/null +++ b/test/fixtures/install/install-from-offline-mirror/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "fake-fbkpm-dependency": "2.0.1" + } +} diff --git a/test/fixtures/install/install-with-save-no-offline-mirror/.npmrc b/test/fixtures/install/install-with-save-no-offline-mirror/.npmrc new file mode 100644 index 0000000000..de83e686d2 --- /dev/null +++ b/test/fixtures/install/install-with-save-no-offline-mirror/.npmrc @@ -0,0 +1 @@ +kpm-offline-mirror-not=./mirror-for-offline diff --git a/test/fixtures/install/install-with-save-offline-mirror/.npmrc b/test/fixtures/install/install-with-save-offline-mirror/.npmrc new file mode 100644 index 0000000000..d912bd785c --- /dev/null +++ b/test/fixtures/install/install-with-save-offline-mirror/.npmrc @@ -0,0 +1 @@ +kpm-offline-mirror=./mirror-for-offline diff --git a/test/package-resolver.js b/test/package-resolver.js index de7bc45465..739365a2e4 100644 --- a/test/package-resolver.js +++ b/test/package-resolver.js @@ -26,7 +26,7 @@ function addTest(pattern, registry = "npm") { let loc = await makeTemp(); await fs.mkdirp(path.join(loc, "node_modules")); - await fs.mkdirp(path.join(loc, constants.MODULE_DIRECTORY)); + await fs.mkdirp(path.join(loc, constants.MODULE_CACHE_DIRECTORY)); let config = new Config(reporter, { cwd: loc,