Move tar outside of dependencies

package.json

@@ -3,7 +3,10 @@
   "version": "1.0.0-dev",
   "description": "NodeJS/browser bindings to the aws-c-* libraries",
   "homepage": "https://github.com/awslabs/aws-crt-nodejs",
-  "repository": "github:awslabs/aws-crt-nodejs",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/awslabs/aws-crt-nodejs.git"
+  },
   "contributors": [
     "AWS Common Runtime Team <[email protected]>"
   ],
@@ -46,9 +49,7 @@
     "@aws-sdk/util-utf8-browser": "^3.109.0",
     "@httptoolkit/websocket-stream": "^6.0.0",
     "axios": "^0.24.0",
-    "cmake-js": "6.3.0",
     "crypto-js": "^4.0.0",
-    "mqtt": "^4.3.7",
-    "tar": "^6.1.11"
+    "mqtt": "^4.3.7"
   }
 }

scripts/build.js

@@ -2,15 +2,14 @@
  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
-const os = require('os');
 const fs = require("fs");
 const crypto = require('crypto');
 const process = require("process");
 const path = require("path");
 
-const cmake = require("cmake-js");
-const axios = require("axios");
-const tar = require('tar');
+const build_step_tar = require("./build_dependencies/build_step_tar");
+const build_step_cmake = require("./build_dependencies/build_step_cmake");
+
 function rmRecursive(rmPath) {
     let rmBasePath = path.basename(rmPath);
     if (rmBasePath == "." || rmBasePath == "..") {
@@ -36,120 +35,8 @@ function rmRecursive(rmPath) {
     }
 };
 
-function downloadFile(fileUrl, outputLocationPath) {
-    const writer = fs.createWriteStream(outputLocationPath);
-    return axios({
-        method: 'get',
-        url: fileUrl,
-        responseType: 'stream',
-    }).then(response => {
-        return new Promise((resolve, reject) => {
-            response.data.pipe(writer);
-            let error = null;
-            writer.on('error', err => {
-                error = err;
-                writer.close();
-                reject(err);
-            });
-            writer.on('close', () => {
-                if (!error) {
-                    resolve();
-                }
-            });
-        });
-    });
-}
-
-function checkChecksum(url, local_file) {
-    return axios({
-        method: 'get',
-        url: url,
-        responseType: 'text',
-    }).then(response => {
-        return new Promise((resolve, reject) => {
-            const filestream = fs.createReadStream(local_file);
-            const hash = crypto.createHash('sha256');
-            filestream.on('readable', () => {
-                // Only one element is going to be produced by the
-                // hash stream.
-                const data = filestream.read();
-                if (data)
-                    hash.update(data);
-                else {
-                    const checksum = hash.digest("hex")
-                    if (checksum === response.data) {
-                        resolve()
-                    }
-                    else {
-                        reject(new Error("source code checksum mismatch"))
-                    }
-                }
-            });
-        });
-    })
-}
-
-async function fetchNativeCode(url, version, path) {
-    const sourceURL = `${url}/aws-crt-${version}-source.tgz`
-    const tarballPath = path + "source.tgz";
-    await downloadFile(sourceURL, tarballPath);
-    const sourceChecksumURL = `${url}/aws-crt-${version}-source.sha256`;
-    await checkChecksum(sourceChecksumURL, tarballPath);
-    await tar.x({ file: tarballPath, strip: 2, C: nativeSourceDir });
-}
-
-function buildLocally() {
-    const platform = os.platform();
-    let arch = os.arch();
-
-    // Allow cross-compile (so OSX can do arm64 or x64 builds) via:
-    // --target-arch ARCH
-    if (process.argv.includes('--target-arch')) {
-        arch = process.argv[process.argv.indexOf('--target-arch') + 1];
-    }
-
-    // options for cmake.BuildSystem
-    let options = {
-        target: "install",
-        debug: process.argv.includes('--debug'),
-        arch: arch,
-        out: path.join('build', `${platform}-${arch}`),
-        cMakeOptions: {
-            CMAKE_EXPORT_COMPILE_COMMANDS: true,
-            CMAKE_JS_PLATFORM: platform,
-            BUILD_TESTING: 'OFF',
-            CMAKE_INSTALL_PREFIX: 'crt/install',
-            CMAKE_PREFIX_PATH: 'crt/install',
-        }
-    }
-
-    // We need to pass some extra flags to pull off cross-compiling
-    // because cmake-js doesn't set everything we need.
-    //
-    // See the docs on `arch`: https://github.com/cmake-js/cmake-js/blob/v6.1.0/README.md?#runtimes
-    // > Notice: on non-Windows systems the C++ toolset's architecture's gonna be used despite this setting.
-    if (platform === 'darwin') {
-        // What Node calls "x64", Apple calls "x86_64". They both agree on the term "arm64" though.
-        options.cMakeOptions.CMAKE_OSX_ARCHITECTURES = (arch === 'x64') ? 'x86_64' : arch;
-    }
-
-    // Convert any -D arguments to this script to cmake -D arguments
-    for (const arg of process.argv) {
-        if (arg.startsWith('-D')) {
-            const option = arg.substring(2).split('=')
-            options.cMakeOptions[option[0]] = option[1]
-        }
-    }
-
-    // Enable parallel build (ignored by cmake older than 3.12)
-    process.env.CMAKE_BUILD_PARALLEL_LEVEL = `${Math.max(os.cpus().length, 1)}`;
-
-    // Run the build
-    var buildSystem = new cmake.BuildSystem(options);
-    return buildSystem.build();
-}
-
 async function buildFromRemoteSource(tmpPath) {
+
     if (fs.existsSync(nativeSourceDir)) {
         //teardown the local source code
         rmRecursive(nativeSourceDir);
@@ -164,11 +51,12 @@ async function buildFromRemoteSource(tmpPath) {
     }
     let rawData = fs.readFileSync('package.json');
     const version = JSON.parse(rawData)["version"];
-    await fetchNativeCode(host, version, tmpPath);
+    // Get the file using tar, loaded as an install-time dependency (see scripts/build_dependencies)
+    await build_step_tar.performStep(host, version, tmpPath);
     // Clean up temp directory
     rmRecursive(tmpPath);
-    // Kick off local build
-    await buildLocally();
+    // Kick off local build using cmake-js loaded as an install-time dependency (see scripts/build_dependencies)
+    await build_step_cmake.performStep();
     // Local build finished successfully, we don't need source anymore.
     rmRecursive(nativeSourceDir);
 }
@@ -197,6 +85,5 @@ if (checkDoDownload()) {
         throw err;
     }
 } else {
-    // Kick off local build
-    buildLocally();
+    build_step_cmake.performStep();
 }

scripts/build_dependencies/build_step_axios.js

@@ -0,0 +1,106 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+const fs = require("fs");
+const crypto = require('crypto');
+const utils = require('./build_utils');
+
+module.exports = {
+
+    axios: null,
+    clean_up_axios: false,
+    axios_version: "0.24.0",
+
+    /**
+     * Loads the axios library. We want to do this seperate instead of having a performStep function
+     * because the axios library is needed for multiple functions that have different data passed to them.
+     */
+    loadAxios: function () {
+        const workDir = path.join(__dirname, "../../")
+
+        process.chdir(__dirname);
+        if (this.axios == null) {
+            if (utils.npmCheckIfPackageExists("axios", this.axios_version) == true) {
+                this.axios = require("axios");
+            } else {
+                try {
+                    this.clean_up_axios = utils.npmDownloadAndInstallRuntimePackage("axios", this.axios_version);
+                    this.axios = require('axios');
+                } catch (error) {
+                    console.log("ERROR: Could not download axios! Cannot build CRT");
+                    console.log("Please install axios verion " + this.axios_version + " and then run the aws-crt install script again");
+                    process.exit(1);
+                }
+            }
+        }
+        process.chdir(workDir);
+    },
+
+    /**
+     * Downloads the file from the given file URL and places it in the given output location path.
+     * @param {*} fileUrl The file to download
+     * @param {*} outputLocationPath The location to store the downloaded file
+     * @returns A promise for the file download
+     */
+    downloadFile: function (fileUrl, outputLocationPath) {
+        const writer = fs.createWriteStream(outputLocationPath);
+        return this.axios({
+            method: 'get',
+            url: fileUrl,
+            responseType: 'stream',
+        }).then(response => {
+            return new Promise((resolve, reject) => {
+                response.data.pipe(writer);
+                let error = null;
+                writer.on('error', err => {
+                    error = err;
+                    writer.close();
+                    reject(err);
+                });
+                writer.on('close', () => {
+                    if (!error) {
+                        resolve();
+                    }
+                });
+            });
+        });
+    },
+
+    /**
+     * Performs a checksum check on the given file. The checksum is downloaded from the given URL
+     * and then the file given is checked using said checksum.
+     * @param {*} url The URL containing the checksum
+     * @param {*} local_file The file to check
+     * @returns A promise for the result of the check
+     */
+    checkChecksum: function (url, local_file) {
+        return this.axios({
+            method: 'get',
+            url: url,
+            responseType: 'text',
+        }).then(response => {
+            return new Promise((resolve, reject) => {
+                const filestream = fs.createReadStream(local_file);
+                const hash = crypto.createHash('sha256');
+                filestream.on('readable', () => {
+                    // Only one element is going to be produced by the
+                    // hash stream.
+                    const data = filestream.read();
+                    if (data)
+                        hash.update(data);
+                    else {
+                        const checksum = hash.digest("hex")
+                        if (checksum === response.data) {
+                            resolve()
+                        }
+                        else {
+                            reject(new Error("source code checksum mismatch"))
+                        }
+                    }
+                });
+            });
+        })
+    }
+
+}