Add task to cache cargo crates and use it for MacOS/Windows builds (#…

…9841) * Add task to cache cargo crates and use it for MacOS builds * Bug: Taskcluster transforms fail if the toolchain has no env * Windows: Use crate toolchain and skip build/source * Windows: remove unnecessary dep on build/source * Convert WIREGUARD_FOLDER to CMake paths
mozilla-mobile · Sep 11, 2024 · d30504b · d30504b
1 parent 1611944
commit d30504b
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 20 deletions.
diff --git a/taskcluster/kinds/build/macos.yml b/taskcluster/kinds/build/macos.yml
@@ -13,6 +13,7 @@ task-defaults:
     worker-type: b-macos
     fetches:
         toolchain:
+            - cargo-vendor
             - conda-macos
     worker:
         taskcluster-proxy: true

diff --git a/taskcluster/kinds/build/windows.yml b/taskcluster/kinds/build/windows.yml
@@ -10,18 +10,15 @@ task-defaults:
         tier: 1
         platform: windows/x86_64
     worker-type: b-win2022
-    dependencies:
-        build: build-source/vpn
     fetches:
         fetch:
             - win-dev-env
             - win-sentry-cli
             - win-conda
             - wireguard-nt
         toolchain:
+            - cargo-vendor
             - conda-windows-x86_64
-        build:
-            - artifact: mozillavpn-sources.tar.gz
     worker:
         taskcluster-proxy: true
         chain-of-trust: true

diff --git a/taskcluster/kinds/toolchain/cargo.yml b/taskcluster/kinds/toolchain/cargo.yml
@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+cargo-vendor:
+    description: "Vendored Rust Crates"
+    worker-type: b-linux
+    worker:
+        env:
+            FOO: bar    # Bug: taskcluster fails if there is no env key?
+        max-run-time: 3600
+        docker-image: {in-tree: build}
+    treeherder:
+        symbol: TL(cargo-vendor)
+    run:
+        script: cargo_vendor_crates.sh
+        resources:
+            - Cargo.lock
+        toolchain-alias: cargo-vendor
+        toolchain-artifact: public/build/cargo-vendor.tar.xz
diff --git a/taskcluster/kinds/toolchain/kind.yml b/taskcluster/kinds/toolchain/kind.yml
@@ -29,3 +29,4 @@ tasks-from:
     - conda_windows.yml
     - conda_osx.yml
     - flatpak.yml
+    - cargo.yml
diff --git a/taskcluster/scripts/build/macos_build.sh b/taskcluster/scripts/build/macos_build.sh
@@ -73,6 +73,18 @@ then
     npm install -g @sentry/cli
 fi
 
+# Use vendored crates - if available.
+if [ -d ${MOZ_FETCHES_DIR}/cargo-vendor ]; then
+mkdir -p .cargo
+cat << EOF > .cargo/config.toml
+[source.vendored-sources]
+directory = "${MOZ_FETCHES_DIR}/cargo-vendor"
+
+[source.crates-io]
+replace-with = "vendored-sources"
+EOF
+fi
+
 print Y "Configuring the build..."
 if [ -d ${TASK_HOME}/build ]; then
     echo "Found old build-folder, weird!"

diff --git a/taskcluster/scripts/build/windows_clang_cl.ps1 b/taskcluster/scripts/build/windows_clang_cl.ps1
@@ -13,6 +13,11 @@ Set-Location -Path $TASK_WORKDIR
 . "$FETCHES_PATH/QT_OUT/configure_qt.ps1"
 
 
+# Ensure we are working from a full checkout of all submodules. Taskcluster
+# doesn't handle recursive submodules, so we'll need to do it ourselves.
+git -C "$REPO_ROOT_PATH" submodule update --init --recursive
+
+
 # We have not yet removed our VC_Redist strategy. Therefore we rely on the old vsstudio bundle to get us that :)
 # TODO: We need to handle this at some point.
 $env:VCToolsRedistDir=(resolve-path "$FETCHES_PATH/VisualStudio/VC/Redist/MSVC/14.30.30704/").ToString()
@@ -31,15 +36,22 @@ if (Test-Path -Path $SSL_PATH) {
 }
 
 
-# Extract the sources
-$SOURCE_DSC = resolve-path "$FETCHES_PATH/mozillavpn_*.dsc"
-$SOURCE_VERSION = ((select-string $SOURCE_DSC -Pattern '^Version:') -split " ")[1]
-tar -xzvf (resolve-path "$FETCHES_PATH/mozillavpn_$SOURCE_VERSION.orig.tar.gz" -Relative)
-$SOURCE_DIR = resolve-path "$TASK_WORKDIR/mozillavpn-$SOURCE_VERSION"
+## Use vendored rust crates, if present
+if (Test-Path -Path "$FETCHES_PATH\cargo-vendor") {
+    $CARGO_VENDOR_PATH = "$FETCHES_PATH/cargo-vendor" -replace @('\\', '/')
+    New-Item -Path "$REPO_ROOT_PATH\.cargo" -ItemType "directory" -Force
+@"
+[source.vendored-sources]
+directory = "$CARGO_VENDOR_PATH"
+
+[source.crates-io]
+replace-with = "vendored-sources"
+"@ | Out-File -Encoding utf8 $REPO_ROOT_PATH\.cargo\config.toml
+}
 
 
 ## Setup the conda environment
-. $SOURCE_DIR/scripts/utils/call_bat.ps1  $FETCHES_PATH/Scripts/activate.bat
+. $REPO_ROOT_PATH/scripts/utils/call_bat.ps1  $FETCHES_PATH/Scripts/activate.bat
 conda-unpack
 
 # Conda Pack excpets to be run under cmd. therefore it will
@@ -65,7 +77,7 @@ mkdir $TASK_WORKDIR/cmake_build
 $BUILD_DIR =resolve-path "$TASK_WORKDIR/cmake_build"
 
 # Do the generic build
-cmake -S $SOURCE_DIR -B $BUILD_DIR -GNinja `
+cmake -S $REPO_ROOT_PATH -B $BUILD_DIR -GNinja `
         -DCMAKE_BUILD_TYPE=Release `
         -DPYTHON_EXECUTABLE="$CONDA_PREFIX\python.exe" `
         -DGOLANG_BUILD_TOOL="$CONDA_PREFIX\bin\go.exe" `
@@ -91,7 +103,7 @@ Get-ChildItem -Path $TASK_WORKDIR/artifacts
 
 
 Get-command python
-python  $SOURCE_DIR/taskcluster/scripts/get-secret.py -s project/mozillavpn/level-1/sentry -k sentry_debug_file_upload_key -f sentry_debug_file_upload_key
+python  $REPO_ROOT_PATH/taskcluster/scripts/get-secret.py -s project/mozillavpn/level-1/sentry -k sentry_debug_file_upload_key -f sentry_debug_file_upload_key
 $env:SENTRY_AUTH_TOKEN=$(Get-Content sentry_debug_file_upload_key)
 # Are we logged in?
 sentry-cli-Windows-x86_64.exe info

diff --git a/taskcluster/scripts/toolchain/cargo_vendor_crates.sh b/taskcluster/scripts/toolchain/cargo_vendor_crates.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+set -e
+
+cargo vendor --manifest-path $VCS_PATH/Cargo.toml $(pwd)/cargo-vendor
+tar -cJf $UPLOAD_DIR/cargo-vendor.tar.xz cargo-vendor/
diff --git a/windows/wireguard_nt/CMakeLists.txt b/windows/wireguard_nt/CMakeLists.txt
@@ -2,9 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-## Download and extract Wix and friends.
-
-
 set(WIREGUARD_FOLDER $ENV{WIREGUARD_FOLDER} CACHE STRING "Path to Extracted WireGuardNT Folder")
 
 if(NOT WIREGUARD_FOLDER)
@@ -14,17 +11,19 @@ if(NOT WIREGUARD_FOLDER)
 
     file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_BINARY_DIR}/.deps/wg-nt.zip DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/.deps)
 
-    set(WIREGUARD_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/.deps/wireguard-nt) 
+    set(CM_WIREGUARD_FOLDER ${CMAKE_CURRENT_BINARY_DIR}/.deps/wireguard-nt) 
+else()
+    cmake_path(CONVERT ${WIREGUARD_FOLDER} TO_CMAKE_PATH_LIST CM_WIREGUARD_FOLDER)
 endif()
 
 
 
-include_directories(${WIREGUARD_FOLDER}/include)
+include_directories(${CM_WIREGUARD_FOLDER}/include)
 
 install(FILES
-    ${WIREGUARD_FOLDER}/bin/${CMAKE_SYSTEM_PROCESSOR}/wireguard.dll
+    ${CM_WIREGUARD_FOLDER}/bin/${CMAKE_SYSTEM_PROCESSOR}/wireguard.dll
     DESTINATION .
 )
 
-target_include_directories(mozillavpn PUBLIC ${WIREGUARD_FOLDER}/include)
-target_include_directories(shared-sources INTERFACE ${WIREGUARD_FOLDER}/include)
+target_include_directories(mozillavpn PUBLIC ${CM_WIREGUARD_FOLDER}/include)
+target_include_directories(shared-sources INTERFACE ${CM_WIREGUARD_FOLDER}/include)