From b19ad643fb993cf279b2e8742873828e0c781953 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Sun, 16 Aug 2020 14:48:40 -0400 Subject: [PATCH 01/24] WIP --- repos/kaleido/py/setup.py | 23 ++++++++++++++++--- .../linux_scripts/{args.gn => args_arm64.gn} | 2 ++ repos/linux_scripts/args_x64.gn | 9 ++++++++ repos/linux_scripts/build_kaleido | 9 +++++++- repos/linux_scripts/build_kaleido_docker | 11 ++++++--- repos/linux_scripts/fetch_chromium_docker | 7 ++++-- repos/linux_scripts/gclient | 10 ++++++++ repos/linux_scripts/perform_kaleido_build | 8 +++---- 8 files changed, 66 insertions(+), 13 deletions(-) rename repos/linux_scripts/{args.gn => args_arm64.gn} (88%) create mode 100644 repos/linux_scripts/args_x64.gn create mode 100644 repos/linux_scripts/gclient diff --git a/repos/kaleido/py/setup.py b/repos/kaleido/py/setup.py index 95eced3b..6123fd9d 100644 --- a/repos/kaleido/py/setup.py +++ b/repos/kaleido/py/setup.py @@ -127,7 +127,7 @@ def run(self): cmd_obj = self.distribution.get_command_obj('bdist_wheel') # Use current platform as plat_name, but replace linux with manylinux2014 - cmd_obj.plat_name = distutils.util.get_platform().replace("linux-", "manylinux1-") + cmd_obj.plat_name = distutils.util.get_platform() # Handle windows 32-bit cross compilation print(os.environ.get("KALEIDO_ARCH", "x64")) @@ -135,10 +135,27 @@ def run(self): arch = os.environ.get("KALEIDO_ARCH", "x64") if arch == "x86": cmd_obj.plat_name = "win32" - print(cmd_obj.plat_name) + elif arch == "x64": + cmd_obj.plat_name = "win_amd64" + else: + raise ValueError( + "Unsupported architecture {arch} for plat_name {plat_name}".format( + arch=arch, plat_name=cmd_obj.plat_name) + ) + elif cmd_obj.plat_name.startswith("linux"): + arch = os.environ.get("KALEIDO_ARCH", "x64") + if arch == "x64": + cmd_obj.plat_name = "manylinux1-x86_64" + elif arch == "x86": + cmd_obj.plat_name = "manylinux1-i686" + elif arch == "arm64": + cmd_obj.plat_name = "manylinux2014-aarch64" + elif arch == "arm": + cmd_obj.plat_name = "manylinux2014-armv7l" + # Set macos platform to 10.10 to match chromium build target (See build/config/mac/mac_sdk.gni) # rather than Python environment - if cmd_obj.plat_name.startswith("macosx"): + elif cmd_obj.plat_name.startswith("macosx"): cmd_obj.plat_name = "macosx-10.10-x86_64" cmd_obj.python_tag = 'py2.py3' diff --git a/repos/linux_scripts/args.gn b/repos/linux_scripts/args_arm64.gn similarity index 88% rename from repos/linux_scripts/args.gn rename to repos/linux_scripts/args_arm64.gn index 4d66d47f..5474bbeb 100644 --- a/repos/linux_scripts/args.gn +++ b/repos/linux_scripts/args_arm64.gn @@ -5,3 +5,5 @@ is_component_build=false symbol_level=0 blink_symbol_level=0 is_debug=false + +target_cpu="x64" \ No newline at end of file diff --git a/repos/linux_scripts/args_x64.gn b/repos/linux_scripts/args_x64.gn new file mode 100644 index 00000000..5474bbeb --- /dev/null +++ b/repos/linux_scripts/args_x64.gn @@ -0,0 +1,9 @@ +import("//build/args/headless.gn") +enable_nacl=false +is_component_build=false + +symbol_level=0 +blink_symbol_level=0 +is_debug=false + +target_cpu="x64" \ No newline at end of file diff --git a/repos/linux_scripts/build_kaleido b/repos/linux_scripts/build_kaleido index f12e3954..d7d9c7af 100755 --- a/repos/linux_scripts/build_kaleido +++ b/repos/linux_scripts/build_kaleido @@ -1,3 +1,10 @@ +#!/bin/bash +if [ $# -eq 0 ]; then + echo "No architecture provided" + exit 1 +fi +export KALEIDO_ARCH=$1 + # full path to linux_scripots/ directory DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -12,4 +19,4 @@ cp ./README.md ./repos/kaleido/ cp ./LICENSE.txt ./repos/kaleido/ # Perform build using docker -docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.7 /repos/linux_scripts/build_kaleido_docker +docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.7 /repos/linux_scripts/build_kaleido_docker $KALEIDO_ARCH diff --git a/repos/linux_scripts/build_kaleido_docker b/repos/linux_scripts/build_kaleido_docker index f9a0ff83..79170399 100755 --- a/repos/linux_scripts/build_kaleido_docker +++ b/repos/linux_scripts/build_kaleido_docker @@ -1,13 +1,18 @@ #!/bin/bash # docker run -it -v /media/jmmease/SSD1/chromium_build/repos/:/repos jonmmease/chromium-builder:0.5 /repos/build_headless +if [ $# -eq 0 ]; then + echo "No architecture provided" + exit 1 +fi +export KALEIDO_ARCH=$1 cd /repos/src # Make output directory -mkdir -p out/Kaleido_linux +mkdir -p out/Kaleido_linux_$KALEIDO_ARCH -# Write out/Kaleido_linux/args.gn -cp /repos/linux_scripts/args.gn out/Kaleido_linux/args.gn +# Write out/Kaleido_linux_$KALEIDO_ARCH/args.gn +cp /repos/linux_scripts/args_$KALEIDO_ARCH.gn out/Kaleido_linux_$KALEIDO_ARCH/args.gn # 1) Reset headless/BUILD.gn git checkout HEAD -- headless/BUILD.gn diff --git a/repos/linux_scripts/fetch_chromium_docker b/repos/linux_scripts/fetch_chromium_docker index 0ab7e267..127e05a8 100755 --- a/repos/linux_scripts/fetch_chromium_docker +++ b/repos/linux_scripts/fetch_chromium_docker @@ -2,8 +2,11 @@ # docker run -it -v /media/jmmease/SSD1/chromium_build/repos/:/repos jonmmease/chromium-builder:0.5 /repos/fetch_chromium_docker cd /repos -# Remove any prior .gclient file -rm -f .gclient +## Remove any prior .gclient file +#rm -f .gclient + +# Replace any prior .gclient file +cp ./linux_scripts/gclient .gclient echo "fetch chromium" fetch --nohooks chromium diff --git a/repos/linux_scripts/gclient b/repos/linux_scripts/gclient new file mode 100644 index 00000000..3f241623 --- /dev/null +++ b/repos/linux_scripts/gclient @@ -0,0 +1,10 @@ +solutions = [ + { + "url": "https://chromium.googlesource.com/chromium/src.git", + "managed": False, + "name": "src", + "custom_deps": {}, + "custom_vars": {}, + }, +] +target_cpu=["x64", "arm64"] \ No newline at end of file diff --git a/repos/linux_scripts/perform_kaleido_build b/repos/linux_scripts/perform_kaleido_build index 778feecb..339e388d 100755 --- a/repos/linux_scripts/perform_kaleido_build +++ b/repos/linux_scripts/perform_kaleido_build @@ -2,15 +2,15 @@ rm -rf headless/app/scopes cp -r /repos/kaleido/cc/* headless/app/ -# Perform build, result will be out/Kaleido_linux/kaleido -ninja -C out/Kaleido_linux -j 16 kaleido +# Perform build, result will be out/Kaleido_linux_$KALEIDO_ARCH/kaleido +ninja -C out/Kaleido_linux_$KALEIDO_ARCH -j 16 kaleido # Copy build files mkdir -p /repos/build/kaleido/ rm -rf /repos/build/kaleido/* mkdir -p /repos/build/kaleido/bin -cp out/Kaleido_linux/kaleido /repos/build/kaleido/bin -cp -r out/Kaleido_linux/swiftshader/ /repos/build/kaleido/bin +cp out/Kaleido_linux_$KALEIDO_ARCH/kaleido /repos/build/kaleido/bin +cp -r out/Kaleido_linux_$KALEIDO_ARCH/swiftshader/ /repos/build/kaleido/bin # copy shared libraries mkdir -p /repos/build/kaleido/lib/ From 043af206ba29396e82bacd812a791f13e7bde4d6 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 17 Aug 2020 05:33:48 -0400 Subject: [PATCH 02/24] arm build WIP --- repos/linux_scripts/args_arm64.gn | 2 +- repos/linux_scripts/build_kaleido_docker | 8 ++++++++ repos/linux_scripts/checkout_revision_docker | 3 +++ repos/linux_scripts/fetch_chromium_docker | 5 +---- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/repos/linux_scripts/args_arm64.gn b/repos/linux_scripts/args_arm64.gn index 5474bbeb..8e5784da 100644 --- a/repos/linux_scripts/args_arm64.gn +++ b/repos/linux_scripts/args_arm64.gn @@ -6,4 +6,4 @@ symbol_level=0 blink_symbol_level=0 is_debug=false -target_cpu="x64" \ No newline at end of file +target_cpu="arm64" \ No newline at end of file diff --git a/repos/linux_scripts/build_kaleido_docker b/repos/linux_scripts/build_kaleido_docker index 79170399..0c7b81a4 100755 --- a/repos/linux_scripts/build_kaleido_docker +++ b/repos/linux_scripts/build_kaleido_docker @@ -4,15 +4,23 @@ if [ $# -eq 0 ]; then echo "No architecture provided" exit 1 fi + export KALEIDO_ARCH=$1 + cd /repos/src +if [ $KALEIDO_ARCH == arm64 ]; then + echo "Installing sysroot" + python ./build/linux/sysroot_scripts/install-sysroot.py --arch=arm64 +fi + # Make output directory mkdir -p out/Kaleido_linux_$KALEIDO_ARCH # Write out/Kaleido_linux_$KALEIDO_ARCH/args.gn cp /repos/linux_scripts/args_$KALEIDO_ARCH.gn out/Kaleido_linux_$KALEIDO_ARCH/args.gn +gn gen out/Kaleido_linux_$KALEIDO_ARCH # 1) Reset headless/BUILD.gn git checkout HEAD -- headless/BUILD.gn diff --git a/repos/linux_scripts/checkout_revision_docker b/repos/linux_scripts/checkout_revision_docker index 94715a60..89548e79 100755 --- a/repos/linux_scripts/checkout_revision_docker +++ b/repos/linux_scripts/checkout_revision_docker @@ -1,6 +1,9 @@ #!/bin/bash CHROMIUM_TAG="83.0.4103.61" +# Replace any prior .gclient file +cp ./linux_scripts/gclient .gclient + cd /repos/src git reset --hard git checkout tags/$CHROMIUM_TAG diff --git a/repos/linux_scripts/fetch_chromium_docker b/repos/linux_scripts/fetch_chromium_docker index 127e05a8..9cda3678 100755 --- a/repos/linux_scripts/fetch_chromium_docker +++ b/repos/linux_scripts/fetch_chromium_docker @@ -3,10 +3,7 @@ cd /repos ## Remove any prior .gclient file -#rm -f .gclient - -# Replace any prior .gclient file -cp ./linux_scripts/gclient .gclient +rm -f .gclient echo "fetch chromium" fetch --nohooks chromium From ce7600b9c529ac3a0ad38b39a71ffed7a8bae5de Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 17 Aug 2020 07:42:42 -0400 Subject: [PATCH 03/24] Only copy shared objects for x64 build --- repos/linux_scripts/perform_kaleido_build | 30 ++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/repos/linux_scripts/perform_kaleido_build b/repos/linux_scripts/perform_kaleido_build index 339e388d..6edf7fea 100755 --- a/repos/linux_scripts/perform_kaleido_build +++ b/repos/linux_scripts/perform_kaleido_build @@ -15,20 +15,22 @@ cp -r out/Kaleido_linux_$KALEIDO_ARCH/swiftshader/ /repos/build/kaleido/bin # copy shared libraries mkdir -p /repos/build/kaleido/lib/ -# /usr/lib/x86_64-linux-gnu/ -for SO_FILE in libnss3.so libnssutil3.so libnspr4.so libplc4.so libplds4.so libsqlite3.so.0 -do - cp /usr/lib/x86_64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE -done - -# /usr/lib/x86_64-linux-gnu/nss -cp /usr/lib/x86_64-linux-gnu/nss/* /repos/build/kaleido/lib/ - -# /lib/x86_64-linux-gnu/ -for SO_FILE in libexpat.so.1 # libdl.so.2 libpthread.so.0 librt.so.1 libm.so.6 libgcc_s.so.1 libc.so.6 -do - cp /lib/x86_64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE -done +if [ $KALEIDO_ARCH == "x64" ]; then + # /usr/lib/x86_64-linux-gnu/ + for SO_FILE in libnss3.so libnssutil3.so libnspr4.so libplc4.so libplds4.so libsqlite3.so.0 + do + cp /usr/lib/x86_64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE + done + + # /usr/lib/x86_64-linux-gnu/nss + cp /usr/lib/x86_64-linux-gnu/nss/* /repos/build/kaleido/lib/ + + # /lib/x86_64-linux-gnu/ + for SO_FILE in libexpat.so.1 # libdl.so.2 libpthread.so.0 librt.so.1 libm.so.6 libgcc_s.so.1 libc.so.6 + do + cp /lib/x86_64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE + done +fi # fonts mkdir -p /repos/build/kaleido/etc/ From 02d1f5e0495eaf2d6babfe5b73595a65c9ae270b Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 17 Aug 2020 18:16:56 -0400 Subject: [PATCH 04/24] Install sysroots in chromium-builder image --- repos/linux_full_scripts/Dockerfile | 2 +- repos/linux_full_scripts/build_kaleido | 9 ++++++++- repos/linux_full_scripts/build_kaleido_docker | 5 +++++ repos/linux_scripts/Dockerfile | 8 ++++++++ repos/linux_scripts/build_kaleido | 2 +- repos/linux_scripts/build_kaleido_docker | 5 ----- repos/linux_scripts/fetch_chromium | 4 ++-- 7 files changed, 25 insertions(+), 10 deletions(-) diff --git a/repos/linux_full_scripts/Dockerfile b/repos/linux_full_scripts/Dockerfile index 3dcddc46..45e89554 100644 --- a/repos/linux_full_scripts/Dockerfile +++ b/repos/linux_full_scripts/Dockerfile @@ -3,7 +3,7 @@ # Build and run: # docker build -t jonmmease/kaleido-builder:0.6 -f Dockerfile . -FROM jonmmease/chromium-builder:0.7 +FROM jonmmease/chromium-builder:0.8 # Fetch chromium source tree diff --git a/repos/linux_full_scripts/build_kaleido b/repos/linux_full_scripts/build_kaleido index 6af60e13..321b52ea 100755 --- a/repos/linux_full_scripts/build_kaleido +++ b/repos/linux_full_scripts/build_kaleido @@ -1,3 +1,10 @@ +#!/bin/bash +if [ $# -eq 0 ]; then + echo "No architecture provided" + exit 1 +fi +export KALEIDO_ARCH=$1 + # full path to linux_full_scripots/ directory DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -12,4 +19,4 @@ cp ./README.md ./repos/kaleido/ cp ./LICENSE.txt ./repos/kaleido/ # Perform build using docker -docker run -it -v `pwd`/repos/:/repos jonmmease/kaleido-builder:0.7 /repos/linux_full_scripts/build_kaleido_docker \ No newline at end of file +docker run -it -v `pwd`/repos/:/repos jonmmease/kaleido-builder:0.7 /repos/linux_full_scripts/build_kaleido_docker $KALEIDO_ARCH \ No newline at end of file diff --git a/repos/linux_full_scripts/build_kaleido_docker b/repos/linux_full_scripts/build_kaleido_docker index 90a3bcf7..3b51f7fc 100755 --- a/repos/linux_full_scripts/build_kaleido_docker +++ b/repos/linux_full_scripts/build_kaleido_docker @@ -1,4 +1,9 @@ #!/bin/bash +if [ $# -eq 0 ]; then + echo "No architecture provided" + exit 1 +fi +export KALEIDO_ARCH=$1 # perform_kaleido_build expect to be run from src directory cd /src diff --git a/repos/linux_scripts/Dockerfile b/repos/linux_scripts/Dockerfile index 4179445c..0d7da3d9 100644 --- a/repos/linux_scripts/Dockerfile +++ b/repos/linux_scripts/Dockerfile @@ -58,3 +58,11 @@ RUN \ curl -s https://chromium.googlesource.com/chromium/src/+/$CHROMIUM_TAG/build/install-build-deps.sh?format=TEXT \ | base64 -d > install-build-deps.sh && chmod +x ./install-build-deps.sh && \ ./install-build-deps.sh --no-syms --no-arm --no-chromeos-fonts --no-nacl --no-prompt + +RUN \ + curl -s https://chromium.googlesource.com/chromium/src/+/$CHROMIUM_TAG/build/linux/sysroot_scripts/sysroots.json?format=TEXT \ + | base64 -d > sysroots.json + +RUN \ + curl -s https://chromium.googlesource.com/chromium/src/+/$CHROMIUM_TAG/build/linux/sysroot_scripts/install-sysroot.py?format=TEXT \ + | base64 -d > install-sysroot.py && python3 ./install-sysroot.py --all diff --git a/repos/linux_scripts/build_kaleido b/repos/linux_scripts/build_kaleido index d7d9c7af..99839c78 100755 --- a/repos/linux_scripts/build_kaleido +++ b/repos/linux_scripts/build_kaleido @@ -19,4 +19,4 @@ cp ./README.md ./repos/kaleido/ cp ./LICENSE.txt ./repos/kaleido/ # Perform build using docker -docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.7 /repos/linux_scripts/build_kaleido_docker $KALEIDO_ARCH +docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.8 /repos/linux_scripts/build_kaleido_docker $KALEIDO_ARCH diff --git a/repos/linux_scripts/build_kaleido_docker b/repos/linux_scripts/build_kaleido_docker index 0c7b81a4..39e7ef8c 100755 --- a/repos/linux_scripts/build_kaleido_docker +++ b/repos/linux_scripts/build_kaleido_docker @@ -10,11 +10,6 @@ export KALEIDO_ARCH=$1 cd /repos/src -if [ $KALEIDO_ARCH == arm64 ]; then - echo "Installing sysroot" - python ./build/linux/sysroot_scripts/install-sysroot.py --arch=arm64 -fi - # Make output directory mkdir -p out/Kaleido_linux_$KALEIDO_ARCH diff --git a/repos/linux_scripts/fetch_chromium b/repos/linux_scripts/fetch_chromium index b26a25c1..256d757e 100755 --- a/repos/linux_scripts/fetch_chromium +++ b/repos/linux_scripts/fetch_chromium @@ -6,7 +6,7 @@ cd $DIR/../.. echo `pwd` # Download chromium -docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.7 /repos/linux_scripts/fetch_chromium_docker +docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.8 /repos/linux_scripts/fetch_chromium_docker # Checkout revision -docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.7 /repos/linux_scripts/checkout_revision_docker +docker run -it -v `pwd`/repos/:/repos jonmmease/chromium-builder:0.8 /repos/linux_scripts/checkout_revision_docker From 051dc423aaea63e4400d85ad03fd55e715ef6aea Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Mon, 17 Aug 2020 19:22:44 -0400 Subject: [PATCH 05/24] Use parameters in circleci config --- .circleci/config.yml | 111 +++++++++++++++++----------- repos/win_scripts/build_kaleido.ps1 | 2 +- 2 files changed, 67 insertions(+), 46 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ebd40317..ba647d48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,30 +3,69 @@ version: 2.1 orbs: win: circleci/windows@2.4.0 # The Windows orb give you everything you need to start using the Windows executor. -jobs: +commands: linux_build: - machine: - image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run - resource_class: large + parameters: + arch: + default: "x64" + type: string steps: - checkout - run: docker --version - - run: docker pull jonmmease/kaleido-builder:0.7 - - run: ./repos/linux_full_scripts/build_kaleido + - run: docker pull jonmmease/kaleido-builder:0.8 + - run: ./repos/linux_full_scripts/build_kaleido <> - run: sudo chmod -R 777 repos/build - run: sudo chmod -R 777 repos/kaleido/py - run: ls repos/build/kaleido - - run: pushd repos/build && zip -r kaleido_linux.zip ./* + - run: pushd repos/build && zip -r kaleido_linux_<>.zip ./* - run: pushd repos/kaleido/py/ && zip -r kaleido_wheel.zip ./dist - store_artifacts: - path: ./repos/build/kaleido_linux.zip + path: ./repos/build/kaleido_linux_<>.zip - store_artifacts: path: ./repos/kaleido/py/kaleido_wheel.zip - persist_to_workspace: root: ./repos paths: - kaleido/py/ - + + windows_build: + parameters: + arch: + default: "x64" + type: string + steps: + - checkout + - run: + name: Update PowerShell.Archive to fix slash direction + command: Install-Module Microsoft.PowerShell.Archive -MinimumVersion 1.2.3.0 -Repository PSGallery -Force + - attach_workspace: + at: .\repos + - run: .\repos\win_scripts\build_kaleido.ps1 <> + - run: dir .\repos\build\kaleido + - run: dir .\repos\kaleido\py\dist + - store_artifacts: + path: ./repos/build/kaleido_win_<>.zip + - store_artifacts: + path: ./repos/kaleido/py/kaleido_wheel.zip + + +jobs: + linux_build_x64: + machine: + image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run + resource_class: large + steps: + - linux_build: + arch: "x64" + + linux_build_arm64: + machine: + image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run + resource_class: large + steps: + - linux_build: + arch: "arm64" + linux_test_37: docker: - image: continuumio/miniconda3 @@ -157,57 +196,36 @@ jobs: - .gclient_entries - kaleido/version - windows_build: + windows_build_x64: executor: name: win/default size: "xlarge" shell: powershell.exe steps: - - checkout - - run: - name: Update PowerShell.Archive to fix slash direction - command: Install-Module Microsoft.PowerShell.Archive -MinimumVersion 1.2.3.0 -Repository PSGallery -Force - - attach_workspace: - at: .\repos - - run: .\repos\win_scripts\build_kaleido.ps1 x64 - - run: dir .\repos\build\kaleido - - run: dir .\repos\kaleido\py\dist - - store_artifacts: - path: ./repos/build/kaleido_win.zip - - store_artifacts: - path: ./repos/kaleido/py/kaleido_wheel.zip + - windows_build: + arch: "x64" - windows_build-32: + windows_build_x86: executor: name: win/default size: "xlarge" shell: powershell.exe steps: - - checkout - - run: - name: Update PowerShell.Archive to fix slash direction - command: Install-Module Microsoft.PowerShell.Archive -MinimumVersion 1.2.3.0 -Repository PSGallery -Force - - attach_workspace: - at: .\repos - - run: .\repos\win_scripts\build_kaleido.ps1 x86 - - run: dir .\repos\build\kaleido - - run: dir .\repos\kaleido\py\dist - - store_artifacts: - path: ./repos/build/kaleido_win.zip - - store_artifacts: - path: ./repos/kaleido/py/kaleido_wheel.zip + - windows_build: + arch: "x86" workflows: - build-windows: + build-windows-x64: jobs: - windows_fetch - - windows_build: + - windows_build_x64: requires: - windows_fetch - build-windows-32: + + build-windows-x86: jobs: - windows_fetch - - windows_build-32: + - windows_build_x86: requires: - windows_fetch build-mac: @@ -220,13 +238,16 @@ workflows: requires: - mac_build_blink - build-linux: + build-linux-x64: jobs: - - linux_build + - linux_build_x64 - linux_test_37: requires: - - linux_build + - linux_build_x64 - linux_test_27: requires: - - linux_build + - linux_build_x64 + build-linux-arm64: + jobs: + - linux_build_arm64 diff --git a/repos/win_scripts/build_kaleido.ps1 b/repos/win_scripts/build_kaleido.ps1 index e5613211..069383cb 100644 --- a/repos/win_scripts/build_kaleido.ps1 +++ b/repos/win_scripts/build_kaleido.ps1 @@ -100,7 +100,7 @@ cd .. if (Test-Path ..\build\kaleido_win.zip) { Remove-Item -Recurse -Force ..\build\kaleido_win.zip } -Compress-Archive -Path ..\build\kaleido -DestinationPath ..\build\kaleido_win.zip +Compress-Archive -Path ..\build\kaleido -DestinationPath ..\build\kaleido_win_$arch.zip # Build wheel zip archive if (Test-Path ..\kaleido\py\kaleido_wheel.zip) { From a95f3747311cc9db5387436ca549b541613aab93 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 18 Aug 2020 12:11:39 -0400 Subject: [PATCH 06/24] Multiple arch in kaleido-builder image --- .dockerignore | 3 ++- README.md | 8 ++++---- repos/linux_full_scripts/Dockerfile | 27 +++++++++++++------------- repos/linux_full_scripts/build_kaleido | 2 +- repos/linux_scripts/gclient | 2 +- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.dockerignore b/.dockerignore index 1d8d9c3f..f8b9e8e8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,2 @@ - repos/ +repos +!repos/linux_scripts/ diff --git a/README.md b/README.md index ef30946d..db10398d 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ This approach relies on the `jonmmease/kaleido-builder` docker image, and the sc Download docker image ``` -$ docker pull jonmmease/kaleido-builder:0.7 +$ docker pull jonmmease/kaleido-builder:0.8 ``` Build Kaleido @@ -113,7 +113,7 @@ This approach relies on the `jonmmease/chromium-builder` docker image, and the s Download docker image ``` -$ docker pull jonmmease/chromium-builder:0.7 +$ docker pull jonmmease/chromium-builder:0.8 ``` Fetch the Chromium codebase and checkout the specific tag, then sync all dependencies @@ -165,14 +165,14 @@ The `chromium-builder` container mostly follows the instructions at https://chro Build container with: ``` -$ docker build -t jonmmease/chromium-builder:0.7 -f repos/linux_scripts/Dockerfile . +$ docker build -t jonmmease/chromium-builder:0.8 -f repos/linux_scripts/Dockerfile . ``` ## kaleido-builder This container contains a pre-compiled version of chromium source tree. Takes several hours to build! ``` -$ docker build -t jonmmease/kaleido-builder:0.7 -f repos/linux_full_scripts/Dockerfile . +$ docker build -t jonmmease/kaleido-builder:0.8 -f repos/linux_full_scripts/Dockerfile . ``` diff --git a/repos/linux_full_scripts/Dockerfile b/repos/linux_full_scripts/Dockerfile index 45e89554..eef6cc07 100644 --- a/repos/linux_full_scripts/Dockerfile +++ b/repos/linux_full_scripts/Dockerfile @@ -5,12 +5,16 @@ FROM jonmmease/chromium-builder:0.8 +# Update depot_tools in case there are changes in how to fetch code +RUN cd /depot_tools && git pull # Fetch chromium source tree RUN \ cd / \ && fetch --nohooks chromium +COPY ./repos/linux_scripts/gclient /.gclient + # Checkout chromium revision RUN \ cd /src \ @@ -24,17 +28,9 @@ RUN rm -r /src/.git # TODO: remove .git directories from third-party dependencies under src/third_party -# Build create linux out directory -RUN \ - cd /src \ - && mkdir -p out/Kaleido_linux \ - && echo 'import("//build/args/headless.gn")' > out/Kaleido_linux/args.gn \ - && echo 'enable_nacl=false' >> out/Kaleido_linux/args.gn \ - && echo 'is_component_build=false' >> out/Kaleido_linux/args.gn \ - && echo 'symbol_level=0' >> out/Kaleido_linux/args.gn \ - && echo 'blink_symbol_level=0' >> out/Kaleido_linux/args.gn \ - && echo 'is_debug=false' >> out/Kaleido_linux/args.gn - +# Build create x64 linux out directory +RUN mkdir -p /src/out/Kaleido_linux_x64/ +COPY ./repos/linux_scripts/args_x64.gn /src/out/Kaleido_linux_x64/args.gn # Add Kaleido executable RUN echo ' \ @@ -52,6 +48,11 @@ executable("kaleido") { \ # dependencies that Kaleido needs precompiled. RUN \ cd /src \ - && gn gen out/Kaleido_linux \ - && ninja -C out/Kaleido_linux -j 20 headless_example + && gn gen out/Kaleido_linux_x64 \ + && ninja -C out/Kaleido_linux_x64 -j 20 headless_example + +# Initialize project directories for other architectures, but don't precompile +RUN mkdir -p /src/out/Kaleido_linux_arm64/ +COPY ./repos/linux_scripts/args_arm64.gn /src/out/Kaleido_linux_arm64/args.gn +RUN cd /src && gn gen out/Kaleido_linux_arm64 diff --git a/repos/linux_full_scripts/build_kaleido b/repos/linux_full_scripts/build_kaleido index 321b52ea..da6e627b 100755 --- a/repos/linux_full_scripts/build_kaleido +++ b/repos/linux_full_scripts/build_kaleido @@ -19,4 +19,4 @@ cp ./README.md ./repos/kaleido/ cp ./LICENSE.txt ./repos/kaleido/ # Perform build using docker -docker run -it -v `pwd`/repos/:/repos jonmmease/kaleido-builder:0.7 /repos/linux_full_scripts/build_kaleido_docker $KALEIDO_ARCH \ No newline at end of file +docker run -it -v `pwd`/repos/:/repos jonmmease/kaleido-builder:0.8 /repos/linux_full_scripts/build_kaleido_docker $KALEIDO_ARCH \ No newline at end of file diff --git a/repos/linux_scripts/gclient b/repos/linux_scripts/gclient index 3f241623..c2e19d37 100644 --- a/repos/linux_scripts/gclient +++ b/repos/linux_scripts/gclient @@ -7,4 +7,4 @@ solutions = [ "custom_vars": {}, }, ] -target_cpu=["x64", "arm64"] \ No newline at end of file +target_cpu=["x64", "arm", "arm64"] \ No newline at end of file From a5dc30c49546bfb49cea0a2b36d517c9a8dbc8b9 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Tue, 18 Aug 2020 12:28:30 -0400 Subject: [PATCH 07/24] Bundle libraries from sysroot for arm64 --- repos/linux_scripts/perform_kaleido_build | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/repos/linux_scripts/perform_kaleido_build b/repos/linux_scripts/perform_kaleido_build index 6edf7fea..bd272e27 100755 --- a/repos/linux_scripts/perform_kaleido_build +++ b/repos/linux_scripts/perform_kaleido_build @@ -30,6 +30,17 @@ if [ $KALEIDO_ARCH == "x64" ]; then do cp /lib/x86_64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE done + +elif [ $KALEIDO_ARCH == "arm64" ]; then + # /usr/lib/aarch64-linux-gnu/ + for SO_FILE in libnss3.so libnssutil3.so libnspr4.so libplc4.so libplds4.so libsqlite3.so.0 + do + cp /debian_sid_arm64-sysroot/usr/lib/aarch64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE + done + + # /usr/lib/aarch64-linux-gnu/nss + cp /debian_sid_arm64-sysroot/usr/lib/aarch64-linux-gnu/nss/* /repos/build/kaleido/lib/ + fi # fonts From 105832d96da4df752439d7388261f38a2d8e5ca4 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 05:26:37 -0400 Subject: [PATCH 08/24] Use linux fetch then build workflow in circleci Old approach of using the huge docker image containing pre-build source doesn't scale to supporting multiple linux architectures. --- .circleci/config.yml | 46 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba647d48..ebf9f144 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,9 +11,10 @@ commands: type: string steps: - checkout - - run: docker --version - - run: docker pull jonmmease/kaleido-builder:0.8 - - run: ./repos/linux_full_scripts/build_kaleido <> + - attach_workspace: + at: ./repos + - run: docker pull jonmmease/chomium-builder:0.8 + - run: ./repos/linux_scripts/build_kaleido <> - run: sudo chmod -R 777 repos/build - run: sudo chmod -R 777 repos/kaleido/py - run: ls repos/build/kaleido @@ -47,9 +48,34 @@ commands: path: ./repos/build/kaleido_win_<>.zip - store_artifacts: path: ./repos/kaleido/py/kaleido_wheel.zip - - + jobs: + linux_fetch: + parameters: + arch: + default: "x64" + type: string + steps: + - checkout + - run: docker pull jonmmease/chomium-builder:0.8 + - run: + name: Fetch detpot_tools and chromium source tree + command: ./repos/linux_scripts/fetch_chromium + no_output_timeout: 30m + - run: + name: Delete .git directories to save space (skipping angle taken from electron circleci build) + command: | + cd ./repos/src + ( find . -type d -name ".git" -not -path "./third_party/angle/*" ) | xargs rm -rf + - persist_to_workspace: + root: ./repos + paths: + - src + - depot_tools + - .gclient + - .gclient_entries + - kaleido/version + linux_build_x64: machine: image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run @@ -240,7 +266,10 @@ workflows: build-linux-x64: jobs: - - linux_build_x64 + - linux_fetch + - linux_build_x64: + requires: + - linux_fetch - linux_test_37: requires: - linux_build_x64 @@ -250,4 +279,7 @@ workflows: build-linux-arm64: jobs: - - linux_build_arm64 + - linux_fetch + - linux_build_arm64: + requires: + - linux_fetch From 1164df99afefaa0bc88deaa2aae94ec7ab825978 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 05:29:23 -0400 Subject: [PATCH 09/24] Fix circle config error --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ebf9f144..4885f93a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,10 +51,9 @@ commands: jobs: linux_fetch: - parameters: - arch: - default: "x64" - type: string + machine: + image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run + resource_class: large steps: - checkout - run: docker pull jonmmease/chomium-builder:0.8 From 1164b5703212d2674cc8ee074b2e60640da2704e Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 05:29:42 -0400 Subject: [PATCH 10/24] Only download source for architectures we need right now --- repos/linux_scripts/gclient | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/linux_scripts/gclient b/repos/linux_scripts/gclient index c2e19d37..3f241623 100644 --- a/repos/linux_scripts/gclient +++ b/repos/linux_scripts/gclient @@ -7,4 +7,4 @@ solutions = [ "custom_vars": {}, }, ] -target_cpu=["x64", "arm", "arm64"] \ No newline at end of file +target_cpu=["x64", "arm64"] \ No newline at end of file From f01e289578b8ff9ff391d9ab60f7347bee15cde6 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 05:32:29 -0400 Subject: [PATCH 11/24] Fix docker image type --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4885f93a..a79248cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ commands: - checkout - attach_workspace: at: ./repos - - run: docker pull jonmmease/chomium-builder:0.8 + #- run: docker pull jonmmease/chomium-builder:0.8 - run: ./repos/linux_scripts/build_kaleido <> - run: sudo chmod -R 777 repos/build - run: sudo chmod -R 777 repos/kaleido/py @@ -56,7 +56,7 @@ jobs: resource_class: large steps: - checkout - - run: docker pull jonmmease/chomium-builder:0.8 + - run: docker pull jonmmease/chromium-builder:0.8 - run: name: Fetch detpot_tools and chromium source tree command: ./repos/linux_scripts/fetch_chromium From bba3454f2ee982609d47a6f3b2395763b5ef0493 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 09:20:20 -0400 Subject: [PATCH 12/24] sudo remove --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a79248cd..177f8b25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: name: Delete .git directories to save space (skipping angle taken from electron circleci build) command: | cd ./repos/src - ( find . -type d -name ".git" -not -path "./third_party/angle/*" ) | xargs rm -rf + ( find . -type d -name ".git" -not -path "./third_party/angle/*" ) | xargs sudo rm -rf - persist_to_workspace: root: ./repos paths: From 7989ce42755544dfacd8d5988641fc1a7c341eaf Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 10:46:30 -0400 Subject: [PATCH 13/24] Only remove top-level .git directory still getting some permission errors with the recursive version. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 177f8b25..ac7be770 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: name: Delete .git directories to save space (skipping angle taken from electron circleci build) command: | cd ./repos/src - ( find . -type d -name ".git" -not -path "./third_party/angle/*" ) | xargs sudo rm -rf + rm -rf .git - persist_to_workspace: root: ./repos paths: From 43b0a251e374b6d61752bcbf94561fadf1ed9c67 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 11:42:11 -0400 Subject: [PATCH 14/24] sudo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac7be770..e4b72f6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: name: Delete .git directories to save space (skipping angle taken from electron circleci build) command: | cd ./repos/src - rm -rf .git + sudo rm -rf .git - persist_to_workspace: root: ./repos paths: From 7ad597e448de4e88b45c22182a79a0f38af26017 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 15:39:38 -0400 Subject: [PATCH 15/24] Open permissions before persisting workspace --- .circleci/config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4b72f6f..f493fb46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,11 +66,16 @@ jobs: command: | cd ./repos/src sudo rm -rf .git + - run: + name: Open permissions of fetched src directory + command: | + sudo chmod -R 777 ./repos/src + sudo chmod 777 .gclient* + sudo chmod -R 777 kaleido/version - persist_to_workspace: root: ./repos paths: - src - - depot_tools - .gclient - .gclient_entries - kaleido/version From a84ee57faa812a7470a196f9ee39e8d92793f1fe Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 16:57:57 -0400 Subject: [PATCH 16/24] Remove non-existent files --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f493fb46..13769974 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,14 +70,11 @@ jobs: name: Open permissions of fetched src directory command: | sudo chmod -R 777 ./repos/src - sudo chmod 777 .gclient* sudo chmod -R 777 kaleido/version - persist_to_workspace: root: ./repos paths: - src - - .gclient - - .gclient_entries - kaleido/version linux_build_x64: From 425ae477a3d2e628f0db5d0bfc2cda5139b38d93 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 18:04:21 -0400 Subject: [PATCH 17/24] Another missing file --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13769974..6992f5d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,6 @@ jobs: root: ./repos paths: - src - - kaleido/version linux_build_x64: machine: From 0ad6228a04d023565dcc094eeb840081f24dbddb Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 18:05:28 -0400 Subject: [PATCH 18/24] update depot_tools before fetch --- repos/linux_scripts/fetch_chromium_docker | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repos/linux_scripts/fetch_chromium_docker b/repos/linux_scripts/fetch_chromium_docker index 9cda3678..145b0260 100755 --- a/repos/linux_scripts/fetch_chromium_docker +++ b/repos/linux_scripts/fetch_chromium_docker @@ -1,5 +1,10 @@ #!/bin/bash # docker run -it -v /media/jmmease/SSD1/chromium_build/repos/:/repos jonmmease/chromium-builder:0.5 /repos/fetch_chromium_docker + +# Update depot tools for fetch +cd /depot_tools/ +git pull + cd /repos ## Remove any prior .gclient file From 7b2838850f95ebb8cff5e85ceeb2ebeaa51a2175 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Wed, 19 Aug 2020 19:29:24 -0400 Subject: [PATCH 19/24] again --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6992f5d4..255fdb18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,6 @@ jobs: name: Open permissions of fetched src directory command: | sudo chmod -R 777 ./repos/src - sudo chmod -R 777 kaleido/version - persist_to_workspace: root: ./repos paths: From 8d9b3657aff643935815733ee3ae891d35d18ddf Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Aug 2020 05:25:02 -0400 Subject: [PATCH 20/24] Move addition of kaleido target to checkout stage This way we don't need to carry around the .git directory after checkout --- repos/linux_scripts/build_kaleido_docker | 14 -------------- repos/linux_scripts/checkout_revision_docker | 11 +++++++++++ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/repos/linux_scripts/build_kaleido_docker b/repos/linux_scripts/build_kaleido_docker index 39e7ef8c..aa104f6c 100755 --- a/repos/linux_scripts/build_kaleido_docker +++ b/repos/linux_scripts/build_kaleido_docker @@ -17,19 +17,5 @@ mkdir -p out/Kaleido_linux_$KALEIDO_ARCH cp /repos/linux_scripts/args_$KALEIDO_ARCH.gn out/Kaleido_linux_$KALEIDO_ARCH/args.gn gn gen out/Kaleido_linux_$KALEIDO_ARCH -# 1) Reset headless/BUILD.gn -git checkout HEAD -- headless/BUILD.gn - -# 2) Append kaleido section to headless build file (src/headless/BUILD.gn) -echo " -executable(\"kaleido\") { - sources = [ \"app/kaleido.cc\" ] - - deps = [ - \":headless_shell_lib\", - \"//skia\", # we need this to override font render hinting in headless build - ] -}" >> headless/BUILD.gn - # perform_kaleido_build expect to be run from src directory /repos/linux_scripts/perform_kaleido_build diff --git a/repos/linux_scripts/checkout_revision_docker b/repos/linux_scripts/checkout_revision_docker index 89548e79..81acf6b6 100755 --- a/repos/linux_scripts/checkout_revision_docker +++ b/repos/linux_scripts/checkout_revision_docker @@ -10,5 +10,16 @@ git checkout tags/$CHROMIUM_TAG gclient sync -D --force --reset gclient runhooks +# Append kaleido section to headless build file (src/headless/BUILD.gn) +echo " +executable(\"kaleido\") { + sources = [ \"app/kaleido.cc\" ] + + deps = [ + \":headless_shell_lib\", + \"//skia\", # we need this to override font render hinting in headless build + ] +}" >> headless/BUILD.gn + # Build credits python /repos/src/tools/licenses.py credits > /repos/CREDITS.html From 64ef185c695b9eb7e17a8e57337e6edaefd96888 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Aug 2020 05:37:56 -0400 Subject: [PATCH 21/24] Install sysroot during checkout instead of in docker container Sysroots had wrong path when installed in docker container --- repos/linux_scripts/checkout_revision_docker | 3 +++ repos/linux_scripts/perform_kaleido_build | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/repos/linux_scripts/checkout_revision_docker b/repos/linux_scripts/checkout_revision_docker index 81acf6b6..6d8f2706 100755 --- a/repos/linux_scripts/checkout_revision_docker +++ b/repos/linux_scripts/checkout_revision_docker @@ -21,5 +21,8 @@ executable(\"kaleido\") { ] }" >> headless/BUILD.gn +# Install sysroots +/repos/src/build/linux/sysroot_scripts/install-sysroot.py --all + # Build credits python /repos/src/tools/licenses.py credits > /repos/CREDITS.html diff --git a/repos/linux_scripts/perform_kaleido_build b/repos/linux_scripts/perform_kaleido_build index bd272e27..3a1607ac 100755 --- a/repos/linux_scripts/perform_kaleido_build +++ b/repos/linux_scripts/perform_kaleido_build @@ -35,11 +35,11 @@ elif [ $KALEIDO_ARCH == "arm64" ]; then # /usr/lib/aarch64-linux-gnu/ for SO_FILE in libnss3.so libnssutil3.so libnspr4.so libplc4.so libplds4.so libsqlite3.so.0 do - cp /debian_sid_arm64-sysroot/usr/lib/aarch64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE + cp /repos/src/build/linux/debian_sid_arm64-sysroot/usr/lib/aarch64-linux-gnu/$SO_FILE /repos/build/kaleido/lib/$SO_FILE done # /usr/lib/aarch64-linux-gnu/nss - cp /debian_sid_arm64-sysroot/usr/lib/aarch64-linux-gnu/nss/* /repos/build/kaleido/lib/ + cp /repos/src/build/linux/debian_sid_arm64-sysroot/usr/lib/aarch64-linux-gnu/nss/* /repos/build/kaleido/lib/ fi From 18cf3c650858d3d0ca6cb71971bee4d761cc10c9 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Aug 2020 05:38:24 -0400 Subject: [PATCH 22/24] Remove sysroot installation from docker container For next build --- repos/linux_scripts/Dockerfile | 8 -------- 1 file changed, 8 deletions(-) diff --git a/repos/linux_scripts/Dockerfile b/repos/linux_scripts/Dockerfile index 0d7da3d9..4179445c 100644 --- a/repos/linux_scripts/Dockerfile +++ b/repos/linux_scripts/Dockerfile @@ -58,11 +58,3 @@ RUN \ curl -s https://chromium.googlesource.com/chromium/src/+/$CHROMIUM_TAG/build/install-build-deps.sh?format=TEXT \ | base64 -d > install-build-deps.sh && chmod +x ./install-build-deps.sh && \ ./install-build-deps.sh --no-syms --no-arm --no-chromeos-fonts --no-nacl --no-prompt - -RUN \ - curl -s https://chromium.googlesource.com/chromium/src/+/$CHROMIUM_TAG/build/linux/sysroot_scripts/sysroots.json?format=TEXT \ - | base64 -d > sysroots.json - -RUN \ - curl -s https://chromium.googlesource.com/chromium/src/+/$CHROMIUM_TAG/build/linux/sysroot_scripts/install-sysroot.py?format=TEXT \ - | base64 -d > install-sysroot.py && python3 ./install-sysroot.py --all From 6259bee5795fd3d88f184bbf3ea5dba772e752fa Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Thu, 20 Aug 2020 16:00:09 -0400 Subject: [PATCH 23/24] xlarge resource class to see if it finishes before timeout --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 255fdb18..34752fb0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ jobs: linux_build_x64: machine: image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run - resource_class: large + resource_class: xlarge steps: - linux_build: arch: "x64" @@ -86,7 +86,7 @@ jobs: linux_build_arm64: machine: image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run - resource_class: large + resource_class: xlarge steps: - linux_build: arch: "arm64" From 2b4dd162395e9cc3129828cefe726ef2602a3cbb Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Fri, 21 Aug 2020 05:14:37 -0400 Subject: [PATCH 24/24] Remove "full" linux scripts approach Now use the plain Linux approach locally and on CI [ci skip] --- README.md | 33 +++++------ repos/linux_full_scripts/Dockerfile | 58 ------------------- repos/linux_full_scripts/build_kaleido | 22 ------- repos/linux_full_scripts/build_kaleido_docker | 10 ---- 4 files changed, 14 insertions(+), 109 deletions(-) delete mode 100644 repos/linux_full_scripts/Dockerfile delete mode 100755 repos/linux_full_scripts/build_kaleido delete mode 100755 repos/linux_full_scripts/build_kaleido_docker diff --git a/README.md b/README.md index db10398d..13b7e627 100644 --- a/README.md +++ b/README.md @@ -94,23 +94,7 @@ $ cd Kaleido ## Linux There are two approaches to building Kaleido on Linux, both of which rely on Docker. -## Method 1 -This approach relies on the `jonmmease/kaleido-builder` docker image, and the scripts in `repos/linux_full_scripts`, to compile Kaleido. This docker image is over 30GB, but in includes a precompiled instance of the Chromium source tree making it possible to compile Kaleido in just a few 10s of seconds. The downside of this approach is that the chromium source tree is not visible outside of the docker image so it may be difficult for development environments to index it. This is the approach used for Continuous Integration on Linux. - -Download docker image -``` -$ docker pull jonmmease/kaleido-builder:0.8 -``` - -Build Kaleido - -``` -$ /repos/linux_full_scripts/build_kaleido -``` - -### Method 2 -This approach relies on the `jonmmease/chromium-builder` docker image, and the scripts in `repos/linux_scripts`, to download the chromium source to a local folder and then build it. This takes longer to get set up than method 2 because the Chromium source tree must be compiled from scratch, but it downloads a copy of the chromium source tree to `repos/src` which makes it possible for development environments like CLion to index the Chromium codebase to provide code completion. - +The Linux build relies on the `jonmmease/chromium-builder` docker image, and the scripts in `repos/linux_scripts`, to download the chromium source to a local folder and then build it. Download docker image ``` $ docker pull jonmmease/chromium-builder:0.8 @@ -126,7 +110,13 @@ Then build the `kaleido` application to `repos/build/kaleido`, and bundle shared create the Python wheel under `repos/kaleido/py/dist/` ``` -$ /repos/linux_scripts/build_kaleido +$ /repos/linux_scripts/build_kaleido x64 +``` + +The above command will build Kaleido for the 64-bit Intel architecture. Kaleido can also be build for 64-bit ARM architecture with + +``` +$ /repos/linux_scripts/build_kaleido arm64 ``` ## MacOS @@ -155,7 +145,12 @@ $ /repos/win_scripts/fetch_chromium.ps1 Then build Kaleido to `repos/build/kaleido`. The build step will also create the Python wheel under `repos/kaleido/py/dist/` ``` -$ /repos/mac_scripts/build_kaleido.ps1 +$ /repos/mac_scripts/build_kaleido.ps1 x64 +``` + +The above commnad will generate a 64-bit build. A 32-bit build can be generated using +``` +$ /repos/mac_scripts/build_kaleido.ps1 x86 ``` # Building Docker containers diff --git a/repos/linux_full_scripts/Dockerfile b/repos/linux_full_scripts/Dockerfile deleted file mode 100644 index eef6cc07..00000000 --- a/repos/linux_full_scripts/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# CLion remote docker environment (How to build docker container, run and stop it) -# -# Build and run: -# docker build -t jonmmease/kaleido-builder:0.6 -f Dockerfile . - -FROM jonmmease/chromium-builder:0.8 - -# Update depot_tools in case there are changes in how to fetch code -RUN cd /depot_tools && git pull - -# Fetch chromium source tree -RUN \ - cd / \ - && fetch --nohooks chromium - -COPY ./repos/linux_scripts/gclient /.gclient - -# Checkout chromium revision -RUN \ - cd /src \ - && git checkout tags/$CHROMIUM_TAG \ - && gclient sync -D --force --reset - -RUN gclient runhooks - -# Remove .git directory to save space -RUN rm -r /src/.git - -# TODO: remove .git directories from third-party dependencies under src/third_party - -# Build create x64 linux out directory -RUN mkdir -p /src/out/Kaleido_linux_x64/ -COPY ./repos/linux_scripts/args_x64.gn /src/out/Kaleido_linux_x64/args.gn - -# Add Kaleido executable -RUN echo ' \ -executable("kaleido") { \ - sources = [ "app/kaleido.cc" ] \ - deps = [ \ - ":headless_shell_lib", \ - "//skia", \ - ] \ -} \ -' >> /src/headless/BUILD.gn - - -# Perform build using headless_example. This will make sure we have all of the -# dependencies that Kaleido needs precompiled. -RUN \ - cd /src \ - && gn gen out/Kaleido_linux_x64 \ - && ninja -C out/Kaleido_linux_x64 -j 20 headless_example - - -# Initialize project directories for other architectures, but don't precompile -RUN mkdir -p /src/out/Kaleido_linux_arm64/ -COPY ./repos/linux_scripts/args_arm64.gn /src/out/Kaleido_linux_arm64/args.gn -RUN cd /src && gn gen out/Kaleido_linux_arm64 diff --git a/repos/linux_full_scripts/build_kaleido b/repos/linux_full_scripts/build_kaleido deleted file mode 100755 index da6e627b..00000000 --- a/repos/linux_full_scripts/build_kaleido +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -if [ $# -eq 0 ]; then - echo "No architecture provided" - exit 1 -fi -export KALEIDO_ARCH=$1 - -# full path to linux_full_scripots/ directory -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# cd to the top-level project directory -cd $DIR/../.. - -# Update version based on git tags -python ./repos/version/build_pep440_version.py - -# Copy README and LICENSE to kaleido directory so it's available in the docker container -cp ./README.md ./repos/kaleido/ -cp ./LICENSE.txt ./repos/kaleido/ - -# Perform build using docker -docker run -it -v `pwd`/repos/:/repos jonmmease/kaleido-builder:0.8 /repos/linux_full_scripts/build_kaleido_docker $KALEIDO_ARCH \ No newline at end of file diff --git a/repos/linux_full_scripts/build_kaleido_docker b/repos/linux_full_scripts/build_kaleido_docker deleted file mode 100755 index 3b51f7fc..00000000 --- a/repos/linux_full_scripts/build_kaleido_docker +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -if [ $# -eq 0 ]; then - echo "No architecture provided" - exit 1 -fi -export KALEIDO_ARCH=$1 - -# perform_kaleido_build expect to be run from src directory -cd /src -/repos/linux_scripts/perform_kaleido_build \ No newline at end of file