Linux arm64 support (#35)

Use the plain Linux approach locally and on CI

.circleci/config.yml

@@ -3,30 +3,94 @@ version: 2.1
 orbs:
   win: circleci/[email protected] # The Windows orb give you everything you need to start using the Windows executor.
 
-jobs:
+commands:
   linux_build:
-    machine: 
-      image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run
-    resource_class: large
+    parameters:
+      arch:
+        default: "x64"
+        type: string
     steps:
       - checkout
-      - run: docker --version
-      - run: docker pull jonmmease/kaleido-builder:0.7
-      - run: ./repos/linux_full_scripts/build_kaleido
+      - attach_workspace:
+          at: ./repos
+      #- run: docker pull jonmmease/chomium-builder:0.8
+      - run: ./repos/linux_scripts/build_kaleido <<parameters.arch>>
       - run: sudo chmod -R 777 repos/build
       - run: sudo chmod -R 777 repos/kaleido/py
       - run: ls repos/build/kaleido
-      - run: pushd repos/build && zip -r kaleido_linux.zip ./*
+      - run: pushd repos/build && zip -r kaleido_linux_<<parameters.arch>>.zip ./*
       - run: pushd repos/kaleido/py/ && zip -r kaleido_wheel.zip ./dist
       - store_artifacts:
-          path: ./repos/build/kaleido_linux.zip
+          path: ./repos/build/kaleido_linux_<<parameters.arch>>.zip
       - store_artifacts:
           path: ./repos/kaleido/py/kaleido_wheel.zip
       - persist_to_workspace:
           root: ./repos
           paths:
             - kaleido/py/
+
+  windows_build:
+    parameters:
+      arch:
+        default: "x64"
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: Update PowerShell.Archive to fix slash direction
+          command: Install-Module Microsoft.PowerShell.Archive -MinimumVersion 1.2.3.0 -Repository PSGallery -Force
+      - attach_workspace:
+          at: .\repos
+      - run: .\repos\win_scripts\build_kaleido.ps1 <<parameters.arch>>
+      - run: dir .\repos\build\kaleido
+      - run: dir .\repos\kaleido\py\dist
+      - store_artifacts:
+          path: ./repos/build/kaleido_win_<<parameters.arch>>.zip
+      - store_artifacts:
+          path: ./repos/kaleido/py/kaleido_wheel.zip
+
+jobs:
+  linux_fetch:
+    machine: 
+      image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run
+    resource_class: large
+    steps:
+      - checkout
+      - run: docker pull jonmmease/chromium-builder:0.8
+      - run:
+          name: Fetch detpot_tools and chromium source tree
+          command: ./repos/linux_scripts/fetch_chromium
+          no_output_timeout: 30m
+      - run:
+          name: Delete .git directories to save space (skipping angle taken from electron circleci build)
+          command: |
+            cd ./repos/src
+            sudo rm -rf .git
+      - run:
+          name: Open permissions of fetched src directory
+          command: |
+            sudo chmod -R 777 ./repos/src
+      - persist_to_workspace:
+          root: ./repos
+          paths:
+            - src
 
+  linux_build_x64:
+    machine: 
+      image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run
+    resource_class: xlarge
+    steps:
+      - linux_build:
+          arch: "x64"
+
+  linux_build_arm64:
+    machine: 
+      image: ubuntu-1604:201903-01 # the primary container, where your job's commands are run
+    resource_class: xlarge
+    steps:
+      - linux_build:
+          arch: "arm64"
+
   linux_test_37:
     docker:
       - image: continuumio/miniconda3
@@ -157,57 +221,36 @@ jobs:
             - .gclient_entries
             - kaleido/version
 
-  windows_build:
+  windows_build_x64:
     executor:
       name: win/default
       size: "xlarge"
       shell: powershell.exe
     steps:
-      - checkout
-      - run:
-          name: Update PowerShell.Archive to fix slash direction
-          command: Install-Module Microsoft.PowerShell.Archive -MinimumVersion 1.2.3.0 -Repository PSGallery -Force
-      - attach_workspace:
-          at: .\repos
-      - run: .\repos\win_scripts\build_kaleido.ps1 x64
-      - run: dir .\repos\build\kaleido
-      - run: dir .\repos\kaleido\py\dist
-      - store_artifacts:
-          path: ./repos/build/kaleido_win.zip
-      - store_artifacts:
-          path: ./repos/kaleido/py/kaleido_wheel.zip
+      - windows_build:
+          arch: "x64"
 
-  windows_build-32:
+  windows_build_x86:
     executor:
       name: win/default
       size: "xlarge"
       shell: powershell.exe
     steps:
-      - checkout
-      - run:
-          name: Update PowerShell.Archive to fix slash direction
-          command: Install-Module Microsoft.PowerShell.Archive -MinimumVersion 1.2.3.0 -Repository PSGallery -Force
-      - attach_workspace:
-          at: .\repos
-      - run: .\repos\win_scripts\build_kaleido.ps1 x86
-      - run: dir .\repos\build\kaleido
-      - run: dir .\repos\kaleido\py\dist
-      - store_artifacts:
-          path: ./repos/build/kaleido_win.zip
-      - store_artifacts:
-          path: ./repos/kaleido/py/kaleido_wheel.zip
+      - windows_build:
+          arch: "x86"
 
 workflows:
-  build-windows:
+  build-windows-x64:
     jobs:
       - windows_fetch
-      - windows_build:
+      - windows_build_x64:
           requires:
             - windows_fetch
-  build-windows-32:
+
+  build-windows-x86:
     jobs:
       - windows_fetch
-      - windows_build-32:
+      - windows_build_x86:
           requires:
             - windows_fetch
   build-mac:
@@ -220,13 +263,22 @@ workflows:
           requires:
             - mac_build_blink
 
-  build-linux:
+  build-linux-x64:
     jobs:
-      - linux_build
+      - linux_fetch
+      - linux_build_x64:
+          requires:
+            - linux_fetch
       - linux_test_37:
           requires:
-            - linux_build
+            - linux_build_x64
       - linux_test_27:
           requires:
-            - linux_build
+            - linux_build_x64
 
+  build-linux-arm64:
+    jobs:
+      - linux_fetch
+      - linux_build_arm64:
+          requires:
+            - linux_fetch

.dockerignore

@@ -1 +1,2 @@
- repos/
+repos
+!repos/linux_scripts/

README.md

@@ -94,26 +94,10 @@ $ cd Kaleido
 ## Linux
 There are two approaches to building Kaleido on Linux, both of which rely on Docker.
 
-## Method 1
-This approach relies on the `jonmmease/kaleido-builder` docker image, and the scripts in `repos/linux_full_scripts`, to compile Kaleido.  This docker image is over 30GB, but in includes a precompiled instance of the Chromium source tree making it possible to compile Kaleido in just a few 10s of seconds. The downside of this approach is that the chromium source tree is not visible outside of the docker image so it may be difficult for development environments to index it. This is the approach used for Continuous Integration on Linux.
-
+The Linux build relies on the `jonmmease/chromium-builder` docker image, and the scripts in `repos/linux_scripts`, to download the chromium source to a local folder and then build it.
 Download docker image
 ```
-$ docker pull jonmmease/kaleido-builder:0.7
-```
-
-Build Kaleido
-
-```
-$ /repos/linux_full_scripts/build_kaleido
-```
-
-### Method 2
-This approach relies on the `jonmmease/chromium-builder` docker image, and the scripts in `repos/linux_scripts`, to download the chromium source to a local folder and then build it.  This takes longer to get set up than method 2 because the Chromium source tree must be compiled from scratch, but it downloads a copy of the chromium source tree to `repos/src` which makes it possible for development environments like CLion to index the Chromium codebase to provide code completion.
-
-Download docker image
-```
-$ docker pull jonmmease/chromium-builder:0.7
+$ docker pull jonmmease/chromium-builder:0.8
 ```
 
 Fetch the Chromium codebase and checkout the specific tag, then sync all dependencies
@@ -126,7 +110,13 @@ Then build the `kaleido` application to `repos/build/kaleido`, and bundle shared
 create the Python wheel under `repos/kaleido/py/dist/`
 
 ```
-$ /repos/linux_scripts/build_kaleido
+$ /repos/linux_scripts/build_kaleido x64
+```
+
+The above command will build Kaleido for the 64-bit Intel architecture. Kaleido can also be build for 64-bit ARM architecture with
+
+```
+$ /repos/linux_scripts/build_kaleido arm64
 ```
 
 ## MacOS
@@ -155,7 +145,12 @@ $ /repos/win_scripts/fetch_chromium.ps1
 
 Then build Kaleido to `repos/build/kaleido`. The build step will also create the Python wheel under `repos/kaleido/py/dist/`
 ```
-$ /repos/mac_scripts/build_kaleido.ps1
+$ /repos/mac_scripts/build_kaleido.ps1 x64
+```
+
+The above commnad will generate a 64-bit build. A 32-bit build can be generated using 
+```
+$ /repos/mac_scripts/build_kaleido.ps1 x86
 ```
 
 # Building Docker containers
@@ -165,14 +160,14 @@ The `chromium-builder` container mostly follows the instructions at https://chro
 Build container with:
 
 ```
-$ docker build -t jonmmease/chromium-builder:0.7 -f repos/linux_scripts/Dockerfile .
+$ docker build -t jonmmease/chromium-builder:0.8 -f repos/linux_scripts/Dockerfile .
 ```
 
 ## kaleido-builder
 This container contains a pre-compiled version of chromium source tree. Takes several hours to build!
 
 ```
-$ docker build -t jonmmease/kaleido-builder:0.7 -f repos/linux_full_scripts/Dockerfile .
+$ docker build -t jonmmease/kaleido-builder:0.8 -f repos/linux_full_scripts/Dockerfile .
 ```
 
 

repos/kaleido/py/setup.py

@@ -127,18 +127,35 @@ def run(self):
         cmd_obj = self.distribution.get_command_obj('bdist_wheel')
 
         # Use current platform as plat_name, but replace linux with manylinux2014
-        cmd_obj.plat_name = distutils.util.get_platform().replace("linux-", "manylinux1-")
+        cmd_obj.plat_name = distutils.util.get_platform()
 
         # Handle windows 32-bit cross compilation
         print(os.environ.get("KALEIDO_ARCH", "x64"))
         if cmd_obj.plat_name.startswith("win-"):
             arch = os.environ.get("KALEIDO_ARCH", "x64")
             if arch == "x86":
                 cmd_obj.plat_name = "win32"
-        print(cmd_obj.plat_name)
+            elif arch == "x64":
+                cmd_obj.plat_name = "win_amd64"
+            else:
+                raise ValueError(
+                    "Unsupported architecture {arch} for plat_name {plat_name}".format(
+                        arch=arch, plat_name=cmd_obj.plat_name)
+                )
+        elif cmd_obj.plat_name.startswith("linux"):
+            arch = os.environ.get("KALEIDO_ARCH", "x64")
+            if arch == "x64":
+                cmd_obj.plat_name = "manylinux1-x86_64"
+            elif arch == "x86":
+                cmd_obj.plat_name = "manylinux1-i686"
+            elif arch == "arm64":
+                cmd_obj.plat_name = "manylinux2014-aarch64"
+            elif arch == "arm":
+                cmd_obj.plat_name = "manylinux2014-armv7l"
+
         # Set macos platform to 10.10 to match chromium build target (See build/config/mac/mac_sdk.gni)
         # rather than Python environment
-        if cmd_obj.plat_name.startswith("macosx"):
+        elif cmd_obj.plat_name.startswith("macosx"):
             cmd_obj.plat_name = "macosx-10.10-x86_64"
 
         cmd_obj.python_tag = 'py2.py3'