Add tests for Jade hww using qemu emulator

JamieDriver · JamieDriver · commit db17fed8ff11 · 2021-11-22T16:08:04.000Z
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -54,6 +54,17 @@ device_matrix_template: &DEVICE_MATRIX_TEMPLATE
       - tar -xvf "mcu.tar.gz"
       - wget -nv "https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/bitcoind_builder/bitcoin/bitcoin.tar.gz"
       - tar -xvf "bitcoin.tar.gz"
+  - env:
+      DEVICE: --jade
+    depends_on:
+      - Jade Sim Builder
+      - dist_builder
+      - bitcoind_builder
+    fetch_sim_script:
+      - wget -nv "https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/Jade Sim Builder/sim/jade.tar.gz"
+      - tar -xvf "jade.tar.gz"
+      - wget -nv "https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/bitcoind_builder/bitcoin/bitcoin.tar.gz"
+      - tar -xvf "bitcoin.tar.gz"
   - env:
       DEVICE: --ledger
     depends_on:
@@ -186,6 +197,18 @@ task:
   sim_artifacts:
     path: "mcu.tar.gz"
 
+task:
+  env:
+    DEVICE: --jade
+  name: Jade Sim Builder
+  sim_work_cache:
+    folder: test/work/jade
+  build_script:
+    - cd test; ./setup_environment.sh $DEVICE; cd ..
+    - tar -czf jade.tar.gz test/work/jade/simulator
+  sim_artifacts:
+    path: "jade.tar.gz"
+
 task:
   env:
     DEVICE: --ledger
diff --git a/ci/cirrus.Dockerfile b/ci/cirrus.Dockerfile
@@ -24,13 +24,15 @@ RUN apt-get install -y \
     libdb-dev \
     libdb++-dev \
     libevent-dev \
+    libgcrypt20-dev \
     libnewlib-arm-none-eabi \
     libsdl2-dev \
     libsdl2-image-dev \
     libssl-dev \
     libtool \
     libudev-dev \
     libusb-1.0-0-dev \
+    ninja-build \
     pkg-config \
     protobuf-compiler \
     qemu-user-static
@@ -48,7 +50,7 @@ ENV PATH="/root/.cargo/bin:$PATH"
 # as needed.
 # e.g.,
 # docker build -f ci/cirrus.Dockerfile -t hwi_test .
-# docker run -it --entrypoint /bin/bash hwi_tst
+# docker run -it --entrypoint /bin/bash hwi_test
 # cd test; poetry run ./run_tests.py --ledger --coldcard --interface=cli --device-only
 ####################
 
@@ -68,6 +70,7 @@ ENV PATH="/root/.cargo/bin:$PATH"
 #RUN cd test; ./setup_environment.sh --bitbox01
 #RUN cd test; ./setup_environment.sh --ledger
 #RUN cd test; ./setup_environment.sh --keepkey
+#RUN cd test; ./setup_environment.sh --jade
 #RUN cd test; ./setup_environment.sh --bitcoind
 #
 ## Once everything has been built, put rest of files in place
diff --git a/hwilib/devices/jade.py b/hwilib/devices/jade.py
@@ -54,8 +54,12 @@
     parse_multisig
 )
 
+import logging
 import os
 
+# The test emulator port
+SIMULATOR_PATH = 'tcp:127.0.0.1:2222'
+
 JADE_DEVICE_IDS = [(0x10c4, 0xea60)]
 HAS_NETWORKING = hasattr(jade, '_http_request')
 
@@ -127,20 +131,27 @@ def __init__(self, path: str, password: str = '', expert: bool = False, timeout:
         self.jade = JadeAPI.create_serial(path, timeout=timeout)
         self.jade.connect()
 
-        # Push some host entropy into jade
-        self.jade.add_entropy(os.urandom(32))
+        verinfo = self.jade.get_version_info()
+        uninitialized = verinfo['JADE_STATE'] not in ['READY', 'TEMP']
 
-        # Ensure Jade unlocked
-        authenticated = False
-        while not authenticated:
-            if not HAS_NETWORKING and self.jade.get_version_info()['JADE_STATE'] not in ['READY', 'TEMP']:
-                # Wallet not initialised nor do we have networking dependencies
+        if path == SIMULATOR_PATH:
+            if uninitialized:
+                # Connected to simulator but it appears to have no wallet set
+                raise DeviceNotReadyError('Use JadeAPI.set_[seed|mnemonic] to set simulator wallet')
+        else:
+            if uninitialized and not HAS_NETWORKING:
+                # Wallet not initialised/unlocked nor do we have networking dependencies
                 # User must use 'Emergency Restore' feature to enter mnemonic on Jade hw
                 raise DeviceNotReadyError('Use "Emergency Restore" feature on Jade hw to enter wallet mnemonic')
 
+            # Push some host entropy into jade
+            self.jade.add_entropy(os.urandom(32))
+
             # Authenticate the user - this may require a PIN and pinserver interaction
             # (if we have required networking dependencies)
-            authenticated = self.jade.auth_user(self._network())
+            authenticated = False
+            while not authenticated:
+                authenticated = self.jade.auth_user(self._network())
 
     # Retrieves the public key at the specified BIP 32 derivation path
     @jade_exception
@@ -477,26 +488,42 @@ def toggle_passphrase(self) -> bool:
 def enumerate(password: str = '') -> List[Dict[str, Any]]:
     results = []
 
+    def _get_device_entry(device_model: str, device_path: str) -> Dict[str, Any]:
+        d_data: Dict[str, Any] = {}
+        d_data['type'] = 'jade'
+        d_data['model'] = device_model
+        d_data['path'] = device_path
+        d_data['needs_pin_sent'] = False
+        d_data['needs_passphrase_sent'] = False
+
+        client = None
+        with handle_errors(common_err_msgs['enumerate'], d_data):
+            client = JadeClient(device_path, password, timeout=1)
+            d_data['fingerprint'] = client.get_master_fingerprint().hex()
+
+        if client:
+            client.close()
+
+        return d_data
+
     # Jade is not really an HID device, it shows as a serial/com port device.
     # Scan com ports looking for the relevant vid and pid, and use 'path' to
     # hold the path to the serial port device, eg. /dev/ttyUSB0
     for devinfo in list_ports.comports():
         if (devinfo.vid, devinfo.pid) in JADE_DEVICE_IDS:
-            d_data: Dict[str, Any] = {}
-            d_data['type'] = 'jade'
-            d_data['model'] = 'jade'
-            d_data['path'] = devinfo.device
-            d_data['needs_pin_sent'] = False
-            d_data['needs_passphrase_sent'] = False
-
-            client = None
-            with handle_errors(common_err_msgs['enumerate'], d_data):
-                client = JadeClient(devinfo.device, password, timeout=1)
-                d_data['fingerprint'] = client.get_master_fingerprint().hex()
-
-            if client:
-                client.close()
-
-            results.append(d_data)
+            results.append(_get_device_entry('jade', devinfo.device))
+
+    # If we can connect to the simulator, add it too
+    try:
+        with JadeAPI.create_serial(SIMULATOR_PATH, timeout=1) as jade:
+            verinfo = jade.get_version_info()
+
+        if verinfo is not None:
+            results.append(_get_device_entry('jade_simulator', SIMULATOR_PATH))
+
+    except Exception as e:
+        # If we get any sort of error do not add the simulator
+        logging.debug(f'Failed to connect to Jade simulator at {SIMULATOR_PATH}')
+        logging.debug(e)
 
     return results
diff --git a/test/run_tests.py b/test/run_tests.py
@@ -15,6 +15,7 @@
 from test_ledger import ledger_test_suite
 from test_digitalbitbox import digitalbitbox_test_suite
 from test_keepkey import keepkey_test_suite
+from test_jade import jade_test_suite
 from test_udevrules import TestUdevRulesInstaller
 
 parser = argparse.ArgumentParser(description='Setup the testing environment and run automated tests')
@@ -38,6 +39,10 @@
 keepkey_group.add_argument('--no-keepkey', dest='keepkey', help='Do not run Keepkey test with emulator', action='store_false')
 keepkey_group.add_argument('--keepkey', dest='keepkey', help='Run Keepkey test with emulator', action='store_true')
 
+jade_group = parser.add_mutually_exclusive_group()
+jade_group.add_argument('--no-jade', dest='jade', help='Do not run Jade test with emulator', action='store_false')
+jade_group.add_argument('--jade', dest='jade', help='Run Jade test with emulator', action='store_true')
+
 dbb_group = parser.add_mutually_exclusive_group()
 dbb_group.add_argument('--no_bitbox01', dest='bitbox01', help='Do not run Digital Bitbox test with simulator', action='store_false')
 dbb_group.add_argument('--bitbox01', dest='bitbox01', help='Run Digital Bitbox test with simulator', action='store_true')
@@ -48,14 +53,15 @@
 parser.add_argument('--keepkey-path', dest='keepkey_path', help='Path to Keepkey emulator', default='work/keepkey-firmware/bin/kkemu')
 parser.add_argument('--bitbox01-path', dest='bitbox01_path', help='Path to Digital Bitbox simulator', default='work/mcu/build/bin/simulator')
 parser.add_argument('--ledger-path', dest='ledger_path', help='Path to Ledger emulator', default='work/speculos/speculos.py')
+parser.add_argument('--jade-path', dest='jade_path', help='Path to Jade qemu emulator', default='work/jade/simulator')
 
 parser.add_argument('--all', help='Run tests on all existing simulators', default=False, action='store_true')
 parser.add_argument('--bitcoind', help='Path to bitcoind', default='work/bitcoin/src/bitcoind')
 parser.add_argument('--interface', help='Which interface to send commands over', choices=['library', 'cli', 'bindist', 'stdin'], default='library')
 
 parser.add_argument("--device-only", help="Only run device tests", action="store_true")
 
-parser.set_defaults(trezor_1=None, trezor_t=None, coldcard=None, keepkey=None, bitbox01=None, ledger=None)
+parser.set_defaults(trezor_1=None, trezor_t=None, coldcard=None, keepkey=None, bitbox01=None, ledger=None, jade=None)
 
 args = parser.parse_args()
 
@@ -80,6 +86,7 @@
     args.keepkey = True if args.keepkey is None else args.keepkey
     args.bitbox01 = True if args.bitbox01 is None else args.bitbox01
     args.ledger = True if args.ledger is None else args.ledger
+    args.jade = True if args.jade is None else args.jade
 else:
     # Default all false unless overridden
     args.trezor_1 = False if args.trezor_1 is None else args.trezor_1
@@ -88,8 +95,9 @@
     args.keepkey = False if args.keepkey is None else args.keepkey
     args.bitbox01 = False if args.bitbox01 is None else args.bitbox01
     args.ledger = False if args.ledger is None else args.ledger
+    args.jade = False if args.jade is None else args.jade
 
-if args.trezor_1 or args.trezor_t or args.coldcard or args.ledger or args.keepkey or args.bitbox01:
+if args.trezor_1 or args.trezor_t or args.coldcard or args.ledger or args.keepkey or args.bitbox01 or args.jade:
     # Start bitcoind
     rpc, userpass = start_bitcoind(args.bitcoind)
 
@@ -105,5 +113,7 @@
         success &= keepkey_test_suite(args.keepkey_path, rpc, userpass, args.interface)
     if success and args.ledger:
         success &= ledger_test_suite(args.ledger_path, rpc, userpass, args.interface)
+    if success and args.jade:
+        success &= jade_test_suite(args.jade_path, rpc, userpass, args.interface)
 
 sys.exit(not success)
diff --git a/test/setup_environment.sh b/test/setup_environment.sh
@@ -26,6 +26,10 @@ while [[ $# -gt 0 ]]; do
         build_keepkey=1
         shift
         ;;
+        --jade)
+        build_jade=1
+        shift
+        ;;
         --bitcoind)
         build_bitcoind=1
         shift
@@ -37,6 +41,7 @@ while [[ $# -gt 0 ]]; do
         build_bitbox01=1
         build_ledger=1
         build_keepkey=1
+        build_jade=1
         build_bitcoind=1
         shift
         ;;
@@ -258,6 +263,101 @@ if [[ -n ${build_ledger} ]]; then
     cd ..
 fi
 
+if [[ -n ${build_jade} ]]; then
+    mkdir -p jade
+    cd jade
+
+    # Clone Blockstream Jade firmware if it doesn't exist, or update it if it does
+    if [ ! -d "jade" ]; then
+        git clone --recursive --branch master https://github.com/Blockstream/Jade.git ./jade
+        cd jade
+    else
+        cd jade
+        git fetch
+
+        # Determine if we need to pull. From https://stackoverflow.com/a/3278427
+        UPSTREAM=${1:-'@{u}'}
+        LOCAL=$(git rev-parse @)
+        REMOTE=$(git rev-parse "$UPSTREAM")
+        BASE=$(git merge-base @ "$UPSTREAM")
+
+        if [ $LOCAL = $REMOTE ]; then
+            echo "Up-to-date"
+        elif [ $LOCAL = $BASE ]; then
+            git pull
+        fi
+        git submodule update --recursive --init
+    fi
+
+    # Deduce the relevant versions of esp-idf and qemu to use
+    ESP_IDF_BRANCH=$(grep "ARG ESP_IDF_BRANCH=" Dockerfile | cut -d\= -f2)
+    ESP_IDF_COMMIT=$(grep "ARG ESP_IDF_COMMIT=" Dockerfile | cut -d\= -f2)
+    ESP_QEMU_BRANCH=$(grep "ARG ESP_QEMU_BRANCH=" Dockerfile | cut -d\= -f2)
+    ESP_QEMU_COMMIT=$(grep "ARG ESP_QEMU_COMMIT=" Dockerfile | cut -d\= -f2)
+    cd ..
+
+    # Build the qemu emulator
+    if [ ! -d "qemu" ]; then
+        git clone --depth 1 --branch ${ESP_QEMU_BRANCH} --single-branch --recursive https://github.com/espressif/qemu.git ./qemu
+        cd qemu
+        ./configure --target-list=xtensa-softmmu \
+            --enable-gcrypt \
+            --enable-debug --enable-sanitizers \
+            --disable-strip --disable-user \
+            --disable-capstone --disable-vnc \
+            --disable-sdl --disable-gtk
+    else
+        cd qemu
+        git fetch
+    fi
+    git checkout ${ESP_QEMU_COMMIT}
+    git submodule update --recursive --init
+    ninja -C build
+    cd ..
+
+    # Build the esp-idf toolchain
+    if [ ! -d "esp-idf" ]; then
+        git clone --depth=1 --branch ${ESP_IDF_BRANCH} --single-branch --recursive https://github.com/espressif/esp-idf.git ./esp-idf
+        cd esp-idf
+    else
+        cd esp-idf
+        git fetch
+    fi
+    git checkout ${ESP_IDF_COMMIT}
+    git submodule update --recursive --init
+
+    # Install the isp-idf tools in a given location (otherwise defauts to user home dir)
+    IDF_TOOLS_PATH=$(pwd)/tools
+    ./install.sh
+    . ./export.sh
+    cd ..
+
+    # Build Blockstream Jade firmware configured for the emulator
+    cd jade
+    rm -fr sdkconfig
+    cp configs/sdkconfig_qemu.defaults sdkconfig.defaults
+    idf.py all
+
+    # Make the qemu flash image
+    esptool.py --chip esp32 merge_bin --fill-flash-size 4MB -o main/qemu/flash_image.bin \
+    --flash_mode dio --flash_freq 40m --flash_size 4MB \
+    0x9000 build/partition_table/partition-table.bin \
+    0xe000 build/ota_data_initial.bin \
+    0x1000 build/bootloader/bootloader.bin \
+    0x10000 build/jade.bin
+    cd ..
+
+    # Extract the minimal artifacts required to run the emulator
+    rm -fr simulator
+    mkdir simulator
+    cp qemu/build/qemu-system-xtensa simulator/
+    cp -R qemu/pc-bios simulator/
+    cp jade/main/qemu/flash_image.bin simulator/
+    cp jade/main/qemu/qemu_efuse.bin simulator/
+
+    cd ..
+fi
+
 if [[ -n ${build_bitcoind} ]]; then
     # Clone bitcoind if it doesn't exist, or update it if it does
     bitcoind_setup_needed=false
diff --git a/test/test_device.py b/test/test_device.py
@@ -20,10 +20,10 @@
 SUPPORTS_MS_DISPLAY = {'trezor_1', 'keepkey', 'coldcard', 'trezor_t'}
 SUPPORTS_XPUB_MS_DISPLAY = {'trezor_1', 'trezor_t'}
 SUPPORTS_UNSORTED_MS = {"trezor_1", "trezor_t"}
-SUPPORTS_MIXED = {'coldcard', 'trezor_1', 'digitalbitbox', 'keepkey', 'trezor_t'}
-SUPPORTS_MULTISIG = {'ledger', 'trezor_1', 'digitalbitbox', 'keepkey', 'coldcard', 'trezor_t'}
-SUPPORTS_EXTERNAL = {'ledger', 'trezor_1', 'digitalbitbox', 'keepkey', 'coldcard', 'trezor_t'}
-SUPPORTS_OP_RETURN = {'ledger', 'digitalbitbox', 'trezor_1', 'trezor_t', 'keepkey'}
+SUPPORTS_MIXED = {'coldcard', 'trezor_1', 'digitalbitbox', 'keepkey', 'trezor_t', 'jade'}
+SUPPORTS_MULTISIG = {'ledger', 'trezor_1', 'digitalbitbox', 'keepkey', 'coldcard', 'trezor_t', 'jade'}
+SUPPORTS_EXTERNAL = {'ledger', 'trezor_1', 'digitalbitbox', 'keepkey', 'coldcard', 'trezor_t', 'jade'}
+SUPPORTS_OP_RETURN = {'ledger', 'digitalbitbox', 'trezor_1', 'trezor_t', 'keepkey', 'jade'}
 
 # Class for emulator control
 class DeviceEmulator():
diff --git a/test/test_jade.py b/test/test_jade.py
