Merge pull request #1763 from phargogh/feature/1580-automate-codesign…

…ing-in-release

Automate Code Signing of Windows Binary

.github/workflows/build-and-test.yml

@@ -430,21 +430,20 @@ jobs:
           WORKBENCH_BINARY=$(find "$(pwd)/workbench/dist" -type f -name 'invest_*.dmg' | head -n 1)
           make WORKBENCH_BIN_TO_SIGN="$WORKBENCH_BINARY" codesign_mac
 
-      #- name: Sign binaries (Windows)
-      #  if: github.event_name != 'pull_request' && matrix.os == 'windows-latest' # secrets not available in PR
-      #  env:
-      #    CERT_FILE: Stanford-natcap-code-signing-cert-expires-2024-01-26.p12
-      #    CERT_PASS: ${{ secrets.WINDOWS_CODESIGN_CERT_PASS }}
-      #  run: |
-      #    # figure out the path to signtool.exe (it keeps changing with SDK updates)
-      #    SIGNTOOL_PATH=$(find 'C:\\Program Files (x86)\\Windows Kits\\10' -type f -name 'signtool.exe*' | head -n 1)
-      #    WORKBENCH_BINARY=$(find "$(pwd)/workbench/dist" -type f -name 'invest_*.exe' | head -n 1)
-      #    make WORKBENCH_BIN_TO_SIGN="$WORKBENCH_BINARY" SIGNTOOL="$SIGNTOOL_PATH" codesign_windows
-
       - name: Deploy artifacts to GCS
         if: github.event_name != 'pull_request'
         run: make deploy
 
+      # This relies on the file existing on GCP, so it must be run after `make
+      # deploy` is called.
+      - name: Queue windows binaries for signing
+        if: github.event_name != 'pull_request' && matrix.os == 'windows-latest' # secrets not available in PR
+        env:
+          ACCESS_TOKEN: ${{ secrets.CODESIGN_QUEUE_ACCESS_TOKEN }}
+        run: |
+          cd codesigning
+          bash enqueue-current-windows-installer.sh
+
       - name: Upload workbench binary artifact
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/release-part-2.yml

@@ -72,14 +72,16 @@ jobs:
           rm -rf artifacts/Wheel*
 
           # download each artifact separately so that the command will fail if any is missing
-          for artifact in Workbench-Windows-binary \
-                          Workbench-macOS-binary \
+          for artifact in Workbench-macOS-binary \
                           InVEST-sample-data \
                           InVEST-user-guide
           do
             gh run download $RUN_ID --dir artifacts --name "$artifact"
           done
 
+          # download the signed windows workbench file from GCS
+          wget --directory-prefix=artifacts https://storage.googleapis.com/releases.naturalcapitalproject.org/invest/${{ env.VERSION }}/workbench/invest_${{ env.VERSION }}_workbench_win32_x64.exe
+
           # We build one sdist per combination of OS and python version, so just
           # download and unzip all of them into an sdists directory so we can
           # just grab the first one.  This approach is more flexible to changes

HISTORY.rst

@@ -45,6 +45,9 @@ Unreleased Changes
     * Now testing and building against Python 3.13.
       No longer testing and building with Python 3.8, which reached EOL.
       https://github.com/natcap/invest/issues/1755
+    * InVEST's windows binaries are now distributed once again with a valid
+      signature, signed by Stanford University.
+      https://github.com/natcap/invest/issues/1580
 * Annual Water Yield
     * Fixed an issue where the model would crash if the valuation table was
       provided, but the demand table was not.  Validation will now warn about

Makefile

@@ -356,10 +356,8 @@ codesign_mac:
 	codesign --timestamp --verbose --sign Stanford $(WORKBENCH_BIN_TO_SIGN)
 
 codesign_windows:
-	$(GSUTIL) cp gs://stanford_cert/$(CERT_FILE) $(BUILD_DIR)/$(CERT_FILE)
 	"$(SIGNTOOL)" sign -fd SHA256 -f $(BUILD_DIR)/$(CERT_FILE) -p $(CERT_PASS) $(WORKBENCH_BIN_TO_SIGN)
 	"$(SIGNTOOL)" timestamp -tr http://timestamp.sectigo.com -td SHA256 $(WORKBENCH_BIN_TO_SIGN)
-	$(RM) $(BUILD_DIR)/$(CERT_FILE)
 	@echo "Installer was signed with signtool"
 
 deploy:

codesigning/Makefile

@@ -0,0 +1,21 @@
+.PHONY: deploy-cloudfunction deploy-worker
+
+deploy-cloudfunction:
+	gcloud functions deploy \
+		--project natcap-servers \
+		codesigning-queue \
+		--memory=256Mi \
+		--trigger-http \
+		--gen2 \
+		--region us-west1 \
+		--allow-unauthenticated \
+		--entry-point main \
+		--runtime python312 \
+		--source gcp-cloudfunc/
+
+# NOTE: This must be executed from a computer that has SSH access to ncp-inkwell.
+deploy-worker:
+	cd signing-worker && ansible-playbook \
+		--ask-become-pass \
+		--inventory-file inventory.ini \
+		playbook.yml

codesigning/README.md

@@ -0,0 +1,74 @@
+# InVEST Codesigning Service
+
+This directory contains all of the functional code and configuration (minus a
+few secrets) that are needed to deploy our code-signing service.  There are
+three key components to this service:
+
+1. A cloud function (`gcp-cloudfunc/') that handles a google cloud
+   storage-backed cloud function that operates as a high-latency queue.
+2. A script (`enqueue-binary.py`) that will enqueue a binary that already
+   exists on one of our GCS buckets.
+3. A `systemd` service that runs on a debian:bookworm machine and periodically
+   polls the cloud function to dequeue the next item to sign.
+
+## Deploying the Cloud Function
+
+The necessary `gcloud` deployment configuration can be executed with
+
+```bash
+$ make deploy-cloudfunction
+```
+
+### Secrets
+
+The current deployment process requires you to manually create an environment
+variable, ``ACCESS_TOKEN``, that contains the secret token shared by the cloud
+function, systemd service and enqueue script.
+
+## Deploying the Systemd Service
+
+To deploy the systemd service, you will need to be on a computer that has ssh
+access to `ncp-inkwell`, which is a computer that has a yubikey installed in
+it.  This computer is assumed to run debian:bookworm at this time.  To deploy
+(non-secret) changes to ncp-inkwell, run this in an environment where
+`ansible-playbook` is available (`pip install ansible` to install):
+
+```bash
+$ make deploy-worker
+```
+
+### Secrets
+
+The systemd service requires several secrets to be available in the codesigning
+workspace, which is located at `/opt/natcap-codesign':
+
+* `/opt/natcap-codesign/pass.txt` is a plain text file containing only the PIN
+  for the yubikey
+* `/opt/natcap-codesign/access_token.txt` is a plain text file containing the
+  access token shared with the cloud function, systemd service and enqueue script.
+* `/opt/natcap-codesign/slack_token.txt` is a plain text file containing the
+  slack token used to post messages to our slack workspace.
+* `/opt/natcap-codesign/natcap-servers-1732552f0202.json` is a GCP service
+  account key used to authenticate to google cloud storage.  This file must be
+  available in the `gcp-cloudfunc/` directory at the time of deployment.
+
+
+## Future Work
+
+### Authenticate to the function with Identity Federation
+
+The cloud function has access controlled by a secret token, which is not ideal.
+Instead, we should be using github/GCP identity federation to control access.
+
+### Trigger the function with GCS Events
+
+GCP Cloud Functions have the ability to subscribe to bucket events, which
+should allow us to subscribe very specifically to just those `finalize` events
+that apply to the Windows workbench binaries.  Doing so will require reworking this cloud function into 2 cloud functions:
+
+1. An endpoint for ncp-inkwell to poll for the next binary to sign
+2. A cloud function that subscribes to GCS bucket events and enqueues the binary to sign.
+
+Relevant docs include:
+* https://cloud.google.com/functions/docs/writing/write-event-driven-functions#cloudevent-example-python
+

codesigning/enqueue-binary.py

@@ -0,0 +1,28 @@
+"""Enqueue a windows binary for signing.
+
+To call this script, you need to set the ACCESS_TOKEN environment variable from
+the software team secrets store.
+
+Example invocation:
+
+    $ ACCESS_TOKEN=abcs1234 python3 enqueue-binary.py <gs:// uri to binary on gcs>
+"""
+
+import os
+import sys
+
+import requests
+
+DATA = {
+    'token': os.environ['ACCESS_TOKEN'],
+    'action': 'enqueue',
+    'url': sys.argv[1].replace(
+        'gs://', 'https://storage.googleapis.com/'),
+}
+response = requests.post(
+    'https://us-west1-natcap-servers.cloudfunctions.net/codesigning-queue',
+    json=DATA
+)
+if response.status_code >= 400:
+    print(response.text)
+    sys.exit(1)

codesigning/enqueue-current-windows-installer.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env sh
+#
+# Run this script to enqueue the windows binary for this current version of the
+# InVEST windows workbench installer for code signing.
+#
+# NOTE: this script must be run from the directory containing this script.
+
+version=$(python -m setuptools_scm)
+url_base=$(make -C .. --no-print-directory print-DIST_URL_BASE | awk ' { print $3 } ')
+url="${url_base}/workbench/invest_${version}_workbench_win32_x64.exe"
+
+echo "Enqueuing URL ${url}"
+python enqueue-binary.py "${url}"

codesigning/gcp-cloudfunc/main.py

@@ -0,0 +1,180 @@
+import contextlib
+import datetime
+import json
+import logging
+import os
+import time
+from urllib.parse import unquote
+
+import functions_framework
+import google.cloud.logging  # pip install google-cloud-logging
+import requests
+from flask import jsonify
+from google.cloud import storage  # pip install google-cloud-storage
+
+GOOGLE_PREFIX = 'https://storage.googleapis.com'
+CODESIGN_DATA_BUCKET = 'natcap-codesigning'
+LOG_CLIENT = google.cloud.logging.Client()
+LOG_CLIENT.setup_logging()
+
+
+@contextlib.contextmanager
+def get_lock():
+    """Acquire a GCS-based mutex.
+
+    This requires that the bucket we are using has versioning.
+    """
+    storage_client = storage.Client()
+    bucket = storage_client.bucket(CODESIGN_DATA_BUCKET)
+
+    lock_obtained = False
+    n_tries = 100
+    for i in range(n_tries):
+        lockfile = bucket.blob('mutex.lock')
+        if not lockfile.generation:
+            lockfile.upload_from_string(
+                f"Lock acquired {datetime.datetime.now().isoformat()}")
+            lock_obtained = True
+            break
+        else:
+            time.sleep(0.1)
+
+    if not lock_obtained:
+        raise RuntimeError(f'Could not obtain lock after {n_tries} tries')
+
+    try:
+        yield
+    finally:
+        lockfile.delete()
+
+
+@functions_framework.http
+def main(request):
+    """Handle requests to this GCP Cloud Function.
+
+    All requests must be POST requests and have a JSON body with the following
+    attributes:
+
+        * token: a secret token that matches the ACCESS_TOKEN environment
+            variable that is defined in the cloud function configuration.
+        * action: either 'enqueue' or 'dequeue'
+
+    If the action is 'enqueue', the request must also have a 'url' attribute.
+    The 'url' attribute, when provided, must be a URL to a file that meets
+    these requirements:
+        * The URL must be a publicly accessible URL
+        * The URL must be a file that ends in '.exe'
+        * The URL must be located in either the releases bucket, or else
+            in the dev builds bucket.  It doesn't necessarily have to be an
+            InVEST binary.
+        * The URL must be a file that is not older than June 1, 2024
+        * The URL must be a file that is not already in the queue
+        * The URL should be a file that is not already signed (if the file has
+            already been signed, its signature will be overwritten)
+    """
+    data = request.get_json()
+    if data['token'] != os.environ['ACCESS_TOKEN']:
+        logging.info('Rejecting request due to invalid token')
+        return jsonify('Invalid token'), 403
+
+    if request.method != 'POST':
+        logging.info('Rejecting request due to invalid HTTP method')
+        return jsonify('Invalid request method'), 405
+
+    storage_client = storage.Client()
+    bucket = storage_client.bucket(CODESIGN_DATA_BUCKET)
+
+    logging.debug(f'Data POSTed: {data}')
+
+    if data['action'] == 'dequeue':
+        with get_lock():
+            queuefile = bucket.blob('queue.json')
+            queue_dict = json.loads(queuefile.download_as_string())
+            try:
+                next_file_url = queue_dict['queue'].pop(0)
+            except IndexError:
+                # No items in the queue!
+                logging.info('No binaries are currently queued for signing')
+                return jsonify('No items in the queue'), 204
+
+            queuefile.upload_from_string(json.dumps(queue_dict))
+
+        data = {
+            'https-url': next_file_url,
+            'basename': os.path.basename(next_file_url),
+            'gs-uri': unquote(next_file_url.replace(
+                f'{GOOGLE_PREFIX}/', 'gs://')),
+        }
+        logging.info(f'Dequeued {next_file_url}')
+        return jsonify(data)
+
+    elif data['action'] == 'enqueue':
+        url = data['url']
+        logging.info(f'Attempting to enqueue url {url}')
+
+        if not url.endswith('.exe'):
+            logging.info("Rejecting URL because it doesn't end in .exe")
+            return jsonify('Invalid URL to sign'), 400
+
+        if not url.startswith(GOOGLE_PREFIX):
+            logging.info(f'Rejecting URL because it does not start with {GOOGLE_PREFIX}')
+            return jsonify('Invalid host'), 400
+
+        if not url.startswith((
+                f'{GOOGLE_PREFIX}/releases.naturalcapitalproject.org/',
+                f'{GOOGLE_PREFIX}/natcap-dev-build-artifacts/')):
+            logging.info('Rejecting URL because the bucket is incorrect')
+            return jsonify("Invalid target bucket"), 400
+
+        # Remove http character quoting
+        url = unquote(url)
+
+        binary_bucket_name, *binary_obj_paths = url.replace(
+            GOOGLE_PREFIX + '/', '').split('/')
+        codesign_bucket = storage_client.bucket(CODESIGN_DATA_BUCKET)
+
+        # If the file does not exist at this URL, reject it.
+        response = requests.head(url)
+        if response.status_code >= 400:
+            logging.info('Rejecting URL because it does not exist')
+            return jsonify('Requested file does not exist'), 403
+
+        # If the file is too old, reject it.  Trying to avoid a
+        # denial-of-service by invoking the service with very old files.
+        # I just pulled June 1 out of thin air as a date that is a little while
+        # ago, but not so long ago that we could suddenly have many files
+        # enqueued.
+        mday, mmonth, myear = response.headers['Last-Modified'].split(' ')[1:4]
+        modified_time = datetime.datetime.strptime(
+            ' '.join((mday, mmonth, myear)), '%d %b %Y')
+        if modified_time < datetime.datetime(year=2024, month=6, day=1):
+            logging.info('Rejecting URL because it is too old')
+            return jsonify('File is too old'), 400
+
+        response = requests.head(f'{url}.signature')
+        if response.status_code != 404:
+            logging.info('Rejecting URL because it has already been signed.')
+            return jsonify('File has already been signed'), 204
+
+        with get_lock():
+            # Since the file has not already been signed, add the file to the
+            # queue
+            queuefile = codesign_bucket.blob('queue.json')
+            if not queuefile.exists():
+                queue_dict = {'queue': []}
+            else:
+                queue_dict = json.loads(queuefile.download_as_string())
+
+            if url not in queue_dict['queue']:
+                queue_dict['queue'].append(url)
+            else:
+                return jsonify(
+                    'File is already in the queue', 200, 'application/json')
+
+            queuefile.upload_from_string(json.dumps(queue_dict))
+
+        logging.info(f'Enqueued {url}')
+        return jsonify("OK"), 200
+
+    else:
+        return jsonify('Invalid action request'), 405

codesigning/gcp-cloudfunc/requirements.txt

@@ -0,0 +1,5 @@
+google-cloud-storage
+google-cloud-logging
+functions-framework==3.*
+flask
+requests