From 1ef37ac5a4455a89fb211fabb3be14893d971758 Mon Sep 17 00:00:00 2001 From: Cristian Lara Date: Tue, 26 Nov 2024 17:29:29 -0600 Subject: [PATCH 1/2] [Actions] Update publish website workflow Dramatically simplify the process of deploying and versioning. We remove the shell script and perform the fewer steps from within github actions. A big change here is the move to using docusaurus's built-in versioning system. Before, we were abusing docusaurus's system to take a snapshot of the entire website. For now we are only handling our md docs, but in future commits we will handle versioning tutorials and sphinx. --- .github/workflows/deploy_on_release.yml | 79 ++++---- .github/workflows/publish_website.yml | 56 ++++++ scripts/build_docs.sh | 74 ++++---- scripts/publish_site.sh | 241 ------------------------ 4 files changed, 132 insertions(+), 318 deletions(-) create mode 100644 .github/workflows/publish_website.yml delete mode 100755 scripts/publish_site.sh diff --git a/.github/workflows/deploy_on_release.yml b/.github/workflows/deploy_on_release.yml index 26f712956a..46083579b1 100644 --- a/.github/workflows/deploy_on_release.yml +++ b/.github/workflows/deploy_on_release.yml @@ -8,47 +8,46 @@ on: jobs: - tests-and-coverage: - name: Test & Coverage - uses: ./.github/workflows/reusable_test.yml - with: - use_latest_pytorch_gpytorch: false - secrets: inherit + # tests-and-coverage: + # name: Test & Coverage + # uses: ./.github/workflows/reusable_test.yml + # with: + # use_latest_pytorch_gpytorch: false + # secrets: inherit - package-deploy-pypi: - name: Package and deploy to pypi.org - runs-on: ubuntu-latest - permissions: - id-token: write # This is required for PyPI OIDC authentication. - needs: tests-and-coverage - steps: - - uses: actions/checkout@v4 - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install dependencies - run: | - pip install .[test] - pip install --upgrade build setuptools setuptools_scm wheel - - name: Build packages (wheel and source distribution) - run: | - python -m build --sdist --wheel - - name: Verify packages - run: | - ./scripts/verify_py_packages.sh - - name: Deploy to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true + # package-deploy-pypi: + # name: Package and deploy to pypi.org + # runs-on: ubuntu-latest + # permissions: + # id-token: write # This is required for PyPI OIDC authentication. + # needs: tests-and-coverage + # steps: + # - uses: actions/checkout@v4 + # - name: Fetch all history for all tags and branches + # run: git fetch --prune --unshallow + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: "3.10" + # - name: Install dependencies + # run: | + # pip install .[test] + # pip install --upgrade build setuptools setuptools_scm wheel + # - name: Build packages (wheel and source distribution) + # run: | + # python -m build --sdist --wheel + # - name: Verify packages + # run: | + # ./scripts/verify_py_packages.sh + # - name: Deploy to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # verbose: true - publish-versioned-website: - name: Publish versioned website - needs: package-deploy-pypi - uses: ./.github/workflows/reusable_website.yml + version-and-publish-website: + # needs: package-deploy-pypi + name: Version and Publish website + uses: ./.github/workflows/publish_website.yml with: - publish_versioned_website: true - release_tag: ${{ github.event.release.tag_name }} + new_version: ${{ github.event.release.tag_name }} secrets: inherit diff --git a/.github/workflows/publish_website.yml b/.github/workflows/publish_website.yml new file mode 100644 index 0000000000..78d0654ab1 --- /dev/null +++ b/.github/workflows/publish_website.yml @@ -0,0 +1,56 @@ +name: Publish Website + +on: + workflow_call: + inputs: + new_version: + required: false + type: string + run_tutorials: + required: false + type: boolean + default: false + workflow_dispatch: + push: + branches: [ main ] + + +jobs: + + publish-website: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies + run: | + pip install ."[dev, tutorials]" + # - if: ${{ inputs.run_tutorials }} + # name: Run Tutorials + # run: | + # python scripts/run_tutorials.py -w $(pwd) + # - if: ${{ inputs.new_version }} + # name: Create new docusaurus version + # run: | + # git config --global user.name "github-actions[bot]" + # git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # python3 scripts/convert_ipynb_to_mdx.py --clean + # cd website + # yarn + # yarn docusaurus docs:version ${{ inputs.new_version }} + + # git add --all + # git commit -m "Create version ${{ inputs.new_version }} of site in Docusaurus" + # git push --force origin HEAD:main + - name: Build website + run: | + bash scripts/make_docs.sh -b + - name: Deploy + uses: JamesIves/github-pages-deploy-action@releases/v4 + with: + branch: gh-pages # The branch the action should deploy to. + folder: website/build # The folder the action should deploy. diff --git a/scripts/build_docs.sh b/scripts/build_docs.sh index b5db11dd0f..780d75ed23 100755 --- a/scripts/build_docs.sh +++ b/scripts/build_docs.sh @@ -37,14 +37,14 @@ while getopts 'bho' flag; do esac done -if [[ $ONLY_DOCUSAURUS == false ]]; then - echo "-----------------------------------" - echo "Generating API reference via Sphinx" - echo "-----------------------------------" - cd sphinx || exit - make html - cd .. || exit -fi +# if [[ $ONLY_DOCUSAURUS == false ]]; then +# echo "-----------------------------------" +# echo "Generating API reference via Sphinx" +# echo "-----------------------------------" +# cd sphinx || exit +# make html +# cd .. || exit +# fi echo "-----------------------------------" echo "Getting Docusaurus deps" @@ -52,43 +52,43 @@ echo "-----------------------------------" cd website || exit yarn -if [[ $ONLY_DOCUSAURUS == false ]]; then - # run script to parse html generated by sphinx - echo "--------------------------------------------" - echo "Parsing Sphinx docs and moving to Docusaurus" - echo "--------------------------------------------" - cd .. - mkdir -p "website/pages/api/" +# if [[ $ONLY_DOCUSAURUS == false ]]; then +# # run script to parse html generated by sphinx +# echo "--------------------------------------------" +# echo "Parsing Sphinx docs and moving to Docusaurus" +# echo "--------------------------------------------" +# cd .. +# mkdir -p "website/pages/api/" - cwd=$(pwd) - python scripts/parse_sphinx.py -i "${cwd}/sphinx/build/html/" -o "${cwd}/website/pages/api/" +# cwd=$(pwd) +# python scripts/parse_sphinx.py -i "${cwd}/sphinx/build/html/" -o "${cwd}/website/pages/api/" - SPHINX_JS_DIR='sphinx/build/html/_static/' - DOCUSAURUS_JS_DIR='website/static/js/' +# SPHINX_JS_DIR='sphinx/build/html/_static/' +# DOCUSAURUS_JS_DIR='website/static/js/' - mkdir -p $DOCUSAURUS_JS_DIR +# mkdir -p $DOCUSAURUS_JS_DIR - # move JS files from /sphinx/build/html/_static/*: - cp "${SPHINX_JS_DIR}documentation_options.js" "${DOCUSAURUS_JS_DIR}documentation_options.js" - cp "${SPHINX_JS_DIR}doctools.js" "${DOCUSAURUS_JS_DIR}doctools.js" - cp "${SPHINX_JS_DIR}language_data.js" "${DOCUSAURUS_JS_DIR}language_data.js" - cp "${SPHINX_JS_DIR}searchtools.js" "${DOCUSAURUS_JS_DIR}searchtools.js" +# # move JS files from /sphinx/build/html/_static/*: +# cp "${SPHINX_JS_DIR}documentation_options.js" "${DOCUSAURUS_JS_DIR}documentation_options.js" +# cp "${SPHINX_JS_DIR}doctools.js" "${DOCUSAURUS_JS_DIR}doctools.js" +# cp "${SPHINX_JS_DIR}language_data.js" "${DOCUSAURUS_JS_DIR}language_data.js" +# cp "${SPHINX_JS_DIR}searchtools.js" "${DOCUSAURUS_JS_DIR}searchtools.js" - # searchindex.js is not static util - cp "sphinx/build/html/searchindex.js" "${DOCUSAURUS_JS_DIR}searchindex.js" +# # searchindex.js is not static util +# cp "sphinx/build/html/searchindex.js" "${DOCUSAURUS_JS_DIR}searchindex.js" - # copy module sources - cp -r "sphinx/build/html/_sources/" "website/static/_sphinx-sources/" +# # copy module sources +# cp -r "sphinx/build/html/_sources/" "website/static/_sphinx-sources/" - echo "-----------------------------------" - echo "Generating tutorials" - echo "-----------------------------------" - mkdir -p "website/_tutorials" - mkdir -p "website/static/files" - python scripts/parse_tutorials.py -w "${cwd}" +# echo "-----------------------------------" +# echo "Generating tutorials" +# echo "-----------------------------------" +# mkdir -p "website/_tutorials" +# mkdir -p "website/static/files" +# python scripts/parse_tutorials.py -w "${cwd}" - cd website || exit -fi # end of not only Docusaurus block +# cd website || exit +# fi # end of not only Docusaurus block if [[ $BUILD_STATIC == true ]]; then echo "-----------------------------------" diff --git a/scripts/publish_site.sh b/scripts/publish_site.sh deleted file mode 100755 index fa5ef90d68..0000000000 --- a/scripts/publish_site.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/bin/bash -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -set -e - -usage() { - echo "Usage: $0 [-d] [-v VERSION]" - echo "" - echo "Build and push updated BoTorch site. Will either update latest or bump stable version." - echo "" - echo " -d Use Docusaurus bot GitHub credentials. If not specified, will use default GitHub credentials." - echo " -v=VERSION Build site for new library version. If not specified, will update latest version." - echo "" - exit 1 -} - -DOCUSAURUS_BOT=false -VERSION=false - -while getopts 'dhv:' option; do - case "${option}" in - d) - DOCUSAURUS_BOT=true - ;; - h) - usage - ;; - v) - VERSION=${OPTARG} - ;; - *) - usage - ;; - esac -done - -if [[ $VERSION != false ]]; then - # Strip any leading "v" in the VERSION string - VERSION=$(echo "$VERSION" | sed "s/v//"); -fi - -# Function to get absolute filename -fullpath() { - echo "$(cd "$(dirname "$1")" || exit; pwd -P)/$(basename "$1")" -} - -# Current directory (needed for cleanup later) -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# Make temporary directory -WORK_DIR=$(mktemp -d) -cd "${WORK_DIR}" || exit - -if [[ $DOCUSAURUS_BOT == true ]]; then - # Assumes docusaurus bot credentials have been stored in ~/.netrc, e.g. via - git config --global user.email "docusaurus-bot@users.noreply.github.com" - git config --global user.name "BoTorch website deployment script" - echo "machine github.com login docusaurus-bot password ${DOCUSAURUS_PUBLISH_TOKEN}" > ~/.netrc - - # Clone both main & gh-pages branches - git clone https://docusaurus-bot@github.com/pytorch/botorch.git botorch-main - git clone --branch gh-pages https://docusaurus-bot@github.com/pytorch/botorch.git botorch-gh-pages -else - git clone https://github.com/pytorch/botorch.git botorch-main - git clone --branch gh-pages https://github.com/pytorch/botorch.git botorch-gh-pages -fi - -# A few notes about the script below: -# * Docusaurus versioning was designed to *only* version the markdown -# files in the docs/ subdirectory. We are repurposing parts of Docusaurus -# versioning, but snapshotting the entire site. Versions of the site are -# stored in the v/ subdirectory on gh-pages: -# -# --gh-pages/ -# |-- api/ -# |-- css/ -# |-- docs/ -# | ... -# |-- v/ -# | |-- 1.0.1/ -# | |-- 1.0.2/ -# | | ... -# | |-- latest/ -# | .. -# |-- versions.html -# -# * The stable version is in the top-level directory. It is also -# placed into the v/ subdirectory so that it does not need to -# be built again when the version is augmented. -# * We want to support serving / building the Docusaurus site locally -# without any versions. This means that we have to keep versions.js -# outside of the website/ subdirectory. -# * We do not want to have a tracked file that contains all versions of -# the site or the latest version. Instead, we determine this at runtime. -# We use what's on gh-pages in the versions subdirectory as the -# source of truth for available versions and use the latest tag on -# the main branch as the source of truth for the latest version. - -if [[ $VERSION == false ]]; then - echo "-----------------------------------------" - echo "Updating latest (main) version of site " - echo "-----------------------------------------" - - # Populate _versions.json from existing versions; this is used - # by versions.js & needed to build the site (note that we don't actually - # use versions.js for latest build, but we do need versions.js - # in website/pages in order to use docusaurus-versions) - CMD="import os, json; " - CMD+="vs = [v for v in os.listdir('botorch-gh-pages/v') if v != 'latest' and not v.startswith('.')]; " - CMD+="print(json.dumps(vs))" - python3 -c "$CMD" > botorch-main/website/_versions.json - - # Move versions.js to website subdirectory. - # This is the page you see when click on version in navbar. - cp botorch-main/scripts/versions.js botorch-main/website/pages/en/versions.js - cd botorch-main/website || exit - - # Replace baseUrl (set to /v/latest/) & disable Algolia - CONFIG_FILE=$(fullpath "siteConfig.js") - python3 ../scripts/patch_site_config.py -f "${CONFIG_FILE}" -b "/v/latest/" --disable_algolia - - # Tag site with "latest" version - yarn - yarn run version latest - - # Build site - cd .. || exit - ./scripts/build_docs.sh -b - rm -rf website/build/botorch/docs/next # don't need this - - # Move built site to gh-pages (but keep old versions.js) - cd "${WORK_DIR}" || exit - cp botorch-gh-pages/v/latest/versions.html versions.html - rm -rf botorch-gh-pages/v/latest - mv botorch-main/website/build/botorch botorch-gh-pages/v/latest - # versions.html goes both in top-level and under en/ (default language) - cp versions.html botorch-gh-pages/v/latest/versions.html - cp versions.html botorch-gh-pages/v/latest/en/versions.html - - # Push changes to gh-pages - cd botorch-gh-pages || exit - git add . - if [[ -z $(git diff --cached --exit-code) ]]; then - # Exit gracefully if there's nothing to commit. - echo "Nothing to commit" - else - # Commit & push if there's something to commit. - git commit -m 'Update latest version of site' - git push - fi -else - echo "-----------------------------------------" - echo "Building new version ($VERSION) of site " - echo "-----------------------------------------" - - # Checkout main branch with specified tag - cd botorch-main || exit - git fetch --tags - git checkout "v${VERSION}" - - # Populate _versions.json from existing versions; this contains a list - # of versions present in gh-pages (excluding latest). This is then used - # to populate versions.js (which forms the page that people see when they - # click on version number in navbar). - # Note that this script doesn't allow building a version of the site that - # is already on gh-pages. - CMD="import os, json; " - CMD+="vs = [v for v in os.listdir('../botorch-gh-pages/v') if v != 'latest' and not v.startswith('.')]; " - CMD+="assert '${VERSION}' not in vs, '${VERSION} is already on gh-pages.'; " - CMD+="vs.append('${VERSION}'); " - CMD+="print(json.dumps(vs))" - python3 -c "$CMD" > website/_versions.json - - cp scripts/versions.js website/pages/en/versions.js - - # Tag site as 'stable' - cd website || exit - yarn - yarn run version stable - - # Build new version of site (this will be stable, default version) - cd .. || exit - ./scripts/build_docs.sh -b - - # Move built site to new folder (new-site) & carry over old versions - # from existing gh-pages - cd "${WORK_DIR}" || exit - rm -rf botorch-main/website/build/botorch/docs/next # don't need this - mv botorch-main/website/build/botorch new-site - mv botorch-gh-pages/v new-site/v - - # Build new version of site (to be placed in v/$VERSION/) - # the only thing that changes here is the baseUrl (for nav purposes) - # we build this now so that in the future, we can just bump version and not move - # previous stable to versions - cd botorch-main/website || exit - - # Replace baseUrl (set to /v/$VERSION/) & disable Algolia - CONFIG_FILE=$(fullpath "siteConfig.js") - python3 ../scripts/patch_site_config.py -f "${CONFIG_FILE}" -b "/v/${VERSION}/" --disable_algolia - - # Tag exact version & build site - yarn run version "${VERSION}" - cd .. || exit - # Only run Docusaurus (skip tutorial build & Sphinx) - ./scripts/build_docs.sh -b -o - rm -rf website/build/botorch/docs/next # don't need this - rm -rf website/build/botorch/docs/stable # or this - mv website/build/botorch "../new-site/v/${VERSION}" - - # Need to run script to update versions.js for previous versions in - # new-site/v with the newly built versions.js. Otherwise, - # the versions.js for older versions in versions subdirectory - # won't be up-to-date and will not have a way to navigate back to - # newer versions. This is the only part of the old versions that - # needs to be updated when a new version is built. - cd "${WORK_DIR}" || exit - python3 botorch-main/scripts/update_versions_html.py -p "${WORK_DIR}" - - # move contents of newsite to botorch-gh-pages, preserving commit history - rm -rfv ./botorch-gh-pages/* - rsync -avh ./new-site/ ./botorch-gh-pages/ - cd botorch-gh-pages || exit - git add --all - if [[ -z $(git diff --cached --exit-code) ]]; then - # Exit gracefully if there's nothing to commit. - echo "Nothing to commit" - else - # Commit & push if there's something to commit. - git commit -m "Publish version ${VERSION} of site" - git push - fi -fi - -# Clean up -echo "Cleaning up temporary directories." -cd "${SCRIPT_DIR}" || exit -rm -rf "${WORK_DIR}" From 70046fc9c848f65d1d923f75dc9a1dedbae3290f Mon Sep 17 00:00:00 2001 From: Cristian Lara Date: Tue, 26 Nov 2024 17:32:39 -0600 Subject: [PATCH 2/2] [Docusaurus] Update baseUrl This is necessary to host from my personal website. --- website/docusaurus.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 94581a3810..fd1bf90d97 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -6,7 +6,7 @@ module.exports={ "title": "BoTorch", "tagline": "Bayesian Optimization in PyTorch", "url": "https://botorch.org", - "baseUrl": "/", + "baseUrl": "/botorch", "organizationName": "pytorch", "projectName": "botorch", "scripts": [