resolve merge conflicts, fix pylint errors

.github/workflows/release.yml

@@ -32,108 +32,61 @@ on:
   release:
     types: [published]
 
-
-env:
-  HOMEBREW_NO_ANALYTICS: "ON" # Make Homebrew installation a little quicker
-  HOMEBREW_NO_AUTO_UPDATE: "ON"
-  HOMEBREW_NO_BOTTLE_SOURCE_FALLBACK: "ON"
-  HOMEBREW_NO_GITHUB_API: "ON"
-  HOMEBREW_NO_INSTALL_CLEANUP: "ON"
-  CIBW_SKIP: "pp* *i686*" # skip building for PyPy
-  CIBW_ARCHS_MACOS: x86_64
-  CIBW_ARCHS_LINUX: x86_64  # ppc64le # uncomment to enable powerPC build
-  CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix)/opt/make/libexec/gnubin:$PATH"
-  MACOSX_DEPLOYMENT_TARGET: "10.09"
-
-
 jobs:
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-22.04, macos-12]
-
+  build_dists:
+    name: Build Distributions
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
 
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel>=2.12.3
+      - name: Install build
+        run: python -m pip install 'build>=1.2.2,<2'
 
       - name: Install build-essentials
-        if: contains(matrix.os, 'ubuntu')
         run: |
           sudo add-apt-repository ppa:ubuntu-toolchain-r/test
           sudo apt-get update
-          sudo apt-get install -y build-essential
-          sudo apt-get install -y wget
+          sudo apt-get install -y build-essential wget
 
-      - name: Install GNU make for MacOS
-        if: contains(matrix.os, 'macos')
-        run: brew install make || true
+      - name: Build Distributions
+        run: python -m build .
 
-      - name: list target wheels
-        run: |
-          python -m cibuildwheel . --print-build-identifiers
-
-      - name: Build wheels
-        run: python -m cibuildwheel --output-dir wheelhouse
-        env:
-          CIBW_ENVIRONMENT_MACOS: PATH="$(brew --prefix)/opt/make/libexec/gnubin:$PATH"
-          MACOSX_DEPLOYMENT_TARGET: "10.09"
-
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
-          path: ./wheelhouse/*.whl
-
-
-  build_sdist:
-    name: Build source distribution
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        name: Install Python
-        with:
-          python-version: '3.9'
-
-      - name: Build sdist
-        run: |
-          python -m pip install cmake>=3.13
-          python setup.py sdist
-
-      - uses: actions/upload-artifact@v2
-        with:
-          path: dist/*.tar.gz
+          name: distributables
+          path: ./dist/*
 
 
   upload_pypi:
-    needs: [build_wheels, build_sdist]
-    runs-on: ubuntu-latest
+    needs: [build_dists]
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
-          name: artifact
+          name: distributables
           path: dist
 
       - uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI }}
-          #repository_url: https://test.pypi.org/legacy/
-
+          # repository-url: https://test.pypi.org/legacy/
 
   createPullRequest:
-    runs-on: ubuntu-latest
+    needs: [upload_pypi]
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: Create pull request
         run: |
-          gh pr create -B develop -H master --title 'Merge master into develop' --body 'This PR brings develop up to date with master for release.'
+          gh pr create -B develop \
+                       -H master \
+                       --title 'Merge master into develop' \
+                       --body 'This PR brings develop up to date with master for release.'
         env:
             GH_TOKEN: ${{ github.token }}

.github/workflows/run_tests.yml

@@ -49,12 +49,12 @@ env:
 
 jobs:
   run_tests:
-    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}, RedisAI ${{ matrix.rai }}
+    name: Run tests ${{ matrix.subset }} with ${{ matrix.os }}, Python ${{ matrix.py_v}}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        subset: [backends, slow_tests, group_a, group_b]
+        subset: [backends, slow_tests, group_a, group_b, dragon]
         os: [macos-12, macos-14, ubuntu-22.04] # Operating systems
         compiler: [8] # GNU compiler version
         rai: [1.2.7] # Redis AI versions
@@ -63,9 +63,6 @@ jobs:
           - os: macos-14
             py_v: "3.9"
 
-    env:
-      SMARTSIM_REDISAI: ${{ matrix.rai }}
-
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -117,15 +114,26 @@ jobs:
       - name: Install SmartSim (with ML backends)
         run: |
           python -m pip install git+https://github.com/CrayLabs/SmartRedis.git@develop#egg=smartredis
-          python -m pip install .[dev,mypy,ml]
+          python -m pip install .[dev,mypy]
 
-      - name: Install ML Runtimes with Smart (with pt, tf, and onnx support)
-        if: contains( matrix.os, 'ubuntu' ) || contains( matrix.os, 'macos-12')
-        run: smart build --device cpu --onnx -v
+      - name: Install ML Runtimes
+        if: matrix.subset != 'dragon'
+        run: smart build --device cpu -v
 
-      - name: Install ML Runtimes with Smart (no ONNX,TF on Apple Silicon)
-        if: contains( matrix.os, 'macos-14' )
-        run: smart build --device cpu --no_tf -v
+
+      - name: Install ML Runtimes (with dragon)
+        if: matrix.subset == 'dragon'
+        env:
+          SMARTSIM_DRAGON_TOKEN: ${{ secrets.DRAGON_TOKEN }}
+        run: |
+          if [ -n "${SMARTSIM_DRAGON_TOKEN}" ]; then
+            smart build --device cpu -v --dragon-repo dragonhpc/dragon-nightly --dragon-version 0.10
+          else
+            smart build --device cpu -v --dragon
+          fi
+          SP=$(python -c 'import site; print(site.getsitepackages()[0])')/smartsim/_core/config/dragon/.env
+          LLP=$(cat $SP | grep LD_LIBRARY_PATH | awk '{split($0, array, "="); print array[2]}')
+          echo "LD_LIBRARY_PATH=$LLP:$LD_LIBRARY_PATH" >> $GITHUB_ENV
 
       - name: Run mypy
         run: |
@@ -151,9 +159,16 @@ jobs:
           echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
           py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ ./tests/backends
 
+      # Run pytest (dragon subtests)
+      - name: Run Dragon Pytest
+        if: (matrix.subset == 'dragon' && matrix.os == 'ubuntu-22.04')
+        run: |
+          echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
+          dragon -s py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests
+
       # Run pytest (test subsets)
       - name: Run Pytest
-        if: "!contains(matrix.subset, 'backends')" # if not running backend tests
+        if: (matrix.subset != 'backends' && matrix.subset != 'dragon') # if not running backend tests or dragon tests
         run: |
           echo "SMARTSIM_LOG_LEVEL=debug" >> $GITHUB_ENV
           py.test -s --import-mode=importlib -o log_cli=true --cov=$(smart site) --cov-report=xml --cov-config=./tests/test_configs/cov/local_cov.cfg --ignore=tests/full_wlm/ -m ${{ matrix.subset }} ./tests

.gitignore

@@ -12,6 +12,7 @@ tests/test_output
 # Dependencies
 smartsim/_core/.third-party
 smartsim/_core/.dragon
+smartsim/_core/build
 
 # Docs
 _build

.wci.yml

@@ -22,8 +22,8 @@
     language: Python
 
     release:
-      version: 0.7.0
-      date: 2024-05-14
+      version: 0.8.0
+      date: 2024-09-25
 
     documentation:
       general: https://www.craylabs.org/docs/overview.html

Makefile

@@ -150,11 +150,11 @@ tutorials-dev:
 	@docker compose build tutorials-dev
 	@docker run -p 8888:8888 smartsim-tutorials:dev-latest
 
-# help: tutorials-prod                 - Build and start a docker container to run the tutorials (v0.7.0)
+# help: tutorials-prod                 - Build and start a docker container to run the tutorials (v0.8.0)
 .PHONY: tutorials-prod
 tutorials-prod:
 	@docker compose build tutorials-prod
-	@docker run -p 8888:8888 smartsim-tutorials:v0.7.0
+	@docker run -p 8888:8888 smartsim-tutorials:v0.8.0
 
 
 # help:
@@ -164,22 +164,22 @@ tutorials-prod:
 # help: test                           - Run all tests
 .PHONY: test
 test:
-	@python -m pytest --ignore=tests/full_wlm/
+	@python -m pytest --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 # help: test-verbose                   - Run all tests verbosely
 .PHONY: test-verbose
 test-verbose:
-	@python -m pytest -vv --ignore=tests/full_wlm/
+	@python -m pytest -vv --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 # help: test-debug                     - Run all tests with debug output
 .PHONY: test-debug
 test-debug:
-	@SMARTSIM_LOG_LEVEL=developer python -m pytest -s -o log_cli=true -vv --ignore=tests/full_wlm/
+	@SMARTSIM_LOG_LEVEL=developer python -m pytest -s -o log_cli=true -vv --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 # help: test-cov                       - Run all tests with coverage
 .PHONY: test-cov
 test-cov:
-	@python -m pytest -vv --cov=./smartsim --cov-config=${COV_FILE} --ignore=tests/full_wlm/
+	@python -m pytest -vv --cov=./smartsim --cov-config=${COV_FILE} --ignore=tests/full_wlm/ --ignore=tests/dragon_wlm
 
 
 # help: test-full                      - Run all WLM tests with Python coverage (full test suite)
@@ -192,3 +192,8 @@ test-full:
 .PHONY: test-wlm
 test-wlm:
 	@python -m pytest -vv tests/full_wlm/ tests/on_wlm
+
+# help: test-dragon                   - Run dragon-specific tests
+.PHONY: test-dragon
+test-dragon:
+	@dragon pytest tests/dragon_wlm

README.md

@@ -643,11 +643,11 @@ from C, C++, Fortran and Python with the SmartRedis Clients:
     <tr>
       <td rowspan="3">1.2.7</td>
       <td>PyTorch</td>
-      <td>2.0.1</td>
+      <td>2.1.0</td>
     </tr>
     <tr>
       <td>TensorFlow\Keras</td>
-      <td>2.13.1</td>
+      <td>2.15.0</td>
     </tr>
     <tr>
       <td>ONNX</td>

doc/_static/version_names.json

@@ -1,7 +1,8 @@
 {
     "version_names":[
         "develop (unstable)",
-        "0.7.0 (stable)",
+        "0.8.0 (stable)",
+        "0.7.0",
         "0.6.2",
         "0.6.1",
         "0.6.0",
@@ -15,6 +16,7 @@
     "version_urls": [
         "https://www.craylabs.org/develop/overview.html",
         "https://www.craylabs.org/docs/overview.html",
+        "https://www.craylabs.org/docs/versions/0.7.0/overview.html",
         "https://www.craylabs.org/docs/versions/0.6.2/overview.html",
         "https://www.craylabs.org/docs/versions/0.6.1/overview.html",
         "https://www.craylabs.org/docs/versions/0.6.0/overview.html",