Add initial Rust implementation (google#22)

.github/workflows/maturin_ci.yml

@@ -0,0 +1,86 @@
+# This file is autogenerated by maturin v1.7.1
+# To update, run
+#
+#    maturin generate-ci github --pytest --platform all
+#
+name: Maturin CI
+on:
+  push:
+    paths:
+      - '**/*.py'
+      - '**/*.rs'
+      - 'pytest.ini'
+  merge_group:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      matrix:
+        platform:
+          - runner: ubuntu-latest
+            target: x86_64
+          - runner: ubuntu-latest
+            target: x86
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist --find-interpreter
+          sccache: 'true'
+          manylinux: auto
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-linux-${{ matrix.platform.target }}
+          path: dist
+      - name: pytest
+        if: ${{ startsWith(matrix.platform.target, 'x86_64') }}
+        shell: bash
+        run: |
+          set -e
+          python3 -m venv .venv
+          source .venv/bin/activate
+          pip install sedpack --find-links dist --force-reinstall
+          pip install pytest
+          pytest
+      - name: pytest
+        if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }}
+        uses: uraimo/run-on-arch-action@v2
+        with:
+          arch: ${{ matrix.platform.target }}
+          distro: ubuntu22.04
+          githubToken: ${{ github.token }}
+          install: |
+            apt-get update
+            apt-get install -y --no-install-recommends python3 python3-pip
+            pip3 install -U pip pytest
+          run: |
+            set -e
+            pip3 install sedpack --find-links dist --force-reinstall
+            pytest
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-sdist
+          path: dist

.github/workflows/pytest.yml

@@ -3,6 +3,7 @@ on:
   push:
     paths:
       - '**/*.py'
+      - '**/*.rs'
       - 'pytest.ini'
   merge_group:
   pull_request:
@@ -49,6 +50,7 @@ jobs:
         # Start by "installing" sedpack to be sure all dependencies are listed
         run: |
           pip install -r test_requirements.txt
+          pip install --editable .
           echo "PYTHONPATH=./src:$PYTHONPATH" >> $GITHUB_ENV
       - name: Running unit tests
         run: |
@@ -91,6 +93,7 @@ jobs:
         # Start by "installing" sedpack to be sure all dependencies are listed
         run: |
           pip install -r test_requirements.txt
+          pip install --editable .
           echo "PYTHONPATH=./src:$PYTHONPATH" >> $GITHUB_ENV
       - name: Install workflow dependencies
         run: pip install --upgrade pytest coverage

README.md

@@ -24,6 +24,14 @@ To use this library you need to have a working version of [TensorFlow
 3.  Install the package in development mode: `python3 -m pip install --editable
     .` (short `pip install -e .` or legacy `python setup.py develop`)
 
+#### Rust install
+
+-   Activate your Python virtual environment
+-   [Install Rust](https://www.rust-lang.org/tools/install)
+-   Run `maturin develop --release`
+-   Run `python -m pytest` from the project root directory -- no tests should
+    be skipped
+
 ### Update dependencies
 
 Make sure to have: `sudo apt install python3 python3-pip python3-venv` and

project-words.txt

@@ -1,8 +1,14 @@
+Bursztein
 Elems
+Karel
+Král
 MNIST
+Picod
 SCAAML
 asyncstdlib
+bldr
 byteswap
+cdylib
 compresslevel
 crossentropy
 diutils
@@ -13,11 +19,16 @@ itemsize
 kwarguments
 ndarray
 newbyteorder
+perfcounters
 pickleable
+pyarray
+pyfunction
+pymodule
 savez
 sedpack
 setaf
 shardfile
+subclassing
 tfdata
 tfdataset
 tfrec

pyproject.toml

@@ -1,9 +1,13 @@
 [build-system]
-requires = ["setuptools>=61.0", "setuptools-scm"]
-build-backend = "setuptools.build_meta"
+requires = ["maturin>=1.7,<2.0"]
+build-backend = "maturin"
 
-[tool.setuptools.packages.find]
-where = ["src"]
+[tool.maturin]
+python-source = "src"
+manifest-path = "rust/Cargo.toml"
+features = ["pyo3/extension-module"]
+# Implemented in Rust:
+module-name = "sedpack._sedpack_rs"
 
 [project]
 name = "sedpack"
@@ -45,16 +49,10 @@ dependencies = [
     "tqdm",
 ]
 
-[tool.setuptools.dynamic]
-version = {attr = "sedpack.__version__"}
-
 [project.optional-dependencies]
 
 [project.scripts]
 
 [project.urls]
 "Homepage" = "https://github.com/google/sedpack"
 "Bug Tracker" = "https://github.com/google/sedpack"
-
-[tool.setuptools.package-data]
-sedpack = ["py.typed"]