From 7ae5720d4fb915dbdc3a1c658eb8a6de8bc77d9e Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Mon, 20 May 2024 10:46:14 +0200 Subject: [PATCH 01/29] Extend CI to test with onnxruntime==1.18.0 Signed-off-by: Xavier Dupre --- .azure-pipelines/linux-conda-CI.yml | 22 ++++++++++++++++++---- .azure-pipelines/win32-conda-CI.yml | 15 ++++++++++++--- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index f5bdae960..acdba04cf 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -14,20 +14,34 @@ jobs: strategy: matrix: - Py311-Onnx160-Rt171-Skl140: + Py312-Onnx160-Rt180-Skl142: do.bench: '0' - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 + python.version: '3.12' # onnxruntime-extensions needs to be update to switch to 3.12 numpy.version: '>=1.21.1' scipy.version: '>=1.7.0' onnx.version: 'onnx==1.16.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.17.1' - sklearn.version: '>=1.4.0' + onnxrt.version: 'onnxruntime==1.18.0' + sklearn.version: '>=1.4.2' pandas.version: '' lgbm.version: '' onnxcc.version: '>=1.8.1' run.example: '1' + Py311-Onnx160-Rt173-Skl140: + do.bench: '0' + python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 + numpy.version: '>=1.21.1' + scipy.version: '>=1.7.0' + onnx.version: 'onnx==1.16.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' + onnx.target_opset: '' + onnxrt.version: 'onnxruntime==1.17.3' + sklearn.version: '==1.4.0' + pandas.version: '' + lgbm.version: '' + onnxcc.version: '>=1.8.1' + run.example: '0' + Py311-Onnx150-Rt170-Skl140: do.bench: '0' python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index 6d9952f9f..4a774d749 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -13,15 +13,24 @@ jobs: vmImage: 'windows-latest' strategy: matrix: - Py311-Onnx160-Rt171-Skl140: + Py312-Onnx160-Rt180-Skl142: + python.version: '3.12' # onnxruntime-extensions needs to be update to switch to 3.12 + onnx.version: 'onnx==1.16.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' + onnx.target_opset: '' + numpy.version: 'numpy>=1.22.3' + scipy.version: 'scipy' + onnxrt.version: 'onnxruntime==1.18.0' + onnxcc.version: 'onnxconverter-common>=1.8.1' + sklearn.version: '==1.4.2' + Py311-Onnx160-Rt173-Skl140: python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 onnx.version: 'onnx==1.16.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' onnx.target_opset: '' numpy.version: 'numpy>=1.22.3' scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.17.1' + onnxrt.version: 'onnxruntime==1.17.3' onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '>=1.4.0' + sklearn.version: '==1.4.0' Py311-Onnx150-Rt170-Skl140: python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 onnx.version: 'onnx==1.15.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' From 25b2199bcb23f75febef9c229e425c9fedab6d19 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Mon, 20 May 2024 11:34:14 +0200 Subject: [PATCH 02/29] update doc Signed-off-by: Xavier Dupre --- .azure-pipelines/linux-conda-CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index acdba04cf..19d36de2f 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -26,7 +26,7 @@ jobs: pandas.version: '' lgbm.version: '' onnxcc.version: '>=1.8.1' - run.example: '1' + run.example: '0' Py311-Onnx160-Rt173-Skl140: do.bench: '0' @@ -40,7 +40,7 @@ jobs: pandas.version: '' lgbm.version: '' onnxcc.version: '>=1.8.1' - run.example: '0' + run.example: '1' Py311-Onnx150-Rt170-Skl140: do.bench: '0' From 9e39c80375169dd97c054a07a7d10cddfc36d518 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Mon, 20 May 2024 12:35:41 +0200 Subject: [PATCH 03/29] simplify pipelines Signed-off-by: Xavier Dupre --- .azure-pipelines/linux-CI-nightly.yml | 127 ------------------------- .azure-pipelines/linux-conda-CI.yml | 2 +- .azure-pipelines/win32-CI-nightly.yml | 128 -------------------------- .azure-pipelines/win32-conda-CI.yml | 2 +- 4 files changed, 2 insertions(+), 257 deletions(-) delete mode 100644 .azure-pipelines/linux-CI-nightly.yml delete mode 100644 .azure-pipelines/win32-CI-nightly.yml diff --git a/.azure-pipelines/linux-CI-nightly.yml b/.azure-pipelines/linux-CI-nightly.yml deleted file mode 100644 index c232ae702..000000000 --- a/.azure-pipelines/linux-CI-nightly.yml +++ /dev/null @@ -1,127 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - Py39-Onnx1102-SkLatest: - python.version: '3.9' - numpy.version: '>=1.18.3' - onnx.version: '==1.10.2' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx190-SkLatest: - python.version: '3.9' - numpy.version: '>=1.18.3' - onnx.version: '==1.9.0' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx181-SkLatest: - python.version: '3.9' - numpy.version: '>=1.18.3' - onnx.version: '==1.8.1' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py38-Onnx170-Sk0242: - python.version: '3.8' - numpy.version: '>=1.18.3' - onnx.version: '==1.7.0' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.24.2' - Py37-Onnx160-Sk0232: - python.version: '3.7' - numpy.version: '==1.18.1' - onnx.version: '==1.6.0' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.23.2' - maxParallel: 3 - - steps: - - script: | - sudo install -d -m 0777 /home/vsts/.conda/envs - displayName: Fix Conda permissions - - - task: CondaEnvironment@1 - inputs: - createCustomEnvironment: true - environmentName: 'py$(python.version)' - packageSpecs: 'python=$(python.version)' - - - script: | - conda config --set always_yes yes --set changeps1 no - conda install -c conda-forge numpy$(numpy.version) - conda install protobuf - python -m pip install --upgrade pip - displayName: 'Install environment' - - - script: | - conda install numpy$(numpy.version) - pip install openpyxl cerberus - displayName: 'install numpy' - - - script: | - conda install scikit-learn$(sklearn.version) - displayName: 'install scikit-learn' - - - script: | - pip install onnx$(onnx.version) - displayName: 'install onnx' - - - script: | - pip install flatbuffers - pip install $(onnxrt.version) - displayName: 'install onnxruntime' - - - script: | - pip install -r requirements.txt - git clone https://github.com/microsoft/onnxconverter-common - cd onnxconverter-common - pip install -e . - cd .. - pip install -r requirements-dev.txt - pip install pytest - displayName: 'install requirements' - - - script: | - pip install -e . - displayName: 'install' - - - script: | - echo "---------------" - pip show numpy - echo "---------------" - pip show onnx - echo "---------------" - pip show onnxruntime - echo "---------------" - pip show onnxconverter-common - echo "---------------" - pip show scikit-learn - echo "---------------" - displayName: 'version' - - - script: | - pytest tests --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest' - - - script: | - python -m pip install openpyxl - python tests/benchmark.py - displayName: 'benchmark' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testBenchmarkFiles: 'TESTDUMP/*.xlsx' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index 19d36de2f..b702581cf 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -4,7 +4,7 @@ # https://docs.microsoft.com/azure/devops/pipelines/languages/python trigger: -- master +- main jobs: diff --git a/.azure-pipelines/win32-CI-nightly.yml b/.azure-pipelines/win32-CI-nightly.yml deleted file mode 100644 index 3206260fc..000000000 --- a/.azure-pipelines/win32-CI-nightly.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- master - -jobs: - -- job: 'Test' - pool: - vmImage: 'windows-latest' - strategy: - matrix: - Py39-Onnx1102-nightlyRT-SkLatest: - python.version: '3.9' - onnx.version: 'onnx==1.10.2' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx190-nightlyRT-SkLatest: - python.version: '3.9' - onnx.version: 'onnx==1.9.1' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py39-Onnx180-nightlyRT-SkLatest: - python.version: '3.9' - onnx.version: 'onnx==1.8.1' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '>=1.0' - Py38-Onnx170-nightlyRT-Sk0242: - python.version: '3.8' - onnx.version: 'onnx==1.7.0' - numpy.version: 'numpy>=1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.24.2' - Py37-Onnx160-nightlyRT-Npy1180-Sk0232: - python.version: '3.7' - onnx.version: 'onnx==1.6.0' - numpy.version: 'numpy==1.18.1' - scipy.version: 'scipy' - onnxrt.version: '-i https://test.pypi.org/simple/ ort-nightly' - sklearn.version: '==0.23.2' - maxParallel: 3 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - - script: conda create --yes --quiet --name py$(python.version) -c conda-forge python=$(python.version) protobuf - displayName: Create Anaconda environment - - - script: | - call activate py$(python.version) - python -m pip install --upgrade pip - conda install $(numpy.version) $(scipy.version) - pip install $(onnx.version) - pip install openpyxl cerberus - displayName: 'Install numpy, scipy, onnx' - - - script: | - call activate py$(python.version) - pip install -r requirements.txt - git clone https://github.com/microsoft/onnxconverter-common - cd onnxconverter-common - pip install -e . - cd .. - pip install -r requirements-dev.txt - pip install pytest - displayName: 'install requirements' - - - script: | - call activate py$(python.version) - pip install $(onnx.version) - pip install flatbuffers - pip install $(onnxrt.version) - displayName: 'install onnx, onnxruntime' - - - script: | - call activate py$(python.version) - conda install scikit-learn$(sklearn.version) - displayName: 'install scikit-learn' - - - script: | - call activate py$(python.version) - pip install -e . - displayName: 'install skl2onnx' - - - script: | - call activate py$(python.version) - python -c "import numpy;print('numpy:',numpy.__version__)" - python -c "import scipy;print('scipy:',scipy.__version__)" - python -c "import sklearn;print('sklearn:',sklearn.__version__)" - python -c "import onnx;print('onnx:',onnx.__version__)" - python -c "import onnxconverter_common;print('onnxconverter-common:',onnxconverter_common.__version__)" - python -c "import onnxruntime;print('onnxruntime:',onnxruntime.__version__)" - displayName: 'version' - - - script: | - call activate py$(python.version) - pytest tests --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest' - - - script: | - call activate py$(python.version) - python -m pip install openpyxl - python tests/benchmark.py - displayName: 'benchmark' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testBenchmarkFiles: 'TESTDUMP/*.xlsx' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index 4a774d749..009e24436 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -4,7 +4,7 @@ # https://docs.microsoft.com/azure/devops/pipelines/languages/python trigger: -- master +- main jobs: From d28747aaa25fa84666ac7d3ae6f15152932ac272 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Mon, 20 May 2024 12:40:43 +0200 Subject: [PATCH 04/29] rename master into main Signed-off-by: Xavier Dupre --- docs/examples/plot_backend.py | 2 +- docs/examples/plot_benchmark_pipeline.py | 2 +- docs/examples/plot_cast_transformer.py | 2 +- docs/examples/plot_complex_pipeline.py | 2 +- docs/examples/plot_custom_model.py | 2 +- docs/examples/plot_intermediate_outputs.py | 2 +- docs/examples/plot_investigate_pipeline.py | 2 +- docs/examples/plot_onnx_operators.py | 10 +++++----- docs/examples/plot_pipeline.py | 2 +- docs/examples/plot_pipeline_lightgbm.py | 4 ++-- docs/examples/plot_pipeline_xgboost.py | 4 ++-- docs/exts/github_link.py | 2 +- docs/index.rst | 10 +++++----- docs/pipeline.rst | 8 ++++---- docs/supported.rst | 4 ++-- docs/tutorial/plot_cbegin_opset.py | 2 +- docs/tutorial/plot_gexternal_lightgbm.py | 4 ++-- docs/tutorial/plot_gexternal_lightgbm_reg.py | 4 ++-- docs/tutorial/plot_gexternal_xgboost.py | 4 ++-- docs/tutorial/plot_jcustom_syntax.py | 4 ++-- skl2onnx/algebra/automation.py | 4 ++-- skl2onnx/algebra/onnx_ops.py | 4 ++-- skl2onnx/common/_topology.py | 2 +- .../operator_converters/calibrated_classifier_cv.py | 4 ++-- skl2onnx/operator_converters/local_outlier_factor.py | 2 +- skl2onnx/operator_converters/naive_bayes.py | 2 +- skl2onnx/operator_converters/ovr_decision_function.py | 2 +- .../operator_converters/support_vector_machines.py | 6 +++--- skl2onnx/operator_converters/text_vectoriser.py | 4 ++-- skl2onnx/sklapi/sklearn_text.py | 6 +++--- skl2onnx/sklapi/woe_transformer_onnx.py | 2 +- tests/test_utils/reference_implementation_afe.py | 2 +- 32 files changed, 58 insertions(+), 58 deletions(-) diff --git a/docs/examples/plot_backend.py b/docs/examples/plot_backend.py index 3599ff5c4..3b7643b33 100644 --- a/docs/examples/plot_backend.py +++ b/docs/examples/plot_backend.py @@ -11,7 +11,7 @@ .. index:: backend *ONNX Runtime* extends the -`onnx backend API `_ to run predictions using this runtime. Let's use the API to compute the prediction diff --git a/docs/examples/plot_benchmark_pipeline.py b/docs/examples/plot_benchmark_pipeline.py index 5634cfcf2..4428d2ea7 100644 --- a/docs/examples/plot_benchmark_pipeline.py +++ b/docs/examples/plot_benchmark_pipeline.py @@ -15,7 +15,7 @@ `Pipelining: chaining a PCA and a logistic regression `_. There is one change because -`ONNX-ML Imputer `_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_cast_transformer.py b/docs/examples/plot_cast_transformer.py index 82cfef35e..66439329b 100644 --- a/docs/examples/plot_cast_transformer.py +++ b/docs/examples/plot_cast_transformer.py @@ -113,7 +113,7 @@ def maxdiff(a1, a2): # by ``(x / y)`` and this division must happen in double. # By default, the *sklearn-onnx* assumes every # computer should happen in float. `ONNX 1.7 specifications -# `_ # does not support double scaling (input and output does, # but not the parameters). The solution needs to diff --git a/docs/examples/plot_complex_pipeline.py b/docs/examples/plot_complex_pipeline.py index f9404d944..8457bdf11 100644 --- a/docs/examples/plot_complex_pipeline.py +++ b/docs/examples/plot_complex_pipeline.py @@ -24,7 +24,7 @@ `_. There is one change because `ONNX-ML Imputer -`_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_custom_model.py b/docs/examples/plot_custom_model.py index 5a35811c0..94e7faf19 100644 --- a/docs/examples/plot_custom_model.py +++ b/docs/examples/plot_custom_model.py @@ -354,7 +354,7 @@ def predictable_tsne_converter(scope, operator, container): name = scope.get_unique_operator_name("Scaler") # The parameter follows the specifications of ONNX - # https://github.com/onnx/onnx/blob/master/docs/Operators-ml.md#ai.onnx.ml.Scaler + # https://github.com/onnx/onnx/blob/main/docs/Operators-ml.md#ai.onnx.ml.Scaler attrs = dict( name=name, scale=op.inv_std_.ravel().astype(numpy.float32), diff --git a/docs/examples/plot_intermediate_outputs.py b/docs/examples/plot_intermediate_outputs.py index 5c3b6493f..2408ca659 100644 --- a/docs/examples/plot_intermediate_outputs.py +++ b/docs/examples/plot_intermediate_outputs.py @@ -20,7 +20,7 @@ `_. There is one change because `ONNX-ML Imputer -`_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_investigate_pipeline.py b/docs/examples/plot_investigate_pipeline.py index 2f532d139..1ca787e27 100644 --- a/docs/examples/plot_investigate_pipeline.py +++ b/docs/examples/plot_investigate_pipeline.py @@ -17,7 +17,7 @@ compose/plot_digits_pipe.html>`_. There is one change because `ONNX-ML Imputer -`_ does not handle string type. This cannot be part of the final ONNX pipeline and must be removed. Look for comment starting with ``---`` below. diff --git a/docs/examples/plot_onnx_operators.py b/docs/examples/plot_onnx_operators.py index cd7d7af1b..48fa0fbc9 100644 --- a/docs/examples/plot_onnx_operators.py +++ b/docs/examples/plot_onnx_operators.py @@ -15,12 +15,12 @@ the backend used to compute the predictions. It is better to look first if the existing operators can be used. The list is available on *github* and gives the `basic operators -`_ +`_ and others `dedicated to machine learning -`_. +`_. *ONNX* has a Python API which can be used to define an *ONNX* graph: `PythonAPIOverview.md -`_. +`_. But it is quite verbose and makes it difficult to describe big graphs. *sklearn-onnx* implements a nicer way to test *ONNX* operators. @@ -29,11 +29,11 @@ Let's try the example given by ONNX documentation: `ONNX Model Using Helper Functions -`_. It relies on *protobuf* whose definition can be found on github `onnx.proto -`_. +`_. """ import onnxruntime import numpy diff --git a/docs/examples/plot_pipeline.py b/docs/examples/plot_pipeline.py index 6eae7ed14..1f230dd8e 100644 --- a/docs/examples/plot_pipeline.py +++ b/docs/examples/plot_pipeline.py @@ -48,7 +48,7 @@ # Draw a model with ONNX # ++++++++++++++++++++++ # We use `net_drawer.py -# `_ +# `_ # included in *onnx* package. # We use *onnx* to load the model # in a different way than before. diff --git a/docs/examples/plot_pipeline_lightgbm.py b/docs/examples/plot_pipeline_lightgbm.py index fe89480a6..63b8df391 100644 --- a/docs/examples/plot_pipeline_lightgbm.py +++ b/docs/examples/plot_pipeline_lightgbm.py @@ -64,11 +64,11 @@ # # The converter is implemented in *onnxmltools*: # `onnxmltools...LightGbm.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. ############################################## diff --git a/docs/examples/plot_pipeline_xgboost.py b/docs/examples/plot_pipeline_xgboost.py index 272369cc2..720373aa3 100644 --- a/docs/examples/plot_pipeline_xgboost.py +++ b/docs/examples/plot_pipeline_xgboost.py @@ -79,11 +79,11 @@ # # The converter is implemented in *onnxmltools*: # `onnxmltools...XGBoost.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. ############################################## diff --git a/docs/exts/github_link.py b/docs/exts/github_link.py index 9d87a68de..a57d24dd1 100644 --- a/docs/exts/github_link.py +++ b/docs/exts/github_link.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Source: https://github.com/scikit-learn/scikit-learn/blob/ -# master/doc/sphinxext/github_link.py +# main/doc/sphinxext/github_link.py from operator import attrgetter import inspect import subprocess diff --git a/docs/index.rst b/docs/index.rst index 8d49467ab..62f5fa066 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,10 +9,10 @@ sklearn-onnx: Convert your scikit-learn model into ONNX :widths: 5 5 * - Linux - Windows - * - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-linux-conda-ci?branchName=master - :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=master - - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-win32-conda-ci?branchName=master - :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=master + * - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-linux-conda-ci?branchName=main + :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=main + - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-win32-conda-ci?branchName=main + :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=main *sklearn-onnx* enables you to convert models from @@ -43,7 +43,7 @@ or submit a new one. Sources are available on The converter can convert a model for a specific version of ONNX. Every ONNX release is labelled with an opset number returned by function `onnx_opset_version -`_. +`_. This function returns the default value for parameter target opset (parameter *target_opset*) if it is not specified when converting the model. Every operator is versioned. diff --git a/docs/pipeline.rst b/docs/pipeline.rst index 8fbb42ce9..7b3b3e02f 100644 --- a/docs/pipeline.rst +++ b/docs/pipeline.rst @@ -8,7 +8,7 @@ Convert a pipeline *skl2onnx* converts any machine learning pipeline into *ONNX* pipelines. Every transformer or predictors is converted into one or multiple nodes into the *ONNX* graph. -Any `ONNX backend `_ +Any `ONNX backend `_ can then use this graph to compute equivalent outputs for the same inputs. .. _l-complex-pipeline: @@ -148,8 +148,8 @@ order: * **converter(scope, operator, container):** The converter converts the transformers or predictors into *ONNX* nodes. Each node can an *ONNX* - `operator `_ or - `ML operator `_ or + `operator `_ or + `ML operator `_ or custom *ONNX* operators. As *sklearn-onnx* may convert pipelines with model coming from other libraries, @@ -228,7 +228,7 @@ isolate the source of the differences. The function may then be used to investigate each component independently. The following piece of code is extracted from unit test `test_investigate.py `_ and converts +blob/main/tests/test_investigate.py>`_ and converts a pipeline and each of its components independently. :: diff --git a/docs/supported.rst b/docs/supported.rst index bda1942cb..147387343 100644 --- a/docs/supported.rst +++ b/docs/supported.rst @@ -46,9 +46,9 @@ easy to insert into a graph. These operators get dynamically added and the list depends on the installed *ONNX* package. The documentation for these operators can be found on github: `ONNX Operators.md -`_ +`_ and `ONNX-ML Operators -`_. +`_. Associated to `onnxruntime `_, the mapping makes it easier to easily check the output of the *ONNX* operators on any data as shown diff --git a/docs/tutorial/plot_cbegin_opset.py b/docs/tutorial/plot_cbegin_opset.py index b0c0bfb38..b5341a7bc 100644 --- a/docs/tutorial/plot_cbegin_opset.py +++ b/docs/tutorial/plot_cbegin_opset.py @@ -103,7 +103,7 @@ def get_domain_opset(onx): ######################################## # It shows that the model cannot be converted for opset # below 5. Operator `Reshape `_ changed in +# onnx/blob/main/docs/Operators.md#Reshape>`_ changed in # opset 5: a parameter became an input. The converter # does not support *opset < 5* because runtimes usually do not. # diff --git a/docs/tutorial/plot_gexternal_lightgbm.py b/docs/tutorial/plot_gexternal_lightgbm.py index 30a3db914..afdfc0eda 100644 --- a/docs/tutorial/plot_gexternal_lightgbm.py +++ b/docs/tutorial/plot_gexternal_lightgbm.py @@ -54,11 +54,11 @@ # # The converter is implemented in :epkg:`onnxmltools`: # `onnxmltools...LightGbm.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. update_registered_converter( diff --git a/docs/tutorial/plot_gexternal_lightgbm_reg.py b/docs/tutorial/plot_gexternal_lightgbm_reg.py index 19f4d4e03..000958584 100644 --- a/docs/tutorial/plot_gexternal_lightgbm_reg.py +++ b/docs/tutorial/plot_gexternal_lightgbm_reg.py @@ -69,11 +69,11 @@ # # The converter is implemented in :epkg:`onnxmltools`: # `onnxmltools...LightGbm.py -# `_. # and the shape calculator: # `onnxmltools...Regressor.py -# `_. diff --git a/docs/tutorial/plot_gexternal_xgboost.py b/docs/tutorial/plot_gexternal_xgboost.py index adde4cf15..d05d2d158 100644 --- a/docs/tutorial/plot_gexternal_xgboost.py +++ b/docs/tutorial/plot_gexternal_xgboost.py @@ -73,11 +73,11 @@ # # The converter is implemented in :epkg:`onnxmltools`: # `onnxmltools...XGBoost.py -# `_. # and the shape calculator: # `onnxmltools...Classifier.py -# `_. update_registered_converter( diff --git a/docs/tutorial/plot_jcustom_syntax.py b/docs/tutorial/plot_jcustom_syntax.py index e56fc8912..595526c77 100644 --- a/docs/tutorial/plot_jcustom_syntax.py +++ b/docs/tutorial/plot_jcustom_syntax.py @@ -9,9 +9,9 @@ There are two ways to write a converter. The first one is less verbose and easier to understand (see `k_means.py `_). +main/skl2onnx/operator_converters/k_means.py>`_). The other is very verbose (see `ada_boost.py `_ +sklearn-onnx/blob/main/skl2onnx/operator_converters/ada_boost.py>`_ for an example). The first way is used in :ref:`l-plot-custom-converter`. diff --git a/skl2onnx/algebra/automation.py b/skl2onnx/algebra/automation.py index 6f8c47a74..187be25c5 100644 --- a/skl2onnx/algebra/automation.py +++ b/skl2onnx/algebra/automation.py @@ -192,7 +192,7 @@ def process_documentation(doc): if doc is None: doc = "" doc = textwrap.dedent(doc) - main_docs_url = "https://github.com/onnx/onnx/blob/master/" + main_docs_url = "https://github.com/onnx/onnx/blob/main/" rep = { "[the doc](IR.md)": "`ONNX <{0}docs/IR.md>`_", "[the doc](Broadcasting.md)": ( @@ -227,7 +227,7 @@ def process_documentation(doc): return "\n".join(lines) def build_doc_url(sch): - doc_url = "https://github.com/onnx/onnx/blob/master/docs/Operators" + doc_url = "https://github.com/onnx/onnx/blob/main/docs/Operators" if "ml" in sch.domain: doc_url += "-ml" doc_url += ".md" diff --git a/skl2onnx/algebra/onnx_ops.py b/skl2onnx/algebra/onnx_ops.py index 0eb8db69d..4517e1963 100644 --- a/skl2onnx/algebra/onnx_ops.py +++ b/skl2onnx/algebra/onnx_ops.py @@ -164,9 +164,9 @@ def dynamic_class_creation(cache=False): Automatically generates classes for each of the operators module *onnx* defines and described at `Operators - `_ + `_ and `Operators - `_. """ cache_dir = cache_folder() diff --git a/skl2onnx/common/_topology.py b/skl2onnx/common/_topology.py index 1197cb8b4..1be1db166 100644 --- a/skl2onnx/common/_topology.py +++ b/skl2onnx/common/_topology.py @@ -1507,7 +1507,7 @@ def convert_topology( raise RuntimeError( "Parameter target_opset {} > {} is higher than the " "version of the installed onnx package. See " - "https://github.com/onnx/onnx/blob/master/docs/" + "https://github.com/onnx/onnx/blob/main/docs/" "Versioning.md#released-versions" ".".format(onnx_target_opset, found) ) diff --git a/skl2onnx/operator_converters/calibrated_classifier_cv.py b/skl2onnx/operator_converters/calibrated_classifier_cv.py index 808958b95..3ca087df4 100644 --- a/skl2onnx/operator_converters/calibrated_classifier_cv.py +++ b/skl2onnx/operator_converters/calibrated_classifier_cv.py @@ -242,7 +242,7 @@ def convert_calibrated_classifier_base_estimator( # In the following graph, variable names are in lower case characters only # and operator names are in upper case characters. We borrow operator names # from the official ONNX spec: - # https://github.com/onnx/onnx/blob/master/docs/Operators.md + # https://github.com/onnx/onnx/blob/main/docs/Operators.md # All variables are followed by their shape in []. # # Symbols: @@ -473,7 +473,7 @@ def convert_sklearn_calibrated_classifier_cv( # In the following graph, variable names are in lower case characters only # and operator names are in upper case characters. We borrow operator names # from the official ONNX spec: - # https://github.com/onnx/onnx/blob/master/docs/Operators.md + # https://github.com/onnx/onnx/blob/main/docs/Operators.md # All variables are followed by their shape in []. # # Symbols: diff --git a/skl2onnx/operator_converters/local_outlier_factor.py b/skl2onnx/operator_converters/local_outlier_factor.py index 2dcd0398a..5a5beda8d 100644 --- a/skl2onnx/operator_converters/local_outlier_factor.py +++ b/skl2onnx/operator_converters/local_outlier_factor.py @@ -55,7 +55,7 @@ def convert_sklearn_local_outlier_factor( if options["optim"] == "cdist": warnings.warn( "Option p=%r may not be compatible with the runtime. " - "See https://github.com/microsoft/onnxruntime/blob/master/" + "See https://github.com/microsoft/onnxruntime/blob/main/" "docs/ContribOperators.md#com.microsoft.CDist." ) kwargs["p"] = op.p diff --git a/skl2onnx/operator_converters/naive_bayes.py b/skl2onnx/operator_converters/naive_bayes.py index e1a63d3ae..1e1af6a80 100644 --- a/skl2onnx/operator_converters/naive_bayes.py +++ b/skl2onnx/operator_converters/naive_bayes.py @@ -319,7 +319,7 @@ def convert_sklearn_naive_bayes( # Note: In the following graph, variable names are in lower case # characters only and operator names are in upper case characters. # We borrow operator names from the official ONNX spec: - # https://github.com/onnx/onnx/blob/master/docs/Operators.md + # https://github.com/onnx/onnx/blob/main/docs/Operators.md # All variables are followed by their shape in []. # # Symbols: diff --git a/skl2onnx/operator_converters/ovr_decision_function.py b/skl2onnx/operator_converters/ovr_decision_function.py index a45834598..fb014c8b3 100644 --- a/skl2onnx/operator_converters/ovr_decision_function.py +++ b/skl2onnx/operator_converters/ovr_decision_function.py @@ -25,7 +25,7 @@ def convert_sklearn_ovr_decision_function( ): # Applies _ovr_decision_function. # See https://github.com/scikit-learn/scikit-learn/blob/ - # master/sklearn/utils/multiclass.py#L407: + # main/sklearn/utils/multiclass.py#L407: # :: # def _ovr_decision_function(predictions, confidences, n_classes): # diff --git a/skl2onnx/operator_converters/support_vector_machines.py b/skl2onnx/operator_converters/support_vector_machines.py index 6f57771d2..ea5160123 100644 --- a/skl2onnx/operator_converters/support_vector_machines.py +++ b/skl2onnx/operator_converters/support_vector_machines.py @@ -39,7 +39,7 @@ def convert_sklearn_svm_regressor( *onnxruntime* and *scikit-learn* do not return the same raw scores. *scikit-learn* returns aggregated scores as a *matrix[N, C]* coming from `_ovr_decision_function - `_. *onnxruntime* returns the raw score from *svm* algorithm as a *matrix[N, (C(C-1)/2]*. """ @@ -167,7 +167,7 @@ def convert_sklearn_svm_classifier( *onnxruntime* and *scikit-learn* do not return the same raw scores. *scikit-learn* returns aggregated scores as a *matrix[N, C]* coming from `_ovr_decision_function - `_. *onnxruntime* returns the raw score from *svm* algorithm as a *matrix[N, (C(C-1)/2]*. """ @@ -285,7 +285,7 @@ def convert_sklearn_svm_classifier( ): # Applies _ovr_decision_function. # See https://github.com/scikit-learn/scikit-learn/blob/ - # master/sklearn/utils/multiclass.py#L407: + # main/sklearn/utils/multiclass.py#L407: # :: # _ovr_decision_function(dec < 0, -dec, len(self.classes_)) diff --git a/skl2onnx/operator_converters/text_vectoriser.py b/skl2onnx/operator_converters/text_vectoriser.py index 2601f4812..e00c819a8 100644 --- a/skl2onnx/operator_converters/text_vectoriser.py +++ b/skl2onnx/operator_converters/text_vectoriser.py @@ -20,7 +20,7 @@ def _intelligent_split(text, op, tokenizer, existing): to name ngrams. ``'a b'`` could be ``('a ', 'b')`` or ``('a', ' b')``. See `ngram sequence - `_. """ if op.analyzer == "word": @@ -173,7 +173,7 @@ def convert_sklearn_text_vectorizer( to a custom tokenizer based on `python wrapper for re2 `_ or its sources `pyre2 `_ - (`syntax `_). + (`syntax `_). If the regular expression is not specified and if the instance of TfidfVectorizer is using the default pattern ``(?u)\\\\b\\\\w\\\\w+\\\\b``, it is replaced by diff --git a/skl2onnx/sklapi/sklearn_text.py b/skl2onnx/sklapi/sklearn_text.py index 6f81caae0..b6b1b90fe 100644 --- a/skl2onnx/sklapi/sklearn_text.py +++ b/skl2onnx/sklapi/sklearn_text.py @@ -15,7 +15,7 @@ class NGramsMixin(VectorizerMixin): """ Overloads method `_word_ngrams - `_ + `_ to get tuples instead of string in member `vocabulary_ `_. of `TfidfVectorizer` or :epkg:`CountVectorizer`. @@ -104,7 +104,7 @@ def _fix_vocabulary(expected, new_voc): class TraceableCountVectorizer(CountVectorizer, NGramsMixin): """ Inherits from :class:`NGramsMixin` which overloads method `_word_ngrams - `_ + `_ to keep more information about n-grams but still produces the same outputs than `CountVectorizer`. @@ -178,7 +178,7 @@ def fit_transform(self, X, y=None): class TraceableTfidfVectorizer(TfidfVectorizer, NGramsMixin): """ Inherits from :class:`NGramsMixin` which overloads method `_word_ngrams - `_ + `_ to keep more information about n-grams but still produces the same outputs than `TfidfVectorizer`. diff --git a/skl2onnx/sklapi/woe_transformer_onnx.py b/skl2onnx/sklapi/woe_transformer_onnx.py index 18c966318..231e22ade 100644 --- a/skl2onnx/sklapi/woe_transformer_onnx.py +++ b/skl2onnx/sklapi/woe_transformer_onnx.py @@ -224,7 +224,7 @@ def _update_feature(self, feature): def onnx_attributes(self): """ See `TreeEnsembleRegressor - `_. """ atts = dict( diff --git a/tests/test_utils/reference_implementation_afe.py b/tests/test_utils/reference_implementation_afe.py index 2275ea67a..499f4e399 100644 --- a/tests/test_utils/reference_implementation_afe.py +++ b/tests/test_utils/reference_implementation_afe.py @@ -49,7 +49,7 @@ def _run(self, data, indices): the output has still two like a matrix with one row. The implementation follows what :epkg:`onnxruntime` does in `array_feature_extractor.cc - `_. """ res = _array_feature_extrator(data, indices) From e418fbc49258c30cd0af17710e114c099b12dfc3 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:19:45 +0200 Subject: [PATCH 05/29] action Signed-off-by: Xavier Dupre --- .azure-pipelines/linux-conda-CI.yml | 409 ---------------------------- .azure-pipelines/win32-conda-CI.yml | 239 ---------------- .github/workflows/linux-ci.yml | 124 +++++++++ docs/requirements.txt | 5 +- requirements-dev.txt | 11 +- 5 files changed, 129 insertions(+), 659 deletions(-) delete mode 100644 .azure-pipelines/linux-conda-CI.yml delete mode 100644 .azure-pipelines/win32-conda-CI.yml create mode 100644 .github/workflows/linux-ci.yml diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml deleted file mode 100644 index b702581cf..000000000 --- a/.azure-pipelines/linux-conda-CI.yml +++ /dev/null @@ -1,409 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- main - -jobs: - -- job: 'CI_Linux' - pool: - vmImage: 'ubuntu-latest' - strategy: - matrix: - - Py312-Onnx160-Rt180-Skl142: - do.bench: '0' - python.version: '3.12' # onnxruntime-extensions needs to be update to switch to 3.12 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.16.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.18.0' - sklearn.version: '>=1.4.2' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx160-Rt173-Skl140: - do.bench: '0' - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.16.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.17.3' - sklearn.version: '==1.4.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '1' - - Py311-Onnx150-Rt170-Skl140: - do.bench: '0' - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.15.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.17.0' - sklearn.version: '>=1.4.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx150-Rt163-Skl140: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.15.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.16.3' - sklearn.version: '==1.4.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx150-Rt161-Skl132: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.15.0' # -i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.16.1' - sklearn.version: '==1.3.2' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx141-Rt160-Skl131: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.16.0' - sklearn.version: '==1.3.1' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py311-Onnx140-Rt151-Skl130: - do.bench: '0' - python.version: '3.11' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.14.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.15.1' - sklearn.version: '==1.3.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx140-Rt140-Skl122: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.14.0' - sklearn.version: '==1.2.2' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx130-Rt140-Skl121: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.2' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.14.0' - sklearn.version: '==1.2.1' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx130-Rt131-Skl120: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.13.1' - sklearn.version: '==1.2.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx120-Rt131-Skl120: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.13.1' - sklearn.version: '==1.2.0' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py310-Onnx120-Rt1201-Skl11: - do.bench: '0' - python.version: '3.10' - numpy.version: '>=1.21.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.12.1' - sklearn.version: '==1.1.3' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - Py39-Onnx120-Rt1111-Skl11: - do.bench: '0' - python.version: '3.9' - numpy.version: '>=1.21.0,<1.23.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.11.1' - sklearn.version: '==1.1.3' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - Py39-Onnx1110-Rt1111-Skl11: - do.bench: '0' - python.version: '3.9' - numpy.version: '>=1.21.0,<1.23.0' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.11.0' - onnx.target_opset: '' - onnxrt.version: 'onnxruntime==1.11.1' - sklearn.version: '==1.1.3' - pandas.version: '' - lgbm.version: '' - onnxcc.version: '>=1.8.1' - run.example: '0' - - maxParallel: 3 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - addToPath: true - architecture: 'x64' - displayName: 'Use Python $(python.version)' - - - script: | - sudo apt-get install -y language-pack-en - sudo locale-gen en_US.UTF-8 - sudo update-locale LANG=en_US.UTF-8 - displayName: 'Install packages' - - - script: | - pip install "numpy$(numpy.version)" - pip install "scipy$(scipy.version)" - pip install protobuf - python -m pip install --upgrade pip - displayName: 'Install environment' - - - script: | - if [ '$(sklearn.version)' == 'git' ] - then - git clone https://github.com/scikit-learn/scikit-learn.git --recursive - python setup.py install - else - pip install "scikit-learn$(sklearn.version)" - fi - displayName: 'install scikit-learn' - - - script: | - pip install $(onnx.version) $(onnxrt.version) - displayName: 'install onnxruntime, onnx' - - - script: | - if [ '$(onnxcc.version)' == 'git' ] - then - pip install git+https://github.com/microsoft/onnxconverter-common --no-deps - else - pip install onnxconverter-common$(onnxcc.version) --no-deps - fi - displayName: 'install onnxconverter-common' - - - script: | - pip install -r requirements.txt - pip install -r requirements-dev.txt - displayName: 'install requirements' - - - script: | - pip show onnx - displayName: 'onnx version' - - - script: | - pip install $(onnx.version) $(onnxrt.version) "numpy$(numpy.version)" scikit-learn$(sklearn.version) scipy$(scipy.version) "pandas$(pandas.version)" - displayName: install onnxruntime, onnx, numpy, scikit-learn, pandas again # to make sure the proper version is installed - - - script: | - pip install -e . - displayName: 'install' - - - script: | - echo "---------------" - pip show numpy - echo "---------------" - pip show pandas - echo "---------------" - pip show scipy - echo "---------------" - pip show pandas - echo "---------------" - pip show onnx - echo "---------------" - pip show onnxruntime - echo "---------------" - pip show onnxconverter-common - echo "---------------" - pip show scikit-learn - echo "---------------" - pip show onnxmltools - echo "---------------" - displayName: 'version' - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - cd tests - python -c "from numpy import __version__;print('numpy', __version__)" - python -c "from pandas import __version__;print('pandas', __version__)" - python -c "from scipy import __version__;print('scipy', __version__)" - python -c "from sklearn import __version__;print('sklearn', __version__)" - python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" - python -c "from onnx import __version__;print('onnx', __version__)" - python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" - python -c "import test_utils;print('max_onnxruntime_opset', test_utils.max_onnxruntime_opset())" - python -c "import test_utils;print('TARGET OPSET, IR', test_utils.TARGET_OPSET, test_utils.TARGET_IR)" - python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" - cd .. - displayName: 'target_opset' - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - pytest tests --maxfail=10 --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml - displayName: 'pytest' - - - script: | - # some of this is triggering the following error when importing scipy on python 3.10 - # ImportError: /lib/x86_64-linux-gnu/libstdc++.so.6: version `GLIBCXX_3.4.29' - conda install -c conda-forge "lightgbm${lgbm.version}" "xgboost<2" --no-deps - pip install hummingbird-ml hummingbird "xgboost<2" lightgbm - pip install --no-deps git+https://github.com/microsoft/onnxconverter-common.git - pip install onnxmltools - displayName: 'install onnxmltools' - condition: eq(variables['run.example'], '1') - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - pytest tests_onnxmltools --durations=0 --basetemp=temp --maxfail=5 - displayName: 'pytest-onnxmltools' - condition: eq(variables['run.example'], '1') - - - script: | - python -m pip install ruff - ruff skl2onnx tests tests_onnxmltools - displayName: 'ruff' - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - if [ '$(do.bench)' == '1' ] - then - pip install openpyxl - coverage run --include=skl2onnx/** tests/benchmark.py - coverage report -m - coverage html - fi - displayName: 'coverage' - - - script: | - pip install -r docs/requirements.txt - pip uninstall -y skl2onnx - python setup.py install - pip install --no-deps git+https://github.com/microsoft/onnxconverter-common.git - pip install --no-deps git+https://github.com/onnx/onnxmltools.git - displayName: 'install for documentation' - condition: eq(variables['run.example'], '1') - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - pytest docs/tests --durations=0 --basetemp=temp --doctest-modules -v -v --maxfail=5 - displayName: 'run documentation examples' - condition: eq(variables['run.example'], '1') - - # dot cannot be found but is missing to build the documentation - #- script: | - # if [ '$(onnx.target_opset)' != '' ] - # then - # export TEST_TARGET_OPSET="$(onnx.target_opset)" - # fi - # conda install -c anaconda graphviz - # python -m sphinx -j2 -v -N -T -b html -d build/doctrees docs dist/html - # displayName: 'build documentation' - # condition: eq(variables['run.example'], '1') - - - script: | - if [ '$(onnx.target_opset)' != '' ] - then - export TEST_TARGET_OPSET="$(onnx.target_opset)" - fi - if [ '$(do.bench)' == '1' ] - then - python tests/benchmark.py - fi - displayName: 'benchmark' - condition: eq(variables['run.example'], '0') - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testBenchmarkFiles: 'TESTDUMP/*.xlsx' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml deleted file mode 100644 index 009e24436..000000000 --- a/.azure-pipelines/win32-conda-CI.yml +++ /dev/null @@ -1,239 +0,0 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python - -trigger: -- main - -jobs: - -- job: 'CI_Windows' - pool: - vmImage: 'windows-latest' - strategy: - matrix: - Py312-Onnx160-Rt180-Skl142: - python.version: '3.12' # onnxruntime-extensions needs to be update to switch to 3.12 - onnx.version: 'onnx==1.16.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.18.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.4.2' - Py311-Onnx160-Rt173-Skl140: - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - onnx.version: 'onnx==1.16.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.17.3' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.4.0' - Py311-Onnx150-Rt170-Skl140: - python.version: '3.11' # onnxruntime-extensions needs to be update to switch to 3.12 - onnx.version: 'onnx==1.15.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.17.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '>=1.4.0' - Py311-Onnx150-Rt163-Skl140: - python.version: '3.11' - onnx.version: 'onnx==1.15.0' # '-i https://test.pypi.org/simple/ onnx==1.15.0rc2' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.16.3' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.4.0' - Py311-Onnx150-Rt163-Skl131: - python.version: '3.11' - onnx.version: 'onnx==1.15.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.16.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.3.1' - Py311-Onnx141-Rt160-Skl131: - python.version: '3.11' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.16.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '>=1.3.1' - Py311-Onnx140-Rt151-Skl130: - python.version: '3.11' - onnx.version: 'onnx==1.14.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.15.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.3.0' - Py310-Onnx140-Rt140-Skl122: - python.version: '3.10' - onnx.version: 'onnx==1.14.1' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.14.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.2' - Py310-Onnx130-Rt140-Skl121: - python.version: '3.10' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.14.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.2' - Py310-Onnx130-Rt131-Skl120: - python.version: '3.10' - onnx.version: 'onnx==1.13.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.13.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.0' - Py310-Onnx120-Rt131-Skl120: - python.version: '3.10' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.13.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.2.0' - Py310-Onnx120-Rt120-Skl11: - python.version: '3.10' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.12.0' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.1.3' - Py39-Onnx120-Rt1111-Skl11: - python.version: '3.9' - onnx.version: 'onnx==1.12.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.11.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.1.3' - Py39-Onnx1110-Rt1111-Skl11: - python.version: '3.9' - onnx.version: 'onnx==1.11.0' - onnx.target_opset: '' - numpy.version: 'numpy>=1.22.3' - scipy.version: 'scipy' - onnxrt.version: 'onnxruntime==1.11.1' - onnxcc.version: 'onnxconverter-common>=1.8.1' - sklearn.version: '==1.1.3' - - maxParallel: 3 - - steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Add conda to PATH - - - script: conda create --yes --quiet --name skl2onnxEnvironment $(numpy.version) "$(scipy.version)" pandas python=$(python.version) pytest - displayName: Create Anaconda environment - - - script: | - call activate skl2onnxEnvironment - python -m pip install --upgrade pip - conda install -y -c conda-forge protobuf - displayName: 'Install protobuf' - - - script: | - call activate skl2onnxEnvironment - python -m pip install typing-extensions>=3.6.2.1 - python -m pip install $(onnx.version) - displayName: 'Install onnx' - - - script: | - call activate skl2onnxEnvironment - python -m pip install $(onnxrt.version) - displayName: 'Install onnxruntime' - - - script: | - call activate skl2onnxEnvironment - python -c "import numpy" - displayName: 'check numpy' - - - script: | - call activate skl2onnxEnvironment - # latest version of scikit-learn are not available on conda - pip install scikit-learn$(sklearn.version) - displayName: 'install scikit-learn' - - - script: | - call activate skl2onnxEnvironment - python -c "import numpy" - displayName: 'check numpy' - - - script: | - call activate skl2onnxEnvironment - python -m pip install ruff - ruff skl2onnx tests tests_onnxmltools - displayName: 'ruff' - - - script: | - call activate skl2onnxEnvironment - pip install $(onnxcc.version) --no-deps - displayName: 'install onnxconverter-common' - - - script: | - call activate skl2onnxEnvironment - pip install -r requirements.txt - pip install -r requirements-dev.txt - displayName: 'install requirements' - - - script: | - call activate skl2onnxEnvironment - pip install -e . - displayName: 'install skl2onnx' - - - script: | - call activate skl2onnxEnvironment - python -c "import numpy" - displayName: 'check numpy' - - - script: | - call activate skl2onnxEnvironment - if "$(onnx.target_opset)" neq "" set TEST_TARGET_OPSET=$(onnx.target_opset) - cd tests - python -c "from numpy import __version__;print('numpy', __version__)" - python -c "from scipy import __version__;print('scipy', __version__)" - python -c "from sklearn import __version__;print('sklearn', __version__)" - python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" - python -c "from onnx import __version__;print('onnx', __version__)" - python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" - python -c "import test_utils;print('max_onnxruntime_opset', test_utils.max_onnxruntime_opset())" - python -c "import test_utils;print('TARGET OPSET, IR', test_utils.TARGET_OPSET, test_utils.TARGET_IR)" - python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" - cd .. - displayName: 'target_opset' - - - script: | - call activate skl2onnxEnvironment - if "$(onnx.target_opset)" neq "" set TEST_TARGET_OPSET=$(onnx.target_opset) - python -m pytest tests --durations=0 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml --maxfail=10 - displayName: 'pytest' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testCoverageFiles: 'htmlcov/**' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml new file mode 100644 index 000000000..2997ba46a --- /dev/null +++ b/.github/workflows/linux-ci.yml @@ -0,0 +1,124 @@ +name: Documentation and Code Coverage + +on: + push: + pull_request: + types: + - closed + branches: + - main + +jobs: + run: + name: CI ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python_version: ['3.12', '3.11', '3.10', '3.9'] + include: + - python_version: '3.12' + documentation: 1 + numpy.version: '>=1.21.1' + scipy.version: '>=1.7.0' + onnx.version: 'onnx==1.16.0' + onnxrt.version: 'onnxruntime==1.18.0' + sklearn.version: '>=1.4.2' + - python_version: '3.11' + documentation: 0 + numpy.version: '>=1.21.1' + scipy.version: '>=1.7.0' + onnx.version: 'onnx<1.16.0' + onnxrt.version: 'onnxruntime==1.17.3' + sklearn.version: '==1.3.2' + - python_version: '3.10' + documentation: 0 + numpy.version: '>=1.21.1' + scipy.version: '>=1.7.0' + onnx.version: 'onnx==1.14.1' + onnxrt.version: 'onnxruntime==1.16.0' + sklearn.version: '==1.2.2' + - python_version: '3.9' + documentation: 0 + numpy.version: '>=1.21.1' + scipy.version: '>=1.7.0' + onnx.version: 'onnx==1.13.0' + onnxrt.version: 'onnxruntime==1.14.0' + sklearn.version: '==1.1.3' + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + - uses: tlylt/install-graphviz@v1 + + - name: Install pandoc + if: matrix.documentation == 1 + run: sudo apt-get install -y pandoc + + - name: Install requirements + run: python -m pip install -r requirements.txt + + - name: Install requirements dev + run: python -m pip install -r requirements-dev.txt + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install specific versions + run: pip install $(onnx.version) $(onnxrt.version) "numpy$(numpy.version)" scikit-learn$(sklearn.version) scipy$(scipy.version) "pandas$(pandas.version)" + + - name: Install + run: pip install -e . + + - name: versions + run: | + echo "---------------" + pip show numpy + echo "---------------" + pip show pandas + echo "---------------" + pip show scipy + echo "---------------" + pip show pandas + echo "---------------" + pip show onnx + echo "---------------" + pip show onnxruntime + echo "---------------" + pip show onnxconverter-common + echo "---------------" + pip show scikit-learn + echo "---------------" + pip show onnxmltools + echo "---------------" + + - name: Generate coverage report + if: matrix.documentation == 1 + run: | + pytest tests --maxfail=10 --durations=10 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml + + - name: documentation test + run: | + pip install -r docs/requirements.txt + pytest docs/tests --durations=10 --basetemp=temp --doctest-modules -v -v --maxfail=5 + python tests/benchmark.py + + - name: build + run: python setup.py bdist_wheel + + - uses: actions/upload-artifact@v3 + with: + path: ./dist/** diff --git a/docs/requirements.txt b/docs/requirements.txt index 58c0069ce..431f63cc9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,7 +5,7 @@ coverage flake8 furo joblib -lightgbm<4.0 +lightgbm loky matplotlib mlinsights>=0.3.631 @@ -20,7 +20,6 @@ pyinstrument pyod pytest pytest-cov -scikit-learn>=1.1 skl2onnx sphinx sphinxcontrib-blockdiag @@ -29,4 +28,4 @@ sphinx-runpython tabulate tqdm wheel -xgboost<2 +xgboost diff --git a/requirements-dev.txt b/requirements-dev.txt index 7d137d5d0..2a100a0ad 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ # tests black onnxruntime-extensions +onnxscript pandas py-cpuinfo pybind11 @@ -9,13 +10,7 @@ pytest-cov ruff wheel -# docs -furo -onnxscript -sphinx -sphinxcontrib-blockdiag -tqdm - # docs/examples -lightgbm<4.0 +xgboost +lightgbm matplotlib From 802b5e29fd91731246b0560a603486a9d16ec21e Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:32:19 +0200 Subject: [PATCH 06/29] ci Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 2997ba46a..3fdb599ec 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -1,4 +1,4 @@ -name: Documentation and Code Coverage +name: CI Linux on: push: @@ -45,7 +45,6 @@ jobs: onnx.version: 'onnx==1.13.0' onnxrt.version: 'onnxruntime==1.14.0' sklearn.version: '==1.1.3' - runs-on: ${{ matrix.os }} steps: - name: Checkout repository From b3ac78ecee9dbd38929079d1e4976b92e8cdacc3 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:35:06 +0200 Subject: [PATCH 07/29] ci Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 3fdb599ec..eaae34ab5 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -1,13 +1,5 @@ name: CI Linux - -on: - push: - pull_request: - types: - - closed - branches: - - main - +on: [push, pull_request] jobs: run: name: CI ${{ matrix.os }} From 47b68e188892c0664b97ba441a3cb1e91dd193ca Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:40:04 +0200 Subject: [PATCH 08/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index eaae34ab5..e6c3d5793 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -69,7 +69,7 @@ jobs: ${{ runner.os }}- - name: Install specific versions - run: pip install $(onnx.version) $(onnxrt.version) "numpy$(numpy.version)" scikit-learn$(sklearn.version) scipy$(scipy.version) "pandas$(pandas.version)" + run: pip install ${{ onnx.version }} ${{ onnxrt.version }} "numpy${{ numpy.version }}" "scikit-learn${{ sklearn.version}}" "scipy${{ scipy.version }}" - name: Install run: pip install -e . From a747472fc3cd105a259f130d8cbb11681c251d09 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:48:56 +0200 Subject: [PATCH 09/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index e6c3d5793..5051d22df 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -69,7 +69,7 @@ jobs: ${{ runner.os }}- - name: Install specific versions - run: pip install ${{ onnx.version }} ${{ onnxrt.version }} "numpy${{ numpy.version }}" "scikit-learn${{ sklearn.version}}" "scipy${{ scipy.version }}" + run: pip install ${{ matrix.onnx.version }} ${{ matrix.onnxrt.version }} "numpy${{ matrix.numpy.version }}" "scikit-learn${{ matrix.sklearn.version}}" "scipy${{ matrix.scipy.version }}" - name: Install run: pip install -e . From c7f5423aad467001c859c84b9d329bd83a4746d8 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:51:42 +0200 Subject: [PATCH 10/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 5051d22df..b6c74728f 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -11,32 +11,32 @@ jobs: include: - python_version: '3.12' documentation: 1 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.16.0' - onnxrt.version: 'onnxruntime==1.18.0' - sklearn.version: '>=1.4.2' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.16.0' + onnxrt_version: 'onnxruntime==1.18.0' + sklearn_version: '>=1.4.2' - python_version: '3.11' documentation: 0 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx<1.16.0' - onnxrt.version: 'onnxruntime==1.17.3' - sklearn.version: '==1.3.2' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.16.0' + onnxrt_version: 'onnxruntime==1.17.3' + sklearn_version: '==1.3.2' - python_version: '3.10' documentation: 0 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.14.1' - onnxrt.version: 'onnxruntime==1.16.0' - sklearn.version: '==1.2.2' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.14.1' + onnxrt_version: 'onnxruntime==1.16.0' + sklearn_version: '==1.2.2' - python_version: '3.9' documentation: 0 - numpy.version: '>=1.21.1' - scipy.version: '>=1.7.0' - onnx.version: 'onnx==1.13.0' - onnxrt.version: 'onnxruntime==1.14.0' - sklearn.version: '==1.1.3' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.13.0' + onnxrt_version: 'onnxruntime==1.14.0' + sklearn_version: '==1.1.3' steps: - name: Checkout repository @@ -69,7 +69,7 @@ jobs: ${{ runner.os }}- - name: Install specific versions - run: pip install ${{ matrix.onnx.version }} ${{ matrix.onnxrt.version }} "numpy${{ matrix.numpy.version }}" "scikit-learn${{ matrix.sklearn.version}}" "scipy${{ matrix.scipy.version }}" + run: pip install ${{ matrix.onnx_version }} ${{ matrix.onnxrt_version }} "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" - name: Install run: pip install -e . From c86a45f6473e2a2fb296dcc8296048ca5aa7606f Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:53:49 +0200 Subject: [PATCH 11/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index b6c74728f..43b4c2e48 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -2,7 +2,7 @@ name: CI Linux on: [push, pull_request] jobs: run: - name: CI ${{ matrix.os }} + name: Python ${{ matrix.python_version }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -69,7 +69,7 @@ jobs: ${{ runner.os }}- - name: Install specific versions - run: pip install ${{ matrix.onnx_version }} ${{ matrix.onnxrt_version }} "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" + run: pip install "${{ matrix.onnx_version }}" "${{ matrix.onnxrt_version }}" "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" - name: Install run: pip install -e . From 1f10b033a10df6ef564c97d8e7683faa953d06c2 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 12:58:02 +0200 Subject: [PATCH 12/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 43b4c2e48..2861f5045 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -76,25 +76,14 @@ jobs: - name: versions run: | - echo "---------------" - pip show numpy - echo "---------------" - pip show pandas - echo "---------------" - pip show scipy - echo "---------------" - pip show pandas - echo "---------------" - pip show onnx - echo "---------------" - pip show onnxruntime - echo "---------------" - pip show onnxconverter-common - echo "---------------" - pip show scikit-learn - echo "---------------" - pip show onnxmltools - echo "---------------" + python -c "from numpy import __version__;print('numpy', __version__)" + python -c "from pandas import __version__;print('pandas', __version__)" + python -c "from scipy import __version__;print('scipy', __version__)" + python -c "from sklearn import __version__;print('sklearn', __version__)" + python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" + python -c "from onnx import __version__;print('onnx', __version__)" + python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" + python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" - name: Generate coverage report if: matrix.documentation == 1 From f4a54208561472a5019353198197effd7312b6b5 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 13:01:43 +0200 Subject: [PATCH 13/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 2861f5045..e1af9d3ec 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -10,14 +10,14 @@ jobs: python_version: ['3.12', '3.11', '3.10', '3.9'] include: - python_version: '3.12' - documentation: 1 + documentation: 0 numpy_version: '>=1.21.1' scipy_version: '>=1.7.0' onnx_version: 'onnx==1.16.0' onnxrt_version: 'onnxruntime==1.18.0' sklearn_version: '>=1.4.2' - python_version: '3.11' - documentation: 0 + documentation: 1 numpy_version: '>=1.21.1' scipy_version: '>=1.7.0' onnx_version: 'onnx<1.16.0' @@ -85,12 +85,12 @@ jobs: python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" - - name: Generate coverage report - if: matrix.documentation == 1 + - name: Run tests run: | pytest tests --maxfail=10 --durations=10 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml - - name: documentation test + - name: Run documentation tests + if: matrix.documentation == 1 run: | pip install -r docs/requirements.txt pytest docs/tests --durations=10 --basetemp=temp --doctest-modules -v -v --maxfail=5 From d99593689f0e9e72392e2259cb9022bf858f1529 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 13:07:28 +0200 Subject: [PATCH 14/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- tests/test_algebra_onnx_doc.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index e1af9d3ec..1a67942ec 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -2,7 +2,7 @@ name: CI Linux on: [push, pull_request] jobs: run: - name: Python ${{ matrix.python_version }} + name: Python ${{ matrix.python_version }} - sklearn ${{ matrix.sklearn_version }} runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/tests/test_algebra_onnx_doc.py b/tests/test_algebra_onnx_doc.py index 1a2d8b722..2b7bd1471 100644 --- a/tests/test_algebra_onnx_doc.py +++ b/tests/test_algebra_onnx_doc.py @@ -46,6 +46,7 @@ def test_transpose2(self): @unittest.skipIf( sys.platform.startswith("win"), reason="onnx schema are incorrect on Windows" ) + @unittest.skipIf(TARGET_OPSET < 18, reason="not available") def test_doc_onnx(self): rst = get_rst_doc() assert "**Summary**" in rst @@ -53,6 +54,7 @@ def test_doc_onnx(self): @unittest.skipIf( sys.platform.startswith("win"), reason="onnx schema are incorrect on Windows" ) + @unittest.skipIf(TARGET_OPSET < 18, reason="not available") def test_doc_sklearn(self): try: rst = get_rst_doc_sklearn() From 5b3546296c8e4b0013b8a3017525d8b7a2e1f476 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 13:13:17 +0200 Subject: [PATCH 15/29] update CI Signed-off-by: Xavier Dupre --- tests/test_algebra_onnx_doc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_algebra_onnx_doc.py b/tests/test_algebra_onnx_doc.py index 2b7bd1471..da70a236c 100644 --- a/tests/test_algebra_onnx_doc.py +++ b/tests/test_algebra_onnx_doc.py @@ -46,7 +46,7 @@ def test_transpose2(self): @unittest.skipIf( sys.platform.startswith("win"), reason="onnx schema are incorrect on Windows" ) - @unittest.skipIf(TARGET_OPSET < 18, reason="not available") + @unittest.skipIf(TARGET_OPSET <= 20, reason="not available") def test_doc_onnx(self): rst = get_rst_doc() assert "**Summary**" in rst @@ -54,7 +54,7 @@ def test_doc_onnx(self): @unittest.skipIf( sys.platform.startswith("win"), reason="onnx schema are incorrect on Windows" ) - @unittest.skipIf(TARGET_OPSET < 18, reason="not available") + @unittest.skipIf(TARGET_OPSET <= 20, reason="not available") def test_doc_sklearn(self): try: rst = get_rst_doc_sklearn() From b5fd4aba654a785fd6e8e6397d716b4b986d3940 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 13:18:55 +0200 Subject: [PATCH 16/29] update CI Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- tests/test_issues_2024.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 1a67942ec..bc7e4a2be 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -2,7 +2,7 @@ name: CI Linux on: [push, pull_request] jobs: run: - name: Python ${{ matrix.python_version }} - sklearn ${{ matrix.sklearn_version }} + name: Py==${{ matrix.python_version }} - sklearn==${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/tests/test_issues_2024.py b/tests/test_issues_2024.py index 569e6a027..1bdc8abef 100644 --- a/tests/test_issues_2024.py +++ b/tests/test_issues_2024.py @@ -115,7 +115,7 @@ def test_issue_1055(self): assert_almost_equal(expected, got[1], decimal=2) @unittest.skipIf( - pv.Version(ort_version) < pv.Version("1.16.0"), + pv.Version(ort_version) < pv.Version("1.17.3"), reason="opset 19 not implemented", ) def test_issue_1069(self): From e63d138dbe0be0866455b26c1e90060ebb520a9f Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 13:30:04 +0200 Subject: [PATCH 17/29] ci Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- .github/workflows/windows-macos-ci.yml | 87 ++++++++++++++++++++++++++ .gitignore | 1 + 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/windows-macos-ci.yml diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index bc7e4a2be..c778133f0 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -97,7 +97,7 @@ jobs: python tests/benchmark.py - name: build - run: python setup.py bdist_wheel + run: pip wheel . - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/windows-macos-ci.yml b/.github/workflows/windows-macos-ci.yml new file mode 100644 index 000000000..14890adf3 --- /dev/null +++ b/.github/workflows/windows-macos-ci.yml @@ -0,0 +1,87 @@ +name: CI Linux +on: [push, pull_request] +jobs: + run: + name: ${{ matrix.os }} Py==${{ matrix.python_version }} - sklearn==${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python_version: ['3.12', '3.11', '3.10', '3.9'] + include: + - python_version: '3.12' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.16.0' + onnxrt_version: 'onnxruntime==1.18.0' + sklearn_version: '>=1.4.2' + - python_version: '3.11' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.16.0' + onnxrt_version: 'onnxruntime==1.17.3' + sklearn_version: '==1.3.2' + - python_version: '3.10' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx==1.14.1' + onnxrt_version: 'onnxruntime==1.16.0' + sklearn_version: '==1.2.2' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python_version }} + + - uses: tlylt/install-graphviz@v1 + + - name: Install pandoc + if: matrix.documentation == 1 + run: sudo apt-get install -y pandoc + + - name: Install requirements + run: python -m pip install -r requirements.txt + + - name: Install requirements dev + run: python -m pip install -r requirements-dev.txt + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install specific versions + run: pip install "${{ matrix.onnx_version }}" "${{ matrix.onnxrt_version }}" "numpy${{ matrix.numpy_version }}" "scikit-learn${{ matrix.sklearn_version}}" "scipy${{ matrix.scipy_version }}" + + - name: Install + run: pip install -e . + + - name: versions + run: | + python -c "from numpy import __version__;print('numpy', __version__)" + python -c "from pandas import __version__;print('pandas', __version__)" + python -c "from scipy import __version__;print('scipy', __version__)" + python -c "from sklearn import __version__;print('sklearn', __version__)" + python -c "from onnxruntime import __version__;print('onnxruntime', __version__)" + python -c "from onnx import __version__;print('onnx', __version__)" + python -c "from skl2onnx import __max_supported_opset__;print('__max_supported_opset__', __max_supported_opset__)" + python -c "import onnx.defs;print('onnx_opset_version', onnx.defs.onnx_opset_version())" + + - name: Run tests + run: | + pytest tests --maxfail=10 --durations=10 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml + + - name: build + run: pip wheel . + + - uses: actions/upload-artifact@v3 + with: + path: ./dist/** diff --git a/.gitignore b/.gitignore index f2ab678fb..8c9a8ba76 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ build/ # test generated files *.onnx +*.whl .pytest_cache .cache htmlcov From 664c2283724834e91156839f5b1adeec8df732c8 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 13:42:46 +0200 Subject: [PATCH 18/29] fix ci Signed-off-by: Xavier Dupre --- .github/workflows/windows-macos-ci.yml | 2 +- docs/requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows-macos-ci.yml b/.github/workflows/windows-macos-ci.yml index 14890adf3..2c9c1a592 100644 --- a/.github/workflows/windows-macos-ci.yml +++ b/.github/workflows/windows-macos-ci.yml @@ -1,4 +1,4 @@ -name: CI Linux +name: CI Win/Mac on: [push, pull_request] jobs: run: diff --git a/docs/requirements.txt b/docs/requirements.txt index 431f63cc9..2434942f0 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -11,6 +11,7 @@ matplotlib mlinsights>=0.3.631 nbsphinx onnx +onnxmltools onnxruntime pillow py-spy From bbd5a9a20d67905c06a6e4d098ad400d4172391d Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 14:16:20 +0200 Subject: [PATCH 19/29] example Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- .github/workflows/windows-macos-ci.yml | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index c778133f0..108f85925 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -2,7 +2,7 @@ name: CI Linux on: [push, pull_request] jobs: run: - name: Py==${{ matrix.python_version }} - sklearn==${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} + name: Py==${{ matrix.python_version }} - sklearn${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/.github/workflows/windows-macos-ci.yml b/.github/workflows/windows-macos-ci.yml index 2c9c1a592..513eb1b07 100644 --- a/.github/workflows/windows-macos-ci.yml +++ b/.github/workflows/windows-macos-ci.yml @@ -6,15 +6,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] - python_version: ['3.12', '3.11', '3.10', '3.9'] + os: [windows-latest, macos-latest] + python_version: ['3.11', '3.10', '3.9'] include: - - python_version: '3.12' - numpy_version: '>=1.21.1' - scipy_version: '>=1.7.0' - onnx_version: 'onnx==1.16.0' - onnxrt_version: 'onnxruntime==1.18.0' - sklearn_version: '>=1.4.2' - python_version: '3.11' numpy_version: '>=1.21.1' scipy_version: '>=1.7.0' @@ -39,10 +33,6 @@ jobs: - uses: tlylt/install-graphviz@v1 - - name: Install pandoc - if: matrix.documentation == 1 - run: sudo apt-get install -y pandoc - - name: Install requirements run: python -m pip install -r requirements.txt From 357a01595ef62b230d31c574f6da8dc42d1ba6e2 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 14:25:27 +0200 Subject: [PATCH 20/29] remove benchmark Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 108f85925..61d58a193 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -94,7 +94,7 @@ jobs: run: | pip install -r docs/requirements.txt pytest docs/tests --durations=10 --basetemp=temp --doctest-modules -v -v --maxfail=5 - python tests/benchmark.py + # python tests/benchmark.py - name: build run: pip wheel . From 1841af573d611f8615f16b0e708eb1565a4d1f77 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 14:37:19 +0200 Subject: [PATCH 21/29] doc Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 4 ++-- .github/workflows/windows-macos-ci.yml | 4 ++-- README.md | 4 ++-- docs/index.rst | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 61d58a193..5777b6dce 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -60,7 +60,7 @@ jobs: run: python -m pip install -r requirements-dev.txt - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} @@ -99,6 +99,6 @@ jobs: - name: build run: pip wheel . - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: ./dist/** diff --git a/.github/workflows/windows-macos-ci.yml b/.github/workflows/windows-macos-ci.yml index 513eb1b07..aecdd6b03 100644 --- a/.github/workflows/windows-macos-ci.yml +++ b/.github/workflows/windows-macos-ci.yml @@ -40,7 +40,7 @@ jobs: run: python -m pip install -r requirements-dev.txt - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} @@ -72,6 +72,6 @@ jobs: - name: build run: pip wheel . - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: ./dist/** diff --git a/README.md b/README.md index 6ac428186..2b032c3c3 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@

-[![Build Status](https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status%2Fonnx.sklearn-onnx.linux.CI?branchName=refs%2Fpull%2F1020%2Fmerge)](https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=21&branchName=refs%2Fpull%2F1020%2Fmerge) +[![Linux](https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml/badge.svg)](https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml) -[![Build Status](https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status%2Fonnx.sklearn-onnx.win.CI?branchName=refs%2Fpull%2F1020%2Fmerge)](https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=22&branchName=refs%2Fpull%2F1020%2Fmerge) +[![Windows/Macos](https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml/badge.svg)](https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) diff --git a/docs/index.rst b/docs/index.rst index 62f5fa066..4fa04e447 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,10 +9,10 @@ sklearn-onnx: Convert your scikit-learn model into ONNX :widths: 5 5 * - Linux - Windows - * - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-linux-conda-ci?branchName=main - :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=main - - .. image:: https://dev.azure.com/onnxmltools/sklearn-onnx/_apis/build/status/sklearn-onnx-win32-conda-ci?branchName=main - :target: https://dev.azure.com/onnxmltools/sklearn-onnx/_build/latest?definitionId=5?branchName=main + * - .. image:: https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml/badge.svg + :target: https://github.com/onnx/sklearn-onnx/actions/workflows/linux-ci.yml + - .. image:: https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml/badge.svg + :target: https://github.com/onnx/sklearn-onnx/actions/workflows/windows-macos-ci.yml *sklearn-onnx* enables you to convert models from From d3237b73afa8b2ba59132a413552eb47727cb6a8 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 15:48:28 +0200 Subject: [PATCH 22/29] ci Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- .github/workflows/windows-macos-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 5777b6dce..4b244e8c0 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -87,7 +87,7 @@ jobs: - name: Run tests run: | - pytest tests --maxfail=10 --durations=10 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml + pytest tests --maxfail=10 --durations=10 - name: Run documentation tests if: matrix.documentation == 1 diff --git a/.github/workflows/windows-macos-ci.yml b/.github/workflows/windows-macos-ci.yml index aecdd6b03..4db15c2f3 100644 --- a/.github/workflows/windows-macos-ci.yml +++ b/.github/workflows/windows-macos-ci.yml @@ -67,7 +67,7 @@ jobs: - name: Run tests run: | - pytest tests --maxfail=10 --durations=10 --basetemp=temp --doctest-modules --junitxml=junit/test-results.xml + pytest tests --maxfail=10 --durations=10 - name: build run: pip wheel . From 06165f12841efeecad7670d7c36872e39072cbef Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 15:48:43 +0200 Subject: [PATCH 23/29] ci Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 4b244e8c0..e69d25006 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -93,7 +93,7 @@ jobs: if: matrix.documentation == 1 run: | pip install -r docs/requirements.txt - pytest docs/tests --durations=10 --basetemp=temp --doctest-modules -v -v --maxfail=5 + pytest docs/tests --durations=10 --maxfail=5 # python tests/benchmark.py - name: build From 6b4c7e3e6f29f6c483d9f9edd92081faabf7f78c Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 17:07:54 +0200 Subject: [PATCH 24/29] ci Signed-off-by: Xavier Dupre --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2a100a0ad..0da5e225f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,5 +12,5 @@ wheel # docs/examples xgboost -lightgbm +lightgbm; sys_platform != 'darwin' matplotlib From 1dfdd942d97a35a32a69eddcd87fcee3665162e0 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 17:08:20 +0200 Subject: [PATCH 25/29] ci Signed-off-by: Xavier Dupre --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 2434942f0..a14fffedd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,7 +5,7 @@ coverage flake8 furo joblib -lightgbm +lightgbm; sys_platform != 'darwin' loky matplotlib mlinsights>=0.3.631 From 3975ac4711d66d0672bb1f7034e6871ccc384adb Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Tue, 21 May 2024 17:38:48 +0200 Subject: [PATCH 26/29] fix ci Signed-off-by: Xavier Dupre --- .github/workflows/windows-macos-ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/windows-macos-ci.yml b/.github/workflows/windows-macos-ci.yml index 4db15c2f3..2916a3cbe 100644 --- a/.github/workflows/windows-macos-ci.yml +++ b/.github/workflows/windows-macos-ci.yml @@ -2,7 +2,7 @@ name: CI Win/Mac on: [push, pull_request] jobs: run: - name: ${{ matrix.os }} Py==${{ matrix.python_version }} - sklearn==${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} + name: ${{ matrix.os }} py==${{ matrix.python_version }} - sklearn==${{ matrix.sklearn_version }} - ${{ matrix.onnxrt_version }} runs-on: ${{ matrix.os }} strategy: matrix: @@ -18,8 +18,14 @@ jobs: - python_version: '3.10' numpy_version: '>=1.21.1' scipy_version: '>=1.7.0' - onnx_version: 'onnx==1.14.1' - onnxrt_version: 'onnxruntime==1.16.0' + onnx_version: 'onnx<1.15' + onnxrt_version: 'onnxruntime<1.17.0' + sklearn_version: '==1.2.2' + - python_version: '3.9' + numpy_version: '>=1.21.1' + scipy_version: '>=1.7.0' + onnx_version: 'onnx<1.14' + onnxrt_version: 'onnxruntime<1.16.0' sklearn_version: '==1.2.2' steps: From aaa287358bb636782b17608c59e406cd7aff179f Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Wed, 22 May 2024 11:13:46 +0200 Subject: [PATCH 27/29] fix unittest Signed-off-by: Xavier Dupre --- .../test_sklearn_one_hot_encoder_converter.py | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/tests/test_sklearn_one_hot_encoder_converter.py b/tests/test_sklearn_one_hot_encoder_converter.py index 58f72100f..8e224499f 100644 --- a/tests/test_sklearn_one_hot_encoder_converter.py +++ b/tests/test_sklearn_one_hot_encoder_converter.py @@ -2,6 +2,7 @@ """Tests scikit-learn's OneHotEncoder converter.""" import unittest +import sys import packaging.version as pv import numpy from numpy.testing import assert_almost_equal @@ -373,18 +374,9 @@ def test_one_hot_encoder_string_drop_first_2(self): data, model, model_onnx, basename="SklearnOneHotEncoderStringDropFirst2" ) - @unittest.skipIf( - onnx_opset_version() < 19, reason="missing ops in reference implementation" - ) - @ignore_warnings(category=RuntimeWarning) - def test_shape_inference(self): + def _shape_inference(self, engine): cat_columns_openings = ["cat_1", "cat_2"] - num_columns_openings = [ - "num_1", - "num_2", - "num_3", - "num_4", - ] + num_columns_openings = ["num_1", "num_2", "num_3", "num_4"] regression_aperturas = LinearRegression() @@ -450,23 +442,33 @@ def test_shape_inference(self): ] ) - # ReferenceEvaluator - with self.subTest(engine="onnx"): - ref = ReferenceEvaluator(model_onnx) - res = ref.run(None, feeds) - self.assertEqual(1, len(res)) - self.assertEqual(expected.shape, res[0].shape) - assert_almost_equal(expected, res[0]) - # onnxruntime - with self.subTest(engine="onnxruntime"): - sess = InferenceSession( + if engine == "onnxruntime": + ref = InferenceSession( model_onnx.SerializeToString(), providers=["CPUExecutionProvider"] ) - res = sess.run(None, feeds) - self.assertEqual(1, len(res)) - self.assertEqual(expected.shape, res[0].shape) - assert_almost_equal(expected, res[0]) + else: + ref = ReferenceEvaluator(model_onnx) + + res = ref.run(None, feeds) + self.assertEqual(1, len(res)) + self.assertEqual(expected.shape, res[0].shape) + assert_almost_equal(expected, res[0]) + + @unittest.skipIf( + onnx_opset_version() < 19, reason="missing ops in reference implementation" + ) + @ignore_warnings(category=RuntimeWarning) + @unittest.skipIf(sys.platform == "darwin", "interesting discrepancy") + def test_shape_inference_onnx(self): + self._shape_inference("onnx") + + @unittest.skipIf( + onnx_opset_version() < 19, reason="missing ops in reference implementation" + ) + @ignore_warnings(category=RuntimeWarning) + def test_shape_inference_onnxruntime(self): + self._shape_inference("onnxruntime") @unittest.skipIf( not one_hot_encoder_supports_drop(), From 92f965cecbb038b959679c6fea8242382ab579e6 Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Wed, 22 May 2024 11:45:26 +0200 Subject: [PATCH 28/29] fix ci Signed-off-by: Xavier Dupre --- .github/workflows/linux-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index e69d25006..d86d794e7 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -15,7 +15,7 @@ jobs: scipy_version: '>=1.7.0' onnx_version: 'onnx==1.16.0' onnxrt_version: 'onnxruntime==1.18.0' - sklearn_version: '>=1.4.2' + sklearn_version: '==1.4.2' - python_version: '3.11' documentation: 1 numpy_version: '>=1.21.1' From 218a1e793b6e5a4e696dfce4cae4b6665910356a Mon Sep 17 00:00:00 2001 From: Xavier Dupre Date: Wed, 22 May 2024 11:53:21 +0200 Subject: [PATCH 29/29] fix ci Signed-off-by: Xavier Dupre --- tests/test_sklearn_one_hot_encoder_converter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_sklearn_one_hot_encoder_converter.py b/tests/test_sklearn_one_hot_encoder_converter.py index 8e224499f..5073c4c1d 100644 --- a/tests/test_sklearn_one_hot_encoder_converter.py +++ b/tests/test_sklearn_one_hot_encoder_converter.py @@ -467,6 +467,7 @@ def test_shape_inference_onnx(self): onnx_opset_version() < 19, reason="missing ops in reference implementation" ) @ignore_warnings(category=RuntimeWarning) + @unittest.skipIf(sys.platform == "darwin", "interesting discrepancy") def test_shape_inference_onnxruntime(self): self._shape_inference("onnxruntime")