From 04025131b708b5c05e4a52c98028c7337b25272c Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 18:21:54 +0200 Subject: [PATCH 1/9] create other light benchmarks to parralelize the work --- .../light_benchmark_nch_min_hull.yml | 44 +++++++++++++++++++ .../workflows/light_benchmark_nch_qaoacv.yml | 44 +++++++++++++++++++ benchmarks/light_benchmark.py | 36 --------------- 3 files changed, 88 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/light_benchmark_nch_min_hull.yml create mode 100644 .github/workflows/light_benchmark_nch_qaoacv.yml diff --git a/.github/workflows/light_benchmark_nch_min_hull.yml b/.github/workflows/light_benchmark_nch_min_hull.yml new file mode 100644 index 00000000..230f6cbc --- /dev/null +++ b/.github/workflows/light_benchmark_nch_min_hull.yml @@ -0,0 +1,44 @@ +name: Light Benchmark + +on: + pull_request: + paths: + - 'pyriemann_qiskit/**' + - 'examples/**' + - '.github/workflows/light_benchmark_nch_min_hull.yml' + - 'benchmarks/light_benchmark_nch_min_hull.py' + - 'setup.py' + +jobs: + light_benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: light_benchmark.yml + - name: Install dependencies + run: | + pip install .[docs] + - name: Run benchmark script (PR) + id: run-benchmark-pr + run: | + python benchmarks/light_benchmark.py pr + - uses: actions/checkout@v4 + with: + ref: 'main' + - name: Install dependencies + run: | + pip install .[docs] + - name: Run benchmark script (main) + id: run-benchmark-main + run: | + python benchmarks/light_benchmark.py ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL}} + - name: Compare performance + run: | + echo ${{steps.run-benchmark-main.outputs.success}} + if [[ "${{steps.run-benchmark-main.outputs.success}}" == "0" ]]; then + exit 1 + fi diff --git a/.github/workflows/light_benchmark_nch_qaoacv.yml b/.github/workflows/light_benchmark_nch_qaoacv.yml new file mode 100644 index 00000000..ac888f16 --- /dev/null +++ b/.github/workflows/light_benchmark_nch_qaoacv.yml @@ -0,0 +1,44 @@ +name: Light Benchmark + +on: + pull_request: + paths: + - 'pyriemann_qiskit/**' + - 'examples/**' + - '.github/workflows/light_benchmark_nch_qaoacv.yml' + - 'benchmarks/light_benchmark_qaoacv.py' + - 'setup.py' + +jobs: + light_benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: light_benchmark.yml + - name: Install dependencies + run: | + pip install .[docs] + - name: Run benchmark script (PR) + id: run-benchmark-pr + run: | + python benchmarks/light_benchmark.py pr + - uses: actions/checkout@v4 + with: + ref: 'main' + - name: Install dependencies + run: | + pip install .[docs] + - name: Run benchmark script (main) + id: run-benchmark-main + run: | + python benchmarks/light_benchmark.py ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL_QAOACV}} + - name: Compare performance + run: | + echo ${{steps.run-benchmark-main.outputs.success}} + if [[ "${{steps.run-benchmark-main.outputs.success}}" == "0" ]]; then + exit 1 + fi diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index 41e7cc71..27abf248 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -115,42 +115,6 @@ LDA(solver="lsqr", shrinkage="auto"), ) -pipelines["NCH_MIN_HULL"] = make_pipeline( - XdawnCovariances( - nfilter=3, - estimator="lwf", - xdawn_estimator="scm", - ), - QuanticNCH( - n_hulls_per_class=1, - n_samples_per_hull=3, - n_jobs=12, - subsampling="min", - quantum=False, - shots=100, - ), -) - - -pipelines["NCH_MIN_HULL_QAOACV"] = make_pipeline( - XdawnCovariances( - nfilter=3, - estimator="lwf", - xdawn_estimator="scm", - ), - QuanticNCH( - n_hulls_per_class=1, - n_samples_per_hull=3, - n_jobs=12, - subsampling="min", - quantum=True, - # Provide create_mixer to force QAOA-CV optimization - create_mixer=create_mixer_rotational_X_gates(0), - shots=100, - qaoa_optimizer=SPSA(maxiter=500), - ), -) - ############################################################################## # Compute score # -------------- From 2c86fdf052dbf9c5d1fad4be1a7a5e7dd6ef166d Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 18:28:45 +0200 Subject: [PATCH 2/9] missing scripts --- .../light_benchmark_nch_min_hull.yml | 2 +- .../workflows/light_benchmark_nch_qaoacv.yml | 2 +- benchmarks/light_benchmark.py | 3 - benchmarks/light_benchmark_nch_min_hull.py | 140 +++++++++++++++++ benchmarks/light_benchmark_nch_qaoacv.py | 143 ++++++++++++++++++ 5 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 benchmarks/light_benchmark_nch_min_hull.py create mode 100644 benchmarks/light_benchmark_nch_qaoacv.py diff --git a/.github/workflows/light_benchmark_nch_min_hull.yml b/.github/workflows/light_benchmark_nch_min_hull.yml index 230f6cbc..eb8e46e3 100644 --- a/.github/workflows/light_benchmark_nch_min_hull.yml +++ b/.github/workflows/light_benchmark_nch_min_hull.yml @@ -1,4 +1,4 @@ -name: Light Benchmark +name: Light Benchmark NCH-MIN-HULL on: pull_request: diff --git a/.github/workflows/light_benchmark_nch_qaoacv.yml b/.github/workflows/light_benchmark_nch_qaoacv.yml index ac888f16..22f6450e 100644 --- a/.github/workflows/light_benchmark_nch_qaoacv.yml +++ b/.github/workflows/light_benchmark_nch_qaoacv.yml @@ -1,4 +1,4 @@ -name: Light Benchmark +name: Light Benchmark NCH-QAOACV on: pull_request: diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index 27abf248..d4d03a63 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -19,14 +19,11 @@ from moabb.paradigms import P300 from pyriemann.estimation import XdawnCovariances, Shrinkage from pyriemann.tangentspace import TangentSpace -from pyriemann_qiskit.classification import QuanticNCH from pyriemann_qiskit.pipelines import ( QuantumClassifierWithDefaultRiemannianPipeline, QuantumMDMWithRiemannianPipeline, ) from pyriemann_qiskit.utils import distance, mean # noqa -from pyriemann_qiskit.utils.hyper_params_factory import create_mixer_rotational_X_gates -from qiskit_algorithms.optimizers import SPSA from sklearn.decomposition import PCA from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA from sklearn.metrics import balanced_accuracy_score diff --git a/benchmarks/light_benchmark_nch_min_hull.py b/benchmarks/light_benchmark_nch_min_hull.py new file mode 100644 index 00000000..660eff58 --- /dev/null +++ b/benchmarks/light_benchmark_nch_min_hull.py @@ -0,0 +1,140 @@ +""" +==================================================================== +Light Benchmark +==================================================================== + +This benchmark is a non-regression performance test, intended +to run on Ci with each PRs. + +""" +# Author: Gregoire Cattan +# Modified from plot_classify_P300_bi.py of pyRiemann +# License: BSD (3-clause) + +import sys +import warnings + +from moabb import set_log_level +from moabb.datasets import bi2012 +from moabb.paradigms import P300 +from pyriemann.estimation import XdawnCovariances, Shrinkage +from pyriemann.tangentspace import TangentSpace +from pyriemann_qiskit.classification import QuanticNCH +from pyriemann_qiskit.pipelines import ( + QuantumClassifierWithDefaultRiemannianPipeline, + QuantumMDMWithRiemannianPipeline, +) +from pyriemann_qiskit.utils import distance, mean # noqa +from pyriemann_qiskit.utils.hyper_params_factory import create_mixer_rotational_X_gates +from qiskit_algorithms.optimizers import SPSA +from sklearn.decomposition import PCA +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA +from sklearn.metrics import balanced_accuracy_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import LabelEncoder + + +print(__doc__) + +############################################################################## +# getting rid of the warnings about the future +warnings.simplefilter(action="ignore", category=FutureWarning) +warnings.simplefilter(action="ignore", category=RuntimeWarning) + +warnings.filterwarnings("ignore") + +set_log_level("info") + +############################################################################## +# Prepare data +# ------------- +# +############################################################################## + +paradigm = P300(resample=128) + +dataset = bi2012() # MOABB provides several other P300 datasets + +X, y, _ = paradigm.get_data(dataset, subjects=[1]) + +# Reduce the dataset size for Ci +_, X, _, y = train_test_split(X, y, test_size=0.7, random_state=42, stratify=y) + +y = LabelEncoder().fit_transform(y) + +# Separate into train and test +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.33, random_state=42, stratify=y +) + +############################################################################## +# Create Pipelines +# ---------------- +# +# Pipelines must be a dict of sklearn pipeline transformer. +# +############################################################################## + +pipelines = {} + +pipelines["NCH_MIN_HULL"] = make_pipeline( + XdawnCovariances( + nfilter=3, + estimator="lwf", + xdawn_estimator="scm", + ), + QuanticNCH( + n_hulls_per_class=1, + n_samples_per_hull=3, + n_jobs=12, + subsampling="min", + quantum=False, + shots=100, + ), +) + +############################################################################## +# Compute score +# -------------- +# +############################################################################## + +scores = {} + +for key, pipeline in pipelines.items(): + pipeline.fit(X_train, y_train) + y_pred = pipeline.predict(X_test) + score = balanced_accuracy_score(y_test, y_pred) + scores[key] = score + +print("Scores: ", scores) + +############################################################################## +# Compare scores between PR and main branches +# ------------------------------------------- +# +############################################################################## + + +def set_output(key: str, value: str): + print(f"::set-output name={key}::{value}") # noqa: E231 + + +is_pr = sys.argv[1] == "pr" + +if is_pr: + for key, score in scores.items(): + set_output(key, score) +else: + success = True + i = 0 + for key, score in scores.items(): + i = i + 1 + pr_score = sys.argv[i] + pr_score_trun = int(float(pr_score) * 100) + score_trun = int(score * 100) + better_pr_score = pr_score_trun >= score_trun + success = success and better_pr_score + print(f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}") + set_output("success", "1" if success else "0") diff --git a/benchmarks/light_benchmark_nch_qaoacv.py b/benchmarks/light_benchmark_nch_qaoacv.py new file mode 100644 index 00000000..2bd92fdb --- /dev/null +++ b/benchmarks/light_benchmark_nch_qaoacv.py @@ -0,0 +1,143 @@ +""" +==================================================================== +Light Benchmark +==================================================================== + +This benchmark is a non-regression performance test, intended +to run on Ci with each PRs. + +""" +# Author: Gregoire Cattan +# Modified from plot_classify_P300_bi.py of pyRiemann +# License: BSD (3-clause) + +import sys +import warnings + +from moabb import set_log_level +from moabb.datasets import bi2012 +from moabb.paradigms import P300 +from pyriemann.estimation import XdawnCovariances, Shrinkage +from pyriemann.tangentspace import TangentSpace +from pyriemann_qiskit.classification import QuanticNCH +from pyriemann_qiskit.pipelines import ( + QuantumClassifierWithDefaultRiemannianPipeline, + QuantumMDMWithRiemannianPipeline, +) +from pyriemann_qiskit.utils import distance, mean # noqa +from pyriemann_qiskit.utils.hyper_params_factory import create_mixer_rotational_X_gates +from qiskit_algorithms.optimizers import SPSA +from sklearn.decomposition import PCA +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA +from sklearn.metrics import balanced_accuracy_score +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import LabelEncoder + + +print(__doc__) + +############################################################################## +# getting rid of the warnings about the future +warnings.simplefilter(action="ignore", category=FutureWarning) +warnings.simplefilter(action="ignore", category=RuntimeWarning) + +warnings.filterwarnings("ignore") + +set_log_level("info") + +############################################################################## +# Prepare data +# ------------- +# +############################################################################## + +paradigm = P300(resample=128) + +dataset = bi2012() # MOABB provides several other P300 datasets + +X, y, _ = paradigm.get_data(dataset, subjects=[1]) + +# Reduce the dataset size for Ci +_, X, _, y = train_test_split(X, y, test_size=0.7, random_state=42, stratify=y) + +y = LabelEncoder().fit_transform(y) + +# Separate into train and test +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.33, random_state=42, stratify=y +) + +############################################################################## +# Create Pipelines +# ---------------- +# +# Pipelines must be a dict of sklearn pipeline transformer. +# +############################################################################## + +pipelines = {} + +pipelines["NCH_MIN_HULL_QAOACV"] = make_pipeline( + XdawnCovariances( + nfilter=3, + estimator="lwf", + xdawn_estimator="scm", + ), + QuanticNCH( + n_hulls_per_class=1, + n_samples_per_hull=3, + n_jobs=12, + subsampling="min", + quantum=True, + # Provide create_mixer to force QAOA-CV optimization + create_mixer=create_mixer_rotational_X_gates(0), + shots=100, + qaoa_optimizer=SPSA(maxiter=100), + ), +) + +############################################################################## +# Compute score +# -------------- +# +############################################################################## + +scores = {} + +for key, pipeline in pipelines.items(): + pipeline.fit(X_train, y_train) + y_pred = pipeline.predict(X_test) + score = balanced_accuracy_score(y_test, y_pred) + scores[key] = score + +print("Scores: ", scores) + +############################################################################## +# Compare scores between PR and main branches +# ------------------------------------------- +# +############################################################################## + + +def set_output(key: str, value: str): + print(f"::set-output name={key}::{value}") # noqa: E231 + + +is_pr = sys.argv[1] == "pr" + +if is_pr: + for key, score in scores.items(): + set_output(key, score) +else: + success = True + i = 0 + for key, score in scores.items(): + i = i + 1 + pr_score = sys.argv[i] + pr_score_trun = int(float(pr_score) * 100) + score_trun = int(score * 100) + better_pr_score = pr_score_trun >= score_trun + success = success and better_pr_score + print(f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}") + set_output("success", "1" if success else "0") From 70e863be353747c852ff19217702ca848dcdc16b Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 18:30:46 +0200 Subject: [PATCH 3/9] correct cache_key --- .github/workflows/light_benchmark_nch_min_hull.yml | 6 +++--- .github/workflows/light_benchmark_nch_qaoacv.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/light_benchmark_nch_min_hull.yml b/.github/workflows/light_benchmark_nch_min_hull.yml index eb8e46e3..464722ea 100644 --- a/.github/workflows/light_benchmark_nch_min_hull.yml +++ b/.github/workflows/light_benchmark_nch_min_hull.yml @@ -1,4 +1,4 @@ -name: Light Benchmark NCH-MIN-HULL +name: Light Benchmark on: pull_request: @@ -10,7 +10,7 @@ on: - 'setup.py' jobs: - light_benchmark: + light_benchmark_nch_min_hull: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -18,7 +18,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/pip - key: light_benchmark.yml + key: light_benchmark_nch_min_hull.yml - name: Install dependencies run: | pip install .[docs] diff --git a/.github/workflows/light_benchmark_nch_qaoacv.yml b/.github/workflows/light_benchmark_nch_qaoacv.yml index 22f6450e..83aef72e 100644 --- a/.github/workflows/light_benchmark_nch_qaoacv.yml +++ b/.github/workflows/light_benchmark_nch_qaoacv.yml @@ -1,4 +1,4 @@ -name: Light Benchmark NCH-QAOACV +name: Light Benchmark on: pull_request: @@ -10,7 +10,7 @@ on: - 'setup.py' jobs: - light_benchmark: + light_benchmark_nch_qaoa: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -18,7 +18,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/pip - key: light_benchmark.yml + key: light_benchmark_nch_qaoacv.yml - name: Install dependencies run: | pip install .[docs] From ec9a9b948b588056717d470b6c7e9b5e8e8e8a56 Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 18:32:10 +0200 Subject: [PATCH 4/9] fix wrong file in main branch --- .github/workflows/light_benchmark_nch_min_hull.yml | 4 ++-- .github/workflows/light_benchmark_nch_qaoacv.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/light_benchmark_nch_min_hull.yml b/.github/workflows/light_benchmark_nch_min_hull.yml index 464722ea..38a073f7 100644 --- a/.github/workflows/light_benchmark_nch_min_hull.yml +++ b/.github/workflows/light_benchmark_nch_min_hull.yml @@ -25,7 +25,7 @@ jobs: - name: Run benchmark script (PR) id: run-benchmark-pr run: | - python benchmarks/light_benchmark.py pr + python benchmarks/light_benchmark_nch_min_hull.py pr - uses: actions/checkout@v4 with: ref: 'main' @@ -35,7 +35,7 @@ jobs: - name: Run benchmark script (main) id: run-benchmark-main run: | - python benchmarks/light_benchmark.py ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL}} + python benchmarks/light_benchmark_nch_min_hull.py ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL}} - name: Compare performance run: | echo ${{steps.run-benchmark-main.outputs.success}} diff --git a/.github/workflows/light_benchmark_nch_qaoacv.yml b/.github/workflows/light_benchmark_nch_qaoacv.yml index 83aef72e..de4290db 100644 --- a/.github/workflows/light_benchmark_nch_qaoacv.yml +++ b/.github/workflows/light_benchmark_nch_qaoacv.yml @@ -25,7 +25,7 @@ jobs: - name: Run benchmark script (PR) id: run-benchmark-pr run: | - python benchmarks/light_benchmark.py pr + python benchmarks/light_benchmark_nch_qaoacv.py pr - uses: actions/checkout@v4 with: ref: 'main' @@ -35,7 +35,7 @@ jobs: - name: Run benchmark script (main) id: run-benchmark-main run: | - python benchmarks/light_benchmark.py ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL_QAOACV}} + python benchmarks/light_benchmark_nch_qaoacv.py ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL_QAOACV}} - name: Compare performance run: | echo ${{steps.run-benchmark-main.outputs.success}} From bd3e14e5106d45790b8972f92c59e4d44e5de709 Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 21:35:28 +0200 Subject: [PATCH 5/9] factorize code in base module --- benchmarks/lb_base.py | 90 ++++++++++++++++++++++ benchmarks/light_benchmark.py | 75 ++---------------- benchmarks/light_benchmark_nch_min_hull.py | 85 ++------------------ benchmarks/light_benchmark_nch_qaoacv.py | 83 ++------------------ 4 files changed, 107 insertions(+), 226 deletions(-) create mode 100644 benchmarks/lb_base.py diff --git a/benchmarks/lb_base.py b/benchmarks/lb_base.py new file mode 100644 index 00000000..d1d91fa9 --- /dev/null +++ b/benchmarks/lb_base.py @@ -0,0 +1,90 @@ +""" +==================================================================== +Light Benchmark +==================================================================== + +Common script to run light benchmarks + +""" +# Author: Gregoire Cattan +# Modified from plot_classify_P300_bi.py of pyRiemann +# License: BSD (3-clause) + +import sys +import warnings + +from moabb import set_log_level +from moabb.datasets import bi2012 +from moabb.paradigms import P300 +from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA +from sklearn.metrics import balanced_accuracy_score +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import LabelEncoder + + +print(__doc__) + +############################################################################## +# getting rid of the warnings about the future +warnings.simplefilter(action="ignore", category=FutureWarning) +warnings.simplefilter(action="ignore", category=RuntimeWarning) + +warnings.filterwarnings("ignore") + +set_log_level("info") + +############################################################################## +# Prepare data +# ------------- +# +############################################################################## + +def _set_output(key: str, value: str): + print(f"::set-output name={key}::{value}") # noqa: E231 + +def run(pipelines): + paradigm = P300(resample=128) + + dataset = bi2012() # MOABB provides several other P300 datasets + + X, y, _ = paradigm.get_data(dataset, subjects=[1]) + + # Reduce the dataset size for Ci + _, X, _, y = train_test_split(X, y, test_size=0.7, random_state=42, stratify=y) + + y = LabelEncoder().fit_transform(y) + + # Separate into train and test + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.33, random_state=42, stratify=y + ) + + # Compute scores + scores = {} + + for key, pipeline in pipelines.items(): + pipeline.fit(X_train, y_train) + y_pred = pipeline.predict(X_test) + score = balanced_accuracy_score(y_test, y_pred) + scores[key] = score + + print("Scores: ", scores) + + # Compare scores between PR and main branches + is_pr = sys.argv[1] == "pr" + + if is_pr: + for key, score in scores.items(): + _set_output(key, score) + else: + success = True + i = 0 + for key, score in scores.items(): + i = i + 1 + pr_score = sys.argv[i] + pr_score_trun = int(float(pr_score) * 100) + score_trun = int(score * 100) + better_pr_score = pr_score_trun >= score_trun + success = success and better_pr_score + print(f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}") + _set_output("success", "1" if success else "0") diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index d4d03a63..4cb567a8 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -11,12 +11,9 @@ # Modified from plot_classify_P300_bi.py of pyRiemann # License: BSD (3-clause) -import sys import warnings from moabb import set_log_level -from moabb.datasets import bi2012 -from moabb.paradigms import P300 from pyriemann.estimation import XdawnCovariances, Shrinkage from pyriemann.tangentspace import TangentSpace from pyriemann_qiskit.pipelines import ( @@ -26,11 +23,8 @@ from pyriemann_qiskit.utils import distance, mean # noqa from sklearn.decomposition import PCA from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA -from sklearn.metrics import balanced_accuracy_score -from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import LabelEncoder - +from .lb_base import run print(__doc__) @@ -43,28 +37,6 @@ set_log_level("info") -############################################################################## -# Prepare data -# ------------- -# -############################################################################## - -paradigm = P300(resample=128) - -dataset = bi2012() # MOABB provides several other P300 datasets - -X, y, _ = paradigm.get_data(dataset, subjects=[1]) - -# Reduce the dataset size for Ci -_, X, _, y = train_test_split(X, y, test_size=0.7, random_state=42, stratify=y) - -y = LabelEncoder().fit_transform(y) - -# Separate into train and test -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.33, random_state=42, stratify=y -) - ############################################################################## # Create Pipelines # ---------------- @@ -113,46 +85,9 @@ ) ############################################################################## -# Compute score -# -------------- -# -############################################################################## - -scores = {} - -for key, pipeline in pipelines.items(): - pipeline.fit(X_train, y_train) - y_pred = pipeline.predict(X_test) - score = balanced_accuracy_score(y_test, y_pred) - scores[key] = score - -print("Scores: ", scores) - +# Compute score and compare with PR branch +# ------------------------------------------ +# ############################################################################## -# Compare scores between PR and main branches -# ------------------------------------------- -# -############################################################################## - - -def set_output(key: str, value: str): - print(f"::set-output name={key}::{value}") # noqa: E231 - - -is_pr = sys.argv[1] == "pr" -if is_pr: - for key, score in scores.items(): - set_output(key, score) -else: - success = True - i = 0 - for key, score in scores.items(): - i = i + 1 - pr_score = sys.argv[i] - pr_score_trun = int(float(pr_score) * 100) - score_trun = int(score * 100) - better_pr_score = pr_score_trun >= score_trun - success = success and better_pr_score - print(f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}") - set_output("success", "1" if success else "0") +run(pipelines) diff --git a/benchmarks/light_benchmark_nch_min_hull.py b/benchmarks/light_benchmark_nch_min_hull.py index 660eff58..f229db9c 100644 --- a/benchmarks/light_benchmark_nch_min_hull.py +++ b/benchmarks/light_benchmark_nch_min_hull.py @@ -11,29 +11,15 @@ # Modified from plot_classify_P300_bi.py of pyRiemann # License: BSD (3-clause) -import sys import warnings from moabb import set_log_level -from moabb.datasets import bi2012 -from moabb.paradigms import P300 -from pyriemann.estimation import XdawnCovariances, Shrinkage -from pyriemann.tangentspace import TangentSpace +from pyriemann.estimation import XdawnCovariances from pyriemann_qiskit.classification import QuanticNCH -from pyriemann_qiskit.pipelines import ( - QuantumClassifierWithDefaultRiemannianPipeline, - QuantumMDMWithRiemannianPipeline, -) from pyriemann_qiskit.utils import distance, mean # noqa -from pyriemann_qiskit.utils.hyper_params_factory import create_mixer_rotational_X_gates -from qiskit_algorithms.optimizers import SPSA -from sklearn.decomposition import PCA -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA -from sklearn.metrics import balanced_accuracy_score -from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import LabelEncoder +from .lb_base import run print(__doc__) @@ -46,28 +32,6 @@ set_log_level("info") -############################################################################## -# Prepare data -# ------------- -# -############################################################################## - -paradigm = P300(resample=128) - -dataset = bi2012() # MOABB provides several other P300 datasets - -X, y, _ = paradigm.get_data(dataset, subjects=[1]) - -# Reduce the dataset size for Ci -_, X, _, y = train_test_split(X, y, test_size=0.7, random_state=42, stratify=y) - -y = LabelEncoder().fit_transform(y) - -# Separate into train and test -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.33, random_state=42, stratify=y -) - ############################################################################## # Create Pipelines # ---------------- @@ -95,46 +59,9 @@ ) ############################################################################## -# Compute score -# -------------- -# -############################################################################## - -scores = {} - -for key, pipeline in pipelines.items(): - pipeline.fit(X_train, y_train) - y_pred = pipeline.predict(X_test) - score = balanced_accuracy_score(y_test, y_pred) - scores[key] = score - -print("Scores: ", scores) - +# Compute score and compare with PR branch +# ------------------------------------------ +# ############################################################################## -# Compare scores between PR and main branches -# ------------------------------------------- -# -############################################################################## - - -def set_output(key: str, value: str): - print(f"::set-output name={key}::{value}") # noqa: E231 - - -is_pr = sys.argv[1] == "pr" -if is_pr: - for key, score in scores.items(): - set_output(key, score) -else: - success = True - i = 0 - for key, score in scores.items(): - i = i + 1 - pr_score = sys.argv[i] - pr_score_trun = int(float(pr_score) * 100) - score_trun = int(score * 100) - better_pr_score = pr_score_trun >= score_trun - success = success and better_pr_score - print(f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}") - set_output("success", "1" if success else "0") +run(pipelines) diff --git a/benchmarks/light_benchmark_nch_qaoacv.py b/benchmarks/light_benchmark_nch_qaoacv.py index 2bd92fdb..e1b3287c 100644 --- a/benchmarks/light_benchmark_nch_qaoacv.py +++ b/benchmarks/light_benchmark_nch_qaoacv.py @@ -11,29 +11,17 @@ # Modified from plot_classify_P300_bi.py of pyRiemann # License: BSD (3-clause) -import sys import warnings from moabb import set_log_level -from moabb.datasets import bi2012 -from moabb.paradigms import P300 -from pyriemann.estimation import XdawnCovariances, Shrinkage -from pyriemann.tangentspace import TangentSpace +from pyriemann.estimation import XdawnCovariances from pyriemann_qiskit.classification import QuanticNCH -from pyriemann_qiskit.pipelines import ( - QuantumClassifierWithDefaultRiemannianPipeline, - QuantumMDMWithRiemannianPipeline, -) from pyriemann_qiskit.utils import distance, mean # noqa from pyriemann_qiskit.utils.hyper_params_factory import create_mixer_rotational_X_gates from qiskit_algorithms.optimizers import SPSA -from sklearn.decomposition import PCA -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA -from sklearn.metrics import balanced_accuracy_score -from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import LabelEncoder +from .lb_base import run print(__doc__) @@ -46,28 +34,6 @@ set_log_level("info") -############################################################################## -# Prepare data -# ------------- -# -############################################################################## - -paradigm = P300(resample=128) - -dataset = bi2012() # MOABB provides several other P300 datasets - -X, y, _ = paradigm.get_data(dataset, subjects=[1]) - -# Reduce the dataset size for Ci -_, X, _, y = train_test_split(X, y, test_size=0.7, random_state=42, stratify=y) - -y = LabelEncoder().fit_transform(y) - -# Separate into train and test -X_train, X_test, y_train, y_test = train_test_split( - X, y, test_size=0.33, random_state=42, stratify=y -) - ############################################################################## # Create Pipelines # ---------------- @@ -98,46 +64,9 @@ ) ############################################################################## -# Compute score -# -------------- -# -############################################################################## - -scores = {} - -for key, pipeline in pipelines.items(): - pipeline.fit(X_train, y_train) - y_pred = pipeline.predict(X_test) - score = balanced_accuracy_score(y_test, y_pred) - scores[key] = score - -print("Scores: ", scores) - +# Compute score and compare with PR branch +# ------------------------------------------ +# ############################################################################## -# Compare scores between PR and main branches -# ------------------------------------------- -# -############################################################################## - - -def set_output(key: str, value: str): - print(f"::set-output name={key}::{value}") # noqa: E231 - - -is_pr = sys.argv[1] == "pr" -if is_pr: - for key, score in scores.items(): - set_output(key, score) -else: - success = True - i = 0 - for key, score in scores.items(): - i = i + 1 - pr_score = sys.argv[i] - pr_score_trun = int(float(pr_score) * 100) - score_trun = int(score * 100) - better_pr_score = pr_score_trun >= score_trun - success = success and better_pr_score - print(f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}") - set_output("success", "1" if success else "0") +run(pipelines) From 40064fda97f3db30ca6ff63bfb298d5026cb5e57 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 19:35:56 +0000 Subject: [PATCH 6/9] [pre-commit.ci] auto fixes from pre-commit.com hooks --- benchmarks/lb_base.py | 8 ++++++-- benchmarks/light_benchmark.py | 2 +- benchmarks/light_benchmark_nch_min_hull.py | 2 +- benchmarks/light_benchmark_nch_qaoacv.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/benchmarks/lb_base.py b/benchmarks/lb_base.py index d1d91fa9..8e047441 100644 --- a/benchmarks/lb_base.py +++ b/benchmarks/lb_base.py @@ -39,8 +39,10 @@ # ############################################################################## + def _set_output(key: str, value: str): - print(f"::set-output name={key}::{value}") # noqa: E231 + print(f"::set-output name={key}::{value}") # noqa: E231 + def run(pipelines): paradigm = P300(resample=128) @@ -86,5 +88,7 @@ def run(pipelines): score_trun = int(score * 100) better_pr_score = pr_score_trun >= score_trun success = success and better_pr_score - print(f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}") + print( + f"{key}: {pr_score_trun} (PR) >= {score_trun} (main): {better_pr_score}" + ) _set_output("success", "1" if success else "0") diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index 4cb567a8..2f2a1079 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -87,7 +87,7 @@ ############################################################################## # Compute score and compare with PR branch # ------------------------------------------ -# +# ############################################################################## run(pipelines) diff --git a/benchmarks/light_benchmark_nch_min_hull.py b/benchmarks/light_benchmark_nch_min_hull.py index f229db9c..4bfeb1c3 100644 --- a/benchmarks/light_benchmark_nch_min_hull.py +++ b/benchmarks/light_benchmark_nch_min_hull.py @@ -61,7 +61,7 @@ ############################################################################## # Compute score and compare with PR branch # ------------------------------------------ -# +# ############################################################################## run(pipelines) diff --git a/benchmarks/light_benchmark_nch_qaoacv.py b/benchmarks/light_benchmark_nch_qaoacv.py index e1b3287c..aef5dfc0 100644 --- a/benchmarks/light_benchmark_nch_qaoacv.py +++ b/benchmarks/light_benchmark_nch_qaoacv.py @@ -66,7 +66,7 @@ ############################################################################## # Compute score and compare with PR branch # ------------------------------------------ -# +# ############################################################################## run(pipelines) From d80cecb4bf407f80a0923edf8faa139ea8558e86 Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 21:36:42 +0200 Subject: [PATCH 7/9] flake8 --- benchmarks/lb_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/lb_base.py b/benchmarks/lb_base.py index 8e047441..2fae2681 100644 --- a/benchmarks/lb_base.py +++ b/benchmarks/lb_base.py @@ -16,7 +16,6 @@ from moabb import set_log_level from moabb.datasets import bi2012 from moabb.paradigms import P300 -from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA from sklearn.metrics import balanced_accuracy_score from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder From bf08abf69cb4f60adb46966cdab2e75111f19aab Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 21:42:22 +0200 Subject: [PATCH 8/9] remove relative import --- benchmarks/light_benchmark.py | 2 +- benchmarks/light_benchmark_nch_min_hull.py | 2 +- benchmarks/light_benchmark_nch_qaoacv.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/light_benchmark.py b/benchmarks/light_benchmark.py index 2f2a1079..ef42e9db 100644 --- a/benchmarks/light_benchmark.py +++ b/benchmarks/light_benchmark.py @@ -24,7 +24,7 @@ from sklearn.decomposition import PCA from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA from sklearn.pipeline import make_pipeline -from .lb_base import run +from lb_base import run print(__doc__) diff --git a/benchmarks/light_benchmark_nch_min_hull.py b/benchmarks/light_benchmark_nch_min_hull.py index 4bfeb1c3..027dd951 100644 --- a/benchmarks/light_benchmark_nch_min_hull.py +++ b/benchmarks/light_benchmark_nch_min_hull.py @@ -19,7 +19,7 @@ from pyriemann_qiskit.utils import distance, mean # noqa from sklearn.pipeline import make_pipeline -from .lb_base import run +from lb_base import run print(__doc__) diff --git a/benchmarks/light_benchmark_nch_qaoacv.py b/benchmarks/light_benchmark_nch_qaoacv.py index aef5dfc0..b1b36033 100644 --- a/benchmarks/light_benchmark_nch_qaoacv.py +++ b/benchmarks/light_benchmark_nch_qaoacv.py @@ -21,7 +21,7 @@ from qiskit_algorithms.optimizers import SPSA from sklearn.pipeline import make_pipeline -from .lb_base import run +from lb_base import run print(__doc__) From fdd693ca8c60558e6b5f5d3140f45374869cdc9e Mon Sep 17 00:00:00 2001 From: Gregoire Cattan Date: Tue, 1 Oct 2024 22:01:29 +0200 Subject: [PATCH 9/9] fix for light_benchmark workflow --- .github/workflows/light_benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/light_benchmark.yml b/.github/workflows/light_benchmark.yml index fcb5c242..fa32a4d5 100644 --- a/.github/workflows/light_benchmark.yml +++ b/.github/workflows/light_benchmark.yml @@ -35,7 +35,7 @@ jobs: - name: Run benchmark script (main) id: run-benchmark-main run: | - python benchmarks/light_benchmark.py ${{steps.run-benchmark-pr.outputs.RG_QSVM}} ${{steps.run-benchmark-pr.outputs.RG_VQC}} ${{steps.run-benchmark-pr.outputs.QMDM_mean}} ${{steps.run-benchmark-pr.outputs.QMDM_dist}} ${{steps.run-benchmark-pr.outputs.RG_LDA}} ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL}} ${{steps.run-benchmark-pr.outputs.NCH_MIN_HULL_QAOACV}} + python benchmarks/light_benchmark.py ${{steps.run-benchmark-pr.outputs.RG_QSVM}} ${{steps.run-benchmark-pr.outputs.RG_VQC}} ${{steps.run-benchmark-pr.outputs.QMDM_mean}} ${{steps.run-benchmark-pr.outputs.QMDM_dist}} ${{steps.run-benchmark-pr.outputs.RG_LDA}} - name: Compare performance run: | echo ${{steps.run-benchmark-main.outputs.success}}