From 42927365457efae020c71abe426f3880c584169b Mon Sep 17 00:00:00 2001 From: andreygetmanov Date: Wed, 28 Jun 2023 20:16:58 +0300 Subject: [PATCH 1/3] - added results for multimodal benchmark --- docs/source/benchmarks/index.rst | 1 + docs/source/benchmarks/multimodal.rst | 12 + docs/source/benchmarks/multimodal_res.html | 318 +++++++++++++++++++++ 3 files changed, 331 insertions(+) create mode 100644 docs/source/benchmarks/multimodal.rst create mode 100644 docs/source/benchmarks/multimodal_res.html diff --git a/docs/source/benchmarks/index.rst b/docs/source/benchmarks/index.rst index 4c98330cc3..f5ca672253 100644 --- a/docs/source/benchmarks/index.rst +++ b/docs/source/benchmarks/index.rst @@ -8,3 +8,4 @@ We make a comparison with existing state-of-the-art AutoML libraries on various forecasting tabular + multimodal \ No newline at end of file diff --git a/docs/source/benchmarks/multimodal.rst b/docs/source/benchmarks/multimodal.rst new file mode 100644 index 0000000000..647c9cbe25 --- /dev/null +++ b/docs/source/benchmarks/multimodal.rst @@ -0,0 +1,12 @@ +Multimodal data +------------ + +Here are overall classification and regression problems results on multimodal datasets across state-of-the-art AutoML frameworks +using `AutoML Multimodal Benchmark `__ test suite: + +.. raw:: html + :file: multimodal_res.html + + +The results are obtained using sever based on Xeon Cascadelake (2900MHz) +with 12 cores and 24GB memory for experiments with the local infrastructure. diff --git a/docs/source/benchmarks/multimodal_res.html b/docs/source/benchmarks/multimodal_res.html new file mode 100644 index 0000000000..5ef172f1fd --- /dev/null +++ b/docs/source/benchmarks/multimodal_res.html @@ -0,0 +1,318 @@ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + framework + + AutoGluon + + FEDOT +
+ Dataset name + + Metric name + + +
+ prod + + accuracy + + 0.895 + + 0.897 +
+ salary + + accuracy + + 0.528 + + 0.402 +
+ airbnb + + accuracy + + 0.466 + + 0.434 +
+ channel + + accuracy + + 0.550 + + 0.542 +
+ wine + + accuracy + + 0.842 + + 0.784 +
+ imdb + + auc + + 0.874 + + 0.872 +
+ fake + + auc + + 0.968 + + 0.958 +
+ kick + + auc + + 0.775 + + 0.752 +
+ jigsaw + + auc + + 0.915 + + 0.998 +
+ qaa + + r2 + + 0.383 + + 0.504 +
+ qaq + + r2 + + 0.426 + + 0.554 +
+ book + + r2 + + 0.600 + + 0.994 +
+ jc + + r2 + + 0.612 + + 0.619 +
+ cloth + + r2 + + 0.654 + + 0.733 +
+ ae + + r2 + + 0.979 + + 0.974 +
+ pop + + r2 + + 0.020 + + 0.020 +
+ house + + r2 + + 0.943 + + 0.928 +
+ mercari + + r2 + + 0.569 + + 0.520 +
+
From 29bb0e8da7b28f9b5153069583befc5c655b812f Mon Sep 17 00:00:00 2001 From: andreygetmanov Date: Wed, 28 Jun 2023 20:57:18 +0300 Subject: [PATCH 2/3] - minor changes --- docs/source/benchmarks/index.rst | 2 +- docs/source/benchmarks/multimodal_res.csv | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 docs/source/benchmarks/multimodal_res.csv diff --git a/docs/source/benchmarks/index.rst b/docs/source/benchmarks/index.rst index f5ca672253..65fa989668 100644 --- a/docs/source/benchmarks/index.rst +++ b/docs/source/benchmarks/index.rst @@ -8,4 +8,4 @@ We make a comparison with existing state-of-the-art AutoML libraries on various forecasting tabular - multimodal \ No newline at end of file + multimodal diff --git a/docs/source/benchmarks/multimodal_res.csv b/docs/source/benchmarks/multimodal_res.csv new file mode 100644 index 0000000000..84a98160a5 --- /dev/null +++ b/docs/source/benchmarks/multimodal_res.csv @@ -0,0 +1,19 @@ +Dataset name,Metric name,AutoGluon,FEDOT +prod,accuracy,0.895,0.897 +salary,accuracy,0.528,0.402 +airbnb,accuracy,0.466,0.434 +channel,accuracy,0.550,0.542 +wine,accuracy,0.842,0.784 +imdb,auc,0.874,0.872 +fake,auc,0.968,0.958 +kick,auc,0.775,0.752 +jigsaw,auc,0.915,0.998 +qaa,r2,0.383,0.504 +qaq,r2,0.426,0.554 +book,r2,0.6,0.994 +jc,r2,0.612,0.619 +cloth,r2,0.654,0.733 +ae,r2,0.979,0.974 +pop,r2,0.02,0.02 +house,r2,0.943,0.928 +mercari,r2,0.569,0.52 From 200843060a156217f3ab4347f6e5d18eb5afb2a6 Mon Sep 17 00:00:00 2001 From: andreygetmanov Date: Wed, 28 Jun 2023 21:05:29 +0300 Subject: [PATCH 3/3] - minor changes --- docs/source/benchmarks/multimodal.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/benchmarks/multimodal.rst b/docs/source/benchmarks/multimodal.rst index 647c9cbe25..43fef2561e 100644 --- a/docs/source/benchmarks/multimodal.rst +++ b/docs/source/benchmarks/multimodal.rst @@ -1,5 +1,5 @@ Multimodal data ------------- +--------------- Here are overall classification and regression problems results on multimodal datasets across state-of-the-art AutoML frameworks using `AutoML Multimodal Benchmark `__ test suite: