diff --git a/ipynb/Examples/CodeSizeComparison/CodeSizeComparison.ipynb b/ipynb/Examples/CodeSizeComparison/CodeSizeComparison.ipynb index 21e0f9d3f..5f60e7ec8 100644 --- a/ipynb/Examples/CodeSizeComparison/CodeSizeComparison.ipynb +++ b/ipynb/Examples/CodeSizeComparison/CodeSizeComparison.ipynb @@ -600,22 +600,21 @@ "text": [ "INFO - Loading environment cache from file\n", "INFO - Successfully initialized cache\n", - "INFO - Loading extensions.py (User)\n", - "INFO - [session-354] Processing stage LOAD\n", - "INFO - [session-354] Processing stage BUILD\n", - "INFO - [session-354] Processing stage COMPILE\n", - "INFO - [session-354] Processing stage RUN\n", + "INFO - [session-4] Processing stage LOAD\n", + "INFO - [session-4] Processing stage BUILD\n", + "INFO - [session-4] Processing stage COMPILE\n", + "INFO - [session-4] Processing stage RUN\n", "INFO - All runs completed successfuly!\n", "INFO - Postprocessing session report\n", - "INFO - [session-354] Done processing runs\n", + "INFO - [session-4] Done processing runs\n", "INFO - Report:\n", - " Session Run Model Frontend Framework Backend Platform Target Cycles Total ROM Total RAM ROM read-only ROM code ROM misc RAM data RAM zero-init data Features Config Postprocesses Comment\n", - "0 354 0 aww tflite tflm tflmi mlif spike 47517926 155265 36220 63661 90396 1208 2816 33404 [] {'tflite.use_inout_data': False, 'tflite.visua... [] -\n", - "1 354 1 aww tflite tflm tflmi mlif spike 16472429 164716 36220 63664 99844 1208 2816 33404 [muriscvnn] {'tflite.use_inout_data': False, 'tflite.visua... [] -\n", - "2 354 2 aww tflite tflm tflmi mlif spike 16530136 166843 36228 63661 101974 1208 2824 33404 [cmsisnn] {'tflite.use_inout_data': False, 'tflite.visua... [] -\n", - "3 354 3 resnet tflite tflm tflmi mlif spike 155009373 199501 68992 102493 95800 1208 2820 66172 [] {'tflite.use_inout_data': False, 'tflite.visua... [] -\n", - "4 354 4 resnet tflite tflm tflmi mlif spike 62430142 203378 68976 102496 99674 1208 2804 66172 [muriscvnn] {'tflite.use_inout_data': False, 'tflite.visua... [] -\n", - "5 354 5 resnet tflite tflm tflmi mlif spike 62563663 205381 68984 102493 101680 1208 2812 66172 [cmsisnn] {'tflite.use_inout_data': False, 'tflite.visua... [] -\n" + " Session Run Model Frontend Framework Backend Platform Target Total Cycles Total Instructions Total CPI Validation Total ROM Total RAM ROM read-only ROM code ROM misc RAM data RAM zero-init data Features Config Postprocesses Comment\n", + "0 4 0 aww tflite tflm tflmi mlif spike 45594676 45594676 1.0 True 153225 36168 63557 89652 16 2800 33368 [] {'aww.output_shapes': {'Identity': [1, 12]}, '... [] -\n", + "1 4 1 aww tflite tflm tflmi mlif spike 15248124 15248124 1.0 True 179322 36176 63560 115746 16 2808 33368 [muriscvnn] {'aww.output_shapes': {'Identity': [1, 12]}, '... [] -\n", + "2 4 2 aww tflite tflm tflmi mlif spike 16597146 16597146 1.0 True 179525 36176 63557 115952 16 2808 33368 [cmsisnn] {'aww.output_shapes': {'Identity': [1, 12]}, '... [] -\n", + "3 4 3 resnet tflite tflm tflmi mlif spike 145098497 145098497 1.0 True 200371 68940 102405 97950 16 2804 66136 [] {'resnet.output_shapes': {'Identity_int8': [1,... [] -\n", + "4 4 4 resnet tflite tflm tflmi mlif spike 54977756 54977756 1.0 True 217830 68932 102408 115406 16 2796 66136 [muriscvnn] {'resnet.output_shapes': {'Identity_int8': [1,... [] -\n", + "5 4 5 resnet tflite tflm tflmi mlif spike 63907632 63907632 1.0 True 218937 68932 102405 116516 16 2796 66136 [cmsisnn] {'resnet.output_shapes': {'Identity_int8': [1,... [] -\n" ] } ], @@ -644,30 +643,29 @@ "text": [ "INFO - Loading environment cache from file\n", "INFO - Successfully initialized cache\n", - "INFO - Loading extensions.py (User)\n", - "INFO - [session-355] Processing stage LOAD\n", - "INFO - [session-355] Processing stage BUILD\n", - "INFO - [session-355] Processing stage COMPILE\n", - "INFO - [session-355] Processing stage RUN\n", - "INFO - [session-355] Processing stage POSTPROCESS\n", + "INFO - [session-5] Processing stage LOAD\n", + "INFO - [session-5] Processing stage BUILD\n", + "INFO - [session-5] Processing stage COMPILE\n", + "INFO - [session-5] Processing stage RUN\n", + "INFO - [session-5] Processing stage POSTPROCESS\n", "INFO - All runs completed successfuly!\n", "INFO - Postprocessing session report\n", - "INFO - [session-355] Done processing runs\n", + "INFO - [session-5] Done processing runs\n", "INFO - Report:\n", - " Model Cycles Total ROM ROM read-only ROM code Features\n", - "0 aww 47517926 155265 63661 90396 []\n", - "1 aww 16472429 164716 63664 99844 [muriscvnn]\n", - "2 aww 16530136 166843 63661 101974 [cmsisnn]\n", - "3 resnet 155009373 199501 102493 95800 []\n", - "4 resnet 62430142 203378 102496 99674 [muriscvnn]\n", - "5 resnet 62563663 205381 102493 101680 [cmsisnn]\n" + " Model Total ROM ROM read-only ROM code Features\n", + "0 aww 153225 63557 89652 []\n", + "1 aww 179322 63560 115746 [muriscvnn]\n", + "2 aww 179525 63557 115952 [cmsisnn]\n", + "3 resnet 200371 102405 97950 []\n", + "4 resnet 217830 102408 115406 [muriscvnn]\n", + "5 resnet 218937 102405 116516 [cmsisnn]\n" ] } ], "source": [ "!python -m mlonmcu.cli.main flow run aww resnet -b tflmi -t spike \\\n", " --feature-gen _ --feature-gen muriscvnn --feature-gen cmsisnn \\\n", - " --postprocess filter_cols --config filter_cols.keep=\"Model,Cycles,Features,Total ROM,ROM read-only,ROM code, ROM misc\"" + " --postprocess filter_cols --config filter_cols.keep=\"Model,Total Cycles,Features,Total ROM,ROM read-only,ROM code, ROM misc\"" ] }, { @@ -691,30 +689,29 @@ "text": [ "INFO - Loading environment cache from file\n", "INFO - Successfully initialized cache\n", - "INFO - Loading extensions.py (User)\n", - "INFO - [session-356] Processing stage LOAD\n", - "INFO - [session-356] Processing stage BUILD\n", - "INFO - [session-356] Processing stage COMPILE\n", - "INFO - [session-356] Processing stage RUN\n", - "INFO - [session-356] Processing stage POSTPROCESS\n", + "INFO - [session-6] Processing stage LOAD\n", + "INFO - [session-6] Processing stage BUILD\n", + "INFO - [session-6] Processing stage COMPILE\n", + "INFO - [session-6] Processing stage RUN\n", + "INFO - [session-6] Processing stage POSTPROCESS\n", "INFO - All runs completed successfuly!\n", "INFO - Postprocessing session report\n", - "INFO - [session-356] Done processing runs\n", + "INFO - [session-6] Done processing runs\n", "INFO - Report:\n", - " Model Cycles Total ROM ROM read-only ROM code Features\n", - "0 aww 153513868 144895 63701 79986 []\n", - "1 aww 16588118 157816 63704 92904 [muriscvnn]\n", - "2 aww 16645825 159943 63701 95034 [cmsisnn]\n", - "3 resnet 687837633 184631 102533 80890 []\n", - "4 resnet 62514256 192488 102536 88744 [muriscvnn]\n", - "5 resnet 62642467 194491 102533 90750 [cmsisnn]\n" + " Model Total ROM ROM read-only ROM code Features\n", + "0 aww 142627 63521 79090 []\n", + "1 aww 161266 63524 97726 [muriscvnn]\n", + "2 aww 172819 63521 109282 [cmsisnn]\n", + "3 resnet 183107 102369 80722 []\n", + "4 resnet 195608 102372 93220 [muriscvnn]\n", + "5 resnet 205073 102369 102688 [cmsisnn]\n" ] } ], "source": [ "!python -m mlonmcu.cli.main flow run aww resnet -b tflmi -t spike --config mlif.optimize=s \\\n", " --feature-gen _ --feature-gen muriscvnn --feature-gen cmsisnn \\\n", - " --postprocess filter_cols --config filter_cols.keep=\"Model,Cycles,Features,Total ROM,ROM read-only,ROM code, ROM misc\"" + " --postprocess filter_cols --config filter_cols.keep=\"Model,Total Cycles,Features,Total ROM,ROM read-only,ROM code, ROM misc\"" ] }, { @@ -722,7 +719,7 @@ "id": "0de7b57a", "metadata": {}, "source": [ - "Well this looks better, but not optimal. One issue here is, that CMSIS-NN lacks an possibility to pass over the optimization flags from another CMake project. Hence in the end only the non CMSIS-NN/muRISCV-NN code was compiled with `-Os`." + "~~Well this looks better, but not optimal. One issue here is, that CMSIS-NN lacks an possibility to pass over the optimization flags from another CMake project. Hence in the end only the non CMSIS-NN/muRISCV-NN code was compiled with `-Os`.~~ (fixed in new version of MLonMCU)" ] }, { @@ -743,7 +740,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 7, "id": "78c53970", "metadata": {}, "outputs": [], @@ -765,7 +762,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 12, "id": "2f6c5e5a", "metadata": {}, "outputs": [], @@ -779,7 +776,7 @@ "FEATURES = [[], [\"cmsisnn\"], [\"muriscvnn\"]]\n", "CONFIG = {\n", " \"mlif.optimize\": \"s\",\n", - " \"filter_cols.keep\": [\"Model\", \"Cycles\", \"ROM code\", \"Features\"]\n", + " \"filter_cols.keep\": [\"Model\", \"Total Cycles\", \"ROM code\", \"Features\"]\n", "}" ] }, @@ -793,7 +790,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 13, "id": "ebe63369", "metadata": {}, "outputs": [ @@ -803,11 +800,10 @@ "text": [ "INFO - Loading environment cache from file\n", "INFO - Successfully initialized cache\n", - "INFO - Loading extensions.py (User)\n", - "INFO - [session-360] Processing all stages\n", + "INFO - [session-8] Processing all stages\n", "INFO - All runs completed successfuly!\n", "INFO - Postprocessing session report\n", - "INFO - [session-360] Done processing runs\n" + "INFO - [session-8] Done processing runs\n" ] }, { @@ -832,7 +828,7 @@ " \n", " \n", " Model\n", - " Cycles\n", + " Total Cycles\n", " ROM code\n", " Features\n", " \n", @@ -841,43 +837,22 @@ " \n", " 0\n", " aww\n", - " 153513868\n", - " 79986\n", + " 172565059\n", + " 79090\n", " []\n", " \n", " \n", " 1\n", " aww\n", - " 16645825\n", - " 95034\n", + " 16711296\n", + " 109282\n", " [cmsisnn]\n", " \n", " \n", " 2\n", " aww\n", - " 16588118\n", - " 92904\n", - " [muriscvnn]\n", - " \n", - " \n", - " 3\n", - " resnet\n", - " 687837633\n", - " 80890\n", - " []\n", - " \n", - " \n", - " 4\n", - " resnet\n", - " 62642467\n", - " 90750\n", - " [cmsisnn]\n", - " \n", - " \n", - " 5\n", - " resnet\n", - " 62514256\n", - " 88744\n", + " 17175092\n", + " 97726\n", " [muriscvnn]\n", " \n", " \n", @@ -885,16 +860,13 @@ "" ], "text/plain": [ - " Model Cycles ROM code Features\n", - "0 aww 153513868 79986 []\n", - "1 aww 16645825 95034 [cmsisnn]\n", - "2 aww 16588118 92904 [muriscvnn]\n", - "3 resnet 687837633 80890 []\n", - "4 resnet 62642467 90750 [cmsisnn]\n", - "5 resnet 62514256 88744 [muriscvnn]" + " Model Total Cycles ROM code Features\n", + "0 aww 172565059 79090 []\n", + "1 aww 16711296 109282 [cmsisnn]\n", + "2 aww 17175092 97726 [muriscvnn]" ] }, - "execution_count": 38, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -926,12 +898,12 @@ "id": "8ac72198", "metadata": {}, "source": [ - "Here we have the report as pandas dataframe. Of course be can also look at relative differences instead:" + "Here we have the report as pandas dataframe. Of course we can also look at relative differences instead:" ] }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 19, "id": "9171e9aa", "metadata": {}, "outputs": [ @@ -957,9 +929,9 @@ " \n", " \n", " Model\n", - " Cycles\n", + " Total Cycles\n", " ROM code\n", - " Cycles (rel.)\n", + " Total Cycles (rel.)\n", " ROM code (rel.)\n", " \n", " \n", @@ -975,67 +947,40 @@ " \n", " default\n", " aww\n", - " 153513868\n", - " 79986\n", + " 172565059\n", + " 79090\n", " 1.000000\n", " 1.000000\n", " \n", " \n", - " default\n", + " cmsisnn\n", " aww\n", - " 16645825\n", - " 95034\n", - " 9.222365\n", - " 0.841657\n", + " 16711296\n", + " 109282\n", + " 10.326252\n", + " 0.723724\n", " \n", " \n", - " default\n", + " muriscvnn\n", " aww\n", - " 16588118\n", - " 92904\n", - " 9.254448\n", - " 0.860953\n", - " \n", - " \n", - " default\n", - " resnet\n", - " 687837633\n", - " 80890\n", - " 1.000000\n", - " 1.000000\n", - " \n", - " \n", - " default\n", - " resnet\n", - " 62642467\n", - " 90750\n", - " 10.980373\n", - " 0.891350\n", - " \n", - " \n", - " default\n", - " resnet\n", - " 62514256\n", - " 88744\n", - " 11.002892\n", - " 0.911498\n", + " 17175092\n", + " 97726\n", + " 10.047402\n", + " 0.809304\n", " \n", " \n", "\n", "" ], "text/plain": [ - " Model Cycles ROM code Cycles (rel.) ROM code (rel.)\n", - "Features \n", - "default aww 153513868 79986 1.000000 1.000000\n", - "default aww 16645825 95034 9.222365 0.841657\n", - "default aww 16588118 92904 9.254448 0.860953\n", - "default resnet 687837633 80890 1.000000 1.000000\n", - "default resnet 62642467 90750 10.980373 0.891350\n", - "default resnet 62514256 88744 11.002892 0.911498" + " Model Total Cycles ROM code Total Cycles (rel.) ROM code (rel.)\n", + "Features \n", + "default aww 172565059 79090 1.000000 1.000000\n", + "cmsisnn aww 16711296 109282 10.326252 0.723724\n", + "muriscvnn aww 17175092 97726 10.047402 0.809304" ] }, - "execution_count": 56, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -1043,13 +988,21 @@ "source": [ "df = report.df\n", "df.set_index('Features', inplace=True)\n", - "df.index = df.index.map(lambda x: str(x)[1:-1] if len(x) > 2 else \"default\")\n", + "df.index = df.index.map(lambda x: x[0] if len(x) > 0 else \"default\")\n", "cycles_firsts = (df.groupby('Model')['Total Cycles'].transform('first'))\n", "rom_firsts = (df.groupby('Model')['ROM code'].transform('first'))\n", "df[\"Total Cycles (rel.)\"] = (1 / (df[\"Total Cycles\"] / cycles_firsts))\n", "df[\"ROM code (rel.)\"] = (1 / (df[\"ROM code\"] / rom_firsts))\n", "df" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d856a0d2-1f81-4dbe-9584-e75bdd7e26c2", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/ipynb/run_notebook.sh b/ipynb/run_notebook.sh index 1b9c7333b..e91e605ff 100755 --- a/ipynb/run_notebook.sh +++ b/ipynb/run_notebook.sh @@ -29,7 +29,7 @@ do shift 2 ;; -e | --environment ) - HOME_="$2" + VENV="$2" shift 2 ;; --skip )