From c114177fedbbd4f9bd383c90c359bfba4c158061 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 9 Sep 2019 19:15:18 +0200 Subject: [PATCH 01/94] Don't ignore database files --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 508a14e1..d28ec650 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *.pyc .ipynb_checkpoints -*.db *.egg-info __pycache__ *.egg-info/ From 29727ace8f16e768d9e0cce328ad9d932af432f3 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 9 Sep 2019 19:25:06 +0200 Subject: [PATCH 02/94] Add github-recommended entries to gitignore --- .gitignore | 135 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d28ec650..2cdadc85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,130 @@ -*.pyc -.ipynb_checkpoints -*.egg-info -__pycache__ -*.egg-info/ +doc/build + +# IDEs .idea/ -doc/build \ No newline at end of file +.vscode/* + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ \ No newline at end of file From 55245b1769e768a01cf4ead4b1ef9d963008f5fc Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Wed, 28 Apr 2021 12:07:28 -0500 Subject: [PATCH 03/94] prototype for simple data description. --- ...c operation of DDH5 and benchmarking.ipynb | 144 +++++++++++++++++- 1 file changed, 139 insertions(+), 5 deletions(-) diff --git a/doc/examples/Basic operation of DDH5 and benchmarking.ipynb b/doc/examples/Basic operation of DDH5 and benchmarking.ipynb index 501672d3..900d02bf 100644 --- a/doc/examples/Basic operation of DDH5 and benchmarking.ipynb +++ b/doc/examples/Basic operation of DDH5 and benchmarking.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:56:44.772347Z", @@ -12,6 +12,9 @@ "outputs": [], "source": [ "import sys\n", + "import re\n", + "from typing import Optional, Dict, List\n", + "\n", "import numpy as np\n", "import h5py\n", "\n", @@ -26,6 +29,137 @@ "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'data': {'unit': 'V', 'axes': ['x', 'y']},\n", + " 'x': {'unit': 'mV'},\n", + " 'y': {'unit': 'mV'},\n", + " 'data_2': {'unit': 'V', 'axes': ['x', 'y']}}" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def string2datastructure(description: str) -> dd.DataDictBase:\n", + " \"\"\"Construct a DataDict from a string description.\n", + " \n", + " Example:\n", + " \"\"\"\n", + " \n", + " description = description.replace(\" \", \"\")\n", + " \n", + " data_name_pattern = r\"[a-zA-Z]+\\w*(\\[\\w*\\])?\"\n", + " pattern = r\"((?<=\\A)|(?<=\\;))\" + data_name_pattern + r\"(\\((\" + data_name_pattern + r\"\\,?)*\\))?\"\n", + " r = re.compile(pattern)\n", + " \n", + " data_fields = []\n", + " while(r.search(description)):\n", + " match = r.search(description)\n", + " if match is None: break\n", + " data_fields.append(description[slice(*match.span())])\n", + " description = description[match.span()[1]:]\n", + " \n", + " dd = dict()\n", + " \n", + " def analyze_field(df):\n", + " has_unit = True if '[' in df and ']' in df else False\n", + " has_dependencies = True if '(' in df and ')' in df else False\n", + " \n", + " name: str = \"\"\n", + " unit: Optional[str] = None\n", + " axes: Optional[List[str]] = None\n", + " \n", + " if has_unit:\n", + " name = df.split('[')[0]\n", + " unit = df.split('[')[1].split(']')[0]\n", + " if has_dependencies:\n", + " axes = df.split('(')[1].split(')')[0].split(',')\n", + " elif has_dependencies:\n", + " name = df.split('(')[0]\n", + " axes = df.split('(')[1].split(')')[0].split(',')\n", + " else:\n", + " name = df\n", + " \n", + " if axes is not None and len(axes) == 0:\n", + " axes = None\n", + " return name, unit, axes\n", + " \n", + " for df in data_fields:\n", + " name, unit, axes = analyze_field(df)\n", + " \n", + " # double specifying is only allowed for independents.\n", + " # if an independent is specified multiple times, units must not collide \n", + " # (but units do not have to be specified more than once)\n", + " if name in dd: \n", + " if 'axes' in dd[name]:\n", + " raise ValueError(f'{name} is specified more than once.')\n", + " if 'unit' in dd[name] and unit is not None and dd[name]['unit'] != unit:\n", + " raise ValueError(f'conflicting units for {name}')\n", + " \n", + " dd[name] = dict()\n", + " if unit is not None:\n", + " dd[name]['unit'] = unit\n", + " \n", + " if axes is not None:\n", + " for ax in axes:\n", + " ax_name, ax_unit, ax_axes = analyze_field(ax)\n", + " \n", + " # we do not allow nested dependencies.\n", + " if ax_axes is not None:\n", + " raise ValueError(f'{ax_name} is independent, may not have dependencies')\n", + " \n", + " # we can add fields implicitly from dependencies. \n", + " # independents may be given both implicitly and explicitly, but only\n", + " # when units don't collide.\n", + " if ax_name not in dd:\n", + " dd[ax_name] = dict()\n", + " if ax_unit is not None:\n", + " dd[ax_name]['unit'] = ax_unit\n", + " else:\n", + " if 'unit' in dd[ax_name] and ax_unit is not None and dd[ax_name]['unit'] != ax_unit:\n", + " raise ValueError(f'conflicting units for {ax_name}')\n", + " \n", + " if 'axes' not in dd[name]:\n", + " dd[name]['axes'] = []\n", + " dd[name]['axes'].append(ax_name)\n", + " \n", + " return dd \n", + " \n", + "\n", + "desc = \"data[V](x, y); data_2[V](x, y); x[mV]; y[mV]\"\n", + "string2datastructure(desc)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -304,9 +438,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python [conda env:plottr-pyqt5]", "language": "python", - "name": "python3" + "name": "conda-env-plottr-pyqt5-py" }, "language_info": { "codemirror_mode": { @@ -318,7 +452,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.9.2" }, "toc": { "base_numbering": 1, @@ -364,5 +498,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From 612be856e584d0ea18ca95ddaa2b25e50a8266c7 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 29 Apr 2021 23:22:32 -0500 Subject: [PATCH 04/94] function for simplified generation of data structures. --- plottr/data/datadict.py | 121 +++++++++++++++++++++++++++++ test/pytest/test_data_dict_base.py | 37 ++++++++- 2 files changed, 157 insertions(+), 1 deletion(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 65072a05..636a9b63 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -5,6 +5,7 @@ """ import warnings import copy as cp +import re import numpy as np from functools import reduce @@ -1291,3 +1292,123 @@ def combine_datadicts(*dicts: DataDict) -> Union[DataDictBase, DataDict]: assert ret is not None ret.validate() return ret + + +def datastructure_from_string(description: str) -> DataDict: + """Construct a DataDict from a string description. + + Examples + -------- + * ``"data[mV](x, y)"`` results in a datadict with one dependent ``data`` with unit ``mV`` and + two independents, ``x`` and ``y``, that do not have units. + + * ``"data_1[mV](x, y); data_2[mA](x); x[mV]; y[nT]"`` results in two dependents, + one of them depening on ``x`` and ``y``, the other only on ``x``. + Note that ``x`` and ``y`` have units. We can (but do not have to) omit them when specifying + the dependencies. + + * ``"data_1[mV](x[mV], y[nT]); data_2[mA](x[mV])"``. Same result as the previous example. + + Rules + ----- + We recognize descriptions of the form ``field1[unit1](ax1, ax2, ...); field1[unit2](...); ...``. + + * field names (like ``field1`` and ``field2`` above) have to start with a letter, and may contain + word characters + * field descriptors consist of the name, optional unit (presence signified by square brackets), + and optional dependencies (presence signified by round brackets). + * dependencies (axes) are implicitly recognized as fields (and thus have the same naming restrictions as field + names) + * axes are separated by commas + * axes may have a unit when specified as dependency, but besides the name, square brackets, and commas no other + symbols are recognized within the round brackets the specify the dependency + * in addition to being specified as dependency for a field, axes may be specified also as additional field without + dependency, for instance to specify the unit (may simplify the string). For example, + ``z1[x, y]; z2[x, y]; x[V]; y[V]`` + * units may only consist of word characters + * use of unexpected characters will result in the ignoring the part that contains the symbol + * the regular expression used to find field descriptors is: + ``((?<=\A)|(?<=\;))[a-zA-Z]+\w*(\[\w*\])?(\(([a-zA-Z]+\w*(\[\w*\])?\,?)*\))?`` + """ + + description = description.replace(" ", "") + + data_name_pattern = r"[a-zA-Z]+\w*(\[\w*\])?" + pattern = r"((?<=\A)|(?<=\;))" + data_name_pattern + r"(\((" + data_name_pattern + r"\,?)*\))?" + r = re.compile(pattern) + + data_fields = [] + while (r.search(description)): + match = r.search(description) + if match is None: break + data_fields.append(description[slice(*match.span())]) + description = description[match.span()[1]:] + + dd = dict() + + def analyze_field(df): + has_unit = True if '[' in df and ']' in df else False + has_dependencies = True if '(' in df and ')' in df else False + + name: str = "" + unit: Optional[str] = None + axes: Optional[List[str]] = None + + if has_unit: + name = df.split('[')[0] + unit = df.split('[')[1].split(']')[0] + if has_dependencies: + axes = df.split('(')[1].split(')')[0].split(',') + elif has_dependencies: + name = df.split('(')[0] + axes = df.split('(')[1].split(')')[0].split(',') + else: + name = df + + if axes is not None and len(axes) == 0: + axes = None + return name, unit, axes + + for df in data_fields: + name, unit, axes = analyze_field(df) + + # double specifying is only allowed for independents. + # if an independent is specified multiple times, units must not collide + # (but units do not have to be specified more than once) + if name in dd: + if 'axes' in dd[name] or axes is not None: + raise ValueError(f'{name} is specified more than once.') + if 'unit' in dd[name] and unit is not None and dd[name]['unit'] != unit: + raise ValueError(f'conflicting units for {name}') + + dd[name] = dict() + if unit is not None: + dd[name]['unit'] = unit + + if axes is not None: + for ax in axes: + ax_name, ax_unit, ax_axes = analyze_field(ax) + + # we do not allow nested dependencies. + if ax_axes is not None: + raise ValueError(f'{ax_name} is independent, may not have dependencies') + + # we can add fields implicitly from dependencies. + # independents may be given both implicitly and explicitly, but only + # when units don't collide. + if ax_name not in dd: + dd[ax_name] = dict() + if ax_unit is not None: + dd[ax_name]['unit'] = ax_unit + else: + if 'unit' in dd[ax_name] and ax_unit is not None and dd[ax_name]['unit'] != ax_unit: + raise ValueError(f'conflicting units for {ax_name}') + + if 'axes' not in dd[name]: + dd[name]['axes'] = [] + dd[name]['axes'].append(ax_name) + + return DataDict(**dd) + + +str2dd = datastructure_from_string diff --git a/test/pytest/test_data_dict_base.py b/test/pytest/test_data_dict_base.py index ae57d283..9e39545a 100644 --- a/test/pytest/test_data_dict_base.py +++ b/test/pytest/test_data_dict_base.py @@ -1,7 +1,7 @@ import pytest import numpy as np from plottr.utils.num import arrays_equal -from plottr.data.datadict import DataDict, DataDictBase +from plottr.data.datadict import DataDict, DataDictBase, str2dd from plottr.data.datadict import combine_datadicts @@ -308,3 +308,38 @@ def test_combine_ddicts(): z=dict(values=z, axes=['x', 'y']), z_0=dict(values=z[::-1], axes=['x', 'y'])) assert combined_dd == expected_dd + +def test_creation_from_string(): + """Test simplified datadict generation""" + str_ok_1 = "z(x,y)" + dd = str2dd(str_ok_1) + assert dd.validate() + assert set(list(dd.keys())) == {'x', 'y', 'z'} + assert dd.axes('z') == ['x', 'y'] + assert dd.axes('x') == [] + + str_ok_2 = "z1[V](x, y); z2[A](y, x); x[mT]" + dd = str2dd(str_ok_2) + assert dd.validate() + assert set(dd.dependents()) == {'z1', 'z2'} + assert dd['z1']['unit'] == 'V' + assert dd['x']['unit'] == 'mT' + + # conflicting units -- should raise ValueError + str_notok = "z[V](x[A], y[T]); x[mA]; y[T]" + with pytest.raises(ValueError): + dd = str2dd(str_notok) + + # cascaded dependency -- should raise ValueError + str_notok = "z(x, y); x(y)" + with pytest.raises(ValueError): + dd = str2dd(str_notok) + + # no error raised, but x(a) is not recognized as valid dependency + str_notok = "z(x(a))" + dd = str2dd(str_notok) + assert dd.validate() + assert dd.dependents() == [] + assert dd.axes() == [] + assert 'z' in dd + From 0805b9b72e1d857509326adf341b39690be35af1 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 29 Apr 2021 23:23:26 -0500 Subject: [PATCH 05/94] fix typo --- plottr/data/datadict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 636a9b63..057d78e5 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -1321,7 +1321,7 @@ def datastructure_from_string(description: str) -> DataDict: names) * axes are separated by commas * axes may have a unit when specified as dependency, but besides the name, square brackets, and commas no other - symbols are recognized within the round brackets the specify the dependency + characters are recognized within the round brackets that specify the dependency * in addition to being specified as dependency for a field, axes may be specified also as additional field without dependency, for instance to specify the unit (may simplify the string). For example, ``z1[x, y]; z2[x, y]; x[V]; y[V]`` From a9c52030208f3947438a996412bbd217bc023a49 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 30 Apr 2021 08:39:41 -0500 Subject: [PATCH 06/94] cleaned up notebook a bit. --- ...c operation of DDH5 and benchmarking.ipynb | 181 ++---------------- 1 file changed, 18 insertions(+), 163 deletions(-) diff --git a/doc/examples/Basic operation of DDH5 and benchmarking.ipynb b/doc/examples/Basic operation of DDH5 and benchmarking.ipynb index 900d02bf..f1d12461 100644 --- a/doc/examples/Basic operation of DDH5 and benchmarking.ipynb +++ b/doc/examples/Basic operation of DDH5 and benchmarking.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 10, + "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:56:44.772347Z", @@ -31,7 +31,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -43,123 +43,18 @@ " 'data_2': {'unit': 'V', 'axes': ['x', 'y']}}" ] }, - "execution_count": 23, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "def string2datastructure(description: str) -> dd.DataDictBase:\n", - " \"\"\"Construct a DataDict from a string description.\n", - " \n", - " Example:\n", - " \"\"\"\n", - " \n", - " description = description.replace(\" \", \"\")\n", - " \n", - " data_name_pattern = r\"[a-zA-Z]+\\w*(\\[\\w*\\])?\"\n", - " pattern = r\"((?<=\\A)|(?<=\\;))\" + data_name_pattern + r\"(\\((\" + data_name_pattern + r\"\\,?)*\\))?\"\n", - " r = re.compile(pattern)\n", - " \n", - " data_fields = []\n", - " while(r.search(description)):\n", - " match = r.search(description)\n", - " if match is None: break\n", - " data_fields.append(description[slice(*match.span())])\n", - " description = description[match.span()[1]:]\n", - " \n", - " dd = dict()\n", - " \n", - " def analyze_field(df):\n", - " has_unit = True if '[' in df and ']' in df else False\n", - " has_dependencies = True if '(' in df and ')' in df else False\n", - " \n", - " name: str = \"\"\n", - " unit: Optional[str] = None\n", - " axes: Optional[List[str]] = None\n", - " \n", - " if has_unit:\n", - " name = df.split('[')[0]\n", - " unit = df.split('[')[1].split(']')[0]\n", - " if has_dependencies:\n", - " axes = df.split('(')[1].split(')')[0].split(',')\n", - " elif has_dependencies:\n", - " name = df.split('(')[0]\n", - " axes = df.split('(')[1].split(')')[0].split(',')\n", - " else:\n", - " name = df\n", - " \n", - " if axes is not None and len(axes) == 0:\n", - " axes = None\n", - " return name, unit, axes\n", - " \n", - " for df in data_fields:\n", - " name, unit, axes = analyze_field(df)\n", - " \n", - " # double specifying is only allowed for independents.\n", - " # if an independent is specified multiple times, units must not collide \n", - " # (but units do not have to be specified more than once)\n", - " if name in dd: \n", - " if 'axes' in dd[name]:\n", - " raise ValueError(f'{name} is specified more than once.')\n", - " if 'unit' in dd[name] and unit is not None and dd[name]['unit'] != unit:\n", - " raise ValueError(f'conflicting units for {name}')\n", - " \n", - " dd[name] = dict()\n", - " if unit is not None:\n", - " dd[name]['unit'] = unit\n", - " \n", - " if axes is not None:\n", - " for ax in axes:\n", - " ax_name, ax_unit, ax_axes = analyze_field(ax)\n", - " \n", - " # we do not allow nested dependencies.\n", - " if ax_axes is not None:\n", - " raise ValueError(f'{ax_name} is independent, may not have dependencies')\n", - " \n", - " # we can add fields implicitly from dependencies. \n", - " # independents may be given both implicitly and explicitly, but only\n", - " # when units don't collide.\n", - " if ax_name not in dd:\n", - " dd[ax_name] = dict()\n", - " if ax_unit is not None:\n", - " dd[ax_name]['unit'] = ax_unit\n", - " else:\n", - " if 'unit' in dd[ax_name] and ax_unit is not None and dd[ax_name]['unit'] != ax_unit:\n", - " raise ValueError(f'conflicting units for {ax_name}')\n", - " \n", - " if 'axes' not in dd[name]:\n", - " dd[name]['axes'] = []\n", - " dd[name]['axes'].append(ax_name)\n", - " \n", - " return dd \n", - " \n", - "\n", "desc = \"data[V](x, y); data_2[V](x, y); x[mV]; y[mV]\"\n", - "string2datastructure(desc)" + "data = dd.str2dd(desc)\n", + "\n", + "data" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "markdown", "metadata": {}, @@ -171,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:57:46.468964Z", @@ -185,22 +80,14 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:57:53.110466Z", "start_time": "2019-07-02T21:57:47.116741Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "71.7 ms ± 1.33 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" - ] - } - ], + "outputs": [], "source": [ "%%timeit\n", "nrows = 10000\n", @@ -231,22 +118,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:57:57.686654Z", "start_time": "2019-07-02T21:57:55.102475Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "310 ms ± 11.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], + "outputs": [], "source": [ "%%timeit\n", "ret_data = dds.datadict_from_hdf5(FN)\n", @@ -268,7 +147,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:58:08.962974Z", @@ -283,22 +162,14 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:58:13.805079Z", "start_time": "2019-07-02T21:58:10.150728Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "452 ms ± 52.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], + "outputs": [], "source": [ "%%timeit\n", "\n", @@ -332,22 +203,14 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T21:59:53.337364Z", "start_time": "2019-07-02T21:59:50.676328Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "3.26 ms ± 163 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" - ] - } - ], + "outputs": [], "source": [ "%%timeit\n", "with h5py.File(FN+'.dd.h5', 'a') as f:\n", @@ -385,22 +248,14 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-07-02T22:00:07.067915Z", "start_time": "2019-07-02T22:00:03.259418Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "460 ms ± 37.5 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" - ] - } - ], + "outputs": [], "source": [ "%%timeit\n", "\n", From a4bc0ae26e0fad5d29e9cd82bf6e6cf44371a255 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sun, 16 May 2021 17:20:28 -0500 Subject: [PATCH 07/94] some playing with pyqtgraph. --- test/gui/pyqtgraph_testing.py | 91 +++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 test/gui/pyqtgraph_testing.py diff --git a/test/gui/pyqtgraph_testing.py b/test/gui/pyqtgraph_testing.py new file mode 100644 index 00000000..5d9695e2 --- /dev/null +++ b/test/gui/pyqtgraph_testing.py @@ -0,0 +1,91 @@ +"""A simple script to play a bit with pyqtgraph plotting. +This has no direct connection to plottr but is just to explore. +""" + +import sys + +import numpy as np +import pyqtgraph as pg + +from plottr import QtWidgets, QtGui, QtCore +from plottr.gui.tools import widgetDialog + +pg.setConfigOption('background', 'w') +pg.setConfigOption('foreground', 'k') + +def image_test(): + app = QtWidgets.QApplication([]) + + # create data + x = np.linspace(0, 10, 51) + y = np.linspace(-4, 4, 51) + xx, yy = np.meshgrid(x, y, indexing='ij') + zz = np.cos(xx)*np.exp(-(yy-1.)**2) + + # layout widget + pgWidget = pg.GraphicsLayoutWidget() + + # main plot + imgPlot = pgWidget.addPlot(title='my image', row=0, col=0) + img = pg.ImageItem() + imgPlot.addItem(img) + + # histogram and colorbar + hist = pg.HistogramLUTItem() + hist.setImageItem(img) + pgWidget.addItem(hist) + hist.gradient.loadPreset('viridis') + + # cut elements + pgWidget2 = pg.GraphicsLayoutWidget() + + # plots for x and y cuts + xplot = pgWidget2.addPlot(row=1, col=0) + yplot = pgWidget2.addPlot(row=0, col=0) + + # add crosshair to main plot + vline = pg.InfiniteLine(angle=90, movable=False, pen='r') + hline = pg.InfiniteLine(angle=0, movable=False, pen='b') + imgPlot.addItem(vline, ignoreBounds=True) + imgPlot.addItem(hline, ignoreBounds=True) + + def crossMoved(event): + pos = event[0].scenePos() + if imgPlot.sceneBoundingRect().contains(pos): + origin = imgPlot.vb.mapSceneToView(pos) + vline.setPos(origin.x()) + hline.setPos(origin.y()) + vidx = np.argmin(np.abs(origin.x()-x)) + hidx = np.argmin(np.abs(origin.y()-y)) + yplot.clear() + yplot.plot(zz[vidx, :], y, + pen=pg.mkPen('r', width=2), + symbol='o', symbolBrush='r', symbolPen=None) + xplot.clear() + xplot.plot(x, zz[:, hidx], + pen=pg.mkPen('b', width=2), + symbol='o', symbolBrush='b', symbolPen=None) + + proxy = pg.SignalProxy(imgPlot.scene().sigMouseClicked, slot=crossMoved) + + dg = widgetDialog(pgWidget, title='pyqtgraph image test') + dg2 = widgetDialog(pgWidget2, title='line cuts') + + # setting the data + img.setImage(zz) + img.setRect(QtCore.QRectF(0, -4, 10, 8.)) + hist.setLevels(zz.min(), zz.max()) + + # formatting + imgPlot.setLabel('left', "Y", units='T') + imgPlot.setLabel('bottom', "X", units='A') + xplot.setLabel('left', 'Z') + xplot.setLabel('bottom', "X", units='A') + yplot.setLabel('left', "Y", units='T') + yplot.setLabel('bottom', "Z") + + if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'): + QtWidgets.QApplication.instance().exec_() + +if __name__ == '__main__': + image_test() \ No newline at end of file From 9ff540309c6ddf1da1af44d2b4b2acdf50f2363b Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sun, 16 May 2021 17:20:42 -0500 Subject: [PATCH 08/94] remove circular import. --- plottr/data/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plottr/data/__init__.py b/plottr/data/__init__.py index 0e6b43a6..34900733 100644 --- a/plottr/data/__init__.py +++ b/plottr/data/__init__.py @@ -1,2 +1 @@ from .datadict import DataDict, MeshgridDataDict -from .datadict_storage import DDH5Writer, datadict_from_hdf5 From c3c9e00a50e05619cb0d10fb9ca6ea6b35249555 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 27 May 2021 18:16:18 -0500 Subject: [PATCH 09/94] first testing of pyqtgraph plot protypes --- plottr/plot/base.py | 10 ++- plottr/plot/pyqtgraph/__init__.py | 0 plottr/plot/pyqtgraph/autoplot.py | 61 +++++++++++++++ test/Untitled.ipynb | 75 +++++++++++++++++++ test/apps/autoplot_app.py | 4 +- .../{mpl_autoplot.py => mpl_figuremaker.py} | 0 test/gui/pyqtgraph_figuremaker.py | 50 +++++++++++++ 7 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 plottr/plot/pyqtgraph/__init__.py create mode 100644 plottr/plot/pyqtgraph/autoplot.py create mode 100644 test/Untitled.ipynb rename test/gui/{mpl_autoplot.py => mpl_figuremaker.py} (100%) create mode 100644 test/gui/pyqtgraph_figuremaker.py diff --git a/plottr/plot/base.py b/plottr/plot/base.py index 02f11ea0..22492d21 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -133,6 +133,12 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: self.dataStructure: Optional[DataDictBase] = None self.dataShapes: Optional[Dict[str, Tuple[int, ...]]] = None self.dataLimits: Optional[Dict[str, Tuple[float, float]]] = None + self.dataChanges: Dict[str, bool] = { + 'dataTypeChanged': False, + 'dataStructureChanged': False, + 'dataShapesChanged': False, + 'dataLimitsChanged': False, + } def setData(self, data: Optional[DataDictBase]) -> None: """Set data. Use this to trigger plotting. @@ -140,7 +146,7 @@ def setData(self, data: Optional[DataDictBase]) -> None: :param data: data to be plotted. """ self.data = data - changes = self.analyzeData(data) + self.dataChanges = self.analyzeData(data) def analyzeData(self, data: Optional[DataDictBase]) -> Dict[str, bool]: """checks data and compares with previous properties. @@ -576,7 +582,7 @@ def previousPlotId(self) -> Optional[int]: return None # Methods to be implemented by inheriting classes - def makeSubPlots(self, nSubPlots: int) -> List[Any]: + def makeSubPlots(self, nSubPlots: int) -> Optional[List[Any]]: """Generate the subplots. Called after all data has been added. Must be implemented by an inheriting class. diff --git a/plottr/plot/pyqtgraph/__init__.py b/plottr/plot/pyqtgraph/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py new file mode 100644 index 00000000..1d8d78f2 --- /dev/null +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -0,0 +1,61 @@ +"""``plottr.plot.pyqtgraph.autoplot`` -- tools for automatic plotting with pyqtgraph. +""" + +import logging +from typing import Dict, List, Tuple, Union, Optional, Any, Type + +import numpy as np + +from pyqtgraph import GraphicsLayoutWidget, GraphicsItem + +from plottr import QtWidgets, QtCore, QtGui +from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ + PlotItem, ComplexRepresentation, determinePlotDataType, \ + PlotWidgetContainer + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class FigureMaker(BaseFM): + """pyqtgraph implementation for :class:`.AutoFigureMaker`. + + """ + # TODO: need to figure out how to reuse widgets when we just update data + + def __init__(self, parentWidget: Optional[QtWidgets.QWidget] = None): + + super().__init__() + self.parentWidget = parentWidget + self.layoutWidget = None + + # re-implementing to get correct type annotation. + def __enter__(self) -> "FigureMaker": + return self + + def makeSubPlots(self, nSubPlots: int) -> List[Any]: + # this is for the case of having only 1d plots + self.layoutWidget = GraphicsLayoutWidget(parent=self.parentWidget) + plotItems = [] + for i in range(nSubPlots): + plotItems.append(self.layoutWidget.addPlot()) + return plotItems + + def formatSubPlot(self, subPlotId: int) -> Any: + pass + + def plot(self, plotItem: PlotItem) -> None: + plots = self.subPlots[plotItem.subPlot].axes + assert isinstance(plots, list) and len(plots) == 1 + assert len(plotItem.data) == 2 + x, y = plotItem.data + return plots[0].plot(x, y) + + + + + + + + + diff --git a/test/Untitled.ipynb b/test/Untitled.ipynb new file mode 100644 index 00000000..1db71ac1 --- /dev/null +++ b/test/Untitled.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "environmental-bermuda", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "polish-fishing", + "metadata": {}, + "outputs": [], + "source": [ + "from plottr import config" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "medical-fairy", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'main': {'matplotlibrc': {'axes.grid': True}}}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "clean-melissa", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:plottr-pyqt5]", + "language": "python", + "name": "conda-env-plottr-pyqt5-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/apps/autoplot_app.py b/test/apps/autoplot_app.py index fc9a3b30..bd1f1b42 100644 --- a/test/apps/autoplot_app.py +++ b/test/apps/autoplot_app.py @@ -154,7 +154,7 @@ def main(dataSrc): if __name__ == '__main__': # src = LineDataMovie(10, 3, 101) # src = ImageDataMovie(50, 2, 501) - # src = ImageDataLiveAcquisition(101, 101, 67) - src = ComplexImage(21, 21) + src = ImageDataLiveAcquisition(101, 101, 67) + # src = ComplexImage(21, 21) src.delay = 0.1 main(src) diff --git a/test/gui/mpl_autoplot.py b/test/gui/mpl_figuremaker.py similarity index 100% rename from test/gui/mpl_autoplot.py rename to test/gui/mpl_figuremaker.py diff --git a/test/gui/pyqtgraph_figuremaker.py b/test/gui/pyqtgraph_figuremaker.py new file mode 100644 index 00000000..dc619a0f --- /dev/null +++ b/test/gui/pyqtgraph_figuremaker.py @@ -0,0 +1,50 @@ +"""A set of simple tests of the pyqtgraph FigureMaker classes.""" + +import numpy as np + +from plottr import QtWidgets +from plottr.gui.tools import widgetDialog +from plottr.plot.pyqtgraph.autoplot import FigureMaker + + +def test_single_line_plot(): + + x = np.linspace(0, 10, 51) + y = np.cos(x) + + with FigureMaker() as fm: + line_1 = fm.addData(x, y) + _ = fm.addData(x, y**2, join=line_1) + + return fm.layoutWidget + +def main(): + app = QtWidgets.QApplication([]) + + widgets = [] + + widgets.append( + test_single_line_plot() + ) + + # wins.append( + # test_multiple_line_plots()) + # wins.append( + # test_multiple_line_plots(single_panel=True)) + # wins.append( + # test_complex_line_plots()) + # wins.append( + # test_complex_line_plots(single_panel=True)) + # wins.append( + # test_complex_line_plots(mag_and_phase_format=True)) + # wins.append( + # test_complex_line_plots(single_panel=True, mag_and_phase_format=True)) + + for w in widgets: + dg = widgetDialog(w) + dg.show() + return app.exec_() + + +if __name__ == '__main__': + main() \ No newline at end of file From 2ab1ca6843f1f2748b23e7e923400426b7876c17 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 31 May 2021 20:04:43 +0200 Subject: [PATCH 10/94] Add disappeared "imported plottr version" log message --- plottr/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plottr/__init__.py b/plottr/__init__.py index 333c55ff..ed16a228 100644 --- a/plottr/__init__.py +++ b/plottr/__init__.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Optional import importlib +import logging import os import sys @@ -20,6 +21,9 @@ __version__ = get_versions()['version'] del get_versions +logger = logging.getLogger(__name__) +logger.info(f"Imported plottr version: {__version__}") + plottrPath = os.path.split(os.path.abspath(__file__))[0] From 07e7adc43469156f8b3f2e9df9ba336830b65394 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Tue, 1 Jun 2021 09:33:30 +0200 Subject: [PATCH 11/94] move setup static content to setup.cfg Also add a few more handly links these will be included on pypi in the sidebar --- setup.cfg | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 44 +------------------------------------------ 2 files changed, 57 insertions(+), 43 deletions(-) diff --git a/setup.cfg b/setup.cfg index fcc11c33..7685eff7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,59 @@ +[metadata] +name = plottr +description = A tool for live plotting and processing data +long_description = file: README.md +long_description_content_type = text/markdown +author = Wolfgang Pfaff +author_email = wolfgangpfff@gmail.com +classifiers = + Development Status :: 3 - Alpha + Intended Audience :: Science/Research + License :: OSI Approved :: MIT License + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Topic :: Scientific/Engineering +license = MIT +url = https://github.com/toolsforexperiments/plottr +project_urls = + Documentation = https://plottr.readthedocs.io + Source = https://github.com/toolsforexperiments/plottr/ + Tracker = https://github.com/toolsforexperiments/plottr/issues + +[options] +packages = find: +python_requires = >=3.7 +install_requires = + pandas>=0.22 + xarray + pyqtgraph>=0.10.0 + matplotlib + numpy + lmfit + h5py>=2.10.0 + qtpy>=1.9.0 + typing-extensions>=3.7.4.3 + packaging>=20.0 + +[options.package_data] +plottr = + resource/gfx/* + +[options.extras_require] +PyQt5 = PyQt5 +PySide2 = PySide2 + +[options.packages.find] +include = + plottr* + + +[options.entry_points] +console_scripts = + plottr-monitr = plottr.apps.monitr:script + plottr-inspectr = plottr.apps.inspectr:script + plottr-autoplot-ddh5 = plottr.apps.autoplot:script + [versioneer] VCS = git style = pep440 diff --git a/setup.py b/setup.py index b084be02..fcbb31fa 100644 --- a/setup.py +++ b/setup.py @@ -1,50 +1,8 @@ -from setuptools import setup, find_packages +from setuptools import setup import versioneer -with open("README.md", "r") as fh: - long_description = fh.read() - setup( - name='plottr', version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), - description='A tool for live plotting and processing data', - long_description=long_description, - long_description_content_type="text/markdown", - author='Wolfgang Pfaff', - author_email='wolfgangpfff@gmail.com', - url='https://github.com/toolsforexperiments/plottr', - packages=find_packages(include=("plottr*",)), - package_data={'plottr': ['resource/gfx/*']}, - install_requires=[ - 'pandas>=0.22', - 'xarray', - 'pyqtgraph>=0.10.0', - 'matplotlib', - 'numpy', - 'lmfit', - 'h5py>=2.10.0', - 'qtpy>=1.9.0', - 'typing-extensions>=3.7.4.3', - "packaging>=20.0", - ], - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Topic :: Scientific/Engineering' - ], - python_requires='>=3.7', - extras_require={'PyQt5': "PyQt5", "PySide2": "PySide2"}, - entry_points={ - "console_scripts": [ - "plottr-monitr = plottr.apps.monitr:script", - "plottr-inspectr = plottr.apps.inspectr:script", - "plottr-autoplot-ddh5 = plottr.apps.autoplot:script", - ], - } ) From 1966ef6f284780188d717e6a1ba3eb88dd9abf48 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Tue, 1 Jun 2021 09:35:57 +0200 Subject: [PATCH 12/94] add 3.9 classifier --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 7685eff7..b71354ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,7 @@ classifiers = Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Topic :: Scientific/Engineering license = MIT url = https://github.com/toolsforexperiments/plottr @@ -47,7 +48,6 @@ PySide2 = PySide2 include = plottr* - [options.entry_points] console_scripts = plottr-monitr = plottr.apps.monitr:script From ec2f7231ed04dab19e69465991b014b461e92a37 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Tue, 1 Jun 2021 09:39:57 +0200 Subject: [PATCH 13/94] add py.typed file to mark as safe for type checking --- plottr/py.typed | 2 ++ setup.cfg | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 plottr/py.typed diff --git a/plottr/py.typed b/plottr/py.typed new file mode 100644 index 00000000..4ed7be9a --- /dev/null +++ b/plottr/py.typed @@ -0,0 +1,2 @@ +this file marks plottr as safe for typechecking +https://mypy.readthedocs.io/en/latest/installed_packages.html#installed-packages diff --git a/setup.cfg b/setup.cfg index b71354ec..4b6246e4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,8 @@ install_requires = [options.package_data] plottr = - resource/gfx/* + resource/gfx/* + py.typed [options.extras_require] PyQt5 = PyQt5 From c9df18062f9eec64f6ee53be2a678553caff4676 Mon Sep 17 00:00:00 2001 From: Akshita07 Date: Wed, 2 Jun 2021 10:26:51 +0200 Subject: [PATCH 14/94] Allow empty dataset if datadict is none --- plottr/data/datadict.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 65072a05..301dbde7 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -1288,6 +1288,10 @@ def combine_datadicts(*dicts: DataDict) -> Union[DataDictBase, DataDict]: dep_axes = [ax_map[ax] for ax in d[d_dep]['axes']] ret[newdep] = d[d_dep] ret[newdep]['axes'] = dep_axes - assert ret is not None - ret.validate() + + if ret is None: + ret = DataDictBase() + else: + ret.validate() + return ret From 97ecd038c1f20e8df5e50e434933d003f5847b7c Mon Sep 17 00:00:00 2001 From: Akshita07 Date: Thu, 3 Jun 2021 10:03:05 +0200 Subject: [PATCH 15/94] update ret to be a empty datadict object --- plottr/data/datadict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 301dbde7..86ac4322 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -1290,7 +1290,7 @@ def combine_datadicts(*dicts: DataDict) -> Union[DataDictBase, DataDict]: ret[newdep]['axes'] = dep_axes if ret is None: - ret = DataDictBase() + ret = DataDict() else: ret.validate() From a9ef0f0d1eb35510aeb127d8e17d4f69056707e9 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 3 Jun 2021 10:11:26 -0500 Subject: [PATCH 16/94] importing config files without modifying sys.path. --- plottr/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plottr/__init__.py b/plottr/__init__.py index ed16a228..9e48d003 100644 --- a/plottr/__init__.py +++ b/plottr/__init__.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Optional import importlib +from importlib.util import spec_from_file_location, module_from_spec import logging import os import sys @@ -57,7 +58,7 @@ def configFiles(fileName: str) -> List[str]: return ret -def config(names: Optional[List[str]] = None, forceReload: bool = True) -> \ +def config(names: Optional[List[str]] = None) -> \ Dict[str, Any]: """Return the plottr configuration as a dictionary. @@ -97,13 +98,11 @@ def config(names: Optional[List[str]] = None, forceReload: bool = True) -> \ filen = f"{modn}.py" this_cfg = {} for filep in configFiles(filen)[::-1]: - path = os.path.split(filep)[0] - sys.path.insert(0, path) - mod = importlib.import_module(modn) - if forceReload: - importlib.reload(mod) + spec = spec_from_file_location(modn, filep) + mod = module_from_spec(spec) + sys.modules[modn] = mod + spec.loader.exec_module(mod) this_cfg.update(getattr(mod, 'config', {})) - sys.path.pop(0) config[name] = this_cfg return config From 6f3cb98285138284642d9470eb0b9d9d4efdb6fa Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 3 Jun 2021 10:43:49 -0500 Subject: [PATCH 17/94] clarify why `formatSubPlot` does not raise an error when not reimplemented. --- plottr/plot/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plottr/plot/base.py b/plottr/plot/base.py index 02f11ea0..434682e6 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -587,11 +587,13 @@ def makeSubPlots(self, nSubPlots: int) -> List[Any]: def formatSubPlot(self, subPlotId: int) -> Any: """Format a subplot. - Must be implemented by an inheriting class. + May be implemented by an inheriting class. + By default, does nothing. :param subPlotId: ID of the subplot. :return: Depends on inheriting class. """ + return None def plot(self, plotItem: PlotItem) -> Any: """Plot an item. From 1d09196dc8236f0154ae72ca70fa8e6fb1bd2037 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 3 Jun 2021 10:49:12 -0500 Subject: [PATCH 18/94] fix mypy issue with importlib. --- plottr/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plottr/__init__.py b/plottr/__init__.py index 9e48d003..8a6b6b9b 100644 --- a/plottr/__init__.py +++ b/plottr/__init__.py @@ -1,5 +1,5 @@ from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Optional -import importlib +from importlib.abc import Loader from importlib.util import spec_from_file_location, module_from_spec import logging import os @@ -101,6 +101,7 @@ def config(names: Optional[List[str]] = None) -> \ spec = spec_from_file_location(modn, filep) mod = module_from_spec(spec) sys.modules[modn] = mod + assert isinstance(spec.loader, Loader) spec.loader.exec_module(mod) this_cfg.update(getattr(mod, 'config', {})) From c12e508d9a93ca4c66308fb557904235eeecd7ff Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 3 Jun 2021 20:29:04 -0500 Subject: [PATCH 19/94] Update plottr/plot/mpl/widgets.py Co-authored-by: Mikhail Astafev --- plottr/plot/mpl/widgets.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plottr/plot/mpl/widgets.py b/plottr/plot/mpl/widgets.py index b477149f..4caf253f 100644 --- a/plottr/plot/mpl/widgets.py +++ b/plottr/plot/mpl/widgets.py @@ -156,8 +156,7 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): self.mplBar = NavBar(self.plot, self) self.addMplBarOptions() - scaling = rint(self.logicalDpiX() / 96.0) - defaultIconSize = 16 * scaling + defaultIconSize = 16 * dpiScalingFactor(self) self.mplBar.setIconSize(QtCore.QSize(defaultIconSize, defaultIconSize)) layout = QtWidgets.QVBoxLayout(self) layout.addWidget(self.plot) @@ -206,4 +205,3 @@ def figureDialog() -> Tuple[Figure, QtWidgets.QDialog]: widget = MPLPlotWidget() return widget.plot.fig, widgetDialog(widget) - From 5359ee49abdd519455912932dec4d59b18e801ee Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 3 Jun 2021 20:47:40 -0500 Subject: [PATCH 20/94] enforcing real in a more sane spot. --- plottr/plot/base.py | 2 +- plottr/plot/mpl/autoplot.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plottr/plot/base.py b/plottr/plot/base.py index 434682e6..3e0cb293 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -452,7 +452,7 @@ def _splitComplexData(self, plotItem: PlotItem) -> List[PlotItem]: data = plotItem.data[-1] # this check avoids a numpy ComplexWarning when we're working with MaskedArray (almost always) - mag_data = np.ma.abs(data) if isinstance(data, np.ma.MaskedArray) else np.abs(data) + mag_data = np.ma.abs(data).real if isinstance(data, np.ma.MaskedArray) else np.abs(data) phase_data = np.angle(data) if label == '': diff --git a/plottr/plot/mpl/autoplot.py b/plottr/plot/mpl/autoplot.py index c7969646..3f1ff2c7 100644 --- a/plottr/plot/mpl/autoplot.py +++ b/plottr/plot/mpl/autoplot.py @@ -125,14 +125,14 @@ def plotLine(self, plotItem: PlotItem) -> Optional[List[Line2D]]: assert len(plotItem.data) == 2 lbl = plotItem.labels[-1] if isinstance(plotItem.labels, list) and len(plotItem.labels) > 0 else '' x, y = plotItem.data - return axes[0].plot(x, y.real, label=lbl, **plotItem.plotOptions) + return axes[0].plot(x, y, label=lbl, **plotItem.plotOptions) def plotImage(self, plotItem: PlotItem) -> Optional[Artist]: assert len(plotItem.data) == 3 x, y, z = plotItem.data axes = self.subPlots[plotItem.subPlot].axes assert isinstance(axes, list) and len(axes) > 0 - im = colorplot2d(axes[0], x, y, z.real, plotType=self.plotType) + im = colorplot2d(axes[0], x, y, z, plotType=self.plotType) cb = self.fig.colorbar(im, ax=axes[0], shrink=0.75, pad=0.02) lbl = plotItem.labels[-1] if isinstance(plotItem.labels, list) and len(plotItem.labels) > 0 else '' cb.set_label(lbl) From ba44f828a2ec5361a33ef869cfa575930650b486 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 3 Jun 2021 20:58:17 -0500 Subject: [PATCH 21/94] fix typing. --- plottr/plot/mpl/widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/plot/mpl/widgets.py b/plottr/plot/mpl/widgets.py index 4caf253f..58629468 100644 --- a/plottr/plot/mpl/widgets.py +++ b/plottr/plot/mpl/widgets.py @@ -156,7 +156,7 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): self.mplBar = NavBar(self.plot, self) self.addMplBarOptions() - defaultIconSize = 16 * dpiScalingFactor(self) + defaultIconSize = int(16 * dpiScalingFactor(self)) self.mplBar.setIconSize(QtCore.QSize(defaultIconSize, defaultIconSize)) layout = QtWidgets.QVBoxLayout(self) layout.addWidget(self.plot) From 02fb934dcc948f696675f8663651d68b7ee0657c Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Fri, 4 Jun 2021 16:50:41 +0200 Subject: [PATCH 22/94] ensure that output of joint_crop2d_rows_cols is always a integer array empty arrays will be float64 by default --- plottr/utils/num.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plottr/utils/num.py b/plottr/utils/num.py index 9745899d..5253a680 100644 --- a/plottr/utils/num.py +++ b/plottr/utils/num.py @@ -351,7 +351,10 @@ def joint_crop2d_rows_cols(*arr: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: xs += _x.tolist() ys += _y.tolist() - return np.array(list(set(xs))), np.array(list(set(ys))) + return ( + np.array(list(set(xs)), dtype=np.int64), + np.array(list(set(ys)), dtype=np.int64), + ) def crop2d_from_xy(arr: np.ndarray, xs: np.ndarray, From c9f4c1896ad247ecc6fdffac9340abb12c66e08e Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Fri, 4 Jun 2021 17:03:29 +0200 Subject: [PATCH 23/94] better tests for crop Ensure that the corner cases of less than one row and no data at all are handled correctly --- test/pytest/test_numtools.py | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/pytest/test_numtools.py b/test/pytest/test_numtools.py index d0742207..9334f845 100644 --- a/test/pytest/test_numtools.py +++ b/test/pytest/test_numtools.py @@ -1,6 +1,7 @@ from collections import OrderedDict import numpy as np +from numpy.testing import assert_array_equal from plottr.utils import num @@ -150,3 +151,61 @@ def test_cropping2d(): assert num.arrays_equal(x, arr[:2, :2]) assert num.arrays_equal(y, arr.T[:2, :2]) assert num.arrays_equal(z, data[:2, :2]) + + +def test_crop2d_noop(): + x = np.arange(1, 10) + y = np.arange(20, 26) + + xx, yy = np.meshgrid(x, y) + + zz = np.random.rand(*xx.shape) + + xxx, yyy, zzz = num.crop2d(xx, yy, zz) + + assert_array_equal(xx, xxx) + assert_array_equal(yy, yyy) + assert_array_equal(zz, zzz) + + +def test_crop_all_nan(): + x = np.arange(1., 10.) + y = np.arange(20., 26.) + + xx, yy = np.meshgrid(x, y) + + xx[:] = np.nan + yy[:] = np.nan + + zz = np.random.rand(*xx.shape) + + xxx, yyy, zzz = num.crop2d(xx, yy, zz) + + assert xxx.shape == (0, 0) + assert yyy.shape == (0, 0) + assert zzz.shape == (0, 0) + + +def test_crop_less_than_one_row(): + x = np.arange(1., 10.) + y = np.arange(20., 26.) + + xx, yy = np.meshgrid(x, y) + + xx[1:, :] = np.nan + xx[0, 5:] = np.nan + yy[1:, :] = np.nan + yy[0:, 5:] = np.nan + + zz = np.random.rand(*xx.shape) + + xxx, yyy, zzz = num.crop2d(xx, yy, zz) + + assert xxx.shape == (1, 5) + assert_array_equal(xxx, xx[0:1, 0:5]) + + assert zzz.shape == (1, 5) + assert_array_equal(yyy, yy[0:1, 0:5]) + + assert zzz.shape == (1, 5) + assert_array_equal(zzz, zz[0:1, 0:5]) From 6dfde9b20426ba1a3173f880b62062c06527b877 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sun, 6 Jun 2021 10:37:28 +0200 Subject: [PATCH 24/94] formatting --- plottr/utils/num.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/utils/num.py b/plottr/utils/num.py index 5253a680..2205cf47 100644 --- a/plottr/utils/num.py +++ b/plottr/utils/num.py @@ -2,7 +2,7 @@ Tools for numerical operations. """ -from typing import Sequence, Tuple, Union, List, Optional +from typing import List, Optional, Sequence, Tuple, Union import numpy as np import pandas as pd From 9bf2df182098dde9295a1a896b332c991f1118dc Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Fri, 4 Jun 2021 20:44:20 +0200 Subject: [PATCH 25/94] fix issue where matplotlib might take color arg for a rgb(a) value if there are exactly 3 or 4 values This happens if a 1,3 or 1,4 element array is passed We can fix this by removing the singleton dim As we are doing a scatter plot that has no value anyway --- plottr/plot/mpl/plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/plot/mpl/plotting.py b/plottr/plot/mpl/plotting.py index c02222b9..1b448f3e 100644 --- a/plottr/plot/mpl/plotting.py +++ b/plottr/plot/mpl/plotting.py @@ -125,7 +125,7 @@ def colorplot2d(ax: Axes, elif plotType is PlotType.colormesh: im = ppcolormesh_from_meshgrid(ax, x, y, z, cmap=cmap, **kw) elif plotType is PlotType.scatter2d: - im = ax.scatter(x, y, c=z, cmap=cmap, **kw) + im = ax.scatter(x.squeeze(), y.squeeze(), c=z.squeeze(), cmap=cmap, **kw) else: im = None From c994a82df99cf33dfcba97c4e656dbd9d178e02c Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sun, 6 Jun 2021 10:36:03 +0200 Subject: [PATCH 26/94] use ravel to always flatten arrays scatter expects 1d arrays we use ravel rather than flatten since there is no need for a copy --- plottr/plot/mpl/plotting.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plottr/plot/mpl/plotting.py b/plottr/plot/mpl/plotting.py index 1b448f3e..917d697f 100644 --- a/plottr/plot/mpl/plotting.py +++ b/plottr/plot/mpl/plotting.py @@ -3,7 +3,7 @@ """ from enum import Enum, auto, unique -from typing import Optional, Tuple, Any, Union +from typing import Any, Optional, Tuple, Union import numpy as np from matplotlib import colors, rcParams @@ -11,8 +11,7 @@ from matplotlib.image import AxesImage from plottr.utils import num -from plottr.utils.num import interp_meshgrid_2d, centers2edges_2d - +from plottr.utils.num import centers2edges_2d, interp_meshgrid_2d __author__ = 'Wolfgang Pfaff' __license__ = 'MIT' @@ -125,7 +124,7 @@ def colorplot2d(ax: Axes, elif plotType is PlotType.colormesh: im = ppcolormesh_from_meshgrid(ax, x, y, z, cmap=cmap, **kw) elif plotType is PlotType.scatter2d: - im = ax.scatter(x.squeeze(), y.squeeze(), c=z.squeeze(), cmap=cmap, **kw) + im = ax.scatter(x.ravel(), y.ravel(), c=z.ravel(), cmap=cmap, **kw) else: im = None From 0d1f63fd4e71307ad20faab4ab374c1d765eb8de Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Sun, 6 Jun 2021 10:56:38 +0200 Subject: [PATCH 27/94] add basic test for scatter of arrays that could be takes as rgba --- test/pytest/test_plotting.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/pytest/test_plotting.py diff --git a/test/pytest/test_plotting.py b/test/pytest/test_plotting.py new file mode 100644 index 00000000..29dc1f75 --- /dev/null +++ b/test/pytest/test_plotting.py @@ -0,0 +1,30 @@ +import matplotlib.pyplot as plt +import numpy as np +from plottr.plot.mpl.plotting import PlotType, colorplot2d + + +def test_colorplot2d_scatter_rgba_error(): + """ + Check that scatter plots are not trying to plot 1x3 and 1x4 + z arrays as rgb(a) colors. + + """ + fig, ax = plt.subplots(1, 1) + x = np.array([[0.0, 11.11111111, 22.22222222, 33.33333333]]) + y = np.array( + [ + [ + 0.0, + 0.0, + 0.0, + 0.0, + ] + ] + ) + z = np.array([[5.08907021, 4.93923391, 5.11400073, 5.0925613]]) + colorplot2d(ax, x, y, z, PlotType.scatter2d) + + x = np.array([[0.0, 11.11111111, 22.22222222]]) + y = np.array([[0.0, 0.0, 0.0]]) + z = np.array([[5.08907021, 4.93923391, 5.11400073]]) + colorplot2d(ax, x, y, z, PlotType.scatter2d) From 7e86fbe09e66e11df470ee411f3aac8697cb06d1 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 7 Jun 2021 16:19:11 +0200 Subject: [PATCH 28/94] dont ignore sphinx make anf bat files --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index 213a7411..83fc3a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -73,9 +73,6 @@ instance/ # Sphinx documentation docs/_build/ -doc/build -doc/make.bat -doc/Makefile # PyBuilder target/ From 07f83dc6fbd7d9b6a7424a8d671abbeac268ceee Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 7 Jun 2021 16:20:42 +0200 Subject: [PATCH 29/94] Ignore qcodes sqlite databsae db files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 83fc3a7a..50b8979d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Qcodes .db files +*.db + # IDEs .idea/ .vscode/* From e628cbc77d18f93b257891e2b7c4661ab875e575 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 7 Jun 2021 16:28:24 +0200 Subject: [PATCH 30/94] Add changelog for 0.7.0 release on 2021-06-08 --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index b2cb055e..867eff47 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,25 @@ You might want to install freshly if you still use the old version. # Recent changes: +## 2021-06-08 + +### Added + +- refactoring the plotting system (#166) +- Add version log message to main ``__init__`` (#175) + +### Fixed + +- Fix crop if less than one row is not nan (#198) +- Fix rgba error (#199) +- Allow empty dataset if datadict is none (#195) + +### Behind the scenes + +- Modernize setup files (#194) +- packaging cleanups (#177) +- upgrade versioneer to 0.19 (#176) + ## 2021-02-16 ### Added From 94a8ca2e2596d44abe91311ce089487bc55b9a19 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 7 Jun 2021 19:33:37 +0200 Subject: [PATCH 31/94] Add minimal versions to dependencies i found out that the conda-forge recipe for plottr has stricter version specifications that the pip, so i decide to sync them https://github.com/conda-forge/plottr-feedstock . @jenshnielsen do those versions look correct to you? --- setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4b6246e4..f3ddd98c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,9 +28,9 @@ install_requires = pandas>=0.22 xarray pyqtgraph>=0.10.0 - matplotlib - numpy - lmfit + matplotlib>=3.0.0 + numpy>=1.12.0 + lmfit>=1.0 h5py>=2.10.0 qtpy>=1.9.0 typing-extensions>=3.7.4.3 @@ -43,7 +43,7 @@ plottr = [options.extras_require] PyQt5 = PyQt5 -PySide2 = PySide2 +PySide2 = PySide2>=5.12 [options.packages.find] include = From c076059a0c47fd5bcfe1cdb5662be164e120a175 Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Tue, 8 Jun 2021 13:18:59 +0200 Subject: [PATCH 32/94] clarify install and mention conda forge --- README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 867eff47..f72a3b69 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,18 @@ https://plottr.readthedocs.io (work in progress...) Plottr is installable from pypi with `pip install plottr` -To install with either PyQt5 or Pyside2 backend you can do -``pip install plottr[PyQt5]`` or ``pip install plottr[Pyside2]`` Note that if -you have installed ``pyqt`` from ``(Ana)Conda`` you should not use any of these -targets but do ``pip install plottr`` +Plottr requires either the PyQt5 or Pyside2 gui framework. +To install with PyQt5 or Pyside2 backend you can do +``pip install plottr[PyQt5]`` or ``pip install plottr[Pyside2]`` + +Note that if you have installed ``pyqt`` from ``(Ana)Conda`` you should not use any of these +targets but do ``pip install plottr`` or install Plottr from conda forge: + +``` +conda config --add channels conda-forge +conda config --set channel_priority strict +conda install plottr +``` To install from source: clone the repo, and install using `pip install -e .` From b3bc2e8902bbdde88325ded91423670c886d523e Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Tue, 15 Jun 2021 11:04:21 +0200 Subject: [PATCH 33/94] spec_from_file_location may return a None So handle that --- plottr/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plottr/__init__.py b/plottr/__init__.py index 8a6b6b9b..1a87d08d 100644 --- a/plottr/__init__.py +++ b/plottr/__init__.py @@ -99,6 +99,8 @@ def config(names: Optional[List[str]] = None) -> \ this_cfg = {} for filep in configFiles(filen)[::-1]: spec = spec_from_file_location(modn, filep) + if spec is None: + raise FileNotFoundError(f"Could not locate spec for {modn}, {filep}") mod = module_from_spec(spec) sys.modules[modn] = mod assert isinstance(spec.loader, Loader) From ec9d1265a3e0bb0528888ed5fca7b7a568b09cfc Mon Sep 17 00:00:00 2001 From: "Jens H. Nielsen" Date: Tue, 15 Jun 2021 11:09:10 +0200 Subject: [PATCH 34/94] add dependabot (copied config from qcodes --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..26f7b01b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: "daily" From 20de371ab8fce95d0ec6b3a0ccaa63fd8daa0efb Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 16 Jul 2021 23:25:01 -0500 Subject: [PATCH 35/94] continued on first working version of pyqtgraph plotting. --- plottr/config/plottrcfg_main.py | 8 +++ plottr/plot/pyqtgraph/__init__.py | 13 ++++ plottr/plot/pyqtgraph/autoplot.py | 115 +++++++++++++++++++++++++----- plottr/plot/pyqtgraph/plots.py | 108 ++++++++++++++++++++++++++++ test/gui/pyqtgraph_figuremaker.py | 110 ++++++++++++++++++++++------ test/gui/pyqtgraph_testing.py | 7 +- 6 files changed, 318 insertions(+), 43 deletions(-) create mode 100644 plottr/plot/pyqtgraph/plots.py diff --git a/plottr/config/plottrcfg_main.py b/plottr/config/plottrcfg_main.py index c86ab9db..c9735569 100644 --- a/plottr/config/plottrcfg_main.py +++ b/plottr/config/plottrcfg_main.py @@ -26,4 +26,12 @@ 'savefig.transparent': False, }, + 'pyqtgraph': { + 'background': 'w', + 'foreground': 'k', + 'line_colors': ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', + '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf'], + 'line_symbols': ['o', ], + 'line_symbol_size': 7, + } } diff --git a/plottr/plot/pyqtgraph/__init__.py b/plottr/plot/pyqtgraph/__init__.py index e69de29b..df602a96 100644 --- a/plottr/plot/pyqtgraph/__init__.py +++ b/plottr/plot/pyqtgraph/__init__.py @@ -0,0 +1,13 @@ +import pyqtgraph as pg + +from plottr import config_entry as getcfg + + +__all__ = [] + + +bg = getcfg('main', 'pyqtgraph', 'background', default='w') +pg.setConfigOption('background', bg) + +fg = getcfg('main', 'pyqtgraph', 'foreground', default='k') +pg.setConfigOption('foreground', fg) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 1d8d78f2..d8798325 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -6,12 +6,13 @@ import numpy as np -from pyqtgraph import GraphicsLayoutWidget, GraphicsItem +from pyqtgraph import GraphicsLayoutWidget, mkPen, mkBrush, HistogramLUTItem, ImageItem -from plottr import QtWidgets, QtCore, QtGui +from plottr import QtWidgets, QtCore, QtGui, config_entry as getcfg from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ PlotItem, ComplexRepresentation, determinePlotDataType, \ PlotWidgetContainer +from .plots import Plot, PlotWithColorbar logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -19,43 +20,121 @@ class FigureMaker(BaseFM): """pyqtgraph implementation for :class:`.AutoFigureMaker`. - """ + # TODO: need to figure out how to reuse widgets when we just update data + # TODO: make scrollable when many figures (set min size)? + # TODO: check for valid plot data? def __init__(self, parentWidget: Optional[QtWidgets.QWidget] = None): - super().__init__() - self.parentWidget = parentWidget - self.layoutWidget = None + self.widget = QtWidgets.QWidget(parentWidget) + + self.layout = QtWidgets.QVBoxLayout() + self.layout.setContentsMargins(0, 0, 0, 0) + self.layout.setSpacing(0) + + self.widget.setLayout(self.layout) + self.widget.setMinimumSize(400, 400) # re-implementing to get correct type annotation. def __enter__(self) -> "FigureMaker": return self def makeSubPlots(self, nSubPlots: int) -> List[Any]: - # this is for the case of having only 1d plots - self.layoutWidget = GraphicsLayoutWidget(parent=self.parentWidget) - plotItems = [] + + plotWidgets = [] # FIXME: needs correct type for i in range(nSubPlots): - plotItems.append(self.layoutWidget.addPlot()) - return plotItems + if max(self.dataDimensionsInSubPlot(i).values()) == 1: + plot = Plot(self.widget) + self.layout.addWidget(plot) + plotWidgets.append([plot]) + + elif max(self.dataDimensionsInSubPlot(i).values()) == 2: + plot = PlotWithColorbar(self.widget) + self.layout.addWidget(plot) + plotWidgets.append([plot]) + + return plotWidgets def formatSubPlot(self, subPlotId: int) -> Any: - pass + if len(self.plotIdsInSubPlot(subPlotId)) == 0: + return - def plot(self, plotItem: PlotItem) -> None: - plots = self.subPlots[plotItem.subPlot].axes - assert isinstance(plots, list) and len(plots) == 1 - assert len(plotItem.data) == 2 - x, y = plotItem.data - return plots[0].plot(x, y) + labels = self.subPlotLabels(subPlotId) + plotwidgets = self.subPlots[subPlotId].axes + assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 + pw = plotwidgets[0] + # label the x axis if there's only one x label + if isinstance(pw, Plot): + if len(set(labels[0])) == 1: + pw.plot.setLabel("bottom", labels[0][0]) + if isinstance(pw, PlotWithColorbar): + if len(set(labels[0])) == 1: + pw.plot.setLabel("bottom", labels[0][0]) + if len(set(labels[1])) == 1: + pw.plot.setLabel('left', labels[1][0]) + if len(set(labels[2])) == 1: + pw.colorbar.setLabel('left', labels[2][0]) + def plot(self, plotItem: PlotItem) -> None: + if plotItem.plotDataType is PlotDataType.unknown: + if len(plotItem.data) == 2: + plotItem.plotDataType = PlotDataType.scatter1d + elif len(plotItem.data) == 3: + plotItem.plotDataType = PlotDataType.scatter2d + + if plotItem.plotDataType in [PlotDataType.scatter1d, PlotDataType.line1d]: + self._1dPlot(plotItem) + elif plotItem.plotDataType == PlotDataType.grid2d: + self._colorPlot(plotItem) + elif plotItem.plotDataType == PlotDataType.scatter2d: + self._scatterPlot2d(plotItem) + else: + raise NotImplementedError('Cannot plot this data.') + + def _1dPlot(self, plotItem): + colors = getcfg('main', 'pyqtgraph', 'line_colors', default=['r', 'b', 'g']) + symbols = getcfg('main', 'pyqtgraph', 'line_symbols', default=['o']) + symbolSize = getcfg('main', 'pyqtgraph', 'line_symbol_size', default=5) + + plotwidgets = self.subPlots[plotItem.subPlot].axes + assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 + pw = plotwidgets[0] + assert len(plotItem.data) == 2 + x, y = plotItem.data + color = colors[self.findPlotIndexInSubPlot(plotItem.id) % len(colors)] + symbol = symbols[self.findPlotIndexInSubPlot(plotItem.id) % len(symbols)] + + if plotItem.plotDataType == PlotDataType.line1d: + return pw.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], + pen=mkPen(color, width=2), + symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) + else: + return pw.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], + pen=None, + symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) + + def _colorPlot(self, plotItem): + plotwidgets = self.subPlots[plotItem.subPlot].axes + assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 + pw = plotwidgets[0] + + assert isinstance(pw, PlotWithColorbar) and len(plotItem.data) == 3 + pw.setImage(*plotItem.data) + + def _scatterPlot2d(self, plotItem): + plotwidgets = self.subPlots[plotItem.subPlot].axes + assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 + pw = plotwidgets[0] + + assert isinstance(pw, PlotWithColorbar) and len(plotItem.data) == 3 + pw.setScatter2d(*plotItem.data) diff --git a/plottr/plot/pyqtgraph/plots.py b/plottr/plot/pyqtgraph/plots.py new file mode 100644 index 00000000..8f934cdc --- /dev/null +++ b/plottr/plot/pyqtgraph/plots.py @@ -0,0 +1,108 @@ +from typing import Any, Optional, List, Tuple + +import numpy as np +import pyqtgraph as pg + +from plottr import QtCore, QtWidgets +from . import * + + +__all__ = ['PlotBase', 'Plot'] + + +class PlotBase(QtWidgets.QWidget): + + def __init__(self, parent=None): + super().__init__(parent) + self.layout = QtWidgets.QVBoxLayout(self) + self.layout.setContentsMargins(0, 0, 0, 0) + self.layout.setSpacing(0) + self.setLayout(self.layout) + + self.graphicsLayout = pg.GraphicsLayoutWidget(self) + self.layout.addWidget(self.graphicsLayout) + + def clearPlot(self): + return + + +class Plot(PlotBase): + + plot: pg.PlotItem + + def __init__(self, parent=None): + super().__init__(parent) + self.plot: pg.PlotItem = self.graphicsLayout.addPlot() + self.plot.addLegend(offset=(5, 5), pen='#999', brush=(255, 255, 255, 150)) + + def clearPlot(self): + self.plot.clear() + + +class PlotWithColorbar(PlotBase): + + plot: pg.PlotItem + colorbar: pg.ColorBarItem + + def __init__(self, parent=None): + super().__init__(parent) + + self.plot: pg.PlotItem = self.graphicsLayout.addPlot() + + cmap = pg.colormap.get('viridis', source='matplotlib') + self.colorbar: pg.ColorBarItem = pg.ColorBarItem(interactive=True, values=(0, 1), + cmap=cmap, width=15) + self.graphicsLayout.addItem(self.colorbar) + + self.img: Optional[pg.ImageItem] = None + self.scatter: Optional[pg.ScatterPlotItem] = None + self.scatterZVals: Optional[np.ndarray] = None + + def clearPlot(self): + self.img = None + self.scatter = None + self.scatterZVals = None + self.plot.clear() + try: + self.colorbar.sigLevelsChanged.disconnect(self._colorScatterPoints) + except TypeError: + pass + + def setImage(self, x: np.ndarray, y: np.ndarray, z: np.ndarray): + self.clearPlot() + + self.img = pg.ImageItem() + self.plot.addItem(self.img) + self.img.setImage(z) + self.img.setRect(QtCore.QRectF(x.min(), y.min(), x.max()-x.min(), y.max()-y.min())) + + self.colorbar.setImageItem(self.img) + self.colorbar.rounding = (z.max()-z.min()) * 1e-2 + self.colorbar.setLevels((z.min(), z.max())) + + def setScatter2d(self, x: np.ndarray, y: np.ndarray, z: np.ndarray): + self.clearPlot() + + self.scatter = pg.ScatterPlotItem() + self.scatter.setData(x=x.flatten(), y=y.flatten(), symbol='o', size=8) + self.plot.addItem(self.scatter) + self.scatterZVals = z + + self.colorbar.setLevels((z.min(), z.max())) + self.colorbar.rounding = (z.max() - z.min()) * 1e-2 + self._colorScatterPoints(self.colorbar) + + self.colorbar.sigLevelsChanged.connect(self._colorScatterPoints) + + def _colorScatterPoints(self, cbar: pg.ColorBarItem): + if self.scatter is not None and self.scatterZVals is not None: + z_norm = self._normalizeColors(self.scatterZVals, cbar.levels()) + colors = self.colorbar.cmap.mapToQColor(z_norm) + self.scatter.setBrush(colors) + + def _normalizeColors(self, z: np.ndarray, levels: Tuple[float, float]): + scale = levels[1] - levels[0] + if scale > 0: + return (z - levels[0]) / scale + else: + return np.ones(z.size()) * 0.5 diff --git a/test/gui/pyqtgraph_figuremaker.py b/test/gui/pyqtgraph_figuremaker.py index dc619a0f..a175e26a 100644 --- a/test/gui/pyqtgraph_figuremaker.py +++ b/test/gui/pyqtgraph_figuremaker.py @@ -4,45 +4,109 @@ from plottr import QtWidgets from plottr.gui.tools import widgetDialog +from plottr.plot.base import PlotDataType, ComplexRepresentation from plottr.plot.pyqtgraph.autoplot import FigureMaker -def test_single_line_plot(): - +def test_basic_line_plot(): x = np.linspace(0, 10, 51) y = np.cos(x) + with FigureMaker() as fm: + line_1 = fm.addData(x, y, labels=['x', 'cos(x)'], + plotDataType=PlotDataType.line1d) + _ = fm.addData(x, y**2, labels=['x', 'cos^2(x)'], + join=line_1, + plotDataType=PlotDataType.scatter1d) + line_2 = fm.addData(x, np.abs(y), labels=['x', '|cos(x)|'], + plotDataType=PlotDataType.line1d) + return fm.widget + + +def test_images(): + x = np.linspace(0, 10, 51) + y = np.linspace(-4, 2, 51) + xx, yy = np.meshgrid(x, y, indexing='ij') + zz = np.cos(xx) * np.exp(-yy**2) + with FigureMaker() as fm: + img1 = fm.addData(xx, yy, zz, labels=['x', 'y', 'fake data'], + plotDataType=PlotDataType.grid2d) + img2 = fm.addData(xx, yy, zz[:, ::-1], labels=['x', 'y', 'fake data (mirror)'], + plotDataType=PlotDataType.grid2d) + return fm.widget + +def test_scatter2d(): + x = np.linspace(0, 10, 21) + y = np.linspace(-4, 2, 21) + xx, yy = np.meshgrid(x, y, indexing='ij') + zz = np.cos(xx) * np.exp(-yy**2) with FigureMaker() as fm: - line_1 = fm.addData(x, y) - _ = fm.addData(x, y**2, join=line_1) + s = fm.addData(xx.flatten(), yy.flatten(), zz.flatten(), labels=['x', 'y', 'fake data'], + plotDataType=PlotDataType.scatter2d) + return fm.widget + + +def test_complex_line_plots(single_panel: bool = False, + mag_and_phase_format: bool = False): + + setpts = np.linspace(0, 10, 101) + data_1 = np.exp(-1j * setpts) + data_2 = np.conjugate(data_1) + + with FigureMaker() as fm: + if mag_and_phase_format: + fm.complexRepresentation = ComplexRepresentation.magAndPhase + + line_1 = fm.addData(setpts, data_1, labels=['x', r'exp(-ix)']) + _ = fm.addData(setpts, data_2, labels=['x', r'exp(ix)'], + join=line_1 if single_panel else None) + + return fm.widget + + +def test_complex_images(mag_and_phase_format: bool = False): + x = np.linspace(0, 10, 51) + y = np.linspace(-4, 2, 51) + xx, yy = np.meshgrid(x, y, indexing='ij') + zz = np.exp(-1j*xx) * np.exp(-yy**2) + with FigureMaker() as fm: + if mag_and_phase_format: + fm.complexRepresentation = ComplexRepresentation.magAndPhase + + img1 = fm.addData(xx, yy, zz, labels=['x', 'y', 'fake data'], + plotDataType=PlotDataType.grid2d) + img2 = fm.addData(xx, yy, np.conjugate(zz), labels=['x', 'y', 'fake data (conjugate)'], + plotDataType=PlotDataType.grid2d) + return fm.widget - return fm.layoutWidget def main(): app = QtWidgets.QApplication([]) - widgets = [] widgets.append( - test_single_line_plot() - ) - - # wins.append( - # test_multiple_line_plots()) - # wins.append( - # test_multiple_line_plots(single_panel=True)) - # wins.append( - # test_complex_line_plots()) - # wins.append( - # test_complex_line_plots(single_panel=True)) - # wins.append( - # test_complex_line_plots(mag_and_phase_format=True)) - # wins.append( - # test_complex_line_plots(single_panel=True, mag_and_phase_format=True)) + test_basic_line_plot()) + widgets.append( + test_images()) + widgets.append( + test_scatter2d()) + widgets.append( + test_complex_line_plots()) + widgets.append( + test_complex_line_plots(single_panel=True)) + widgets.append( + test_complex_line_plots(mag_and_phase_format=True)) + widgets.append( + test_complex_line_plots(single_panel=True, mag_and_phase_format=True)) + widgets.append( + test_complex_images()) + widgets.append( + test_complex_images(mag_and_phase_format=True)) + dgs = [] for w in widgets: - dg = widgetDialog(w) - dg.show() + dgs.append(widgetDialog(w)) + dgs[-1].show() return app.exec_() diff --git a/test/gui/pyqtgraph_testing.py b/test/gui/pyqtgraph_testing.py index 5d9695e2..2c7ccc0d 100644 --- a/test/gui/pyqtgraph_testing.py +++ b/test/gui/pyqtgraph_testing.py @@ -43,6 +43,9 @@ def image_test(): xplot = pgWidget2.addPlot(row=1, col=0) yplot = pgWidget2.addPlot(row=0, col=0) + xplot.addLegend() + yplot.addLegend() + # add crosshair to main plot vline = pg.InfiniteLine(angle=90, movable=False, pen='r') hline = pg.InfiniteLine(angle=0, movable=False, pen='b') @@ -58,11 +61,11 @@ def crossMoved(event): vidx = np.argmin(np.abs(origin.x()-x)) hidx = np.argmin(np.abs(origin.y()-y)) yplot.clear() - yplot.plot(zz[vidx, :], y, + yplot.plot(zz[vidx, :], y, name='vertical cut', pen=pg.mkPen('r', width=2), symbol='o', symbolBrush='r', symbolPen=None) xplot.clear() - xplot.plot(x, zz[:, hidx], + xplot.plot(x, zz[:, hidx], name='horizontal cut', pen=pg.mkPen('b', width=2), symbol='o', symbolBrush='b', symbolPen=None) From c1d5e85e95de20ff6b373777e93217a6dd66d0b1 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 16 Jul 2021 23:25:25 -0500 Subject: [PATCH 36/94] added tool for easier retrieval of config entries. --- plottr/__init__.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/plottr/__init__.py b/plottr/__init__.py index 8a6b6b9b..0374b7ea 100644 --- a/plottr/__init__.py +++ b/plottr/__init__.py @@ -109,5 +109,37 @@ def config(names: Optional[List[str]] = None) -> \ return config +def config_entry(*path: str, default: Optional[Any] = None, + names: Optional[List[str]] = None) -> Any: + """Get a specific config value. + + ..Example: If the config is:: python + + config = { + 'foo' : { + 'bar' : 'spam', + }, + } + + .. then we can get an entry like this:: python + + >>> config_entry('foo', 'bar', default=None) + 'spam' + >>> config_entry('foo', 'bacon') + None + >>> config_entry('foo', 'bar', 'bacon') + None + + :param path: strings denoting the nested keys to the desired value + :param names: see :func:`.config`. + :param default: what to return when key isn't found in the config. + :returns: desired value + """ - + cfg = config(names) + for k in path: + if isinstance(cfg, dict) and k in cfg: + cfg = cfg.get(k) + else: + return default + return cfg From 6539d1d42566b34cc7377712439059ec1b36b817 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 16 Jul 2021 23:26:19 -0500 Subject: [PATCH 37/94] added additional var to track all plot ids, incl joined ones. --- plottr/plot/base.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/plottr/plot/base.py b/plottr/plot/base.py index d3537ed7..bdc8dc4f 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -375,6 +375,9 @@ def __init__(self) -> None: #: ids of all main plot items (does not contain derived/secondary plot items) self.plotIds: List = [] + #: ids of all plot items, incl those who are 'joined' with 'main' plot items. + self.allPlotIds: List = [] + #: how to represent complex data. #: must be set before adding data to the plot to have an effect. self.complexRepresentation = ComplexRepresentation.realAndImag @@ -542,7 +545,7 @@ def addData(self, *data: Union[np.ndarray, np.ma.MaskedArray], :param data: data arrays describing the plot (one or more independents, one dependent) :param join: ID of a plot item the new item should be shown together with in the same subplot :param labels: list of labels for the data arrays - :param plotDataType: what kind of plot data the supplied data contains (not needed, typically) + :param plotDataType: what kind of plot data the supplied data contains. :param plotOptions: options (as kwargs) to be passed to the actual plot functions (depends on the backend) :return: ID of the new plot item. """ @@ -562,13 +565,15 @@ def addData(self, *data: Union[np.ndarray, np.ma.MaskedArray], if labels is None: labels = [''] * len(data) + elif len(labels) < len(data): + labels += [''] * (len(data)-len(labels)) plotItem = PlotItem(list(data), id, subPlotId, plotDataType, labels, plotOptions) for p in self._splitComplexData(plotItem): self.plotItems[p.id] = p - + self.allPlotIds.append(p.id) self.plotIds.append(id) return id @@ -576,11 +581,47 @@ def previousPlotId(self) -> Optional[int]: """Get the ID of the most recently added plot item. :return: the ID. """ + if not len(self.plotIds) > 0: + return None + if len(self.plotIds) > 0: return self.plotIds[-1] else: return None + def findPlotIndexInSubPlot(self, plotId: int) -> int: + """find the index of a plot in its subplot + + :param plotId: plot ID to check + :return: index at which the plot is located in its subplot. + """ + if plotId not in self.allPlotIds: + raise ValueError("Plot ID not found.") + + subPlotId = self.plotItems[plotId].subPlot + itemsInSubPlot = [i for i in self.allPlotIds if self.plotItems[i].subPlot == subPlotId] + return itemsInSubPlot.index(plotId) + + def plotIdsInSubPlot(self, subPlotId: int) -> List[int]: + """return all plot IDs in a given subplot + + :param subPlotId: ID of the subplot + :return: list of plot IDs + """ + itemsInSubPlot = [i for i in self.allPlotIds if self.plotItems[i].subPlot == subPlotId] + return itemsInSubPlot + + def dataDimensionsInSubPlot(self, subPlotId: int) -> Dict[int, int]: + """Determine what the data dimensions are in a subplot. + + :param subPlotId: ID of the subplot + :return: dictionary with plot id as key, data dimension (i.e., number of independents) as value. + """ + ret: Dict[int, int] = {} + for plotId in self.plotIdsInSubPlot(subPlotId): + ret[plotId] = len(self.plotItems[plotId].data) - 1 + return ret + # Methods to be implemented by inheriting classes def makeSubPlots(self, nSubPlots: int) -> Optional[List[Any]]: """Generate the subplots. Called after all data has been added. From f8c95ce425731b803eb049171889b5d145db3b1c Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 16 Jul 2021 23:26:43 -0500 Subject: [PATCH 38/94] tighter layout. --- plottr/gui/tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plottr/gui/tools.py b/plottr/gui/tools.py index 60c590c0..63555dc6 100644 --- a/plottr/gui/tools.py +++ b/plottr/gui/tools.py @@ -24,6 +24,8 @@ def widgetDialog(widget: QtWidgets.QWidget, title: str = '', win = QtWidgets.QDialog() win.setWindowTitle('plottr ' + title) layout = QtWidgets.QVBoxLayout() + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) layout.addWidget(widget) win.setLayout(layout) if show: From f57e92502682de363948a1c156980c39c75b90c1 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 5 Aug 2021 21:58:16 +0200 Subject: [PATCH 39/94] clearer type annotation. --- plottr/plot/mpl/widgets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plottr/plot/mpl/widgets.py b/plottr/plot/mpl/widgets.py index 58629468..027329d8 100644 --- a/plottr/plot/mpl/widgets.py +++ b/plottr/plot/mpl/widgets.py @@ -17,7 +17,7 @@ from plottr import QtWidgets, QtGui, QtCore, config as plottrconfig from plottr.data.datadict import DataDictBase from plottr.gui.tools import widgetDialog, dpiScalingFactor -from ..base import PlotWidget +from ..base import PlotWidget, PlotWidgetContainer class MPLPlot(FCanvas): @@ -146,7 +146,7 @@ class MPLPlotWidget(PlotWidget): Per default, add a canvas and the matplotlib NavBar. """ - def __init__(self, parent: Optional[QtWidgets.QWidget] = None): + def __init__(self, parent: Optional[PlotWidgetContainer] = None): super().__init__(parent=parent) #: the plot widget From 365a4886c0952f12f745e82d36b7f1b32382d82b Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 5 Aug 2021 21:58:57 +0200 Subject: [PATCH 40/94] come clarifications. --- plottr/gui/widgets.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/plottr/gui/widgets.py b/plottr/gui/widgets.py index 0477ea0f..ef6cf20c 100644 --- a/plottr/gui/widgets.py +++ b/plottr/gui/widgets.py @@ -4,7 +4,7 @@ Common GUI widgets that are re-used across plottr. """ from numpy import rint -from typing import Union, List, Tuple, Optional, Type, Sequence, Dict, Any +from typing import Union, List, Tuple, Optional, Type, Sequence, Dict, Any, Type from .tools import dictToTreeWidgetItems, dpiScalingFactor from plottr import QtGui, QtCore, Flowchart, QtWidgets, Signal, Slot @@ -66,21 +66,33 @@ def spinValueChanged(self, val: float) -> None: self.intervalChanged.emit(val) + class PlotWindow(QtWidgets.QMainWindow): """ - Simple MainWindow class for embedding flowcharts and plots. - - All keyword arguments supplied will be propagated to - :meth:`addNodeWidgetFromFlowchart`. + Simple MainWindow class for embedding flowcharts and plots, based on + ``QtWidgets.QMainWindow``. """ + # FIXME: defaulting to MPL should probably be on a higher level. + #: Signal() -- emitted when the window is closed windowClosed = Signal() def __init__(self, parent: Optional[QtWidgets.QMainWindow] = None, fc: Optional[Flowchart] = None, - plotWidgetClass: Optional[Any] = None, + plotWidgetClass: Optional[Type[PlotWidget]] = None, **kw: Any): + """ + Constructor for :class:`.PlotWindow`. + + :param parent: parent widget + :param fc: flowchart with nodes. if given, we will generate node widgets + in this window. + :param plotWidgetClass: class of the plot widget to use. + defaults to :class:`plottr.plot.mpl.AutoPlot`. + :param kw: any keywords will be propagated to + :meth:`addNodeWidgetFromFlowchart`. + """ super().__init__(parent) if plotWidgetClass is None: From 563674dae94fdd5dab86594821aa573956be92da Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 5 Aug 2021 21:59:57 +0200 Subject: [PATCH 41/94] added option for minimum plot size for pyqtgraph plots. --- plottr/config/plottrcfg_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plottr/config/plottrcfg_main.py b/plottr/config/plottrcfg_main.py index c9735569..112b86ba 100644 --- a/plottr/config/plottrcfg_main.py +++ b/plottr/config/plottrcfg_main.py @@ -33,5 +33,6 @@ '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf'], 'line_symbols': ['o', ], 'line_symbol_size': 7, + 'minimum_plot_size': (400, 400), } } From faff82330b8d64eccfe3794ffcc938cd70c47123 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 5 Aug 2021 22:00:35 +0200 Subject: [PATCH 42/94] added backend-independent method for updating the plot. --- plottr/plot/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plottr/plot/base.py b/plottr/plot/base.py index bdc8dc4f..e901bbc3 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -125,7 +125,7 @@ class PlotWidget(QtWidgets.QWidget): Implement a child class for actual plotting. """ - def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: + def __init__(self, parent: Optional[PlotWidgetContainer] = None) -> None: super().__init__(parent=parent) self.data: Optional[DataDictBase] = None @@ -140,6 +140,9 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: 'dataLimitsChanged': False, } + def update(self) -> None: + return None + def setData(self, data: Optional[DataDictBase]) -> None: """Set data. Use this to trigger plotting. From 58abc5db8e501c6bb22c3783735f966aebe815ca Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 5 Aug 2021 22:01:01 +0200 Subject: [PATCH 43/94] added option for testing with pyqtgraph backend. --- test/apps/autoplot_app.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/apps/autoplot_app.py b/test/apps/autoplot_app.py index bd1f1b42..84e69ec2 100644 --- a/test/apps/autoplot_app.py +++ b/test/apps/autoplot_app.py @@ -21,6 +21,8 @@ from plottr.apps.autoplot import autoplot from plottr.data.datadict import DataDictBase, DataDict from plottr.utils import testdata +from plottr.plot.mpl.autoplot import AutoPlot as MPLAutoPlot +from plottr.plot.pyqtgraph.autoplot import AutoPlot as PGAutoPlot plottrlog.enableStreamHandler(True) logger = plottrlog.getLogger('plottr.test.autoplot_app') @@ -136,7 +138,7 @@ def main(dataSrc): plottrlog.LEVEL = logging.INFO app = QtWidgets.QApplication([]) - fc, win = autoplot() + fc, win = autoplot(plotWidgetClass=plotWidgetClass) dataThread = QtCore.QThread() dataSrc.moveToThread(dataThread) @@ -151,10 +153,14 @@ def main(dataSrc): QtWidgets.QApplication.instance().exec_() +# plotWidgetClass = MPLAutoPlot +plotWidgetClass = PGAutoPlot + + if __name__ == '__main__': - # src = LineDataMovie(10, 3, 101) + src = LineDataMovie(100, 3, 1001) # src = ImageDataMovie(50, 2, 501) - src = ImageDataLiveAcquisition(101, 101, 67) + # src = ImageDataLiveAcquisition(101, 101, 67) # src = ComplexImage(21, 21) - src.delay = 0.1 + src.delay = 0.5 main(src) From 0611cf72b5fdbbb24c30b4d4b964b7cc5811b7b5 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 5 Aug 2021 22:03:39 +0200 Subject: [PATCH 44/94] first reasonably working (with live plot) prototype of pyqtgraph autoplot. --- plottr/plot/pyqtgraph/autoplot.py | 175 +++++++++++++++++++++--------- 1 file changed, 121 insertions(+), 54 deletions(-) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index d8798325..01ea63ea 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -9,78 +9,110 @@ from pyqtgraph import GraphicsLayoutWidget, mkPen, mkBrush, HistogramLUTItem, ImageItem from plottr import QtWidgets, QtCore, QtGui, config_entry as getcfg +from plottr.data.datadict import DataDictBase from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ PlotItem, ComplexRepresentation, determinePlotDataType, \ - PlotWidgetContainer -from .plots import Plot, PlotWithColorbar + PlotWidgetContainer, PlotWidget +from .plots import Plot, PlotWithColorbar, PlotBase logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +class _FigureMakerWidget(QtWidgets.QWidget): + + def __init__(self, parent: Optional[QtWidgets.QWidget] = None): + super().__init__(parent=parent) + + self.subPlots: List[PlotBase] = [] + + self.layout = QtWidgets.QVBoxLayout() + self.layout.setContentsMargins(0, 0, 0, 0) + self.layout.setSpacing(0) + self.setLayout(self.layout) + self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', + default=(400, 400))) + + def addPlot(self, plot: PlotBase): + self.layout.addWidget(plot) + self.subPlots.append(plot) + + def clearAllPlots(self): + for p in self.subPlots: + p.clearPlot() + + def deleteAllPlots(self): + for p in self.subPlots: + p.deleteLater() + self.subPlots = [] + + class FigureMaker(BaseFM): """pyqtgraph implementation for :class:`.AutoFigureMaker`. """ - # TODO: need to figure out how to reuse widgets when we just update data # TODO: make scrollable when many figures (set min size)? - # TODO: check for valid plot data? + # TODO: check for valid plot data - def __init__(self, parentWidget: Optional[QtWidgets.QWidget] = None): + def __init__(self, widget: Optional[_FigureMakerWidget] = None, + clearWidget: bool = True, + parentWidget: Optional[QtWidgets.QWidget] = None): super().__init__() - self.widget = QtWidgets.QWidget(parentWidget) - - self.layout = QtWidgets.QVBoxLayout() - self.layout.setContentsMargins(0, 0, 0, 0) - self.layout.setSpacing(0) - self.widget.setLayout(self.layout) - self.widget.setMinimumSize(400, 400) + self.clearWidget: bool = clearWidget + if widget is None: + self.widget = _FigureMakerWidget(parent=parentWidget) + else: + self.widget = widget # re-implementing to get correct type annotation. def __enter__(self) -> "FigureMaker": return self - def makeSubPlots(self, nSubPlots: int) -> List[Any]: + def subPlotFromId(self, subPlotId): + subPlots = self.subPlots[subPlotId].axes + assert isinstance(subPlots, list) and len(subPlots) > 0 and \ + isinstance(subPlots[0], PlotBase) + return subPlots[0] - plotWidgets = [] # FIXME: needs correct type - for i in range(nSubPlots): - if max(self.dataDimensionsInSubPlot(i).values()) == 1: - plot = Plot(self.widget) - self.layout.addWidget(plot) - plotWidgets.append([plot]) + def makeSubPlots(self, nSubPlots: int) -> List[List[PlotBase]]: + if self.clearWidget: + self.widget.deleteAllPlots() - elif max(self.dataDimensionsInSubPlot(i).values()) == 2: - plot = PlotWithColorbar(self.widget) - self.layout.addWidget(plot) - plotWidgets.append([plot]) + for i in range(nSubPlots): + if max(self.dataDimensionsInSubPlot(i).values()) == 1: + plot = Plot(self.widget) + self.widget.addPlot(plot) + elif max(self.dataDimensionsInSubPlot(i).values()) == 2: + plot = PlotWithColorbar(self.widget) + self.widget.addPlot(plot) - return plotWidgets + else: + self.widget.clearAllPlots() + + return self.widget.subPlots def formatSubPlot(self, subPlotId: int) -> Any: if len(self.plotIdsInSubPlot(subPlotId)) == 0: return labels = self.subPlotLabels(subPlotId) - plotwidgets = self.subPlots[subPlotId].axes - - assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 - pw = plotwidgets[0] + subPlot = self.subPlotFromId(subPlotId) # label the x axis if there's only one x label - if isinstance(pw, Plot): + if isinstance(subPlot, Plot): if len(set(labels[0])) == 1: - pw.plot.setLabel("bottom", labels[0][0]) + subPlot.plot.setLabel("bottom", labels[0][0]) - if isinstance(pw, PlotWithColorbar): + if isinstance(subPlot, PlotWithColorbar): if len(set(labels[0])) == 1: - pw.plot.setLabel("bottom", labels[0][0]) + subPlot.plot.setLabel("bottom", labels[0][0]) if len(set(labels[1])) == 1: - pw.plot.setLabel('left', labels[1][0]) + subPlot.plot.setLabel('left', labels[1][0]) if len(set(labels[2])) == 1: - pw.colorbar.setLabel('left', labels[2][0]) + subPlot.colorbar.setLabel('left', labels[2][0]) def plot(self, plotItem: PlotItem) -> None: if plotItem.plotDataType is PlotDataType.unknown: @@ -103,9 +135,7 @@ def _1dPlot(self, plotItem): symbols = getcfg('main', 'pyqtgraph', 'line_symbols', default=['o']) symbolSize = getcfg('main', 'pyqtgraph', 'line_symbol_size', default=5) - plotwidgets = self.subPlots[plotItem.subPlot].axes - assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 - pw = plotwidgets[0] + subPlot = self.subPlotFromId(plotItem.subPlot) assert len(plotItem.data) == 2 x, y = plotItem.data @@ -114,27 +144,64 @@ def _1dPlot(self, plotItem): symbol = symbols[self.findPlotIndexInSubPlot(plotItem.id) % len(symbols)] if plotItem.plotDataType == PlotDataType.line1d: - return pw.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], - pen=mkPen(color, width=2), - symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) + return subPlot.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], + pen=mkPen(color, width=2), + symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) else: - return pw.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], - pen=None, - symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) + return subPlot.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], + pen=None, + symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) def _colorPlot(self, plotItem): - plotwidgets = self.subPlots[plotItem.subPlot].axes - assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 - pw = plotwidgets[0] - - assert isinstance(pw, PlotWithColorbar) and len(plotItem.data) == 3 - pw.setImage(*plotItem.data) + subPlot = self.subPlotFromId(plotItem.subPlot) + assert isinstance(subPlot, PlotWithColorbar) and len(plotItem.data) == 3 + subPlot.setImage(*plotItem.data) def _scatterPlot2d(self, plotItem): - plotwidgets = self.subPlots[plotItem.subPlot].axes - assert isinstance(plotwidgets, list) and len(plotwidgets) > 0 - pw = plotwidgets[0] + subPlot = self.subPlotFromId(plotItem.subPlot) + assert isinstance(subPlot, PlotWithColorbar) and len(plotItem.data) == 3 + subPlot.setScatter2d(*plotItem.data) - assert isinstance(pw, PlotWithColorbar) and len(plotItem.data) == 3 - pw.setScatter2d(*plotItem.data) +class AutoPlot(PlotWidget): + """Widget for automatic plotting with pyqtgraph.""" + + def __init__(self, parent: Optional[PlotWidgetContainer]): + """Constructor for the pyqtgraph auto plot widget. + + :param parent: + """ + super().__init__(parent=parent) + + self.fmWidget: Optional[PlotWidget] = None + self.layout = QtWidgets.QVBoxLayout() + self.layout.setContentsMargins(0, 0, 0, 0) + self.layout.setSpacing(0) + + self.setLayout(self.layout) + self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', default=(400, 400))) + + def setData(self, data: Optional[DataDictBase]) -> None: + super().setData(data) + if self.data is None: + return + + fmKwargs = {'widget': self.fmWidget} + dc = self.dataChanges + if not dc['dataTypeChanged'] and not dc['dataStructureChanged'] \ + and not dc['dataShapesChanged']: + fmKwargs['clearWidget'] = False + else: + fmKwargs['clearWidget'] = True + + with FigureMaker(parentWidget=self, **fmKwargs) as fm: + inds = self.data.axes() + for dep in self.data.dependents(): + dvals = self.data.data_vals(dep) + plotId = fm.addData( + *[np.asanyarray(self.data.data_vals(n)) for n in inds] + [dvals] + ) + + if self.fmWidget is None: + self.fmWidget = fm.widget + self.layout.addWidget(self.fmWidget) From c42971d8846bb1412839827eb70f4f997fc01751 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 5 Aug 2021 22:04:58 +0200 Subject: [PATCH 45/94] - clarified type annotations - make autoplot frontend backend independent. --- plottr/apps/autoplot.py | 16 ++++++++++------ plottr/plot/mpl/autoplot.py | 11 +++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/plottr/apps/autoplot.py b/plottr/apps/autoplot.py index 25870c51..6b04f8b0 100644 --- a/plottr/apps/autoplot.py +++ b/plottr/apps/autoplot.py @@ -6,7 +6,7 @@ import os import time import argparse -from typing import Union, Tuple, Optional, Type, List, Any +from typing import Union, Tuple, Optional, Type, List, Any, Type from packaging import version from .. import QtCore, Flowchart, Signal, Slot, QtWidgets, QtGui @@ -23,7 +23,7 @@ from ..node.grid import DataGridder, GridOption from ..node.tools import linearFlowchart from ..node.node import Node -from ..plot import PlotNode, makeFlowchartWithPlot +from ..plot import PlotNode, makeFlowchartWithPlot, PlotWidget from ..utils.misc import unwrap_optional __author__ = 'Wolfgang Pfaff' @@ -39,7 +39,8 @@ def logger() -> logging.Logger: return logger -def autoplot(inputData: Union[None, DataDictBase] = None) \ +def autoplot(inputData: Union[None, DataDictBase] = None, + plotWidgetClass: Optional[Type[PlotWidget]] = None) \ -> Tuple[Flowchart, 'AutoPlotMainWindow']: """ Sets up a simple flowchart consisting of a data selector, gridder, @@ -63,7 +64,8 @@ def autoplot(inputData: Union[None, DataDictBase] = None) \ } fc = makeFlowchartWithPlot(nodes) - win = AutoPlotMainWindow(fc, widgetOptions=widgetOptions) + win = AutoPlotMainWindow(fc, widgetOptions=widgetOptions, + plotWidgetClass=plotWidgetClass) win.show() if inputData is not None: @@ -130,9 +132,11 @@ def __init__(self, fc: Flowchart, monitor: bool = False, monitorInterval: Union[float, None] = None, loaderName: Optional[str] = None, + plotWidgetClass: Optional[Type[PlotWidget]] = None, **kwargs: Any): - super().__init__(parent, fc=fc, **kwargs) + super().__init__(parent, fc=fc, plotWidgetClass=plotWidgetClass, + **kwargs) self.fc = fc if loaderName is not None: @@ -236,7 +240,7 @@ def setDefaults(self, data: DataDictBase) -> None: self.fc.nodes()['Data selection'].selectedData = selected self.fc.nodes()['Grid'].grid = GridOption.guessShape, {} self.fc.nodes()['Dimension assignment'].dimensionRoles = drs - unwrap_optional(self.plotWidget).plot.draw() + unwrap_optional(self.plotWidget).update() class QCAutoPlotMainWindow(AutoPlotMainWindow): diff --git a/plottr/plot/mpl/autoplot.py b/plottr/plot/mpl/autoplot.py index 3f1ff2c7..0624c32b 100644 --- a/plottr/plot/mpl/autoplot.py +++ b/plottr/plot/mpl/autoplot.py @@ -21,7 +21,7 @@ from .plotting import PlotType, colorplot2d from .widgets import MPLPlotWidget from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ - PlotItem, ComplexRepresentation, determinePlotDataType + PlotItem, ComplexRepresentation, determinePlotDataType, PlotWidgetContainer logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -348,7 +348,7 @@ class AutoPlot(MPLPlotWidget): presented through a toolbar. """ - def __init__(self, parent: Optional[QtWidgets.QWidget] = None): + def __init__(self, parent: Optional[PlotWidgetContainer] = None): super().__init__(parent=parent) self.plotDataType = PlotDataType.unknown @@ -373,6 +373,10 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): self.plotOptionsToolBar.setIconSize(QtCore.QSize(iconSize, iconSize)) self.setMinimumSize(int(640*scaling), int(480*scaling)) + def update(self): + self.plot.draw() + QtCore.QCoreApplication.processEvents() + def setData(self, data: Optional[DataDictBase]) -> None: """Analyses data and determines whether/what to plot. @@ -462,5 +466,4 @@ def _plotData(self) -> None: **kw) self.setMeta(self.data) - self.plot.draw() - QtCore.QCoreApplication.processEvents() + self.update() From 7a1555a4ff9d6ef6d82621278c81b2b26a62f799 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 6 Aug 2021 15:34:04 +0200 Subject: [PATCH 46/94] added missing cosmetic features (labels, etc). added docstrings and type hints. --- plottr/plot/pyqtgraph/autoplot.py | 93 +++++++++++++++++++++++++------ plottr/plot/pyqtgraph/plots.py | 82 ++++++++++++++++++++++----- test/apps/autoplot_app.py | 8 +-- test/gui/pyqtgraph_figuremaker.py | 32 +++++------ 4 files changed, 165 insertions(+), 50 deletions(-) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 01ea63ea..47c2222c 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -1,4 +1,12 @@ -"""``plottr.plot.pyqtgraph.autoplot`` -- tools for automatic plotting with pyqtgraph. +"""Tools for automatic plotting with ``pyqtgraph``. + +Contains a ``pyqtgraph``-specific implementation of :class:`.plot.base.FigureMaker` +(:class:`.FigureMaker`) and :class:`.plot.base.PlotWidget` +(:class:`.AutoPlot`). +``FigureMaker`` can be used to quickly create a figure in a mostly automatic fashion +from known data. +For embedding in GUIs (such as plottr applications), ``AutoPlot`` is the main +object for plotting data automatically using ``pyqtgraph``. """ import logging @@ -6,13 +14,13 @@ import numpy as np -from pyqtgraph import GraphicsLayoutWidget, mkPen, mkBrush, HistogramLUTItem, ImageItem +from pyqtgraph import mkPen from plottr import QtWidgets, QtCore, QtGui, config_entry as getcfg from plottr.data.datadict import DataDictBase from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ PlotItem, ComplexRepresentation, determinePlotDataType, \ - PlotWidgetContainer, PlotWidget + PlotWidgetContainer, PlotWidget, SubPlot from .plots import Plot, PlotWithColorbar, PlotBase logger = logging.getLogger(__name__) @@ -20,8 +28,16 @@ class _FigureMakerWidget(QtWidgets.QWidget): + """Widget that contains all plots generated by :class:`.FigureMaker`. + + Widget has a vertical layout, and plots can be added in a single column. + """ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): + """Constructor for :class:`._FigureMakerWidget`. + + :param parent: parent widget. + """ super().__init__(parent=parent) self.subPlots: List[PlotBase] = [] @@ -33,33 +49,55 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', default=(400, 400))) - def addPlot(self, plot: PlotBase): + def addPlot(self, plot: PlotBase) -> None: + """Add a :class:`.PlotBase` widget. + + :param plot: plot widget + """ self.layout.addWidget(plot) self.subPlots.append(plot) - def clearAllPlots(self): + def clearAllPlots(self) -> None: + """Clear all plot contents.""" for p in self.subPlots: p.clearPlot() def deleteAllPlots(self): + """Delete all subplot widgets.""" for p in self.subPlots: p.deleteLater() self.subPlots = [] class FigureMaker(BaseFM): - """pyqtgraph implementation for :class:`.AutoFigureMaker`. + """``pyqtgraph`` implementation for :class:`.AutoFigureMaker`. + + Please see the base class for more general information. """ # TODO: make scrollable when many figures (set min size)? # TODO: check for valid plot data + widget: _FigureMakerWidget + def __init__(self, widget: Optional[_FigureMakerWidget] = None, clearWidget: bool = True, parentWidget: Optional[QtWidgets.QWidget] = None): + """Constructor for :class:`.FigureMaker`. + + :param widget: a widget produced by an earlier use of this class + may be supplied. This widget will then be used for generating the + plot and avoids deleting and re-creating the widget if no structural + changes to the plots is required (important case is live-plotting of + data that keeps updating). + :param clearWidget: + if ``True`` force re-creation of all plot elements. + :param parentWidget: + parent of the main widget of FigureMaker. + """ super().__init__() - self.clearWidget: bool = clearWidget + self.clearWidget = clearWidget if widget is None: self.widget = _FigureMakerWidget(parent=parentWidget) else: @@ -69,13 +107,19 @@ def __init__(self, widget: Optional[_FigureMakerWidget] = None, def __enter__(self) -> "FigureMaker": return self - def subPlotFromId(self, subPlotId): + def subPlotFromId(self, subPlotId: int) -> SubPlot: + """Get SubPlot from ID.""" subPlots = self.subPlots[subPlotId].axes assert isinstance(subPlots, list) and len(subPlots) > 0 and \ isinstance(subPlots[0], PlotBase) return subPlots[0] def makeSubPlots(self, nSubPlots: int) -> List[List[PlotBase]]: + """Create empty subplots in the widgets. + + If ``clearWidget`` was not set to ``True`` in the constructor, + existing sub plot widgets are cleared, but not deleted and re-created. + """ if self.clearWidget: self.widget.deleteAllPlots() @@ -92,7 +136,8 @@ def makeSubPlots(self, nSubPlots: int) -> List[List[PlotBase]]: return self.widget.subPlots - def formatSubPlot(self, subPlotId: int) -> Any: + def formatSubPlot(self, subPlotId: int) -> None: + """Set labels/legends for the given subplot.""" if len(self.plotIdsInSubPlot(subPlotId)) == 0: return @@ -115,6 +160,7 @@ def formatSubPlot(self, subPlotId: int) -> Any: subPlot.colorbar.setLabel('left', labels[2][0]) def plot(self, plotItem: PlotItem) -> None: + """Plot the given item.""" if plotItem.plotDataType is PlotDataType.unknown: if len(plotItem.data) == 2: plotItem.plotDataType = PlotDataType.scatter1d @@ -164,12 +210,15 @@ def _scatterPlot2d(self, plotItem): class AutoPlot(PlotWidget): - """Widget for automatic plotting with pyqtgraph.""" + """Widget for automatic plotting with pyqtgraph. + + Uses :class:`.FigureMaker` to produce subplots. + """ - def __init__(self, parent: Optional[PlotWidgetContainer]): + def __init__(self, parent: Optional[PlotWidgetContainer]) -> None: """Constructor for the pyqtgraph auto plot widget. - :param parent: + :param parent: plot widget container """ super().__init__(parent=parent) @@ -182,24 +231,36 @@ def __init__(self, parent: Optional[PlotWidgetContainer]): self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', default=(400, 400))) def setData(self, data: Optional[DataDictBase]) -> None: + """Uses :class:`.FigureMaker` to populate the plot(s). + + This method aims to re-use existing plotwidgets if possible -- i.e., + when the type of data (generally speaking) has not changed. + If changes to the data structure are detected, then all subplot + widgets are re-created from scratch, however. + + :param data: input data + :return: ``None`` + """ super().setData(data) if self.data is None: return fmKwargs = {'widget': self.fmWidget} dc = self.dataChanges - if not dc['dataTypeChanged'] and not dc['dataStructureChanged'] \ - and not dc['dataShapesChanged']: + if not dc['dataTypeChanged'] and not dc['dataStructureChanged']: fmKwargs['clearWidget'] = False else: fmKwargs['clearWidget'] = True with FigureMaker(parentWidget=self, **fmKwargs) as fm: - inds = self.data.axes() for dep in self.data.dependents(): + inds = self.data.axes(dep) dvals = self.data.data_vals(dep) + pdt = determinePlotDataType(self.data.extract([dep])) plotId = fm.addData( - *[np.asanyarray(self.data.data_vals(n)) for n in inds] + [dvals] + *[np.asanyarray(self.data.data_vals(n)) for n in inds] + [dvals], + labels=[str(self.data.label(n)) for n in inds] + [str(self.data.label(dep))], + plotDataType=pdt, ) if self.fmWidget is None: diff --git a/plottr/plot/pyqtgraph/plots.py b/plottr/plot/pyqtgraph/plots.py index 8f934cdc..69c4c5ec 100644 --- a/plottr/plot/pyqtgraph/plots.py +++ b/plottr/plot/pyqtgraph/plots.py @@ -1,4 +1,7 @@ -from typing import Any, Optional, List, Tuple +"""Convenience tools for generating ``pyqtgraph`` plots that can +be used in plottr's automatic plotting framework.""" + +from typing import Optional, Tuple, NoReturn import numpy as np import pyqtgraph as pg @@ -11,40 +14,69 @@ class PlotBase(QtWidgets.QWidget): + """A simple convenience widget class as container for ``pyqtgraph`` plots. + + The widget contains a layout that contains a ``GraphicsLayoutWidget``. + This is handy because a plot may contain multiple elements (like an image + and a colorbar). - def __init__(self, parent=None): + This base class should be inherited to use. + """ + + def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: super().__init__(parent) + + #: central layout of the widget. only contains a graphics layout. self.layout = QtWidgets.QVBoxLayout(self) + #: ``pyqtgraph`` graphics layout + self.graphicsLayout = pg.GraphicsLayoutWidget(self) + self.layout.setContentsMargins(0, 0, 0, 0) self.layout.setSpacing(0) self.setLayout(self.layout) - - self.graphicsLayout = pg.GraphicsLayoutWidget(self) self.layout.addWidget(self.graphicsLayout) - def clearPlot(self): - return + def clearPlot(self) -> NoReturn: + """Clear all plot contents (but do not delete plot elements, like axis + spines, insets, etc). + + To be implemented by inheriting classes.""" + raise NotImplementedError class Plot(PlotBase): + """A simple plot with a single ``PlotItem``.""" + #: ``pyqtgraph`` plot item plot: pg.PlotItem - def __init__(self, parent=None): + def __init__(self, parent: Optional[QtWidgets.QWidget] = None): super().__init__(parent) self.plot: pg.PlotItem = self.graphicsLayout.addPlot() - self.plot.addLegend(offset=(5, 5), pen='#999', brush=(255, 255, 255, 150)) + legend = self.plot.addLegend(offset=(5, 5), pen='#999', + brush=(255, 255, 255, 150)) + legend.layout.setContentsMargins(0, 0, 0, 0) + self.plot.showGrid(True, True) - def clearPlot(self): + def clearPlot(self) -> None: + """Clear the plot item.""" self.plot.clear() class PlotWithColorbar(PlotBase): + """Plot containing a plot item and a colorbar item. + Plot is suited for either an image plot (:meth:`.setImage`) or a color + scatter plot (:meth:`.setScatter2D`). + The color scale is displayed in an interactive colorbar. + """ + + #: main plot item plot: pg.PlotItem + #: colorbar colorbar: pg.ColorBarItem - def __init__(self, parent=None): + def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: super().__init__(parent) self.plot: pg.PlotItem = self.graphicsLayout.addPlot() @@ -58,7 +90,8 @@ def __init__(self, parent=None): self.scatter: Optional[pg.ScatterPlotItem] = None self.scatterZVals: Optional[np.ndarray] = None - def clearPlot(self): + def clearPlot(self) -> None: + """Clear the content of the plot.""" self.img = None self.scatter = None self.scatterZVals = None @@ -68,7 +101,17 @@ def clearPlot(self): except TypeError: pass - def setImage(self, x: np.ndarray, y: np.ndarray, z: np.ndarray): + def setImage(self, x: np.ndarray, y: np.ndarray, z: np.ndarray) -> None: + """Set data to be plotted as image. + + Clears the plot before creating a new image item that gets places in the + plot and linked to the colorscale. + + :param x: x coordinates (as 2D meshgrid) + :param y: y coordinates (as 2D meshgrid) + :param z: data values (as 2D meshgrid) + :return: None + """ self.clearPlot() self.img = pg.ImageItem() @@ -80,13 +123,23 @@ def setImage(self, x: np.ndarray, y: np.ndarray, z: np.ndarray): self.colorbar.rounding = (z.max()-z.min()) * 1e-2 self.colorbar.setLevels((z.min(), z.max())) - def setScatter2d(self, x: np.ndarray, y: np.ndarray, z: np.ndarray): + def setScatter2d(self, x: np.ndarray, y: np.ndarray, z: np.ndarray) -> None: + """Set data to be plotted as image. + + Clears the plot before creating a new scatter item (based on flattened + input data) that gets placed in the plot and linked to the colorscale. + + :param x: x coordinates + :param y: y coordinates + :param z: data values + :return: None + """ self.clearPlot() self.scatter = pg.ScatterPlotItem() self.scatter.setData(x=x.flatten(), y=y.flatten(), symbol='o', size=8) self.plot.addItem(self.scatter) - self.scatterZVals = z + self.scatterZVals = z.flatten() self.colorbar.setLevels((z.min(), z.max())) self.colorbar.rounding = (z.max() - z.min()) * 1e-2 @@ -94,6 +147,7 @@ def setScatter2d(self, x: np.ndarray, y: np.ndarray, z: np.ndarray): self.colorbar.sigLevelsChanged.connect(self._colorScatterPoints) + # TODO: this seems crazy slow. def _colorScatterPoints(self, cbar: pg.ColorBarItem): if self.scatter is not None and self.scatterZVals is not None: z_norm = self._normalizeColors(self.scatterZVals, cbar.levels()) diff --git a/test/apps/autoplot_app.py b/test/apps/autoplot_app.py index 84e69ec2..0e1bc301 100644 --- a/test/apps/autoplot_app.py +++ b/test/apps/autoplot_app.py @@ -158,9 +158,9 @@ def main(dataSrc): if __name__ == '__main__': - src = LineDataMovie(100, 3, 1001) - # src = ImageDataMovie(50, 2, 501) + # src = LineDataMovie(20, 3, 31) + # src = ImageDataMovie(10, 2, 101) # src = ImageDataLiveAcquisition(101, 101, 67) - # src = ComplexImage(21, 21) - src.delay = 0.5 + src = ComplexImage(21, 21) + src.delay = 0.1 main(src) diff --git a/test/gui/pyqtgraph_figuremaker.py b/test/gui/pyqtgraph_figuremaker.py index a175e26a..8024883e 100644 --- a/test/gui/pyqtgraph_figuremaker.py +++ b/test/gui/pyqtgraph_figuremaker.py @@ -86,22 +86,22 @@ def main(): widgets.append( test_basic_line_plot()) - widgets.append( - test_images()) - widgets.append( - test_scatter2d()) - widgets.append( - test_complex_line_plots()) - widgets.append( - test_complex_line_plots(single_panel=True)) - widgets.append( - test_complex_line_plots(mag_and_phase_format=True)) - widgets.append( - test_complex_line_plots(single_panel=True, mag_and_phase_format=True)) - widgets.append( - test_complex_images()) - widgets.append( - test_complex_images(mag_and_phase_format=True)) + # widgets.append( + # test_images()) + # widgets.append( + # test_scatter2d()) + # widgets.append( + # test_complex_line_plots()) + # widgets.append( + # test_complex_line_plots(single_panel=True)) + # widgets.append( + # test_complex_line_plots(mag_and_phase_format=True)) + # widgets.append( + # test_complex_line_plots(single_panel=True, mag_and_phase_format=True)) + # widgets.append( + # test_complex_images()) + # widgets.append( + # test_complex_images(mag_and_phase_format=True)) dgs = [] for w in widgets: From 4bb9f9e9dcafb9b638bccd58475b5dde40324fe7 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 12 Aug 2021 20:46:01 -0500 Subject: [PATCH 47/94] small fixes and cleanup. --- plottr/plot/pyqtgraph/plots.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/plottr/plot/pyqtgraph/plots.py b/plottr/plot/pyqtgraph/plots.py index 69c4c5ec..39d6e0e3 100644 --- a/plottr/plot/pyqtgraph/plots.py +++ b/plottr/plot/pyqtgraph/plots.py @@ -7,8 +7,6 @@ import pyqtgraph as pg from plottr import QtCore, QtWidgets -from . import * - __all__ = ['PlotBase', 'Plot'] @@ -23,11 +21,14 @@ class PlotBase(QtWidgets.QWidget): This base class should be inherited to use. """ + #: ``pyqtgraph`` plot item + plot: pg.PlotItem + def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: super().__init__(parent) #: central layout of the widget. only contains a graphics layout. - self.layout = QtWidgets.QVBoxLayout(self) + self.layout = QtWidgets.QHBoxLayout(self) #: ``pyqtgraph`` graphics layout self.graphicsLayout = pg.GraphicsLayoutWidget(self) @@ -47,9 +48,6 @@ def clearPlot(self) -> NoReturn: class Plot(PlotBase): """A simple plot with a single ``PlotItem``.""" - #: ``pyqtgraph`` plot item - plot: pg.PlotItem - def __init__(self, parent: Optional[QtWidgets.QWidget] = None): super().__init__(parent) self.plot: pg.PlotItem = self.graphicsLayout.addPlot() @@ -70,9 +68,6 @@ class PlotWithColorbar(PlotBase): scatter plot (:meth:`.setScatter2D`). The color scale is displayed in an interactive colorbar. """ - - #: main plot item - plot: pg.PlotItem #: colorbar colorbar: pg.ColorBarItem @@ -117,10 +112,10 @@ def setImage(self, x: np.ndarray, y: np.ndarray, z: np.ndarray) -> None: self.img = pg.ImageItem() self.plot.addItem(self.img) self.img.setImage(z) - self.img.setRect(QtCore.QRectF(x.min(), y.min(), x.max()-x.min(), y.max()-y.min())) + self.img.setRect(QtCore.QRectF(x.min(), y.min(), x.max() - x.min(), y.max() - y.min())) self.colorbar.setImageItem(self.img) - self.colorbar.rounding = (z.max()-z.min()) * 1e-2 + self.colorbar.rounding = (z.max() - z.min()) * 1e-2 self.colorbar.setLevels((z.min(), z.max())) def setScatter2d(self, x: np.ndarray, y: np.ndarray, z: np.ndarray) -> None: From dacce9bec04147b331418e6c9d92bf736b7481c0 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 12 Aug 2021 20:47:41 -0500 Subject: [PATCH 48/94] - added option to combine traces. - use enhanced Enum class for options that show up in GUI. --- plottr/plot/base.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/plottr/plot/base.py b/plottr/plot/base.py index e901bbc3..40ef8505 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -7,15 +7,16 @@ from copy import deepcopy from dataclasses import dataclass from enum import Enum, unique, auto +from types import TracebackType from typing import Dict, List, Type, Tuple, Optional, Any, \ OrderedDict as OrderedDictType, Union -from types import TracebackType import numpy as np from .. import Signal, Flowchart, QtWidgets from ..data.datadict import DataDictBase, DataDict, MeshgridDataDict from ..node import Node, linearFlowchart +from ..utils import LabeledOptions __author__ = 'Wolfgang Pfaff' __license__ = 'MIT' @@ -248,21 +249,20 @@ class PlotDataType(Enum): grid2d = auto() -@unique -class ComplexRepresentation(Enum): +class ComplexRepresentation(LabeledOptions): """Options for plotting complex-valued data.""" #: only real - real = auto() + real = "Real" #: real and imaginary - realAndImag = auto() + realAndImag = "Real/Imag" #: real and imaginary, separated - realAndImagSeparate = auto() + realAndImagSeparate = "Real/Imag (split)" #: magnitude and phase - magAndPhase = auto() + magAndPhase = "Mag/Phase" def determinePlotDataType(data: Optional[DataDictBase]) -> PlotDataType: @@ -383,7 +383,10 @@ def __init__(self) -> None: #: how to represent complex data. #: must be set before adding data to the plot to have an effect. - self.complexRepresentation = ComplexRepresentation.realAndImag + self.complexRepresentation: ComplexRepresentation = ComplexRepresentation.realAndImag + + #: whether to combine 1D traces into one plot + self.combineTraces: bool = False def __enter__(self) -> "AutoFigureMaker": return self @@ -553,6 +556,9 @@ def addData(self, *data: Union[np.ndarray, np.ma.MaskedArray], :return: ID of the new plot item. """ + if self.combineTraces and join is None: + join = self.previousPlotId() + id = _generate_auto_dict_key(self.plotItems) # TODO: allow any negative number @@ -569,7 +575,7 @@ def addData(self, *data: Union[np.ndarray, np.ma.MaskedArray], if labels is None: labels = [''] * len(data) elif len(labels) < len(data): - labels += [''] * (len(data)-len(labels)) + labels += [''] * (len(data) - len(labels)) plotItem = PlotItem(list(data), id, subPlotId, plotDataType, labels, plotOptions) From 9a9c353abcf6936103377338a7fc0d59d7799270 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 12 Aug 2021 20:47:56 -0500 Subject: [PATCH 49/94] minor cleanup. --- test/apps/autoplot_app.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/apps/autoplot_app.py b/test/apps/autoplot_app.py index 0e1bc301..6a6eb9c0 100644 --- a/test/apps/autoplot_app.py +++ b/test/apps/autoplot_app.py @@ -20,9 +20,8 @@ from plottr import log as plottrlog from plottr.apps.autoplot import autoplot from plottr.data.datadict import DataDictBase, DataDict -from plottr.utils import testdata -from plottr.plot.mpl.autoplot import AutoPlot as MPLAutoPlot from plottr.plot.pyqtgraph.autoplot import AutoPlot as PGAutoPlot +from plottr.utils import testdata plottrlog.enableStreamHandler(True) logger = plottrlog.getLogger('plottr.test.autoplot_app') @@ -156,11 +155,10 @@ def main(dataSrc): # plotWidgetClass = MPLAutoPlot plotWidgetClass = PGAutoPlot - if __name__ == '__main__': # src = LineDataMovie(20, 3, 31) - # src = ImageDataMovie(10, 2, 101) + src = ImageDataMovie(10, 2, 101) # src = ImageDataLiveAcquisition(101, 101, 67) - src = ComplexImage(21, 21) + # src = ComplexImage(21, 21) src.delay = 0.1 main(src) From 88d2691f57aea6d742c649968e26723da85dc718 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 12 Aug 2021 20:48:19 -0500 Subject: [PATCH 50/94] GUI options for plot preferences. --- plottr/plot/pyqtgraph/autoplot.py | 115 +++++++++++++++++++++++++----- plottr/utils/misc.py | 46 ++++++++++++ 2 files changed, 144 insertions(+), 17 deletions(-) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 47c2222c..5311440c 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -1,8 +1,7 @@ """Tools for automatic plotting with ``pyqtgraph``. Contains a ``pyqtgraph``-specific implementation of :class:`.plot.base.FigureMaker` -(:class:`.FigureMaker`) and :class:`.plot.base.PlotWidget` -(:class:`.AutoPlot`). +(:class:`.FigureMaker`) and :class:`.plot.base.PlotWidget` (:class:`.AutoPlot`). ``FigureMaker`` can be used to quickly create a figure in a mostly automatic fashion from known data. For embedding in GUIs (such as plottr applications), ``AutoPlot`` is the main @@ -10,18 +9,19 @@ """ import logging -from typing import Dict, List, Tuple, Union, Optional, Any, Type +from dataclasses import dataclass +from typing import List, Optional import numpy as np - from pyqtgraph import mkPen -from plottr import QtWidgets, QtCore, QtGui, config_entry as getcfg +from plottr import QtWidgets, QtCore, Signal, Slot, \ + config_entry as getcfg from plottr.data.datadict import DataDictBase +from .plots import Plot, PlotWithColorbar, PlotBase from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ PlotItem, ComplexRepresentation, determinePlotDataType, \ - PlotWidgetContainer, PlotWidget, SubPlot -from .plots import Plot, PlotWithColorbar, PlotBase + PlotWidgetContainer, PlotWidget logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -46,8 +46,6 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): self.layout.setContentsMargins(0, 0, 0, 0) self.layout.setSpacing(0) self.setLayout(self.layout) - self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', - default=(400, 400))) def addPlot(self, plot: PlotBase) -> None: """Add a :class:`.PlotBase` widget. @@ -107,7 +105,7 @@ def __init__(self, widget: Optional[_FigureMakerWidget] = None, def __enter__(self) -> "FigureMaker": return self - def subPlotFromId(self, subPlotId: int) -> SubPlot: + def subPlotFromId(self, subPlotId: int) -> PlotBase: """Get SubPlot from ID.""" subPlots = self.subPlots[subPlotId].axes assert isinstance(subPlots, list) and len(subPlots) > 0 and \ @@ -191,12 +189,12 @@ def _1dPlot(self, plotItem): if plotItem.plotDataType == PlotDataType.line1d: return subPlot.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], - pen=mkPen(color, width=2), - symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) + pen=mkPen(color, width=2), symbol=symbol, symbolBrush=color, + symbolPen=None, symbolSize=symbolSize) else: return subPlot.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], - pen=None, - symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) + pen=None, symbol=symbol, symbolBrush=color, + symbolPen=None, symbolSize=symbolSize) def _colorPlot(self, plotItem): subPlot = self.subPlotFromId(plotItem.subPlot) @@ -223,12 +221,16 @@ def __init__(self, parent: Optional[PlotWidgetContainer]) -> None: super().__init__(parent=parent) self.fmWidget: Optional[PlotWidget] = None + self.figConfig: Optional[FigureConfigToolBar] = None + self.figOptions: FigureOptions = FigureOptions() + self.layout = QtWidgets.QVBoxLayout() self.layout.setContentsMargins(0, 0, 0, 0) self.layout.setSpacing(0) self.setLayout(self.layout) - self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', default=(400, 400))) + self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', + default=(400, 400))) def setData(self, data: Optional[DataDictBase]) -> None: """Uses :class:`.FigureMaker` to populate the plot(s). @@ -245,14 +247,21 @@ def setData(self, data: Optional[DataDictBase]) -> None: if self.data is None: return - fmKwargs = {'widget': self.fmWidget} + fmKwargs = {} # {'widget': self.fmWidget} dc = self.dataChanges if not dc['dataTypeChanged'] and not dc['dataStructureChanged']: fmKwargs['clearWidget'] = False else: fmKwargs['clearWidget'] = True + self._plotData(**fmKwargs) + + def _plotData(self, **kwargs): + with FigureMaker(parentWidget=self, widget=self.fmWidget, + **kwargs) as fm: + + fm.complexRepresentation = self.figOptions.complexRepresentation + fm.combineTraces = self.figOptions.combineLinePlots - with FigureMaker(parentWidget=self, **fmKwargs) as fm: for dep in self.data.dependents(): inds = self.data.axes(dep) dvals = self.data.data_vals(dep) @@ -266,3 +275,75 @@ def setData(self, data: Optional[DataDictBase]) -> None: if self.fmWidget is None: self.fmWidget = fm.widget self.layout.addWidget(self.fmWidget) + self.figConfig = FigureConfigToolBar(self.figOptions, + parent=self) + self.layout.addWidget(self.figConfig) + self.figConfig.optionsChanged.connect(self._refreshPlot) + + @Slot() + def _refreshPlot(self): + self._plotData() + + +@dataclass +class FigureOptions: + """Dataclass that describes the configuration options for the figure.""" + + #: whether to plot all 1D traces into a single panel + combineLinePlots: bool = False + + #: how to represent complex data + complexRepresentation: ComplexRepresentation = ComplexRepresentation.realAndImag + + +class FigureConfigToolBar(QtWidgets.QToolBar): + """Simple toolbar to configure the figure.""" + + # TODO: find better config system that generates GUI automatically and + # links updates easier. + + #: Signal() -- emitted when options have been changed in the GUI. + optionsChanged = Signal() + + def __init__(self, options: FigureOptions, parent: QtWidgets.QWidget = None) -> None: + """Constructor. + + :param options: options object. GUI interaction will make changes + in-place to this object. + :param parent: parent Widget + """ + super().__init__(parent) + + self.options = options + + combineLinePlots = self.addAction("Combine 1D") + combineLinePlots.setCheckable(True) + combineLinePlots.setChecked(self.options.combineLinePlots) + combineLinePlots.triggered.connect( + lambda: self._setOption('combineLinePlots', + combineLinePlots.isChecked()) + ) + + complexOptions = QtWidgets.QMenu(parent=self) + complexGroup = QtWidgets.QActionGroup(complexOptions) + complexGroup.setExclusive(True) + for k in ComplexRepresentation: + a = QtWidgets.QAction(k.label, complexOptions) + a.setCheckable(True) + complexGroup.addAction(a) + complexOptions.addAction(a) + a.setChecked(k == self.options.complexRepresentation) + complexGroup.triggered.connect( + lambda _a: self._setOption('complexRepresentation', + ComplexRepresentation.fromLabel(_a.text())) + ) + complexButton = QtWidgets.QToolButton() + complexButton.setToolButtonStyle(QtCore.Qt.ToolButtonTextOnly) + complexButton.setText('Complex') + complexButton.setPopupMode(QtWidgets.QToolButton.InstantPopup) + complexButton.setMenu(complexOptions) + self.addWidget(complexButton) + + def _setOption(self, option, value): + setattr(self.options, option, value) + self.optionsChanged.emit() diff --git a/plottr/utils/misc.py b/plottr/utils/misc.py index e4dd4166..3755192e 100644 --- a/plottr/utils/misc.py +++ b/plottr/utils/misc.py @@ -3,6 +3,7 @@ Various utility functions. """ +from enum import Enum from typing import List, Tuple, TypeVar, Optional, Sequence @@ -66,3 +67,48 @@ def unwrap_optional(val: Optional[T]) -> T: if val is None: raise ValueError("Expected a not None value but got a None value.") return val + + +class AutoEnum(Enum): + """Enum that with automatically incremented integer values. + + Allows to pass additional arguments in the class variables to the __init__ + method of the instances. + See: https://stackoverflow.com/questions/19330460/how-do-i-put-docstrings-on-enums/19330461#19330461 + """ + + def __new__(cls, *args) -> "AutoEnum": + """creating a new instance. + + :param args: will be passed to __init__. + """ + value = len(cls) + 1 + obj = object.__new__(cls) + obj._value_ = value + return obj + + +class LabeledOptions(AutoEnum): + """Enum with a label for each element. We can find the name from the label + using :meth:`.fromLabel`. + + Example:: + + >>> class Color(LabeledOptions): + ... red = 'Red' + ... blue = 'Blue' + + Here, ``Color.blue`` has value ``2`` and ``Color.fromLabel('Blue')`` returns + ``Color.blue``. + """ + + def __init__(self, label: str) -> None: + self.label = label + + @classmethod + def fromLabel(cls, label: str) -> Optional["LabeledOptions"]: + """Find enum element from label.""" + for k in cls: + if k.label == label: + return k + return None From 1bfb4dbe17b48c9fa218d906c128a6fa0d930387 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 12 Aug 2021 20:54:47 -0500 Subject: [PATCH 51/94] removed file that got commited by accident. --- test/Untitled.ipynb | 75 --------------------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 test/Untitled.ipynb diff --git a/test/Untitled.ipynb b/test/Untitled.ipynb deleted file mode 100644 index 1db71ac1..00000000 --- a/test/Untitled.ipynb +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "environmental-bermuda", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "polish-fishing", - "metadata": {}, - "outputs": [], - "source": [ - "from plottr import config" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "medical-fairy", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'main': {'matplotlibrc': {'axes.grid': True}}}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "config()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "clean-melissa", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python [conda env:plottr-pyqt5]", - "language": "python", - "name": "conda-env-plottr-pyqt5-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 64b5ee36a988ff7c48605a48cb3a606357e18810 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sat, 14 Aug 2021 16:35:48 -0500 Subject: [PATCH 52/94] bugfix -- broke config with earlier commit. --- plottr/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/__init__.py b/plottr/__init__.py index 91f51604..3ff0dc8e 100644 --- a/plottr/__init__.py +++ b/plottr/__init__.py @@ -138,7 +138,7 @@ def config_entry(*path: str, default: Optional[Any] = None, :returns: desired value """ - cfg = config(names) + cfg: Any = config(names) for k in path: if isinstance(cfg, dict) and k in cfg: cfg = cfg.get(k) From fcd7efca5d7112b88ed130987546025ef0af47d6 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sat, 14 Aug 2021 17:14:43 -0500 Subject: [PATCH 53/94] mypy fixes. --- plottr/apps/inspectr.py | 4 ++-- plottr/data/datadict.py | 5 +++++ plottr/data/datadict_storage.py | 9 ++++++--- plottr/plot/base.py | 2 +- plottr/plot/mpl/autoplot.py | 2 +- plottr/plot/pyqtgraph/__init__.py | 4 +++- plottr/plot/pyqtgraph/autoplot.py | 27 ++++++++++++++++----------- plottr/plot/pyqtgraph/plots.py | 8 ++++---- plottr/utils/misc.py | 4 ++-- plottr/utils/num.py | 2 +- 10 files changed, 41 insertions(+), 26 deletions(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index 04cc755b..bc226cac 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -293,14 +293,14 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None, self._plotWindows: Dict[int, WindowDict] = {} self.filepath = dbPath - self.dbdf = None + self.dbdf: Optional[pandas.DataFrame] = None self.monitor = QtCore.QTimer() # flag for determining what has been loaded so far. # * None: nothing opened yet. # * -1: empty DS open. # * any value > 0: run ID from the most recent loading. - self.latestRunId = None + self.latestRunId: Optional[int] = None self.setWindowTitle('Plottr | QCoDeS dataset inspectr') diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 86ac4322..69adaab3 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -61,6 +61,11 @@ def __init__(self, **kw: Any): def __eq__(self, other: object) -> bool: """Check for content equality of two datadicts.""" + + # TODO: require a version that ignores metadata. + # FIXME: proper comparison of arrays for metadata. + # FIXME: arrays can be equal even if dtypes are not + if not isinstance(other, DataDictBase): return NotImplemented diff --git a/plottr/data/datadict_storage.py b/plottr/data/datadict_storage.py index 98a1eacc..ca75d17e 100644 --- a/plottr/data/datadict_storage.py +++ b/plottr/data/datadict_storage.py @@ -38,6 +38,8 @@ DATAFILEXT = '.ddh5' TIMESTRFORMAT = "%Y-%m-%d %H:%M:%S" +# FIXME: need correct handling of dtypes and list/array conversion + class AppendMode(Enum): """How/Whether to append data to existing data.""" @@ -51,7 +53,7 @@ class AppendMode(Enum): def h5ify(obj: Any) -> Any: """ - Convert an object into something that we can assing to an HDF5 attribute. + Convert an object into something that we can assign to an HDF5 attribute. Performs the following conversions: - list/array of strings -> numpy chararray of unicode type @@ -69,7 +71,7 @@ def h5ify(obj: Any) -> Any: obj = np.array(obj) if type(obj) == np.ndarray and obj.dtype.kind == 'U': - return np.chararray.encode(obj, encoding='utf8') + return np.char.encode(obj, encoding='utf8') return obj @@ -80,7 +82,7 @@ def deh5ify(obj: Any) -> Any: return obj.decode() if type(obj) == np.ndarray and obj.dtype.kind == 'S': - return np.chararray.decode(obj) + return np.char.decode(obj) return obj @@ -97,6 +99,7 @@ def set_attr(h5obj: Any, name: str, val: Any) -> None: except TypeError: newval = str(val) h5obj.attrs[name] = h5ify(newval) + print(f"{name} set as string") def add_cur_time_attr(h5obj: Any, name: str = 'creation', diff --git a/plottr/plot/base.py b/plottr/plot/base.py index 40ef8505..33f9ff0b 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -632,7 +632,7 @@ def dataDimensionsInSubPlot(self, subPlotId: int) -> Dict[int, int]: return ret # Methods to be implemented by inheriting classes - def makeSubPlots(self, nSubPlots: int) -> Optional[List[Any]]: + def makeSubPlots(self, nSubPlots: int) -> List[Any]: """Generate the subplots. Called after all data has been added. Must be implemented by an inheriting class. diff --git a/plottr/plot/mpl/autoplot.py b/plottr/plot/mpl/autoplot.py index 0624c32b..978e54d8 100644 --- a/plottr/plot/mpl/autoplot.py +++ b/plottr/plot/mpl/autoplot.py @@ -373,7 +373,7 @@ def __init__(self, parent: Optional[PlotWidgetContainer] = None): self.plotOptionsToolBar.setIconSize(QtCore.QSize(iconSize, iconSize)) self.setMinimumSize(int(640*scaling), int(480*scaling)) - def update(self): + def update(self) -> None: self.plot.draw() QtCore.QCoreApplication.processEvents() diff --git a/plottr/plot/pyqtgraph/__init__.py b/plottr/plot/pyqtgraph/__init__.py index df602a96..a9d16f8b 100644 --- a/plottr/plot/pyqtgraph/__init__.py +++ b/plottr/plot/pyqtgraph/__init__.py @@ -1,9 +1,11 @@ +from typing import List + import pyqtgraph as pg from plottr import config_entry as getcfg -__all__ = [] +__all__: List[str] = [] bg = getcfg('main', 'pyqtgraph', 'background', default='w') diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 5311440c..b7376ab7 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -10,7 +10,7 @@ import logging from dataclasses import dataclass -from typing import List, Optional +from typing import List, Optional, Any import numpy as np from pyqtgraph import mkPen @@ -60,7 +60,7 @@ def clearAllPlots(self) -> None: for p in self.subPlots: p.clearPlot() - def deleteAllPlots(self): + def deleteAllPlots(self) -> None: """Delete all subplot widgets.""" for p in self.subPlots: p.deleteLater() @@ -112,7 +112,7 @@ def subPlotFromId(self, subPlotId: int) -> PlotBase: isinstance(subPlots[0], PlotBase) return subPlots[0] - def makeSubPlots(self, nSubPlots: int) -> List[List[PlotBase]]: + def makeSubPlots(self, nSubPlots: int) -> List[PlotBase]: """Create empty subplots in the widgets. If ``clearWidget`` was not set to ``True`` in the constructor, @@ -174,7 +174,7 @@ def plot(self, plotItem: PlotItem) -> None: else: raise NotImplementedError('Cannot plot this data.') - def _1dPlot(self, plotItem): + def _1dPlot(self, plotItem: PlotItem) -> None: colors = getcfg('main', 'pyqtgraph', 'line_colors', default=['r', 'b', 'g']) symbols = getcfg('main', 'pyqtgraph', 'line_symbols', default=['o']) symbolSize = getcfg('main', 'pyqtgraph', 'line_symbol_size', default=5) @@ -188,20 +188,22 @@ def _1dPlot(self, plotItem): symbol = symbols[self.findPlotIndexInSubPlot(plotItem.id) % len(symbols)] if plotItem.plotDataType == PlotDataType.line1d: - return subPlot.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], + name = plotItem.labels[-1] if isinstance(plotItem.labels, list) else '' + return subPlot.plot.plot(x.flatten(), y.flatten(), name=name, pen=mkPen(color, width=2), symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) else: - return subPlot.plot.plot(x.flatten(), y.flatten(), name=plotItem.labels[-1], + name = plotItem.labels[-1] if isinstance(plotItem.labels, list) else '' + return subPlot.plot.plot(x.flatten(), y.flatten(), name=name, pen=None, symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) - def _colorPlot(self, plotItem): + def _colorPlot(self, plotItem: PlotItem) -> None: subPlot = self.subPlotFromId(plotItem.subPlot) assert isinstance(subPlot, PlotWithColorbar) and len(plotItem.data) == 3 subPlot.setImage(*plotItem.data) - def _scatterPlot2d(self, plotItem): + def _scatterPlot2d(self, plotItem: PlotItem) -> None: subPlot = self.subPlotFromId(plotItem.subPlot) assert isinstance(subPlot, PlotWithColorbar) and len(plotItem.data) == 3 subPlot.setScatter2d(*plotItem.data) @@ -255,7 +257,10 @@ def setData(self, data: Optional[DataDictBase]) -> None: fmKwargs['clearWidget'] = True self._plotData(**fmKwargs) - def _plotData(self, **kwargs): + def _plotData(self, **kwargs: Any) -> None: + if self.data is None: + return + with FigureMaker(parentWidget=self, widget=self.fmWidget, **kwargs) as fm: @@ -281,7 +286,7 @@ def _plotData(self, **kwargs): self.figConfig.optionsChanged.connect(self._refreshPlot) @Slot() - def _refreshPlot(self): + def _refreshPlot(self) -> None: self._plotData() @@ -344,6 +349,6 @@ def __init__(self, options: FigureOptions, parent: QtWidgets.QWidget = None) -> complexButton.setMenu(complexOptions) self.addWidget(complexButton) - def _setOption(self, option, value): + def _setOption(self, option: str, value: Any) -> None: setattr(self.options, option, value) self.optionsChanged.emit() diff --git a/plottr/plot/pyqtgraph/plots.py b/plottr/plot/pyqtgraph/plots.py index 39d6e0e3..10ea9d4a 100644 --- a/plottr/plot/pyqtgraph/plots.py +++ b/plottr/plot/pyqtgraph/plots.py @@ -37,7 +37,7 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: self.setLayout(self.layout) self.layout.addWidget(self.graphicsLayout) - def clearPlot(self) -> NoReturn: + def clearPlot(self) -> None: """Clear all plot contents (but do not delete plot elements, like axis spines, insets, etc). @@ -143,15 +143,15 @@ def setScatter2d(self, x: np.ndarray, y: np.ndarray, z: np.ndarray) -> None: self.colorbar.sigLevelsChanged.connect(self._colorScatterPoints) # TODO: this seems crazy slow. - def _colorScatterPoints(self, cbar: pg.ColorBarItem): + def _colorScatterPoints(self, cbar: pg.ColorBarItem) -> None: if self.scatter is not None and self.scatterZVals is not None: z_norm = self._normalizeColors(self.scatterZVals, cbar.levels()) colors = self.colorbar.cmap.mapToQColor(z_norm) self.scatter.setBrush(colors) - def _normalizeColors(self, z: np.ndarray, levels: Tuple[float, float]): + def _normalizeColors(self, z: np.ndarray, levels: Tuple[float, float]) -> np.ndarray: scale = levels[1] - levels[0] if scale > 0: return (z - levels[0]) / scale else: - return np.ones(z.size()) * 0.5 + return np.ones(z.size) * 0.5 diff --git a/plottr/utils/misc.py b/plottr/utils/misc.py index 3755192e..48d9e21d 100644 --- a/plottr/utils/misc.py +++ b/plottr/utils/misc.py @@ -4,7 +4,7 @@ """ from enum import Enum -from typing import List, Tuple, TypeVar, Optional, Sequence +from typing import List, Tuple, TypeVar, Optional, Sequence, Any def reorder_indices(lst: Sequence[str], target: Sequence[str]) -> Tuple[int, ...]: @@ -77,7 +77,7 @@ class AutoEnum(Enum): See: https://stackoverflow.com/questions/19330460/how-do-i-put-docstrings-on-enums/19330461#19330461 """ - def __new__(cls, *args) -> "AutoEnum": + def __new__(cls, *args: Any) -> "AutoEnum": """creating a new instance. :param args: will be passed to __init__. diff --git a/plottr/utils/num.py b/plottr/utils/num.py index 2205cf47..12ec35de 100644 --- a/plottr/utils/num.py +++ b/plottr/utils/num.py @@ -148,7 +148,7 @@ def array1d_to_meshgrid(arr: Union[List, np.ndarray], def _find_switches(arr: np.ndarray, rth: float = 25, ztol: float = 1e-15) -> np.ndarray: - arr_ = np.ma.MaskedArray(arr, is_invalid(arr)) + arr_: np.ma.MaskedArray = np.ma.MaskedArray(arr, is_invalid(arr)) deltas = arr_[1:] - arr_[:-1] hi = np.percentile(arr[~is_invalid(arr)], 100.-rth) lo = np.percentile(arr[~is_invalid(arr)], rth) From 035788b7a873525cb74e1c62dec8bdb94990be08 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sat, 14 Aug 2021 17:23:50 -0500 Subject: [PATCH 54/94] don't overwrite method with variable. --- plottr/plot/pyqtgraph/plots.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plottr/plot/pyqtgraph/plots.py b/plottr/plot/pyqtgraph/plots.py index 10ea9d4a..22eebafa 100644 --- a/plottr/plot/pyqtgraph/plots.py +++ b/plottr/plot/pyqtgraph/plots.py @@ -28,14 +28,14 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: super().__init__(parent) #: central layout of the widget. only contains a graphics layout. - self.layout = QtWidgets.QHBoxLayout(self) + layout = QtWidgets.QHBoxLayout(self) #: ``pyqtgraph`` graphics layout self.graphicsLayout = pg.GraphicsLayoutWidget(self) - self.layout.setContentsMargins(0, 0, 0, 0) - self.layout.setSpacing(0) - self.setLayout(self.layout) - self.layout.addWidget(self.graphicsLayout) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) + self.setLayout(layout) + layout.addWidget(self.graphicsLayout) def clearPlot(self) -> None: """Clear all plot contents (but do not delete plot elements, like axis From 66ee28e25ef48b84f6ce813dfc06413d6b5d6c0e Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sat, 14 Aug 2021 17:52:42 -0500 Subject: [PATCH 55/94] more mypy fixes. --- plottr/plot/base.py | 2 +- plottr/plot/mpl/autoplot.py | 4 ++-- plottr/plot/pyqtgraph/autoplot.py | 33 ++++++++++++++++--------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/plottr/plot/base.py b/plottr/plot/base.py index 33f9ff0b..82668b12 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -141,7 +141,7 @@ def __init__(self, parent: Optional[PlotWidgetContainer] = None) -> None: 'dataLimitsChanged': False, } - def update(self) -> None: + def updatePlot(self) -> None: return None def setData(self, data: Optional[DataDictBase]) -> None: diff --git a/plottr/plot/mpl/autoplot.py b/plottr/plot/mpl/autoplot.py index 978e54d8..7e16ee59 100644 --- a/plottr/plot/mpl/autoplot.py +++ b/plottr/plot/mpl/autoplot.py @@ -373,7 +373,7 @@ def __init__(self, parent: Optional[PlotWidgetContainer] = None): self.plotOptionsToolBar.setIconSize(QtCore.QSize(iconSize, iconSize)) self.setMinimumSize(int(640*scaling), int(480*scaling)) - def update(self) -> None: + def updatePlot(self) -> None: self.plot.draw() QtCore.QCoreApplication.processEvents() @@ -466,4 +466,4 @@ def _plotData(self) -> None: **kw) self.setMeta(self.data) - self.update() + self.updatePlot() diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index b7376ab7..ed9c8495 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -42,17 +42,17 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): self.subPlots: List[PlotBase] = [] - self.layout = QtWidgets.QVBoxLayout() - self.layout.setContentsMargins(0, 0, 0, 0) - self.layout.setSpacing(0) - self.setLayout(self.layout) + layout = QtWidgets.QVBoxLayout() + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) + self.setLayout(layout) def addPlot(self, plot: PlotBase) -> None: """Add a :class:`.PlotBase` widget. :param plot: plot widget """ - self.layout.addWidget(plot) + self.layout().addWidget(plot) self.subPlots.append(plot) def clearAllPlots(self) -> None: @@ -76,8 +76,6 @@ class FigureMaker(BaseFM): # TODO: make scrollable when many figures (set min size)? # TODO: check for valid plot data - widget: _FigureMakerWidget - def __init__(self, widget: Optional[_FigureMakerWidget] = None, clearWidget: bool = True, parentWidget: Optional[QtWidgets.QWidget] = None): @@ -95,7 +93,9 @@ def __init__(self, widget: Optional[_FigureMakerWidget] = None, """ super().__init__() + self.widget: _FigureMakerWidget self.clearWidget = clearWidget + if widget is None: self.widget = _FigureMakerWidget(parent=parentWidget) else: @@ -118,9 +118,9 @@ def makeSubPlots(self, nSubPlots: int) -> List[PlotBase]: If ``clearWidget`` was not set to ``True`` in the constructor, existing sub plot widgets are cleared, but not deleted and re-created. """ + plot: PlotBase if self.clearWidget: self.widget.deleteAllPlots() - for i in range(nSubPlots): if max(self.dataDimensionsInSubPlot(i).values()) == 1: plot = Plot(self.widget) @@ -222,15 +222,15 @@ def __init__(self, parent: Optional[PlotWidgetContainer]) -> None: """ super().__init__(parent=parent) - self.fmWidget: Optional[PlotWidget] = None + self.fmWidget: Optional[_FigureMakerWidget] = None self.figConfig: Optional[FigureConfigToolBar] = None self.figOptions: FigureOptions = FigureOptions() - self.layout = QtWidgets.QVBoxLayout() - self.layout.setContentsMargins(0, 0, 0, 0) - self.layout.setSpacing(0) + layout = QtWidgets.QVBoxLayout() + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(0) - self.setLayout(self.layout) + self.setLayout(layout) self.setMinimumSize(*getcfg('main', 'pyqtgraph', 'minimum_plot_size', default=(400, 400))) @@ -279,10 +279,10 @@ def _plotData(self, **kwargs: Any) -> None: if self.fmWidget is None: self.fmWidget = fm.widget - self.layout.addWidget(self.fmWidget) + self.layout().addWidget(self.fmWidget) self.figConfig = FigureConfigToolBar(self.figOptions, parent=self) - self.layout.addWidget(self.figConfig) + self.layout().addWidget(self.figConfig) self.figConfig.optionsChanged.connect(self._refreshPlot) @Slot() @@ -310,7 +310,8 @@ class FigureConfigToolBar(QtWidgets.QToolBar): #: Signal() -- emitted when options have been changed in the GUI. optionsChanged = Signal() - def __init__(self, options: FigureOptions, parent: QtWidgets.QWidget = None) -> None: + def __init__(self, options: FigureOptions, + parent: Optional[QtWidgets.QWidget] = None) -> None: """Constructor. :param options: options object. GUI interaction will make changes From 1f78d36861cec0bd2810dfec239c3ceb32e4f8b5 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sat, 14 Aug 2021 19:34:13 -0500 Subject: [PATCH 56/94] nicer widget defaults. --- plottr/apps/autoplot.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plottr/apps/autoplot.py b/plottr/apps/autoplot.py index 6b04f8b0..e920fc07 100644 --- a/plottr/apps/autoplot.py +++ b/plottr/apps/autoplot.py @@ -24,6 +24,7 @@ from ..node.tools import linearFlowchart from ..node.node import Node from ..plot import PlotNode, makeFlowchartWithPlot, PlotWidget +from ..plot.pyqtgraph.autoplot import AutoPlot as PGAutoPlot from ..utils.misc import unwrap_optional __author__ = 'Wolfgang Pfaff' @@ -328,11 +329,19 @@ def autoplotDDH5(filepath: str = '', groupname: str = 'data') \ ('Data selection', DataSelector), ('Grid', DataGridder), ('Dimension assignment', XYSelector), - # ('Subtract average', SubtractAverage), ('plot', PlotNode) ) - win = AutoPlotMainWindow(fc, loaderName='Data loader', monitor=True, + widgetOptions = { + "Data selection": dict(visible=True, + dockArea=QtCore.Qt.TopDockWidgetArea), + "Dimension assignment": dict(visible=True, + dockArea=QtCore.Qt.TopDockWidgetArea), + } + + win = AutoPlotMainWindow(fc, loaderName='Data loader', + widgetOptions=widgetOptions, + monitor=True, monitorInterval=2.0) win.show() From 4f9c190c3c6f6d6a8ca0856e1275ff6687a4a4af Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Sat, 14 Aug 2021 19:34:48 -0500 Subject: [PATCH 57/94] default plot widget from config rather than hardcoded. --- plottr/config/plottrcfg_main.py | 5 +++++ plottr/gui/widgets.py | 10 +++++----- test/apps/autoplot_app.py | 2 ++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/plottr/config/plottrcfg_main.py b/plottr/config/plottrcfg_main.py index 112b86ba..ee0ebad6 100644 --- a/plottr/config/plottrcfg_main.py +++ b/plottr/config/plottrcfg_main.py @@ -1,6 +1,11 @@ from matplotlib import cycler +from plottr.plot.pyqtgraph.autoplot import AutoPlot as PGAutoPlot +from plottr.plot.mpl.autoplot import AutoPlot as MPLAutoPlot config = { + + 'default-plotwidget': MPLAutoPlot, + 'matplotlibrc': { 'axes.grid': True, 'axes.prop_cycle': cycler('color', ['1f77b4', 'ff7f0e', '2ca02c', 'd62728', '9467bd', '8c564b', diff --git a/plottr/gui/widgets.py b/plottr/gui/widgets.py index ef6cf20c..6648f39f 100644 --- a/plottr/gui/widgets.py +++ b/plottr/gui/widgets.py @@ -10,6 +10,7 @@ from plottr import QtGui, QtCore, Flowchart, QtWidgets, Signal, Slot from plottr.node import Node, linearFlowchart from ..plot import PlotNode, PlotWidgetContainer, PlotWidget +from .. import config_entry as getcfg __author__ = 'Wolfgang Pfaff' __license__ = 'MIT' @@ -66,15 +67,12 @@ def spinValueChanged(self, val: float) -> None: self.intervalChanged.emit(val) - class PlotWindow(QtWidgets.QMainWindow): """ Simple MainWindow class for embedding flowcharts and plots, based on ``QtWidgets.QMainWindow``. """ - # FIXME: defaulting to MPL should probably be on a higher level. - #: Signal() -- emitted when the window is closed windowClosed = Signal() @@ -96,8 +94,10 @@ def __init__(self, parent: Optional[QtWidgets.QMainWindow] = None, super().__init__(parent) if plotWidgetClass is None: - from ..plot.mpl import AutoPlot - plotWidgetClass = AutoPlot + plotWidgetClass = getcfg('main', 'default-plotwidget') + + if plotWidgetClass is None: + raise RuntimeError("No PlotWidget has been specified.") self.plotWidgetClass = plotWidgetClass self.plot = PlotWidgetContainer(parent=self) diff --git a/test/apps/autoplot_app.py b/test/apps/autoplot_app.py index 6a6eb9c0..b334d8a6 100644 --- a/test/apps/autoplot_app.py +++ b/test/apps/autoplot_app.py @@ -20,6 +20,7 @@ from plottr import log as plottrlog from plottr.apps.autoplot import autoplot from plottr.data.datadict import DataDictBase, DataDict +from plottr.plot.mpl.autoplot import AutoPlot as MPLAutoPlot from plottr.plot.pyqtgraph.autoplot import AutoPlot as PGAutoPlot from plottr.utils import testdata @@ -154,6 +155,7 @@ def main(dataSrc): # plotWidgetClass = MPLAutoPlot plotWidgetClass = PGAutoPlot +# plotWidgetClass = None if __name__ == '__main__': # src = LineDataMovie(20, 3, 31) From 9573dcd64c78597fc525a965674c9ad83fdab769 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Mon, 16 Aug 2021 15:19:15 -0500 Subject: [PATCH 58/94] better default filename --- plottr/data/datadict_storage.py | 81 +++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/plottr/data/datadict_storage.py b/plottr/data/datadict_storage.py index 98a1eacc..00cf2049 100644 --- a/plottr/data/datadict_storage.py +++ b/plottr/data/datadict_storage.py @@ -16,6 +16,8 @@ """ import os import time +import datetime +import uuid from enum import Enum from typing import Any, Union, Optional, Dict, Type, Collection from types import TracebackType @@ -35,7 +37,7 @@ __author__ = 'Wolfgang Pfaff' __license__ = 'MIT' -DATAFILEXT = '.ddh5' +DATAFILEXT = 'ddh5' TIMESTRFORMAT = "%Y-%m-%d %H:%M:%S" @@ -460,6 +462,9 @@ def groupname(self, val: str) -> None: # Data processing # def process(self, dataIn: Optional[DataDictBase] = None) -> Optional[Dict[str, Any]]: + # TODO: maybe needs an optional way to read only new data from file? + # TODO: implement a threaded version. + if self._filepath is None or self._groupname is None: return None if not os.path.exists(self._filepath): @@ -498,20 +503,21 @@ class DDH5Writer(object): ... with DDH5Writer('./data/', data, name='Test') as writer: ... for x in range(10): ... writer.add_data(x=x, y=x**2) - Data location: ./data/2020-06-05/2020-06-05_0001_Example/2020-06-05_0001_Test.ddh5 + Data location: ./data/2020-06-05/2020-06-05T102345_d11541ca-Test/data.ddh5 :param basedir: The root directory in which data is stored. :meth:`.create_file_structure` is creating the structure inside this root and determines the file name of the data. The default structure implemented here is - ``/YYYY-MM-DD/YYYY-MM-DD__/YYYY-MM-DD__.ddh5``, - where is the automatically increasing number of this dataset in the day - folder and is the value of parameter `name`. To change this, re-implement + ``/YYYY-MM-DD/YYYY-mm-dd_THHMMSS_-/.ddh5``, + where is a short identifier string and is the value of parameter `name`. + To change this, re-implement :meth:`.data_folder` and/or :meth:`.create_file_structure`. :param datadict: initial data object. Must contain at least the structure of the data to be able to use :meth:`add_data` to add data. :param groupname: name of the top-level group in the file container. An existing group of that name will be deleted. :param name: name of this dataset. Used in path/file creation and added as meta data. + :param filename: filename to use. defaults to 'data.ddh5'. """ # TODO: need an operation mode for not keeping data in memory. @@ -520,7 +526,9 @@ class DDH5Writer(object): def __init__(self, basedir: str, datadict: DataDict, groupname: str = 'data', - name: Optional[str] = None): + name: Optional[str] = None, + filename: str = 'data', + filepath: Optional[str] = None): """Constructor for :class:`.DDH5Writer`""" self.basedir = basedir @@ -528,19 +536,20 @@ def __init__(self, basedir: str, self.inserted_rows = 0 self.name = name self.groupname = groupname + self.filename = filename + if self.filename[-(len(DATAFILEXT)+1):] != f".{DATAFILEXT}": + self.filename += f".{DATAFILEXT}" + self.filepath = filepath - self.file_base: Optional[str] = None - self.file_path: Optional[str] = None self.file: Optional[h5py.File] = None self.datadict.add_meta('dataset.name', name) def __enter__(self) -> "DDH5Writer": - self.file_base = self.create_file_structure() - self.file_path = self.file_base + f"{DATAFILEXT}" - print('Data location: ', self.file_path) + self.filepath = self.create_file_structure() + print('Data location: ', self.filepath) - self.file = h5py.File(self.file_path, mode='a', libver='latest') + self.file = h5py.File(self.filepath, mode='a', libver='latest') init_file(self.file, self.groupname) add_cur_time_attr(self.file, name='last_change') add_cur_time_attr(self.file[self.groupname], name='last_change') @@ -563,30 +572,44 @@ def __exit__(self, add_cur_time_attr(self.file[self.groupname], name='close') self.file.close() + def data_folder(self) -> str: + """Return the folder, relative to the data root path, in which data will + be saved. + + Default format: + ``/YYYY-MM-DD/YYYY-mm-ddTHHMMSS_-``. + In this implementation we use the first 8 characters of a UUID as ID. + """ + ID = str(uuid.uuid1()).split('-')[0] + path = os.path.join( + time.strftime("%Y-%m-%d"), + f"{datetime.datetime.now().replace(microsecond=0).isoformat().replace(':', '')}_{ID}-{self.name}" + ) + return path + def create_file_structure(self) -> str: """Determine the filepath and create all subfolders. - :returns: the filepath (without extension) of the data file. + :returns: the filepath of the data file. """ - day_folder_path = os.path.join(self.basedir, time.strftime("%Y-%m-%d")) - os.makedirs(day_folder_path, exist_ok=True) - - filebase = time.strftime("%Y-%m-%d_") - existing_datafolders = [f for f in os.listdir(day_folder_path) - if f[:len(filebase)] == filebase] - prev_idxs = [int(f[len(filebase):len(filebase)+4]) for f in existing_datafolders] - if len(prev_idxs) == 0: - new_idx = 1 + + if self.filepath is None: + data_folder_path = os.path.join( + self.basedir, + self.data_folder() + ) + appendix = '' + idx = 2 + while os.path.exists(data_folder_path+appendix): + appendix = f'-{idx}' + idx += 1 + data_folder_path += appendix + self.filepath = os.path.join(data_folder_path, self.filename) else: - new_idx = max(prev_idxs) + 1 - filebase += f"{new_idx:04}" - if self.name is not None: - filebase += f"_{self.name}" + data_folder_path, self.filename = os.path.split(self.filepath) - data_folder_path = os.path.join(day_folder_path, filebase) os.makedirs(data_folder_path, exist_ok=True) - - return os.path.join(data_folder_path, filebase) + return self.filepath def add_data(self, **kwargs: Any) -> None: """Add data to the file (and the internal `DataDict`). From d40249e6dad3602904f8b93cfc001f5cf2f5e1dc Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Tue, 17 Aug 2021 12:25:08 -0500 Subject: [PATCH 59/94] added tools for better recognition of added data shapes. --- plottr/data/datadict.py | 71 +++++++++++++++++++++++++++++++++++------ plottr/utils/num.py | 2 +- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 6cf30fd6..20b60920 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -142,6 +142,53 @@ def _meta_key_to_name(key: str) -> str: def _meta_name_to_key(name: str) -> str: return meta_name_to_key(name) + @staticmethod + def to_records(**data: Any) -> Dict[str, np.ndarray]: + """Convert data to rows that can be added to the ``DataDict``. + All data is converted to np.array, and the first dimension of all resulting + arrays has the same length (chosen to be the smallest possible number + that does not alter any shapes beyond adding a length-1 dimension as + first dimesion, if necessary). + + If a field is given as ``None``, it will be converted to ``numpy.array([numpy.nan])``. + """ + records: Dict[str, np.ndarray] = {} + + seqtypes = (np.ndarray, tuple, list) + nantypes = (type(None), ) + + for k, v in data.items(): + if isinstance(v, seqtypes): + records[k] = np.array(v) + elif isinstance(v, nantypes): + records[k] = np.array([np.nan]) + else: + records[k] = np.array([v]) + + possible_nrecords = {} + for k, v in records.items(): + possible_nrecords[k] = [1, v.shape[0]] + + commons = [] + for k, v in possible_nrecords.items(): + for n in v: + if n in commons: + continue + is_common = True + for kk, vv in possible_nrecords.items(): + if n not in vv: + is_common = False + if is_common: + commons.append(n) + nrecs = min(commons) + + for k, v in records.items(): + shp = v.shape + if nrecs == 1 and shp[0] > 1: + newshp = tuple([1] + list(shp)) + records[k] = v.reshape(newshp) + return records + def data_items(self) -> Iterator[Tuple[str, Dict[str, Any]]]: """ Generator for data field items. @@ -749,13 +796,19 @@ def add_data(self, **kw: Sequence) -> None: :return: None """ dd = misc.unwrap_optional(self.structure(same_type=True)) - for k, v in kw.items(): - if isinstance(v, list): - dd[k]['values'] = np.array(v) - elif isinstance(v, np.ndarray): - dd[k]['values'] = v - else: - dd[k]['values'] = np.array([v]) + + for k in dd.keys(): + if k not in kw: + kw[k] = None + records = self.to_records(**kw) + for k, v in records.items(): + dd[k]['values'] = v + # if isinstance(v, list): + # dd[k]['values'] = np.array(v) + # elif isinstance(v, np.ndarray): + # dd[k]['values'] = v + # else: + # dd[k]['values'] = np.array([v]) if dd.validate(): records = self.nrecords() @@ -1348,9 +1401,9 @@ def datastructure_from_string(description: str) -> DataDict: data_fields.append(description[slice(*match.span())]) description = description[match.span()[1]:] - dd = dict() + dd: Dict[str, Any] = dict() - def analyze_field(df): + def analyze_field(df: str) -> Tuple[str, Optional[str], Optional[List[str]]]: has_unit = True if '[' in df and ']' in df else False has_dependencies = True if '(' in df and ')' in df else False diff --git a/plottr/utils/num.py b/plottr/utils/num.py index 2205cf47..a9f1577e 100644 --- a/plottr/utils/num.py +++ b/plottr/utils/num.py @@ -148,7 +148,7 @@ def array1d_to_meshgrid(arr: Union[List, np.ndarray], def _find_switches(arr: np.ndarray, rth: float = 25, ztol: float = 1e-15) -> np.ndarray: - arr_ = np.ma.MaskedArray(arr, is_invalid(arr)) + arr_: np.ndarray = np.ma.MaskedArray(arr, is_invalid(arr)) deltas = arr_[1:] - arr_[:-1] hi = np.percentile(arr[~is_invalid(arr)], 100.-rth) lo = np.percentile(arr[~is_invalid(arr)], rth) From 6044e9b02155f35d430d4da49913f6e79ee22336 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Tue, 17 Aug 2021 13:45:29 -0500 Subject: [PATCH 60/94] fixed mypy issues. --- plottr/data/datadict.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 42baf653..25858836 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -789,7 +789,7 @@ def append(self, newdata: "DataDict") -> None: for k, v in newvals.items(): self[k]['values'] = v - def add_data(self, **kw: Sequence) -> None: + def add_data(self, **kw: Any) -> None: # TODO: fill non-given data with nan or none """ Add data to all values. new data must be valid in itself. @@ -816,8 +816,8 @@ def add_data(self, **kw: Sequence) -> None: # dd[k]['values'] = np.array([v]) if dd.validate(): - records = self.nrecords() - if records is not None and records > 0: + nrecords = self.nrecords() + if nrecords is not None and nrecords > 0: self.append(dd) else: for key, val in dd.data_items(): From 1b1d9cd2e5fcaecb715faaca04dc0e8bbb582d39 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Tue, 17 Aug 2021 14:28:09 -0500 Subject: [PATCH 61/94] some testing and minor (mypy) fixes. --- ...c operation of DDH5 and benchmarking.ipynb | 357 ------------------ plottr/data/datadict.py | 16 +- plottr/data/datadict_storage.py | 20 +- 3 files changed, 15 insertions(+), 378 deletions(-) delete mode 100644 doc/examples/Basic operation of DDH5 and benchmarking.ipynb diff --git a/doc/examples/Basic operation of DDH5 and benchmarking.ipynb b/doc/examples/Basic operation of DDH5 and benchmarking.ipynb deleted file mode 100644 index f1d12461..00000000 --- a/doc/examples/Basic operation of DDH5 and benchmarking.ipynb +++ /dev/null @@ -1,357 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T21:56:44.772347Z", - "start_time": "2019-07-02T21:56:44.242727Z" - } - }, - "outputs": [], - "source": [ - "import sys\n", - "import re\n", - "from typing import Optional, Dict, List\n", - "\n", - "import numpy as np\n", - "import h5py\n", - "\n", - "from plottr.data import datadict as dd\n", - "from plottr.data import datadict_storage as dds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'data': {'unit': 'V', 'axes': ['x', 'y']},\n", - " 'x': {'unit': 'mV'},\n", - " 'y': {'unit': 'mV'},\n", - " 'data_2': {'unit': 'V', 'axes': ['x', 'y']}}" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "desc = \"data[V](x, y); data_2[V](x, y); x[mV]; y[mV]\"\n", - "data = dd.str2dd(desc)\n", - "\n", - "data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Simple timing for writing/reading a datadict\n", - "\n", - "## Write" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T21:57:46.468964Z", - "start_time": "2019-07-02T21:57:46.464975Z" - } - }, - "outputs": [], - "source": [ - "FN = './ddh5_test-1'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T21:57:53.110466Z", - "start_time": "2019-07-02T21:57:47.116741Z" - } - }, - "outputs": [], - "source": [ - "%%timeit\n", - "nrows = 10000\n", - "\n", - "x = np.arange(nrows, dtype=np.float)\n", - "y = np.repeat(np.linspace(0., 1., 1001).reshape(1, -1), nrows, 0)\n", - "z = np.arange(y.size, dtype=np.float).reshape(y.shape)\n", - "\n", - "# print(f\"total size = {(x.nbytes + y.nbytes + z.nbytes) * 1e-6} MB\")\n", - "\n", - "data = dd.DataDict(\n", - " x=dict(values=x, unit='nA'), \n", - " y=dict(values=y, unit='nB'),\n", - " z=dict(values=z, unit='nC', axes=['x', 'y']),\n", - ")\n", - "if not data.validate():\n", - " raise ValueError\n", - "\n", - "dds.datadict_to_hdf5(data, FN)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Read back" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T21:57:57.686654Z", - "start_time": "2019-07-02T21:57:55.102475Z" - } - }, - "outputs": [], - "source": [ - "%%timeit\n", - "ret_data = dds.datadict_from_hdf5(FN)\n", - "size = sum([ret_data.data_vals(k).nbytes for k in ['x', 'y', 'z']]) * 1e-6\n", - "# print(f\"total size = {size} MB\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T19:55:32.710489Z", - "start_time": "2019-07-02T19:55:32.706500Z" - } - }, - "source": [ - "## Appending row by row" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T21:58:08.962974Z", - "start_time": "2019-07-02T21:58:08.958987Z" - } - }, - "outputs": [], - "source": [ - "FN = './ddh5_test-2'\n", - "nrows = 100" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T21:58:13.805079Z", - "start_time": "2019-07-02T21:58:10.150728Z" - } - }, - "outputs": [], - "source": [ - "%%timeit\n", - "\n", - "x = np.array([0.])\n", - "y = np.linspace(0., 1., 1001).reshape(1, -1)\n", - "z = np.arange(y.size, dtype=np.float).reshape(y.shape)\n", - "\n", - "data = dd.DataDict(\n", - " x=dict(values=x, unit='nA'), \n", - " y=dict(values=y, unit='nB'),\n", - " z=dict(values=z, unit='nC', axes=['x', 'y']),\n", - ")\n", - "\n", - "dds.datadict_to_hdf5(data, FN, append_mode=dds.AppendMode.none)\n", - "\n", - "for n in range(nrows):\n", - " data = dd.DataDict(\n", - " x=dict(values=np.array([n+1], dtype=np.float), unit='nA'), \n", - " y=dict(values=y, unit='nB'),\n", - " z=dict(values=z, unit='nC', axes=['x', 'y']),\n", - " )\n", - " dds.datadict_to_hdf5(data, FN, append_mode=dds.AppendMode.all)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It's important to note that the bulk of this time is just for opening the files. Below we can see that opening the HDF5 file in append mode takes us around 3 ms." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T21:59:53.337364Z", - "start_time": "2019-07-02T21:59:50.676328Z" - } - }, - "outputs": [], - "source": [ - "%%timeit\n", - "with h5py.File(FN+'.dd.h5', 'a') as f:\n", - " # just do something of no effect.\n", - " dsets = list(f['data'].keys())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Bare HDF5 benchmarking" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## appending row by row, resize every time" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2019-07-02T22:00:07.067915Z", - "start_time": "2019-07-02T22:00:03.259418Z" - } - }, - "outputs": [], - "source": [ - "%%timeit\n", - "\n", - "FN = './hdf5_test.h5'\n", - "\n", - "nrows = 100\n", - "\n", - "x = np.array([0.])\n", - "y = np.linspace(0., 1., 1001).reshape(1, -1)\n", - "z = np.arange(y.size, dtype=np.float).reshape(y.shape)\n", - "\n", - "with h5py.File(FN, 'w', libver='latest') as f: \n", - " grp = f.create_group('data')\n", - " for dn, d in ('x', x), ('y', y), ('z', z):\n", - " grp.create_dataset(dn, maxshape=tuple([None] + list(d.shape[1:])), data=d)\n", - " \n", - "for n in range(nrows): \n", - " with h5py.File(FN, 'a', libver='latest') as f:\n", - " grp = f['data']\n", - " for dn, d in ('x', x), ('y', y), ('z', z):\n", - " ds = grp[dn]\n", - " ds.resize(tuple([ds.shape[0]+1] + list(ds.shape[1:])))\n", - " ds[-1:] = d\n", - " ds.flush()\n", - " f.flush()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python [conda env:plottr-pyqt5]", - "language": "python", - "name": "conda-env-plottr-pyqt5-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.2" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 25858836..17dfe3f7 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -801,19 +801,13 @@ def add_data(self, **kw: Any) -> None: :return: None """ dd = misc.unwrap_optional(self.structure(same_type=True)) + for name, _ in dd.data_items(): + if name not in kw: + kw[name] = None - for k in dd.keys(): - if k not in kw: - kw[k] = None records = self.to_records(**kw) - for k, v in records.items(): - dd[k]['values'] = v - # if isinstance(v, list): - # dd[k]['values'] = np.array(v) - # elif isinstance(v, np.ndarray): - # dd[k]['values'] = v - # else: - # dd[k]['values'] = np.array([v]) + for name, datavals in records.items(): # + dd[name]['values'] = datavals if dd.validate(): nrecords = self.nrecords() diff --git a/plottr/data/datadict_storage.py b/plottr/data/datadict_storage.py index 6202512c..d819efc9 100644 --- a/plottr/data/datadict_storage.py +++ b/plottr/data/datadict_storage.py @@ -101,7 +101,6 @@ def set_attr(h5obj: Any, name: str, val: Any) -> None: except TypeError: newval = str(val) h5obj.attrs[name] = h5ify(newval) - print(f"{name} set as string") def add_cur_time_attr(h5obj: Any, name: str = 'creation', @@ -154,12 +153,12 @@ def datadict_to_hdf5(datadict: DataDict, - `AppendMode.all` : append all data in datadict to file data sets :param swmr_mode: use HDF5 SWMR mode on the file when appending. """ - - if len(basepath) > len(DATAFILEXT) and \ - basepath[-len(DATAFILEXT):] == DATAFILEXT: + ext = f".{DATAFILEXT}" + if len(basepath) > len(ext) and \ + basepath[-len(ext):] == ext: filepath = basepath else: - filepath = basepath + DATAFILEXT + filepath = basepath + ext if not os.path.exists(filepath): init_path(filepath) @@ -297,12 +296,12 @@ def datadict_from_hdf5(basepath: str, :param swmr_mode: if `True`, open HDF5 file in SWMR mode. :return: validated DataDict. """ - - if len(basepath) > len(DATAFILEXT) and \ - basepath[-len(DATAFILEXT):] == DATAFILEXT: + ext = f".{DATAFILEXT}" + if len(basepath) > len(ext) and \ + basepath[-len(ext):] == ext: filepath = basepath else: - filepath = basepath + DATAFILEXT + filepath = basepath + ext if not os.path.exists(filepath): raise ValueError("Specified file does not exist.") @@ -526,8 +525,9 @@ class DDH5Writer(object): # TODO: need an operation mode for not keeping data in memory. # TODO: a mode for working with pre-allocated data - def __init__(self, basedir: str, + def __init__(self, datadict: DataDict, + basedir: str = '.', groupname: str = 'data', name: Optional[str] = None, filename: str = 'data', From 60f603d7047d067678759a780a499c6aa0f31bc7 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Wed, 18 Aug 2021 21:58:47 -0500 Subject: [PATCH 62/94] better name for the Figure class. --- plottr/plot/pyqtgraph/autoplot.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index ed9c8495..57798cdc 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -27,7 +27,7 @@ logger.setLevel(logging.INFO) -class _FigureMakerWidget(QtWidgets.QWidget): +class FigureWidget(QtWidgets.QWidget): """Widget that contains all plots generated by :class:`.FigureMaker`. Widget has a vertical layout, and plots can be added in a single column. @@ -76,7 +76,7 @@ class FigureMaker(BaseFM): # TODO: make scrollable when many figures (set min size)? # TODO: check for valid plot data - def __init__(self, widget: Optional[_FigureMakerWidget] = None, + def __init__(self, widget: Optional[FigureWidget] = None, clearWidget: bool = True, parentWidget: Optional[QtWidgets.QWidget] = None): """Constructor for :class:`.FigureMaker`. @@ -93,11 +93,11 @@ def __init__(self, widget: Optional[_FigureMakerWidget] = None, """ super().__init__() - self.widget: _FigureMakerWidget + self.widget: FigureWidget self.clearWidget = clearWidget if widget is None: - self.widget = _FigureMakerWidget(parent=parentWidget) + self.widget = FigureWidget(parent=parentWidget) else: self.widget = widget @@ -222,7 +222,7 @@ def __init__(self, parent: Optional[PlotWidgetContainer]) -> None: """ super().__init__(parent=parent) - self.fmWidget: Optional[_FigureMakerWidget] = None + self.fmWidget: Optional[FigureWidget] = None self.figConfig: Optional[FigureConfigToolBar] = None self.figOptions: FigureOptions = FigureOptions() From 3de55660b8562aba5831fdcf05ca87de7ed89857 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Wed, 18 Aug 2021 21:59:26 -0500 Subject: [PATCH 63/94] init plot item in the base class. --- plottr/plot/pyqtgraph/plots.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plottr/plot/pyqtgraph/plots.py b/plottr/plot/pyqtgraph/plots.py index 22eebafa..6b409fde 100644 --- a/plottr/plot/pyqtgraph/plots.py +++ b/plottr/plot/pyqtgraph/plots.py @@ -21,9 +21,6 @@ class PlotBase(QtWidgets.QWidget): This base class should be inherited to use. """ - #: ``pyqtgraph`` plot item - plot: pg.PlotItem - def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: super().__init__(parent) @@ -37,6 +34,9 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: self.setLayout(layout) layout.addWidget(self.graphicsLayout) + #: ``pyqtgraph`` plot item + self.plot: pg.PlotItem = self.graphicsLayout.addPlot() + def clearPlot(self) -> None: """Clear all plot contents (but do not delete plot elements, like axis spines, insets, etc). @@ -50,7 +50,6 @@ class Plot(PlotBase): def __init__(self, parent: Optional[QtWidgets.QWidget] = None): super().__init__(parent) - self.plot: pg.PlotItem = self.graphicsLayout.addPlot() legend = self.plot.addLegend(offset=(5, 5), pen='#999', brush=(255, 255, 255, 150)) legend.layout.setContentsMargins(0, 0, 0, 0) @@ -74,8 +73,6 @@ class PlotWithColorbar(PlotBase): def __init__(self, parent: Optional[QtWidgets.QWidget] = None) -> None: super().__init__(parent) - self.plot: pg.PlotItem = self.graphicsLayout.addPlot() - cmap = pg.colormap.get('viridis', source='matplotlib') self.colorbar: pg.ColorBarItem = pg.ColorBarItem(interactive=True, values=(0, 1), cmap=cmap, width=15) From 445e7bc8f654e4cb2b4899095d3b1f3993978310 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Wed, 18 Aug 2021 22:04:34 -0500 Subject: [PATCH 64/94] tests for plotting widgets. --- test/prototyping/autoplot testing.ipynb | 109 +++++++++++++++++++ test/prototyping/collapsible widget.ipynb | 123 ++++++++++++++++++++++ test/prototyping/plottrcfg_main.py | 5 + 3 files changed, 237 insertions(+) create mode 100644 test/prototyping/autoplot testing.ipynb create mode 100644 test/prototyping/collapsible widget.ipynb create mode 100644 test/prototyping/plottrcfg_main.py diff --git a/test/prototyping/autoplot testing.ipynb b/test/prototyping/autoplot testing.ipynb new file mode 100644 index 00000000..3e83e2b4 --- /dev/null +++ b/test/prototyping/autoplot testing.ipynb @@ -0,0 +1,109 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "13fd24a7-5a8d-4045-a04f-245702201dce", + "metadata": {}, + "outputs": [], + "source": [ + "%gui qt\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9f21e8b2-a442-40ae-8b3b-6243bdd71fdd", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "ed548577-e45b-4819-9c10-31c474441e06", + "metadata": {}, + "outputs": [], + "source": [ + "from plottr.data.datadict import DataDict\n", + "from plottr.apps.autoplot import autoplot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "700404b6-962f-4ae4-8826-322cf8d6605a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "aa0f36e4-f67e-4a38-bbfb-ab4bf640e54d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = DataDict(\n", + " x = dict(values=np.arange(10)),\n", + " y = dict(values=np.arange(10)**2, axes=['x'])\n", + ")\n", + "data.validate()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "e68d4a50-7929-4ffa-b584-131603c6b395", + "metadata": {}, + "outputs": [], + "source": [ + "fc, win = autoplot(data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53bd54e4-e1f3-4cb1-be29-c79a9ad1e3c1", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:msmt-pyqt5]", + "language": "python", + "name": "conda-env-msmt-pyqt5-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/prototyping/collapsible widget.ipynb b/test/prototyping/collapsible widget.ipynb new file mode 100644 index 00000000..1e0e9cc3 --- /dev/null +++ b/test/prototyping/collapsible widget.ipynb @@ -0,0 +1,123 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "2421b31e-f5a4-41df-8227-0d62e384732c", + "metadata": {}, + "outputs": [], + "source": [ + "%gui qt" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "50b4f0d5-f9e4-49d6-8d91-af3c5c57faac", + "metadata": {}, + "outputs": [], + "source": [ + "from plottr import QtWidgets, QtCore\n", + "from plottr.gui.tools import widgetDialog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8de6986-660e-4497-8f57-ef1354c1eee4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 134, + "id": "2e3e6a8a-2de2-4d97-8c6a-13199b6a78c4", + "metadata": {}, + "outputs": [], + "source": [ + "def setHExpanding(w):\n", + " p = w.sizePolicy()\n", + " p.setHorizontalPolicy(QtWidgets.QSizePolicy.MinimumExpanding)\n", + " p.setHorizontalStretch(1)\n", + " w.setSizePolicy(p)\n", + " \n", + "def setVExpanding(w):\n", + " p = w.sizePolicy()\n", + " p.setVerticalPolicy(QtWidgets.QSizePolicy.MinimumExpanding)\n", + " p.setVerticalStretch(1)\n", + " w.setSizePolicy(p)\n", + "\n", + "class Collapsible(QtWidgets.QWidget):\n", + " \n", + " def __init__(self, widget, title='', parent=None):\n", + " super().__init__(parent=parent)\n", + " \n", + " self.widget = widget\n", + " self.widget.setParent(self)\n", + " setVExpanding(self.widget)\n", + " \n", + " self.expandedTitle = \"[-] \"+title\n", + " self.collapsedTitle = \"[+] \"+title\n", + " \n", + " self.btn = QtWidgets.QPushButton(self.expandedTitle, parent=self)\n", + " self.btn.setStyleSheet(\"background: white; color: black; border: 0px; \")\n", + " self.btn.setFlat(True)\n", + " self.btn.setCheckable(True)\n", + " self.btn.setChecked(True)\n", + " setHExpanding(self.btn)\n", + " self.btn.clicked.connect(self.onButton)\n", + "\n", + " layout = QtWidgets.QVBoxLayout(self)\n", + " layout.setContentsMargins(0, 0, 0, 0)\n", + " layout.setSpacing(2)\n", + " layout.addWidget(self.btn)\n", + " layout.addWidget(self.widget)\n", + " \n", + " def onButton(self): \n", + " if self.btn.isChecked():\n", + " self.widget.setVisible(True)\n", + " self.btn.setText(self.expandedTitle)\n", + " else:\n", + " self.widget.setVisible(False)\n", + " self.btn.setText(self.collapsedTitle)\n", + " \n", + " \n", + "widget = QtWidgets.QLabel(\"Hello, I am some text.\")\n", + "widget.setStyleSheet(\"background: red; color: white\")\n", + "\n", + "collapsible = Collapsible(widget, title='This is a widget')\n", + "d = widgetDialog(collapsible)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "288cc0d2-3501-4584-8312-833fad5bb4f7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:msmt-pyqt5]", + "language": "python", + "name": "conda-env-msmt-pyqt5-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/prototyping/plottrcfg_main.py b/test/prototyping/plottrcfg_main.py new file mode 100644 index 00000000..ba05dd35 --- /dev/null +++ b/test/prototyping/plottrcfg_main.py @@ -0,0 +1,5 @@ +from plottr.plot.pyqtgraph.autoplot import AutoPlot as PGAutoPlot + +config = { + 'default-plotwidget': PGAutoPlot, +} From 674514c200ce401906a9b2fb42dd3dabd82999e9 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 19 Aug 2021 15:14:36 -0500 Subject: [PATCH 65/94] subplots in figure widget are now collapsible. --- plottr/gui/widgets.py | 62 +++++++++++++++++++++++++++++++ plottr/plot/base.py | 5 ++- plottr/plot/pyqtgraph/autoplot.py | 23 ++++++++---- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/plottr/gui/widgets.py b/plottr/gui/widgets.py index 6648f39f..8e438297 100644 --- a/plottr/gui/widgets.py +++ b/plottr/gui/widgets.py @@ -241,3 +241,65 @@ def loadSnapshot(self, snapshotDict : Optional[dict]) -> None: for i in range(2): self.resizeColumnToContents(i) + +def setHExpanding(w: QtWidgets.QWidget) -> None: + """Set the size policy of a widget such that is expands horizontally.""" + p = w.sizePolicy() + p.setHorizontalPolicy(QtWidgets.QSizePolicy.MinimumExpanding) + p.setHorizontalStretch(1) + w.setSizePolicy(p) + + +def setVExpanding(w: QtWidgets.QWidget) -> None: + """Set the size policy of a widget such that is expands vertically.""" + p = w.sizePolicy() + p.setVerticalPolicy(QtWidgets.QSizePolicy.MinimumExpanding) + p.setVerticalStretch(1) + w.setSizePolicy(p) + + +class Collapsible(QtWidgets.QWidget): + """A wrapper that allow collapsing a widget.""" + + def __init__(self, widget: QtWidgets.QWidget, title: str = '', + parent: Optional[QtWidgets.QWidget] = None) -> None: + """Constructor. + + :param widget: the widget we'd like to collapse. + :param title: title of the widget. will appear on the toolbutton that + we use to trigger collapse/expansion. + :param parent: parent widget. + """ + super().__init__(parent=parent) + + self.widget = widget + self.widget.setParent(self) + setVExpanding(self.widget) + + self.expandedTitle = "[-] " + title + self.collapsedTitle = "[+] " + title + + self.btn = QtWidgets.QPushButton(self.expandedTitle, parent=self) + self.btn.setStyleSheet("""background: white; + color: black; + border: 2px solid white; + text-align: left;""") + self.btn.setFlat(True) + self.btn.setCheckable(True) + self.btn.setChecked(True) + setHExpanding(self.btn) + self.btn.clicked.connect(self._onButton) + + layout = QtWidgets.QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.setSpacing(2) + layout.addWidget(self.btn) + layout.addWidget(self.widget) + + def _onButton(self) -> None: + if self.btn.isChecked(): + self.widget.setVisible(True) + self.btn.setText(self.expandedTitle) + else: + self.widget.setVisible(False) + self.btn.setText(self.collapsedTitle) diff --git a/plottr/plot/base.py b/plottr/plot/base.py index 82668b12..3dcc0aca 100644 --- a/plottr/plot/base.py +++ b/plottr/plot/base.py @@ -557,7 +557,10 @@ def addData(self, *data: Union[np.ndarray, np.ma.MaskedArray], """ if self.combineTraces and join is None: - join = self.previousPlotId() + prev = self.previousPlotId() + if prev is not None: + if len(data) == 2 and len(self.plotItems[prev].data) == 2: + join = prev id = _generate_auto_dict_key(self.plotItems) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 57798cdc..76a17a3d 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -18,6 +18,7 @@ from plottr import QtWidgets, QtCore, Signal, Slot, \ config_entry as getcfg from plottr.data.datadict import DataDictBase +from plottr.gui.widgets import Collapsible from .plots import Plot, PlotWithColorbar, PlotBase from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ PlotItem, ComplexRepresentation, determinePlotDataType, \ @@ -34,25 +35,29 @@ class FigureWidget(QtWidgets.QWidget): """ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): - """Constructor for :class:`._FigureMakerWidget`. + """Constructor for :class:`.FigureMakerWidget`. :param parent: parent widget. """ super().__init__(parent=parent) self.subPlots: List[PlotBase] = [] + self._widgets: List[QtWidgets.QWidget] = [] layout = QtWidgets.QVBoxLayout() layout.setContentsMargins(0, 0, 0, 0) layout.setSpacing(0) self.setLayout(layout) - def addPlot(self, plot: PlotBase) -> None: + def addPlot(self, plot: PlotBase, title: str = '') -> None: """Add a :class:`.PlotBase` widget. :param plot: plot widget + :param title: title of the plot """ - self.layout().addWidget(plot) + w = Collapsible(plot, title=title, parent=self) + self.layout().addWidget(w) + self._widgets.append(w) self.subPlots.append(plot) def clearAllPlots(self) -> None: @@ -62,9 +67,11 @@ def clearAllPlots(self) -> None: def deleteAllPlots(self) -> None: """Delete all subplot widgets.""" - for p in self.subPlots: - p.deleteLater() self.subPlots = [] + for w in self._widgets: + self.layout().removeWidget(w) + w.deleteLater() + self._widgets = [] class FigureMaker(BaseFM): @@ -122,12 +129,14 @@ def makeSubPlots(self, nSubPlots: int) -> List[PlotBase]: if self.clearWidget: self.widget.deleteAllPlots() for i in range(nSubPlots): + labels = [v.labels[-1] for v in self.subPlotItems(i).values() if v.labels is not None] + plotTitle = ", ".join(labels) if max(self.dataDimensionsInSubPlot(i).values()) == 1: plot = Plot(self.widget) - self.widget.addPlot(plot) + self.widget.addPlot(plot, title=plotTitle) elif max(self.dataDimensionsInSubPlot(i).values()) == 2: plot = PlotWithColorbar(self.widget) - self.widget.addPlot(plot) + self.widget.addPlot(plot, title=plotTitle) else: self.widget.clearAllPlots() From e0c6e19b8850f140dfdb5d5fbe47284791ca17c0 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 20 Aug 2021 22:09:30 -0500 Subject: [PATCH 66/94] some cleaning up of the GUI. --- plottr/gui/widgets.py | 6 +++-- plottr/plot/pyqtgraph/autoplot.py | 44 ++++++++++++++++++++----------- test/apps/autoplot_app.py | 8 +++--- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/plottr/gui/widgets.py b/plottr/gui/widgets.py index 8e438297..150706bc 100644 --- a/plottr/gui/widgets.py +++ b/plottr/gui/widgets.py @@ -262,7 +262,8 @@ class Collapsible(QtWidgets.QWidget): """A wrapper that allow collapsing a widget.""" def __init__(self, widget: QtWidgets.QWidget, title: str = '', - parent: Optional[QtWidgets.QWidget] = None) -> None: + parent: Optional[QtWidgets.QWidget] = None, + expanding: bool = True) -> None: """Constructor. :param widget: the widget we'd like to collapse. @@ -274,7 +275,8 @@ def __init__(self, widget: QtWidgets.QWidget, title: str = '', self.widget = widget self.widget.setParent(self) - setVExpanding(self.widget) + if expanding: + setVExpanding(self.widget) self.expandedTitle = "[-] " + title self.collapsedTitle = "[+] " + title diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 76a17a3d..77628d33 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -15,7 +15,7 @@ import numpy as np from pyqtgraph import mkPen -from plottr import QtWidgets, QtCore, Signal, Slot, \ +from plottr import QtWidgets, QtGui, QtCore, Signal, Slot, \ config_entry as getcfg from plottr.data.datadict import DataDictBase from plottr.gui.widgets import Collapsible @@ -42,22 +42,29 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): super().__init__(parent=parent) self.subPlots: List[PlotBase] = [] - self._widgets: List[QtWidgets.QWidget] = [] + + self.title = QtWidgets.QLabel(parent=self) + self.title.setAlignment(QtCore.Qt.AlignHCenter) + + self.split = QtWidgets.QSplitter(parent=self) + self.split.setOrientation(QtCore.Qt.Vertical) layout = QtWidgets.QVBoxLayout() layout.setContentsMargins(0, 0, 0, 0) - layout.setSpacing(0) + layout.setSpacing(2) + layout.addWidget(self.title) + layout.addWidget(self.split) self.setLayout(layout) - def addPlot(self, plot: PlotBase, title: str = '') -> None: + self.setTitle('') + + def addPlot(self, plot: PlotBase) -> None: """Add a :class:`.PlotBase` widget. :param plot: plot widget :param title: title of the plot """ - w = Collapsible(plot, title=title, parent=self) - self.layout().addWidget(w) - self._widgets.append(w) + self.split.addWidget(plot) self.subPlots.append(plot) def clearAllPlots(self) -> None: @@ -67,11 +74,16 @@ def clearAllPlots(self) -> None: def deleteAllPlots(self) -> None: """Delete all subplot widgets.""" + for p in self.subPlots: + p.deleteLater() self.subPlots = [] - for w in self._widgets: - self.layout().removeWidget(w) - w.deleteLater() - self._widgets = [] + + def setTitle(self, title: str = '') -> None: + self.title.setText(title) + if len(title.strip()) == 0: + self.title.setVisible(False) + else: + self.title.setVisible(True) class FigureMaker(BaseFM): @@ -129,15 +141,12 @@ def makeSubPlots(self, nSubPlots: int) -> List[PlotBase]: if self.clearWidget: self.widget.deleteAllPlots() for i in range(nSubPlots): - labels = [v.labels[-1] for v in self.subPlotItems(i).values() if v.labels is not None] - plotTitle = ", ".join(labels) if max(self.dataDimensionsInSubPlot(i).values()) == 1: plot = Plot(self.widget) - self.widget.addPlot(plot, title=plotTitle) + self.widget.addPlot(plot) elif max(self.dataDimensionsInSubPlot(i).values()) == 2: plot = PlotWithColorbar(self.widget) - self.widget.addPlot(plot, title=plotTitle) - + self.widget.addPlot(plot) else: self.widget.clearAllPlots() @@ -294,6 +303,9 @@ def _plotData(self, **kwargs: Any) -> None: self.layout().addWidget(self.figConfig) self.figConfig.optionsChanged.connect(self._refreshPlot) + if self.data.has_meta('title'): + self.fmWidget.setTitle(self.data.meta_val('title')) + @Slot() def _refreshPlot(self) -> None: self._plotData() diff --git a/test/apps/autoplot_app.py b/test/apps/autoplot_app.py index b334d8a6..8704957e 100644 --- a/test/apps/autoplot_app.py +++ b/test/apps/autoplot_app.py @@ -78,7 +78,9 @@ def __init__(self, nreps: int = 1, nsets: int = 2, nx: int = 21): def data(self) -> Iterable[DataDictBase]: for i in range(self.nreps): - yield testdata.get_2d_scalar_cos_data(self.nx, self.nx, self.nsets) + data = testdata.get_2d_scalar_cos_data(self.nx, self.nx, self.nsets) + # data.add_meta('title', "A changing image.") + yield data class ImageDataLiveAcquisition(DataSource): @@ -159,8 +161,8 @@ def main(dataSrc): if __name__ == '__main__': # src = LineDataMovie(20, 3, 31) - src = ImageDataMovie(10, 2, 101) + # src = ImageDataMovie(10, 2, 101) # src = ImageDataLiveAcquisition(101, 101, 67) - # src = ComplexImage(21, 21) + src = ComplexImage(21, 21) src.delay = 0.1 main(src) From 477136f09f88a076cb5c49d618244e8ae8dc64ae Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 20 Aug 2021 22:09:48 -0500 Subject: [PATCH 67/94] removed old file. --- test/prototyping/collapsible widget.ipynb | 123 ---------------------- 1 file changed, 123 deletions(-) delete mode 100644 test/prototyping/collapsible widget.ipynb diff --git a/test/prototyping/collapsible widget.ipynb b/test/prototyping/collapsible widget.ipynb deleted file mode 100644 index 1e0e9cc3..00000000 --- a/test/prototyping/collapsible widget.ipynb +++ /dev/null @@ -1,123 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "2421b31e-f5a4-41df-8227-0d62e384732c", - "metadata": {}, - "outputs": [], - "source": [ - "%gui qt" - ] - }, - { - "cell_type": "code", - "execution_count": 70, - "id": "50b4f0d5-f9e4-49d6-8d91-af3c5c57faac", - "metadata": {}, - "outputs": [], - "source": [ - "from plottr import QtWidgets, QtCore\n", - "from plottr.gui.tools import widgetDialog" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f8de6986-660e-4497-8f57-ef1354c1eee4", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 134, - "id": "2e3e6a8a-2de2-4d97-8c6a-13199b6a78c4", - "metadata": {}, - "outputs": [], - "source": [ - "def setHExpanding(w):\n", - " p = w.sizePolicy()\n", - " p.setHorizontalPolicy(QtWidgets.QSizePolicy.MinimumExpanding)\n", - " p.setHorizontalStretch(1)\n", - " w.setSizePolicy(p)\n", - " \n", - "def setVExpanding(w):\n", - " p = w.sizePolicy()\n", - " p.setVerticalPolicy(QtWidgets.QSizePolicy.MinimumExpanding)\n", - " p.setVerticalStretch(1)\n", - " w.setSizePolicy(p)\n", - "\n", - "class Collapsible(QtWidgets.QWidget):\n", - " \n", - " def __init__(self, widget, title='', parent=None):\n", - " super().__init__(parent=parent)\n", - " \n", - " self.widget = widget\n", - " self.widget.setParent(self)\n", - " setVExpanding(self.widget)\n", - " \n", - " self.expandedTitle = \"[-] \"+title\n", - " self.collapsedTitle = \"[+] \"+title\n", - " \n", - " self.btn = QtWidgets.QPushButton(self.expandedTitle, parent=self)\n", - " self.btn.setStyleSheet(\"background: white; color: black; border: 0px; \")\n", - " self.btn.setFlat(True)\n", - " self.btn.setCheckable(True)\n", - " self.btn.setChecked(True)\n", - " setHExpanding(self.btn)\n", - " self.btn.clicked.connect(self.onButton)\n", - "\n", - " layout = QtWidgets.QVBoxLayout(self)\n", - " layout.setContentsMargins(0, 0, 0, 0)\n", - " layout.setSpacing(2)\n", - " layout.addWidget(self.btn)\n", - " layout.addWidget(self.widget)\n", - " \n", - " def onButton(self): \n", - " if self.btn.isChecked():\n", - " self.widget.setVisible(True)\n", - " self.btn.setText(self.expandedTitle)\n", - " else:\n", - " self.widget.setVisible(False)\n", - " self.btn.setText(self.collapsedTitle)\n", - " \n", - " \n", - "widget = QtWidgets.QLabel(\"Hello, I am some text.\")\n", - "widget.setStyleSheet(\"background: red; color: white\")\n", - "\n", - "collapsible = Collapsible(widget, title='This is a widget')\n", - "d = widgetDialog(collapsible)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "288cc0d2-3501-4584-8312-833fad5bb4f7", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python [conda env:msmt-pyqt5]", - "language": "python", - "name": "conda-env-msmt-pyqt5-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From ccf72ec2140391678bf223fc442dae2b95d0c1b1 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 20 Aug 2021 22:10:44 -0500 Subject: [PATCH 68/94] removed unused imports. --- plottr/plot/pyqtgraph/autoplot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 77628d33..8ac4f590 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -15,10 +15,9 @@ import numpy as np from pyqtgraph import mkPen -from plottr import QtWidgets, QtGui, QtCore, Signal, Slot, \ +from plottr import QtWidgets, QtCore, Signal, Slot, \ config_entry as getcfg from plottr.data.datadict import DataDictBase -from plottr.gui.widgets import Collapsible from .plots import Plot, PlotWithColorbar, PlotBase from ..base import AutoFigureMaker as BaseFM, PlotDataType, \ PlotItem, ComplexRepresentation, determinePlotDataType, \ From 7c534434f47be9195e7e453d8c83a4b586f477d2 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 2 Sep 2021 20:14:49 -0500 Subject: [PATCH 69/94] run each plotting window in a separate process. --- plottr/apps/monitr.py | 50 ++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/plottr/apps/monitr.py b/plottr/apps/monitr.py index 384adc2c..8350fa63 100644 --- a/plottr/apps/monitr.py +++ b/plottr/apps/monitr.py @@ -2,14 +2,14 @@ """ import sys import os -import time import argparse -from typing import List, Optional, Dict +from typing import List, Optional, Dict, Any from functools import partial +import importlib +from multiprocessing import Process from .. import QtCore, QtWidgets, Signal, Slot from ..data.datadict_storage import all_datadicts_from_hdf5 -from ..apps.autoplot import autoplotDDH5 from ..utils.misc import unwrap_optional from .ui.Monitr_UI import Ui_MainWindow @@ -17,6 +17,8 @@ class Monitr(QtWidgets.QMainWindow): + # TODO: keep a list of app processes and monitor them if alive. + #: Signal(object) -- emitted when a valid data file is selected. #: Arguments: #: - a dictionary containing the datadicts found in the file (as top-level groups) @@ -63,6 +65,8 @@ def plotQueuedFiles(self) -> None: if not self.ui.autoPlotNewAction.isChecked(): return + # FIXME: sometimes opening a file will never succeed. + # we should make sure that we don't try reloading it over and over. removeFiles = [] for f in self.newFiles: try: @@ -83,23 +87,13 @@ def plotSelected(self, group: str) -> None: self.plot(unwrap_optional(self.selectedFile), group) def plot(self, filePath: str, group: str) -> None: - fc, win = autoplotDDH5(filePath, group) - plotId = time.time() - while plotId in self.plotDialogs: - plotId += 1e-6 - self.plotDialogs[plotId] = dict( - flowchart=fc, - window=win, + plotApp = 'plottr.apps.autoplot.autoplotDDH5' + process = launchApp(plotApp, filePath, group) + self.plotDialogs[process.pid] = dict( + process=process, path=filePath, group=group, ) - win.windowClosed.connect(lambda: self.onPlotClose(plotId)) - win.show() - - def onPlotClose(self, plotId: float) -> None: - self.plotDialogs[plotId]['flowchart'].deleteLater() - self.plotDialogs[plotId]['window'].deleteLater() - self.plotDialogs.pop(plotId, None) def script() -> int: @@ -119,3 +113,25 @@ def script() -> int: win = Monitr(path, args.refresh_interval) win.show() return app.exec_() + + +def launchApp(appPath: str, filepath: str, group: str, **kwargs: Any) -> Process: + p = Process(target=_runAppStandalone, + args=(appPath, filepath, group), + kwargs=kwargs) + p.start() + p.join(timeout=0) + return p + + +def _runAppStandalone(appPath, filepath, group, **kwargs): + sep = appPath.split('.') + modName = '.'.join(sep[:-1]) + funName = sep[-1] + mod = importlib.import_module(modName) + fun = getattr(mod, funName) + + app = QtWidgets.QApplication([]) + fc, win = fun(filepath, group, **kwargs) + win.show() + return app.exec_() From 561d524e9eb3dddd98861b482a40b6b59d3c88bd Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 3 Sep 2021 10:20:48 -0500 Subject: [PATCH 70/94] - ddh5 loader is now threaded - some mypy fixes. --- plottr/apps/autoplot.py | 24 +++++++---- plottr/apps/monitr.py | 17 ++++---- plottr/data/datadict_storage.py | 73 ++++++++++++++++++++++++++------- 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/plottr/apps/autoplot.py b/plottr/apps/autoplot.py index e920fc07..a3d0ee01 100644 --- a/plottr/apps/autoplot.py +++ b/plottr/apps/autoplot.py @@ -140,10 +140,9 @@ def __init__(self, fc: Flowchart, **kwargs) self.fc = fc + self.loaderNode: Optional[Node] = None if loaderName is not None: self.loaderNode = fc.nodes()[loaderName] - else: - self.loaderNode = None # a flag we use to set reasonable defaults when the first data # is processed @@ -176,6 +175,10 @@ def __init__(self, fc: Flowchart, else: self.monitorToolBar = None + # set some sane defaults any time the data is significantly altered. + if self.loaderNode is not None: + self.loaderNode.dataStructureChanged.connect(self.onChangedLoaderData) + def setMonitorInterval(self, val: float) -> None: if self.monitorToolBar is not None: self.monitorToolBar.setMonitorInterval(val) @@ -196,6 +199,13 @@ def showTime(self) -> None: tstamp = time.strftime("%Y-%m-%d %H:%M:%S") self.status.showMessage(f"loaded: {tstamp}") + @Slot() + def onChangedLoaderData(self) -> None: + assert self.loaderNode is not None + data = self.loaderNode.outputValues()['dataOut'] + if data is not None: + self.setDefaults(self.loaderNode.outputValues()['dataOut']) + @Slot() def refreshData(self) -> None: """ @@ -207,7 +217,7 @@ def refreshData(self) -> None: self.showTime() if not self._initialized and self.loaderNode.nLoadedRecords > 0: - self.setDefaults(self.loaderNode.outputValues()['dataOut']) + self.onChangedLoaderData() self._initialized = True def setInput(self, data: DataDictBase, resetDefaults: bool = True) -> None: @@ -266,10 +276,10 @@ def __init__(self, fc: Flowchart, pathAndId = path, pathAndId[1] self.setWindowTitle(windowTitle) - if pathAndId is not None: + if pathAndId is not None and self.loaderNode is not None: self.loaderNode.pathAndId = pathAndId - if self.loaderNode.nLoadedRecords > 0: + if self.loaderNode is not None and self.loaderNode.nLoadedRecords > 0: self.setDefaults(self.loaderNode.outputValues()['dataOut']) self._initialized = True @@ -342,13 +352,13 @@ def autoplotDDH5(filepath: str = '', groupname: str = 'data') \ win = AutoPlotMainWindow(fc, loaderName='Data loader', widgetOptions=widgetOptions, monitor=True, - monitorInterval=2.0) + monitorInterval=5.0) win.show() fc.nodes()['Data loader'].filepath = filepath fc.nodes()['Data loader'].groupname = groupname win.refreshData() - win.setMonitorInterval(2.0) + win.setMonitorInterval(5.0) return fc, win diff --git a/plottr/apps/monitr.py b/plottr/apps/monitr.py index 8350fa63..ec2a1fbd 100644 --- a/plottr/apps/monitr.py +++ b/plottr/apps/monitr.py @@ -32,7 +32,7 @@ def __init__(self, monitorPath: str = '.', self.ui = Ui_MainWindow() self.ui.setupUi(self) - self.plotDialogs: Dict[float, dict] = {} + self.plotDialogs: Dict[int, dict] = {} self.selectedFile: Optional[str] = None self.newFiles: List[str] = [] @@ -89,17 +89,18 @@ def plotSelected(self, group: str) -> None: def plot(self, filePath: str, group: str) -> None: plotApp = 'plottr.apps.autoplot.autoplotDDH5' process = launchApp(plotApp, filePath, group) - self.plotDialogs[process.pid] = dict( - process=process, - path=filePath, - group=group, - ) + if process.pid is not None: + self.plotDialogs[process.pid] = dict( + process=process, + path=filePath, + group=group, + ) def script() -> int: parser = argparse.ArgumentParser(description='Monitr main application') parser.add_argument("path", help="path to monitor for data", default=None) - parser.add_argument("-r", "--refresh_interval", default=2, + parser.add_argument("-r", "--refresh_interval", default=2, type=float, help="interval at which to look for changes in the " "monitored path (in seconds)") args = parser.parse_args() @@ -124,7 +125,7 @@ def launchApp(appPath: str, filepath: str, group: str, **kwargs: Any) -> Process return p -def _runAppStandalone(appPath, filepath, group, **kwargs): +def _runAppStandalone(appPath: str, filepath: str, group: str, **kwargs: Any) -> Any: sep = appPath.split('.') modName = '.'.join(sep[:-1]) funName = sep[-1] diff --git a/plottr/data/datadict_storage.py b/plottr/data/datadict_storage.py index d819efc9..efb1d2ae 100644 --- a/plottr/data/datadict_storage.py +++ b/plottr/data/datadict_storage.py @@ -25,7 +25,7 @@ import numpy as np import h5py -from plottr import QtGui, Signal, Slot, QtWidgets +from plottr import QtGui, Signal, Slot, QtWidgets, QtCore from ..node import ( Node, NodeWidget, updateOption, updateGuiFromNode, @@ -432,8 +432,11 @@ class DDH5Loader(Node): nodeName = 'DDH5Loader' uiClass = DDH5LoaderWidget useUi = True - nRetries = 5 - retryDelay = 0.01 + + # nRetries = 5 + # retryDelay = 0.01 + + setProcessOptions = Signal(str, str) def __init__(self, name: str): self._filepath: Optional[str] = None @@ -443,6 +446,14 @@ def __init__(self, name: str): self.groupname = 'data' # type: ignore[misc] self.nLoadedRecords = 0 + self.loadingThread = QtCore.QThread() + self.loadingWorker = _Loader(self.filepath, self.groupname) + self.loadingWorker.moveToThread(self.loadingThread) + self.loadingThread.started.connect(self.loadingWorker.loadData) + self.loadingWorker.dataLoaded.connect(self.onThreadComplete) + self.loadingWorker.dataLoaded.connect(lambda x: self.loadingThread.quit()) + self.setProcessOptions.connect(self.loadingWorker.setPathAndGroup) + @property def filepath(self) -> Optional[str]: return self._filepath @@ -464,21 +475,24 @@ def groupname(self, val: str) -> None: # Data processing # def process(self, dataIn: Optional[DataDictBase] = None) -> Optional[Dict[str, Any]]: - # TODO: maybe needs an optional way to read only new data from file? + + # TODO: maybe needs an optional way to read only new data from file? -- can make that an option # TODO: implement a threaded version. + # this is the flow when process is called due to some trigger if self._filepath is None or self._groupname is None: return None if not os.path.exists(self._filepath): return None - try: - data = datadict_from_hdf5(self._filepath, - groupname=self.groupname, - n_retries=self.nRetries, - retry_delay=self.retryDelay) - except OSError: - # TODO needs logging + if not self.loadingThread.isRunning(): + self.loadingWorker.setPathAndGroup(self.filepath, self.groupname) + self.loadingThread.start() + return None + + @Slot(object) + def onThreadComplete(self, data: Optional[DataDict]) -> None: + if data is None: return None title = f"{self.filepath}" @@ -486,11 +500,42 @@ def process(self, dataIn: Optional[DataDictBase] = None) -> Optional[Dict[str, A nrecords = data.nrecords() assert nrecords is not None self.nLoadedRecords = nrecords + self.setOutput(dataOut=data) - if super().process(dataIn=data) is None: - return None + # this makes sure that we analyze the data and emit signals for changes + super().process(dataIn=data) + + +class _Loader(QtCore.QObject): + + nRetries = 5 + retryDelay = 0.01 + + dataLoaded = Signal(object) + + def __init__(self, filepath: Optional[str], groupname: Optional[str]) -> None: + super().__init__() + self.filepath = filepath + self.groupname = groupname + + def setPathAndGroup(self, filepath: Optional[str], groupname: Optional[str]) -> None: + self.filepath = filepath + self.groupname = groupname + + def loadData(self) -> bool: + if self.filepath is None or self.groupname is None: + self.dataLoaded.emit(None) + return True - return dict(dataOut=data) + try: + data = datadict_from_hdf5(self.filepath, + groupname=self.groupname, + n_retries=self.nRetries, + retry_delay=self.retryDelay) + self.dataLoaded.emit(data) + except OSError: + self.dataLoaded.emit(None) + return True class DDH5Writer(object): From e913a9f62f0a64f1aa8f22ef263348c09314f73c Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Fri, 3 Sep 2021 17:46:13 -0500 Subject: [PATCH 71/94] fixing a performance issue. --- plottr/plot/pyqtgraph/autoplot.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 8ac4f590..aadd23a5 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -207,13 +207,16 @@ def _1dPlot(self, plotItem: PlotItem) -> None: if plotItem.plotDataType == PlotDataType.line1d: name = plotItem.labels[-1] if isinstance(plotItem.labels, list) else '' return subPlot.plot.plot(x.flatten(), y.flatten(), name=name, - pen=mkPen(color, width=2), symbol=symbol, symbolBrush=color, - symbolPen=None, symbolSize=symbolSize) + pen=mkPen(color), + symbol=symbol, symbolBrush=color, + symbolPen=None, symbolSize=symbolSize + ) else: name = plotItem.labels[-1] if isinstance(plotItem.labels, list) else '' return subPlot.plot.plot(x.flatten(), y.flatten(), name=name, pen=None, symbol=symbol, symbolBrush=color, - symbolPen=None, symbolSize=symbolSize) + symbolPen=None, symbolSize=symbolSize + ) def _colorPlot(self, plotItem: PlotItem) -> None: subPlot = self.subPlotFromId(plotItem.subPlot) From a196557185c8859de614a8a597036088ed9e2ef4 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 9 Sep 2021 20:01:13 -0500 Subject: [PATCH 72/94] removed unnecessary debug statement. --- plottr/data/datadict_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plottr/data/datadict_storage.py b/plottr/data/datadict_storage.py index ca75d17e..4d20504b 100644 --- a/plottr/data/datadict_storage.py +++ b/plottr/data/datadict_storage.py @@ -99,7 +99,6 @@ def set_attr(h5obj: Any, name: str, val: Any) -> None: except TypeError: newval = str(val) h5obj.attrs[name] = h5ify(newval) - print(f"{name} set as string") def add_cur_time_attr(h5obj: Any, name: str = 'creation', From c47689773bb1f0c3d15b5986e21bc8d3004d814c Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 9 Sep 2021 20:06:51 -0500 Subject: [PATCH 73/94] Update plottr/utils/misc.py only compare lowercase for convenience. Co-authored-by: Mikhail Astafev --- plottr/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/utils/misc.py b/plottr/utils/misc.py index 48d9e21d..436e83c7 100644 --- a/plottr/utils/misc.py +++ b/plottr/utils/misc.py @@ -109,6 +109,6 @@ def __init__(self, label: str) -> None: def fromLabel(cls, label: str) -> Optional["LabeledOptions"]: """Find enum element from label.""" for k in cls: - if k.label == label: + if k.label.lower() == label.lower(): return k return None From efdec91abdc330655a0bf952ff360a0e9b4c0dd0 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 9 Sep 2021 20:18:59 -0500 Subject: [PATCH 74/94] fixed perfomance issue. --- plottr/plot/pyqtgraph/autoplot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/plot/pyqtgraph/autoplot.py b/plottr/plot/pyqtgraph/autoplot.py index 8ac4f590..83c78003 100644 --- a/plottr/plot/pyqtgraph/autoplot.py +++ b/plottr/plot/pyqtgraph/autoplot.py @@ -207,7 +207,7 @@ def _1dPlot(self, plotItem: PlotItem) -> None: if plotItem.plotDataType == PlotDataType.line1d: name = plotItem.labels[-1] if isinstance(plotItem.labels, list) else '' return subPlot.plot.plot(x.flatten(), y.flatten(), name=name, - pen=mkPen(color, width=2), symbol=symbol, symbolBrush=color, + pen=mkPen(color, width=1), symbol=symbol, symbolBrush=color, symbolPen=None, symbolSize=symbolSize) else: name = plotItem.labels[-1] if isinstance(plotItem.labels, list) else '' From 0ffe84ec2d1ade90830e336b83494f9becda3e77 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 9 Sep 2021 20:19:33 -0500 Subject: [PATCH 75/94] removed old debugging statement. --- test/apps/autoplot_app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/apps/autoplot_app.py b/test/apps/autoplot_app.py index 8704957e..32b4d56e 100644 --- a/test/apps/autoplot_app.py +++ b/test/apps/autoplot_app.py @@ -79,7 +79,6 @@ def __init__(self, nreps: int = 1, nsets: int = 2, nx: int = 21): def data(self) -> Iterable[DataDictBase]: for i in range(self.nreps): data = testdata.get_2d_scalar_cos_data(self.nx, self.nx, self.nsets) - # data.add_meta('title', "A changing image.") yield data From 2c4cec75579c1b2f29efb4b869cf17c90f42303d Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Thu, 9 Sep 2021 20:42:14 -0500 Subject: [PATCH 76/94] better signal to connect here. --- plottr/apps/autoplot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/apps/autoplot.py b/plottr/apps/autoplot.py index a3d0ee01..bd1fdd99 100644 --- a/plottr/apps/autoplot.py +++ b/plottr/apps/autoplot.py @@ -177,7 +177,7 @@ def __init__(self, fc: Flowchart, # set some sane defaults any time the data is significantly altered. if self.loaderNode is not None: - self.loaderNode.dataStructureChanged.connect(self.onChangedLoaderData) + self.loaderNode.dataFieldsChanged.connect(self.onChangedLoaderData) def setMonitorInterval(self, val: float) -> None: if self.monitorToolBar is not None: From e69e3fd5ce5b08cf56d9dc1f3fd28014649b0cde Mon Sep 17 00:00:00 2001 From: pfafflabatuiuc <72898175+pfafflabatuiuc@users.noreply.github.com> Date: Fri, 10 Sep 2021 15:20:11 -0500 Subject: [PATCH 77/94] fix in shape recognition when adding data. --- plottr/data/datadict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 17dfe3f7..1a2b353f 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -185,7 +185,7 @@ def to_records(**data: Any) -> Dict[str, np.ndarray]: is_common = False if is_common: commons.append(n) - nrecs = min(commons) + nrecs = max(commons) for k, v in records.items(): shp = v.shape From 44147fb4b3b77095557c61dba4d6f056a5b893ba Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Tue, 14 Sep 2021 17:23:05 -0500 Subject: [PATCH 78/94] bugfix: incorrect use of array. --- plottr/data/datadict.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plottr/data/datadict.py b/plottr/data/datadict.py index 1a2b353f..b2d87b96 100644 --- a/plottr/data/datadict.py +++ b/plottr/data/datadict.py @@ -988,11 +988,10 @@ def remove_invalid_entries(self) -> 'DataDict': except TypeError: pass - idxs.append(_idxs) + idxs.append(_idxs.astype(int)) if len(idxs) > 0: - remove_idxs = reduce(np.intersect1d, - tuple(np.array(idxs).astype(int))) + remove_idxs = reduce(np.intersect1d, tuple(idxs)) for k, v in ret.data_items(): v['values'] = np.delete(v['values'], remove_idxs, axis=0) From a8c50b2d5dfe93914075c6c823a3f622b4720170 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Tue, 14 Sep 2021 21:00:51 -0500 Subject: [PATCH 79/94] bugfix: with very strange data even harmless looking defaults can be an issue. --- plottr/apps/autoplot.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/plottr/apps/autoplot.py b/plottr/apps/autoplot.py index bd1fdd99..30be48e8 100644 --- a/plottr/apps/autoplot.py +++ b/plottr/apps/autoplot.py @@ -248,9 +248,16 @@ def setDefaults(self, data: DataDictBase) -> None: if len(axes) == 1: drs = {axes[0]: 'x-axis'} - self.fc.nodes()['Data selection'].selectedData = selected - self.fc.nodes()['Grid'].grid = GridOption.guessShape, {} - self.fc.nodes()['Dimension assignment'].dimensionRoles = drs + try: + self.fc.nodes()['Data selection'].selectedData = selected + self.fc.nodes()['Grid'].grid = GridOption.guessShape, {} + self.fc.nodes()['Dimension assignment'].dimensionRoles = drs + # FIXME: this is maybe a bit excessive, but trying to set all the defaults + # like this can result in many types of errors. + # a better approach would be to inspect the data better and make sure + # we can set defaults reliably. + except: + pass unwrap_optional(self.plotWidget).update() From 7ed446afb15f17bfe0beab93b0693efe499dcc99 Mon Sep 17 00:00:00 2001 From: Wolfgang Pfaff Date: Tue, 14 Sep 2021 21:01:20 -0500 Subject: [PATCH 80/94] fix: make sure we don't treat differently ordered lists as different content here. --- plottr/node/node.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/plottr/node/node.py b/plottr/node/node.py index 233ad8b3..2069380d 100644 --- a/plottr/node/node.py +++ b/plottr/node/node.py @@ -262,17 +262,28 @@ def process(self, dataIn: Optional[DataDictBase]=None) -> Optional[Dict[str, Opt _structChanged = False _shapesChanged = False - if daxes != self.dataAxes: + if self.dataAxes is None and daxes is not None: + _fieldsChanged = True + _structChanged = True _axesChanged = True - - if daxes != self.dataAxes or ddeps != self.dataDependents: + elif self.dataAxes is not None and daxes is None: + assert daxes is not None and self.dataAxes is not None + if set(daxes) != set(self.dataAxes): + _axesChanged = True + _fieldsChanged = True + _structChanged = True + + if self.dataDependents is None and ddeps is not None: _fieldsChanged = True + _structChanged = True + else: + assert ddeps is not None and self.dataDependents is not None + if set(ddeps) != set(self.dataDependents): + _fieldsChanged = True + _structChanged = True if dtype != self.dataType: _typeChanged = True - - if dtype != self.dataType or daxes != self.dataAxes \ - or ddeps != self.dataDependents: _structChanged = True if dshapes != self.dataShapes: From 090178974bff1442207de0f6747905eb716e79df Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Sun, 7 Nov 2021 22:16:37 +0900 Subject: [PATCH 81/94] Show tag in RunList --- plottr/apps/inspectr.py | 4 +++- plottr/data/qcodes_dataset.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index bc226cac..f3a7814b 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -129,7 +129,8 @@ def __lt__(self, other: "SortableTreeWidgetItem") -> bool: class RunList(QtWidgets.QTreeWidget): """Shows the list of runs for a given date selection.""" - cols = ['Run ID', 'Experiment', 'Sample', 'Name', 'Started', 'Completed', 'Records', 'GUID'] + cols = ['Run ID', 'Tag', 'Experiment', 'Sample', 'Name', 'Started', 'Completed', 'Records', 'GUID'] + tag_dict = {'': '', 'star': '⭐', 'trash': '🗑️'} runSelected = Signal(int) runActivated = Signal(int) @@ -160,6 +161,7 @@ def copy_to_clipboard(self, position: QtCore.QPoint) -> None: def addRun(self, runId: int, **vals: str) -> None: lst = [str(runId)] + lst.append(self.tag_dict[vals.get('tag', '')]) lst.append(vals.get('experiment', '')) lst.append(vals.get('sample', '')) lst.append(vals.get('name', '')) diff --git a/plottr/data/qcodes_dataset.py b/plottr/data/qcodes_dataset.py index bb23ac91..071c1445 100644 --- a/plottr/data/qcodes_dataset.py +++ b/plottr/data/qcodes_dataset.py @@ -51,6 +51,7 @@ class DependentParameterDict(IndependentParameterDict): class DataSetInfoDict(TypedDict): + tag: str experiment: str sample: str name: str @@ -140,6 +141,7 @@ def get_ds_info(ds: 'DataSet', get_structure: bool = True) -> DataSetInfoDict: structure = None data = DataSetInfoDict( + tag=ds.metadata.get('tag', ''), experiment=ds.exp_name, sample=ds.sample_name, name=ds.name, From bc3d0c7103a4268c7201bec39c7d829007fbe23b Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Sun, 7 Nov 2021 22:18:15 +0900 Subject: [PATCH 82/94] Show tag and other metadata in RunInfo --- plottr/apps/inspectr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index f3a7814b..f36e7f0d 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -531,7 +531,8 @@ def setRunSelection(self, runId: int) -> None: for k, v in structure.items(): v.pop('values') contentInfo = {'Data structure': structure, - 'QCoDeS Snapshot': snap} + 'QCoDeS Snapshot': snap, + 'Metadata': ds.metadata} self._sendInfo.emit(contentInfo) @Slot(int) From 47aa34a905899955b46a05524397329029f4edac Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Sun, 7 Nov 2021 22:22:16 +0900 Subject: [PATCH 83/94] Assign shortcut keys for tagging the selected run --- plottr/apps/inspectr.py | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index f36e7f0d..f01e355a 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -366,6 +366,18 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None, refreshAction.triggered.connect(self.refreshDB) fileMenu.addAction(refreshAction) + # action: star/unstar the selected run + self.starAction = QtWidgets.QAction() + self.starAction.setShortcut('Ctrl+Alt+S') + self.starAction.triggered.connect(self.starSelectedRun) + self.addAction(self.starAction) + + # action: trash/untrash the selected run + self.trashAction = QtWidgets.QAction() + self.trashAction.setShortcut('Ctrl+Alt+T') + self.trashAction.triggered.connect(self.trashSelectedRun) + self.addAction(self.trashAction) + # sizing scaledSize = 640 * rint(self.logicalDpiX() / 96.0) self.resize(scaledSize, scaledSize) @@ -545,6 +557,37 @@ def plotRun(self, runId: int) -> None: } win.showTime() + def setTag(self, item: QtWidgets.QTreeWidgetItem, tag: str): + # set tag in the database + assert self.filepath is not None + runId = int(item.text(0)) + ds = load_dataset_from(self.filepath, runId) + ds.add_metadata('tag', tag) + + # set tag in the GUI + tag_char = self.runList.tag_dict[tag] + item.setText(1, tag_char) + + # refresh the RunInfo widget + self.setRunSelection(runId) + + def tagSelectedRun(self, tag: str): + for item in self.runList.selectedItems(): + current_tag_char = item.text(1) + tag_char = self.runList.tag_dict[tag] + if current_tag_char == tag_char: # if already tagged + self.setTag(item, '') # clear tag + else: # if not tagged + self.setTag(item, tag) # set tag + + @Slot() + def starSelectedRun(self): + self.tagSelectedRun('star') + + @Slot() + def trashSelectedRun(self): + self.tagSelectedRun('trash') + class WindowDict(TypedDict): flowchart: Flowchart From 807bc1694ab392c9a0ca4999e32dd8fe9bd398ef Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Sun, 7 Nov 2021 22:24:04 +0900 Subject: [PATCH 84/94] Enable tagging from the right-click menu --- plottr/apps/inspectr.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index f01e355a..5617cee7 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -145,17 +145,29 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None): self.itemActivated.connect(self.activateRun) self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu) - self.customContextMenuRequested.connect(self.copy_to_clipboard) + self.customContextMenuRequested.connect(self.showContextMenu) @Slot(QtCore.QPoint) - def copy_to_clipboard(self, position: QtCore.QPoint) -> None: + def showContextMenu(self, position: QtCore.QPoint) -> None: + model_index = self.indexAt(position) + item = self.itemFromIndex(model_index) + current_tag_char = item.text(1) + menu = QtWidgets.QMenu() + copy_icon = self.style().standardIcon(QtWidgets.QStyle.SP_DialogSaveButton) copy_action = menu.addAction(copy_icon, "Copy") + + star_action = self.window().starAction + star_action.setText('Star' if current_tag_char != self.tag_dict['star'] else 'Unstar') + menu.addAction(star_action) + + trash_action = self.window().trashAction + trash_action.setText('Trash' if current_tag_char != self.tag_dict['trash'] else 'Untrash') + menu.addAction(trash_action) + action = menu.exec_(self.mapToGlobal(position)) if action == copy_action: - model_index = self.indexAt(position) - item = self.itemFromIndex(model_index) QtWidgets.QApplication.clipboard().setText(item.text( model_index.column())) From 83fa5ca37193756019a103037ef46181c05bf6cd Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Tue, 9 Nov 2021 03:06:17 +0900 Subject: [PATCH 85/94] Implement showOnlyStar and showAlsoTrash --- plottr/apps/inspectr.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index 5617cee7..0d7ea0a1 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -185,14 +185,18 @@ def addRun(self, runId: int, **vals: str) -> None: item = SortableTreeWidgetItem(lst) self.addTopLevelItem(item) - def setRuns(self, selection: Dict[int, Dict[str, str]]) -> None: + def setRuns(self, selection: Dict[int, Dict[str, str]], show_only_star: bool, show_also_trash: bool) -> None: self.clear() # disable sorting before inserting values to avoid performance hit self.setSortingEnabled(False) for runId, record in selection.items(): - self.addRun(runId, **record) + tag = record.get('tag', '') + if show_only_star and tag != 'star': + continue + elif show_also_trash or tag != 'trash': + self.addRun(runId, **record) self.setSortingEnabled(True) @@ -362,6 +366,15 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None, self.autoLaunchPlots.setToolTip(tt) self.toolbar.addWidget(self.autoLaunchPlots) + self.showOnlyStarAction = self.toolbar.addAction('⭐') + self.showOnlyStarAction.setToolTip('Show only starred runs') + self.showOnlyStarAction.setCheckable(True) + self.showOnlyStarAction.triggered.connect(self.updateRunList) + self.showAlsoTrashAction = self.toolbar.addAction('🗑️') + self.showAlsoTrashAction.setToolTip('Show also trashed runs') + self.showAlsoTrashAction.setCheckable(True) + self.showAlsoTrashAction.triggered.connect(self.updateRunList) + # menu bar menu = self.menuBar() fileMenu = menu.addMenu('&File') @@ -526,6 +539,15 @@ def monitorTriggered(self) -> None: logger().debug('Refreshing DB') self.refreshDB() + @Slot() + def updateRunList(self) -> None: + if self.dbdf is None: + return + selection = self.dbdf.loc[self.dbdf['started_date'].isin(self._selected_dates)].sort_index(ascending=False) + show_only_star = self.showOnlyStarAction.isChecked() + show_also_trash = self.showAlsoTrashAction.isChecked() + self.runList.setRuns(selection.to_dict(orient='index'), show_only_star, show_also_trash) + ### handling user selections @Slot(list) def setDateSelection(self, dates: Sequence[str]) -> None: @@ -534,7 +556,9 @@ def setDateSelection(self, dates: Sequence[str]) -> None: selection = self.dbdf.loc[self.dbdf['started_date'].isin(dates)].sort_index(ascending=False) old_dates = self._selected_dates if not all(date in old_dates for date in dates): - self.runList.setRuns(selection.to_dict(orient='index')) + show_only_star = self.showOnlyStarAction.isChecked() + show_also_trash = self.showAlsoTrashAction.isChecked() + self.runList.setRuns(selection.to_dict(orient='index'), show_only_star, show_also_trash) else: self.runList.updateRuns(selection.to_dict(orient='index')) self._selected_dates = tuple(dates) From edaa53556134de22231250ff8d10aa2341894e95 Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Tue, 9 Nov 2021 13:07:57 +0900 Subject: [PATCH 86/94] bugfix: set tag in self.dbdf --- plottr/apps/inspectr.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index 0d7ea0a1..9d2dfa4e 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -600,6 +600,9 @@ def setTag(self, item: QtWidgets.QTreeWidgetItem, tag: str): ds = load_dataset_from(self.filepath, runId) ds.add_metadata('tag', tag) + # set tag in self.dbdf + self.dbdf.at[runId, 'tag'] = tag + # set tag in the GUI tag_char = self.runList.tag_dict[tag] item.setText(1, tag_char) From e1fc6509239fb6107313b01c2f37d5e1166e09d3 Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Wed, 10 Nov 2021 00:50:23 +0900 Subject: [PATCH 87/94] Show metadata before snapshot in RunInfo Co-authored-by: Mikhail Astafev --- plottr/apps/inspectr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index 9d2dfa4e..dadccad0 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -579,8 +579,8 @@ def setRunSelection(self, runId: int) -> None: for k, v in structure.items(): v.pop('values') contentInfo = {'Data structure': structure, - 'QCoDeS Snapshot': snap, - 'Metadata': ds.metadata} + 'Metadata': ds.metadata, + 'QCoDeS Snapshot': snap} self._sendInfo.emit(contentInfo) @Slot(int) From 02cf910fca2981c0a11ee3048a12f8855df2f043 Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Wed, 10 Nov 2021 01:03:56 +0900 Subject: [PATCH 88/94] Rename metadata 'tag' to 'inspectr_tag' --- plottr/apps/inspectr.py | 8 ++++---- plottr/data/qcodes_dataset.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index 9d2dfa4e..7ed3c9c0 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -173,7 +173,7 @@ def showContextMenu(self, position: QtCore.QPoint) -> None: def addRun(self, runId: int, **vals: str) -> None: lst = [str(runId)] - lst.append(self.tag_dict[vals.get('tag', '')]) + lst.append(self.tag_dict[vals.get('inspectr_tag', '')]) lst.append(vals.get('experiment', '')) lst.append(vals.get('sample', '')) lst.append(vals.get('name', '')) @@ -192,7 +192,7 @@ def setRuns(self, selection: Dict[int, Dict[str, str]], show_only_star: bool, sh self.setSortingEnabled(False) for runId, record in selection.items(): - tag = record.get('tag', '') + tag = record.get('inspectr_tag', '') if show_only_star and tag != 'star': continue elif show_also_trash or tag != 'trash': @@ -598,10 +598,10 @@ def setTag(self, item: QtWidgets.QTreeWidgetItem, tag: str): assert self.filepath is not None runId = int(item.text(0)) ds = load_dataset_from(self.filepath, runId) - ds.add_metadata('tag', tag) + ds.add_metadata('inspectr_tag', tag) # set tag in self.dbdf - self.dbdf.at[runId, 'tag'] = tag + self.dbdf.at[runId, 'inspectr_tag'] = tag # set tag in the GUI tag_char = self.runList.tag_dict[tag] diff --git a/plottr/data/qcodes_dataset.py b/plottr/data/qcodes_dataset.py index 071c1445..e8feadbf 100644 --- a/plottr/data/qcodes_dataset.py +++ b/plottr/data/qcodes_dataset.py @@ -51,7 +51,6 @@ class DependentParameterDict(IndependentParameterDict): class DataSetInfoDict(TypedDict): - tag: str experiment: str sample: str name: str @@ -62,6 +61,7 @@ class DataSetInfoDict(TypedDict): structure: Optional[DataSetStructureDict] records: int guid: str + inspectr_tag: str # Tools for extracting information on runs in a database @@ -141,7 +141,6 @@ def get_ds_info(ds: 'DataSet', get_structure: bool = True) -> DataSetInfoDict: structure = None data = DataSetInfoDict( - tag=ds.metadata.get('tag', ''), experiment=ds.exp_name, sample=ds.sample_name, name=ds.name, @@ -151,7 +150,8 @@ def get_ds_info(ds: 'DataSet', get_structure: bool = True) -> DataSetInfoDict: started_time=started_time, structure=structure, records=ds.number_of_results, - guid=ds.guid + guid=ds.guid, + inspectr_tag=ds.metadata.get('inspectr_tag', ''), ) return data From afd311a2fe5b345088ffbe6d3485ad5ff8ac44dd Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Wed, 10 Nov 2021 12:18:35 +0900 Subject: [PATCH 89/94] =?UTF-8?q?Rename=20trash=20(=F0=9F=97=91=EF=B8=8F)?= =?UTF-8?q?=20to=20cross=20(=E2=9D=8C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- plottr/apps/inspectr.py | 44 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index a45d1aa9..096902eb 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -130,7 +130,7 @@ class RunList(QtWidgets.QTreeWidget): """Shows the list of runs for a given date selection.""" cols = ['Run ID', 'Tag', 'Experiment', 'Sample', 'Name', 'Started', 'Completed', 'Records', 'GUID'] - tag_dict = {'': '', 'star': '⭐', 'trash': '🗑️'} + tag_dict = {'': '', 'star': '⭐', 'cross': '❌'} runSelected = Signal(int) runActivated = Signal(int) @@ -162,9 +162,9 @@ def showContextMenu(self, position: QtCore.QPoint) -> None: star_action.setText('Star' if current_tag_char != self.tag_dict['star'] else 'Unstar') menu.addAction(star_action) - trash_action = self.window().trashAction - trash_action.setText('Trash' if current_tag_char != self.tag_dict['trash'] else 'Untrash') - menu.addAction(trash_action) + cross_action = self.window().crossAction + cross_action.setText('Cross' if current_tag_char != self.tag_dict['cross'] else 'Uncross') + menu.addAction(cross_action) action = menu.exec_(self.mapToGlobal(position)) if action == copy_action: @@ -185,7 +185,7 @@ def addRun(self, runId: int, **vals: str) -> None: item = SortableTreeWidgetItem(lst) self.addTopLevelItem(item) - def setRuns(self, selection: Dict[int, Dict[str, str]], show_only_star: bool, show_also_trash: bool) -> None: + def setRuns(self, selection: Dict[int, Dict[str, str]], show_only_star: bool, show_also_cross: bool) -> None: self.clear() # disable sorting before inserting values to avoid performance hit @@ -195,7 +195,7 @@ def setRuns(self, selection: Dict[int, Dict[str, str]], show_only_star: bool, sh tag = record.get('inspectr_tag', '') if show_only_star and tag != 'star': continue - elif show_also_trash or tag != 'trash': + elif show_also_cross or tag != 'cross': self.addRun(runId, **record) self.setSortingEnabled(True) @@ -366,14 +366,14 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None, self.autoLaunchPlots.setToolTip(tt) self.toolbar.addWidget(self.autoLaunchPlots) - self.showOnlyStarAction = self.toolbar.addAction('⭐') + self.showOnlyStarAction = self.toolbar.addAction(RunList.tag_dict['star']) self.showOnlyStarAction.setToolTip('Show only starred runs') self.showOnlyStarAction.setCheckable(True) self.showOnlyStarAction.triggered.connect(self.updateRunList) - self.showAlsoTrashAction = self.toolbar.addAction('🗑️') - self.showAlsoTrashAction.setToolTip('Show also trashed runs') - self.showAlsoTrashAction.setCheckable(True) - self.showAlsoTrashAction.triggered.connect(self.updateRunList) + self.showAlsoCrossAction = self.toolbar.addAction(RunList.tag_dict['cross']) + self.showAlsoCrossAction.setToolTip('Show also crossed runs') + self.showAlsoCrossAction.setCheckable(True) + self.showAlsoCrossAction.triggered.connect(self.updateRunList) # menu bar menu = self.menuBar() @@ -397,11 +397,11 @@ def __init__(self, parent: Optional[QtWidgets.QWidget] = None, self.starAction.triggered.connect(self.starSelectedRun) self.addAction(self.starAction) - # action: trash/untrash the selected run - self.trashAction = QtWidgets.QAction() - self.trashAction.setShortcut('Ctrl+Alt+T') - self.trashAction.triggered.connect(self.trashSelectedRun) - self.addAction(self.trashAction) + # action: cross/uncross the selected run + self.crossAction = QtWidgets.QAction() + self.crossAction.setShortcut('Ctrl+Alt+X') + self.crossAction.triggered.connect(self.crossSelectedRun) + self.addAction(self.crossAction) # sizing scaledSize = 640 * rint(self.logicalDpiX() / 96.0) @@ -545,8 +545,8 @@ def updateRunList(self) -> None: return selection = self.dbdf.loc[self.dbdf['started_date'].isin(self._selected_dates)].sort_index(ascending=False) show_only_star = self.showOnlyStarAction.isChecked() - show_also_trash = self.showAlsoTrashAction.isChecked() - self.runList.setRuns(selection.to_dict(orient='index'), show_only_star, show_also_trash) + show_also_cross = self.showAlsoCrossAction.isChecked() + self.runList.setRuns(selection.to_dict(orient='index'), show_only_star, show_also_cross) ### handling user selections @Slot(list) @@ -557,8 +557,8 @@ def setDateSelection(self, dates: Sequence[str]) -> None: old_dates = self._selected_dates if not all(date in old_dates for date in dates): show_only_star = self.showOnlyStarAction.isChecked() - show_also_trash = self.showAlsoTrashAction.isChecked() - self.runList.setRuns(selection.to_dict(orient='index'), show_only_star, show_also_trash) + show_also_cross = self.showAlsoCrossAction.isChecked() + self.runList.setRuns(selection.to_dict(orient='index'), show_only_star, show_also_cross) else: self.runList.updateRuns(selection.to_dict(orient='index')) self._selected_dates = tuple(dates) @@ -624,8 +624,8 @@ def starSelectedRun(self): self.tagSelectedRun('star') @Slot() - def trashSelectedRun(self): - self.tagSelectedRun('trash') + def crossSelectedRun(self): + self.tagSelectedRun('cross') class WindowDict(TypedDict): From 2e9084243f08b791114b4e2601d777db1fefc7d7 Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Wed, 10 Nov 2021 12:32:47 +0900 Subject: [PATCH 90/94] Bugfix: handling of tags not in tag_dict --- plottr/apps/inspectr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index 096902eb..9cefb82d 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -173,7 +173,8 @@ def showContextMenu(self, position: QtCore.QPoint) -> None: def addRun(self, runId: int, **vals: str) -> None: lst = [str(runId)] - lst.append(self.tag_dict[vals.get('inspectr_tag', '')]) + tag = vals.get('inspectr_tag', '') + lst.append(self.tag_dict.get(tag, tag)) # if the tag is not in tag_dict, display in text lst.append(vals.get('experiment', '')) lst.append(vals.get('sample', '')) lst.append(vals.get('name', '')) From fdc3ebd5b937471565c1782a4df103d1a6c5fcfd Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Wed, 10 Nov 2021 12:38:37 +0900 Subject: [PATCH 91/94] Bugfix: behavior when show_only_star and show_also_cross are both enabled --- plottr/apps/inspectr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index 9cefb82d..c6536e5a 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -194,7 +194,7 @@ def setRuns(self, selection: Dict[int, Dict[str, str]], show_only_star: bool, sh for runId, record in selection.items(): tag = record.get('inspectr_tag', '') - if show_only_star and tag != 'star': + if show_only_star and tag == '': continue elif show_also_cross or tag != 'cross': self.addRun(runId, **record) From 3644174906c2023519c8d7f6e0f355d5d828f0ae Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Thu, 11 Nov 2021 10:35:10 +0900 Subject: [PATCH 92/94] Address Mypy errors --- plottr/apps/inspectr.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plottr/apps/inspectr.py b/plottr/apps/inspectr.py index c6536e5a..f9becd0e 100644 --- a/plottr/apps/inspectr.py +++ b/plottr/apps/inspectr.py @@ -594,7 +594,7 @@ def plotRun(self, runId: int) -> None: } win.showTime() - def setTag(self, item: QtWidgets.QTreeWidgetItem, tag: str): + def setTag(self, item: QtWidgets.QTreeWidgetItem, tag: str) -> None: # set tag in the database assert self.filepath is not None runId = int(item.text(0)) @@ -602,6 +602,7 @@ def setTag(self, item: QtWidgets.QTreeWidgetItem, tag: str): ds.add_metadata('inspectr_tag', tag) # set tag in self.dbdf + assert self.dbdf is not None self.dbdf.at[runId, 'inspectr_tag'] = tag # set tag in the GUI @@ -611,7 +612,7 @@ def setTag(self, item: QtWidgets.QTreeWidgetItem, tag: str): # refresh the RunInfo widget self.setRunSelection(runId) - def tagSelectedRun(self, tag: str): + def tagSelectedRun(self, tag: str) -> None: for item in self.runList.selectedItems(): current_tag_char = item.text(1) tag_char = self.runList.tag_dict[tag] @@ -621,11 +622,11 @@ def tagSelectedRun(self, tag: str): self.setTag(item, tag) # set tag @Slot() - def starSelectedRun(self): + def starSelectedRun(self) -> None: self.tagSelectedRun('star') @Slot() - def crossSelectedRun(self): + def crossSelectedRun(self) -> None: self.tagSelectedRun('cross') From 81776e00bf8a3080959a50ae77a07a3aef082402 Mon Sep 17 00:00:00 2001 From: yoshi74ls181 Date: Thu, 11 Nov 2021 12:18:57 +0900 Subject: [PATCH 93/94] Address pytest failure --- test/pytest/test_qcodes_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pytest/test_qcodes_data.py b/test/pytest/test_qcodes_data.py index 18e6c3ba..2ffdaeb9 100644 --- a/test/pytest/test_qcodes_data.py +++ b/test/pytest/test_qcodes_data.py @@ -215,7 +215,8 @@ def test_get_ds_info(experiment): 'name': 'results', 'structure': None, 'records': 0, - 'guid': dataset.guid + 'guid': dataset.guid, + 'inspectr_tag': '' } ds_info = get_ds_info(dataset, get_structure=False) From d26a8833c84cc55253222a4313339f36e7737679 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Thu, 11 Nov 2021 12:19:55 +0100 Subject: [PATCH 94/94] Changelog for 2021-11-11 v0.8.0 --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index f72a3b69..653b385b 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,31 @@ You might want to install freshly if you still use the old version. # Recent changes: +## 2021-11-11 + +### Added + +- Inspectr: tag a run with a star (⭐) or cross (❌) icon, filter by those, + also show dataset metadata next to parameters and snapshot (#229) +- Improvements to monitr: more stability in adding data to ddh5, better + performance by making data loading multithreaded and running reach plot + window in a separate process (#219) +- Added pyqtgraph backend for plotting that can be used instead of matplotlib + (Example for how to select can be found in test/apps/autoplot_app.py) (#215, #218) + +### Fixed + +- Fix/invaliddata: small fixes when data contains a lot of invalid entries (#226) +- Fix in shape recognition when adding data (#220) + +### Behind the scenes + +- Add minimal versions to dependencies (#201) +- Make the .gitignore proper (#73) +- add dependabot (#208) +- Fix typechecking with mypy 0.9xx (#207) +- clarify install instructions wrt qt and mention conda forge (#202) + ## 2021-06-08 ### Added