From 21e0361f72233ff78d881ac83f73733c7be8e75e Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 18 Jan 2023 10:02:09 -0500 Subject: [PATCH] add SYNOP support/pipeline (#304) --- .github/workflows/tests-docker.yml | 9 ++ Dockerfile | 5 +- docs/source/conf.py | 2 +- docs/source/reference/how-wis2box-works.rst | 3 +- docs/source/reference/quickstart.rst | 3 + .../reference/running/data-mappings.rst | 2 +- .../running/data-pipeline-plugins.rst | 10 +- docs/source/reference/running/index.rst | 4 +- .../rou-synoptic-weather-observations.yml | 60 ++++++++++++ tests/data/metadata/station/station_list.csv | 23 +++++ ...1200CCA_C_EDZW_20230117174401_51649529.txt | 5 + ...1200CCB_C_EDZW_20230118094300_52396633.txt | 5 + ...K171200_C_EDZW_20230117120502_51362175.txt | 72 ++++++++++++++ ...K171200_C_EDZW_20230117125200_51396856.txt | 72 ++++++++++++++ ...1800CCA_C_EDZW_20230117184900_51697747.txt | 5 + ...1800CCB_C_EDZW_20230118055302_52230688.txt | 6 ++ ...K171800_C_EDZW_20230117180502_51662689.txt | 75 +++++++++++++++ ...K171800_C_EDZW_20230117181403_51669400.txt | 53 +++++++++++ ...0000CCA_C_EDZW_20230118004301_51967254.txt | 6 ++ ...K180000_C_EDZW_20230118000502_51936144.txt | 76 +++++++++++++++ ...K180000_C_EDZW_20230118001801_51945941.txt | 56 +++++++++++ ...K180600_C_EDZW_20230118060404_52242453.txt | 77 +++++++++++++++ ...K181200_C_EDZW_20230118120404_52514693.txt | 72 ++++++++++++++ tests/integration/test_workflow.py | 10 +- wis2box/data/base.py | 13 +-- wis2box/data/csv2bufr.py | 2 +- wis2box/data/synop2bufr.py | 95 +++++++++++++++++++ wis2box/handler.py | 4 +- wis2box/resources/data-mappings.yml | 9 ++ 29 files changed, 809 insertions(+), 25 deletions(-) create mode 100644 tests/data/metadata/discovery/rou-synoptic-weather-observations.yml create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171200CCA_C_EDZW_20230117174401_51649529.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171200CCB_C_EDZW_20230118094300_52396633.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117125200_51396856.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171800CCA_C_EDZW_20230117184900_51697747.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171800CCB_C_EDZW_20230118055302_52230688.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117181403_51669400.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK180000CCA_C_EDZW_20230118004301_51967254.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118000502_51936144.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118001801_51945941.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK180600_C_EDZW_20230118060404_52242453.txt create mode 100644 tests/data/observations/romania/A_SMRO01YRBK181200_C_EDZW_20230118120404_52514693.txt create mode 100644 wis2box/data/synop2bufr.py diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 06b44ed1..dbc8f1dc 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -69,6 +69,15 @@ jobs: python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA + - name: add Romania data 🇷🇴 + env: + TOPIC_HIERARCHY: rou.rnimh.data.core.weather.surface-based-observations.synop + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/rou-synoptic-weather-observations.yml + TEST_DATA: /data/wis2box/observations/romania + run: | + python3 wis2box-ctl.py execute wis2box metadata discovery publish $DISCOVERY_METADATA + python3 wis2box-ctl.py execute wis2box data add-collection $DISCOVERY_METADATA + python3 wis2box-ctl.py execute wis2box data ingest -th $TOPIC_HIERARCHY -p $TEST_DATA - name: sync stations 🔄 run: | sleep 15 diff --git a/Dockerfile b/Dockerfile index cd869698..4a3f192b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,9 +42,12 @@ RUN echo 'deb http://deb.debian.org/debian bookworm main' >> /etc/apt/sources.li RUN apt-get update -y \ && apt-get install -y -t bookworm libeccodes-data \ && apt-get install -y ${DEBIAN_PACKAGES} \ - # install wis2box dependencies + # install wis2box data pipeline dependencies && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/master.zip \ && pip3 install --no-cache-dir https://github.com/wmo-im/bufr2geojson/archive/refs/tags/v0.4.1.zip \ + && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/master.zip \ + && pip3 install --no-cache-dir https://github.com/wmo-im/synop2bufr/archive/main.zip \ + # install geopython tooling && pip3 install --no-cache-dir https://github.com/geopython/pygeometa/archive/master.zip \ # install shapely && pip3 install cython pygeos==0.13 \ diff --git a/docs/source/conf.py b/docs/source/conf.py index 57fae31a..5ac4cc77 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -35,7 +35,7 @@ project = 'WIS2 in a box' author = 'World Meteorological Organization (WMO)' license = 'This work is licensed under a Creative Commons Attribution 4.0 International License' # noqa -copyright = '2021-2022, ' + author + ' ' + license +copyright = '2021-2023, ' + author + ' ' + license # The full version, including alpha/beta/rc tags diff --git a/docs/source/reference/how-wis2box-works.rst b/docs/source/reference/how-wis2box-works.rst index 12b57f2b..b53d288f 100644 --- a/docs/source/reference/how-wis2box-works.rst +++ b/docs/source/reference/how-wis2box-works.rst @@ -72,7 +72,7 @@ wis2box is built on free and open source (FOSS) technology. Storage,data and metadata storage,`MinIO`_ `Elasticsearch`_,`S3`_ Internal Message Broker,PubSub,`mosquitto`_, `MQTT`_ Public Message Broker,PubSub,`mosquitto`_, `MQTT`_ - Data Management,data processing and publishing,`ecCodes`_ `csv2bufr`_ `bufr2geojson`_ `OWSLib`_ `pygeometa`_,:abbr:`WCMP (WMO Core Metadata Profile)` :abbr:`WMDR (WIGOS Metadata Record)` + Data Management,data processing and publishing,`ecCodes`_ `csv2bufr`_ `bufr2geojson`_ `synop2bufr`_ `OWSLib`_ `pygeometa`_,:abbr:`WCMP (WMO Core Metadata Profile)` :abbr:`WMDR (WIGOS Metadata Record)` API Application,data discovery and access, `pygeoapi`_,`OGC API`_ Web Application,data discovery and visualization,`Vue.js`_ `Leaflet`_,`OGC API`_ @@ -91,6 +91,7 @@ wis2box is built on free and open source (FOSS) technology. .. _`OGC API`: https://ogcapi.ogc.org .. _`ecCodes`: https://confluence.ecmwf.int/display/ECC .. _`csv2bufr`: https://csv2bufr.readthedocs.io +.. _`synop2bufr`: https://github.com/wmo-im/synop2bufr .. _`bufr2geojson`: https://github.com/wmo-im/bufr2geojson .. _`MinIO`: https://dosc.min.io .. _`S3`: https://en.wikipedia.org/wiki/Object_storage diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst index fdcc96c8..01b9f38c 100644 --- a/docs/source/reference/quickstart.rst +++ b/docs/source/reference/quickstart.rst @@ -43,6 +43,7 @@ Publish test discovery metadata: wis2box metadata discovery publish $WIS2BOX_DATADIR/metadata/discovery/mwi-surface-weather-observations.yml wis2box metadata discovery publish $WIS2BOX_DATADIR/metadata/discovery/ita-surface-weather-observations.yml wis2box metadata discovery publish $WIS2BOX_DATADIR/metadata/discovery/dza-surface-weather-observations.yml + wis2box metadata discovery publish $WIS2BOX_DATADIR/metadata/discovery/rou-synoptic-weather-observations.yml Setup observation collections from discovery metadata: @@ -52,6 +53,7 @@ Setup observation collections from discovery metadata: wis2box data add-collection $WIS2BOX_DATADIR/metadata/discovery/mwi-surface-weather-observations.yml wis2box data add-collection $WIS2BOX_DATADIR/metadata/discovery/ita-surface-weather-observations.yml wis2box data add-collection $WIS2BOX_DATADIR/metadata/discovery/dza-surface-weather-observations.yml + wis2box data add-collection $WIS2BOX_DATADIR/metadata/discovery/rou-synoptic-weather-observations.yml Ingest data using the data ingest command to push data to the ``wis2box-incoming`` bucket: @@ -60,6 +62,7 @@ Ingest data using the data ingest command to push data to the ``wis2box-incoming wis2box data ingest --topic-hierarchy mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/malawi wis2box data ingest --topic-hierarchy ita.roma_met_centre.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/italy wis2box data ingest --topic-hierarchy dza.alger_met_centre.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/algeria + wis2box data ingest --topic-hierarchy rou.rnimh.data.core.weather.surface-based-observations.synop --path $WIS2BOX_DATADIR/observations/romania Publish stations: diff --git a/docs/source/reference/running/data-mappings.rst b/docs/source/reference/running/data-mappings.rst index 1b03891c..e8554dcb 100644 --- a/docs/source/reference/running/data-mappings.rst +++ b/docs/source/reference/running/data-mappings.rst @@ -19,7 +19,7 @@ Each plugin is based on the file extension to be detected and processed, with th - ``notify``: whether the plugin should publish a data notification - ``template``: additional argument allowing a mapping template name to be passed to the plugin - ``file-pattern``: additional argument allowing a file pattern to be passed to the plugin -- ``buckets``: the name(s) of the storage bucket(s) that data should be saved to (See :ref:`configuration` for more informaiton on buckets) +- ``buckets``: the name(s) of the storage bucket(s) that data should be saved to (See :ref:`configuration` for more information on buckets) The default data mapping can be overriden by user-defined data mappings with the following steps: diff --git a/docs/source/reference/running/data-pipeline-plugins.rst b/docs/source/reference/running/data-pipeline-plugins.rst index 4acd7c21..b4b1e003 100644 --- a/docs/source/reference/running/data-pipeline-plugins.rst +++ b/docs/source/reference/running/data-pipeline-plugins.rst @@ -17,7 +17,6 @@ Default pipeline plugins wis2box provides a number of data pipeline plugins which users can be used "out of the box". - ``wis2box.data.csv2bufr.ObservationDataCSV2BUFR`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -33,12 +32,19 @@ This plugin converts BUFR observation data into GeoJSON using ``bufr2geojson``. can be used to filter on incoming data based on a regular expression. Consult the bufr2geojson documentation for more information on configuration and templating. - ``wis2box.data.geojson.ObservationDataGeoJSON`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This plugin is for the purposes of publishing GeoJSON data to the API. +``wis2box.data.synop2bufr.SYNOP2BUFR`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This plugin converts SYNOP ASCII data into BUFR using ``synop2bufr``. A ``file-pattern`` can be used +to filter on incoming data based on a regular expression. Note that the regex must contain two groups +(for year and month), which are used as part of synop2bufr processing. Consult the synop2bufr documentation +for more information. .. _`csv2bufr`: https://csv2bufr.readthedocs.io .. _`bufr2geojson`: https://github.com/wmo-im/bufr2geojson +.. _`synop2bufr`: https://github.com/wmo-im/synop2bufr diff --git a/docs/source/reference/running/index.rst b/docs/source/reference/running/index.rst index cd8b24d0..2ab6a9eb 100644 --- a/docs/source/reference/running/index.rst +++ b/docs/source/reference/running/index.rst @@ -1,7 +1,7 @@ .. _running: -Workflows -========= +Running +======= wis2box workflows can be categorized as design time (interactive) or runtime (automated). diff --git a/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml b/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml new file mode 100644 index 00000000..be556c5d --- /dev/null +++ b/tests/data/metadata/discovery/rou-synoptic-weather-observations.yml @@ -0,0 +1,60 @@ +wis2box: + retention: P30D + topic_hierarchy: rou.rnimh.data.core.weather.surface-based-observations.synop + country: rou + centre_id: rnimh + +mcf: + version: 1.0 + +metadata: + identifier: urn:x-wmo:md:rou:rnimh:synoptic-weather-observations + hierarchylevel: dataset + +identification: + title: Synoptic weather observations from Romania + abstract: Synoptic weather observations from Romania + dates: + creation: 2023-01-18 + keywords: + default: + keywords: + - synoptic weather + - temperature + - observations + wmo: + keywords: + - weatherObservations + keywords_type: theme + vocabulary: + name: WMO Category Code + url: https://github.com/wmo-im/wcmp-codelists/blob/main/codelists/WMO_CategoryCode.csv + extents: + spatial: + - bbox: [20.2201924985,43.6884447292,29.62654341,48.2208812526] + crs: 4326 + temporal: + - begin: 2023-01-17 + end: null + resolution: P6H + url: https://example.org/malawi-surface-weather-observations + wmo_data_policy: core + +contact: + pointOfContact: &contact_poc + organization: Romanian National Meteorological Administration + url: https://www.meteoromania.ro + individualname: Firstname Lastname + positionname: Position Name + phone: +40 21 318 32 40 + fax: +40 21 316 31 43 + address: Building A, Șoseaua București-Ploiești 9 + city: Bucharest + administrativearea: Municipality of Bucharest + postalcode: 013686 + country: Romania + email: you@example.org + hoursofservice: 0700h - 1500h UTC + contactinstructions: email + + distributor: *contact_poc diff --git a/tests/data/metadata/station/station_list.csv b/tests/data/metadata/station/station_list.csv index a5e2d287..ef8dd502 100644 --- a/tests/data/metadata/station/station_list.csv +++ b/tests/data/metadata/station/station_list.csv @@ -55,3 +55,26 @@ EL-BAYADH,0-20000-0-60550,60550,Land (fixed),33.71966,1.09484,1361,Algeria,1 USTICA,0-20000-0-16400,16400,Land (fixed),38.7072222222,13.1772222222,242,Italy,6 GROSSETO,0-20000-0-16206,16206,Land (fixed),42.7480555556,11.0588888889,5.4,Italy,6 BENI OUNIF,0-12-0-08BECCN60577,60577,Land (fixed),32.05138,-1.26527,830,Algeria,1 +OCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.7770616258,23.9404602638,503,Romania,6 +BOTOSANI,0-20000-0-15020,15020,Land (fixed),47.7356532437,26.6455501701,161,Romania,6 +IASI,0-20000-0-15090,15090,Land (fixed),47.163333333,27.6272222222,74.29,Romania,6 +CEAHLAU TOACA,0-20000-0-15108,15108,Land (fixed),46.9775099973,25.9499399749,1897,Romania,6 +CLUJ-NAPOCA,0-20000-0-15120,15120,Land (fixed),46.7777705044,23.5713052939,410,Romania,6 +BACAU,0-20000-0-15150,15150,Land (fixed),46.5577777778,26.8966666667,174,Romania,6 +MIERCUREA CIUC,0-20000-0-15170,15170,Land (fixed),46.3713166568,25.7726166755,661,Romania,6 +ARAD,0-20000-0-15200,15200,Land (fixed),46.1335163958,21.3536215174,116.59,Romania,6 +DEVA,0-20000-0-15230,15230,Land (fixed),45.8649230138,22.898806236,240,Romania,6 +SIBIU,0-20000-0-15260,15260,Land (fixed),45.79018,24.036245,450,Romania,6 +VARFU OMU,0-20000-0-15280,15280,Land (fixed),45.4457927989,25.456690976,2504,Romania,6 +CARANSEBES,0-20000-0-15292,15292,Land (fixed),45.41667,22.22917,241,Romania,6 +GALATI,0-20000-0-15310,15310,Land (fixed),45.4729181384,28.0323010582,69,Romania,6 +TULCEA,0-20000-0-15335,15335,Land (fixed),45.1905064849,28.8241607619,4.36,Romania,6 +RAMNICU VALCEA,0-20000-0-15346,15346,Land (fixed),45.0888211225,24.3628139123,237,Romania,6 +BUZAU,0-20000-0-15350,15350,Land (fixed),45.1326632857,26.8517319231,97,Romania,6 +SULINA,0-20000-0-15360,15360,Land (fixed),45.1623111,29.7268286,12.69,Romania,6 +DROBETA-TURNU SEVERIN,0-20000-0-15410,15410,Land (fixed),44.6264587019,22.6260737132,77,Romania,6 +BUCURESTI BANEASA,0-20000-0-15420,15420,Land (fixed),44.5104330044,26.0781904077,90,Romania,6 +CRAIOVA,0-20000-0-15450,15450,Land (fixed),44.3101404313,23.8669847441,192,Romania,6 +CALARASI,0-20000-0-15460,15460,Land (fixed),44.2057385289,27.3383080718,18.72,Romania,6 +ROSIORII DE VEDE,0-20000-0-15470,15470,Land (fixed),44.1072133362,24.9787400713,102.15,Romania,6 +CONSTANTA,0-20000-0-15480,15480,Land (fixed),44.2138143888,28.6455646839,12.8,Romania,6 diff --git a/tests/data/observations/romania/A_SMRO01YRBK171200CCA_C_EDZW_20230117174401_51649529.txt b/tests/data/observations/romania/A_SMRO01YRBK171200CCA_C_EDZW_20230117174401_51649529.txt new file mode 100644 index 00000000..2063a0c0 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171200CCA_C_EDZW_20230117174401_51649529.txt @@ -0,0 +1,5 @@ +SMRO01 YRBK 171200 CCA +AAXX 17121 +15108 01/92 92514 11028 21028 37901 48315 50001 69941 77174 333 48014 +55300 0//// 20270 3//// 69947 91020 911// 92727 92913 96047= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK171200CCB_C_EDZW_20230118094300_52396633.txt b/tests/data/observations/romania/A_SMRO01YRBK171200CCB_C_EDZW_20230118094300_52396633.txt new file mode 100644 index 00000000..74aa40dd --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171200CCB_C_EDZW_20230118094300_52396633.txt @@ -0,0 +1,5 @@ +SMRO01 YRBK 171200 CCB +AAXX 17121 +15280 01/90 92518 11047 21054 37334 47872 51004 60001 74143 333 49070 +55300 0//// 20000 3//// 60007 91026 911// 92956= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt b/tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt new file mode 100644 index 00000000..286a4972 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117120502_51362175.txt @@ -0,0 +1,72 @@ +SMRO01 YRBK 171200 +AAXX 17121 +15015 01597 71702 10057 20036 39390 42628 50004 60021 78082 87300 333 +4/000 +55304 0//// 20643 3//// 69977 91003 91108= +15020 02597 61303 10104 20040 39783 49976 58007 60001 83570 333 4/000 +55308 +0//// 21085 3//// 60007 91005 91106= +15090 02597 72003 10109 20036 39904 49994 56010 60001 83571 333 4/000 +55301 +00256 20589 30542 60007 91005 91107= +15108 01/92 92514 11028 21028 37901 48315 50001 69941 77174 333 48013 +55300 +0//// 20270 3//// 69947 91020 911// 92727 92913 96047= +15120 02598 61904 10106 20016 39496 49978 50003 60001 83360 333 55309 +00649 +21413 30243 60007 91007 91107= +15150 02597 61604 10091 20036 39782 49991 57013 60001 86800 333 4/000 +55308 +0//// 20986 3//// 60007 91006 91109= +15170 01597 81701 10050 20031 39238 42652 53007 69961 76162 885// 333 +55300 +0//// 20206 3//// 69937 91003 91104= +15200 02598 51707 10111 20062 39840 49980 58006 60001 81140 333 4/000 +55310 +0//// 21428 3//// 60007 91010 91111= +15230 01598 70101 10089 20054 39714 40002 58007 60011 70262 87500 333 +55303 +00347 20712 30640 60007 91002 91102= +15260 05599 21502 10088 20029 39467 49998 50004 60001 7000/ 82800 333 +55309 +0//// 21214 3//// 60007 91003 91104 95000= +15280 01/90 92518 11047 21054 37334 47872 51004 60001 74143 333 49070 +55300 +0//// 20000 3//// 60007 91026 911// 92956= +15292 01597 61505 10074 20048 39713 40004 58010 60021 70182 86300 333 +55302 +0//// 20637 3//// 60007 91008 91108 95090= +15310 02998 71903 10111 20066 39942 40026 55008 60001 82038 333 4/000 +55304 +00448 20862 30571 60007 91007 91110= +15335 02597 51705 10124 20085 30028 40035 57013 60001 83530 333 55309 +0//// +21651 3//// 60007 91008 91110= +15346 01596 73401 10059 20047 39746 40035 58003 60021 76162 85570 333 +4/000 +55300 0//// 20415 3//// 69947 91002 91103= +15350 02997 32204 10103 20057 39906 40025 57020 60001 80002 333 4/000 +55308 +0//// 20913 3//// 60007 91006 91106= +15360 02997 61910 10098 20077 30030 40046 57009 60001 86070 222// +06070 20503 +333 55303 ///// 20758 3//// 60007 91011 91112 92447= +15410 01597 80302 10066 20045 39934 40030 57013 60031 70262 8457/ 333 +4/000 +55301 0//// 20600 3//// 60017 91004 91106= +15420 01597 62101 10127 20087 39912 40021 58011 69911 70262 83570 333 +55305 +0//// 20850 3//// 60007 91003 91103= +15450 01596 70203 10078 20054 39792 40025 50001 69911 71086 84530 333 +55306 +00537 21202 30592 69907 91004 91105= +15460 01997 72106 10175 20069 30002 40026 57014 69911 70282 87070 333 +4/000 +55310 0//// 21460 3//// 60007 91011 91111= +15470 02997 51902 10141 20082 39897 40019 58008 60001 85050 333 55310 +0//// +21469 3//// 60007 91003 91103= +15480 05997 42104 10146 20092 30023 40040 57015 60001 7000/ 82031 +222// 06078 +2//// 333 55309 00621 21275 30948 60007 91010 91112 92427= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117125200_51396856.txt b/tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117125200_51396856.txt new file mode 100644 index 00000000..3731cd12 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171200_C_EDZW_20230117125200_51396856.txt @@ -0,0 +1,72 @@ +SMRO01 YRBK 171200 +AAXX 17121 +15015 01597 71702 10057 20036 39390 42628 50004 60021 78082 87300 333 +4/000 +55304 0//// 20643 3//// 69977 91003 91108= +15020 02597 61303 10104 20040 39783 49976 58007 60001 83570 333 4/000 +55308 +0//// 21085 3//// 60007 91005 91106= +15090 02597 72003 10109 20036 39904 49994 56010 60001 83571 333 4/000 +55301 +00256 20589 30542 60007 91005 91107= +15108 01/92 92514 11028 21028 37901 48315 50001 69941 77174 333 48013 +55300 +0//// 20270 3//// 69947 91020 911// 92727 92913 96047= +15120 02598 61904 10106 20016 39496 49978 50003 60001 83360 333 55309 +00649 +21413 30243 60007 91007 91107= +15150 02597 61604 10091 20036 39782 49991 57013 60001 86800 333 4/000 +55308 +0//// 20986 3//// 60007 91006 91109= +15170 01597 81701 10050 20031 39238 42652 53007 69961 76162 885// 333 +55300 +0//// 20206 3//// 69937 91003 91104= +15200 02598 51707 10111 20062 39840 49980 58006 60001 81140 333 4/000 +55310 +0//// 21428 3//// 60007 91010 91111= +15230 01598 70101 10089 20054 39714 40002 58007 60011 70262 87500 333 +55303 +00347 20712 30640 60007 91002 91102= +15260 05599 21502 10088 20029 39467 49998 50004 60001 7000/ 82800 333 +55309 +0//// 21214 3//// 60007 91003 91104 95000= +15280 01/90 92518 11047 21054 37334 47872 51004 60001 74143 333 49070 +55300 +0//// 20000 3//// 60007 91026 911// 92956= +15292 01597 61505 10074 20048 39713 40004 58010 60021 70182 86300 333 +55302 +0//// 20637 3//// 60007 91008 91108 95090= +15310 02998 71903 10111 20066 39942 40026 55008 60001 82038 333 4/000 +55304 +00448 20862 30571 60007 91007 91110= +15335 02597 51705 10124 20085 30028 40035 57013 60001 83530 333 55309 +0//// +21651 3//// 60007 91008 91110= +15346 01596 73401 10059 20047 39746 40035 58003 60021 76162 85570 333 +4/000 +55300 0//// 20415 3//// 69947 91002 91103= +15350 02997 32204 10103 20057 39906 40025 57020 60001 80002 333 4/000 +55308 +0//// 20913 3//// 60007 91006 91106= +15360 02997 61910 10098 20077 30030 40046 57009 60001 86070 222// +06070 20503 +333 55303 20758 3//// 60007 91011 91112 92447= +15410 01597 80302 10066 20045 39934 40030 57013 60031 70262 8457/ 333 +4/000 +55301 0//// 20600 3//// 60017 91004 91106= +15420 01597 62101 10127 20087 39912 40021 58011 69911 70262 83570 333 +55305 +0//// 20850 3//// 60007 91003 91103= +15450 01596 70203 10078 20054 39792 40025 50001 69911 71086 84530 333 +55306 +00537 21202 30592 69907 91004 91105= +15460 01997 72106 10175 20069 30002 40026 57014 69911 70282 87070 333 +4/000 +55310 0//// 21460 3//// 60007 91011 91111= +15470 02997 51902 10141 20082 39897 40019 58008 60001 85050 333 55310 +0//// +21469 3//// 60007 91003 91103= +15480 05997 42104 10146 20092 30023 40040 57015 60001 7000/ 82031 +222// 06078 +2//// 333 55309 00621 21275 30948 60007 91010 91112 92427= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK171800CCA_C_EDZW_20230117184900_51697747.txt b/tests/data/observations/romania/A_SMRO01YRBK171800CCA_C_EDZW_20230117184900_51697747.txt new file mode 100644 index 00000000..e7f71b09 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171800CCA_C_EDZW_20230117184900_51697747.txt @@ -0,0 +1,5 @@ +SMRO01 YRBK 171800 CCA +AAXX 17181 +15090 02997 12101 10084 20048 39928 40020 51019 60002 81030 333 10122 +20059 31010 4/000 55300 10143 20000 30000 60007 91004 91104= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK171800CCB_C_EDZW_20230118055302_52230688.txt b/tests/data/observations/romania/A_SMRO01YRBK171800CCB_C_EDZW_20230118055302_52230688.txt new file mode 100644 index 00000000..1457998d --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171800CCB_C_EDZW_20230118055302_52230688.txt @@ -0,0 +1,6 @@ +SMRO01 YRBK 171800 CCB +AAXX 17181 +15280 01/90 92012 11041 21047 37349 47889 50004 60002 74143 333 11041 +21059 +3//// 49075 55300 0//// 20000 3//// 60007 91020 911// 92818 92946= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt b/tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt new file mode 100644 index 00000000..4f6048c3 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117180502_51662689.txt @@ -0,0 +1,75 @@ +SMRO01 YRBK 171800 +AAXX 17181 +15015 01598 82700 10039 20026 39397 42633 58006 60022 70282 8657/ 333 +10066 +20029 31010 4/000 55300 0//// 20000 3//// 69917 91002 91104= +15020 02997 01503 10071 20018 39811 40008 51023 60002 333 10113 20024 +31012 +4/000 55300 0//// 20000 3//// 60007 91006 91106= +15090 02997 12101 10084 20048 39928 40020 51019 60002 81030 333 10122 +20059 +3/010 4/000 55300 10143 20000 30000 60007 91004 91104= +15108 01299 22710 11024 21040 37925 48339 52022 69972 73674 81540 333 +11021 +21036 3//// 48014 55300 0//// 20000 3//// 60007 91016 911// 92727 +92813 92911= +15120 02997 71303 10079 20027 39497 49984 58006 60002 87070 333 10117 +20049 +31016 55300 10080 20000 30000 60007 91006 91106= +15150 02997 01902 10058 20046 39811 40024 51021 60002 333 10117 20028 +31013 +4/000 55300 0//// 20000 3//// 60007 91004 91104= +15170 01998 23100 10001 21001 39242 42656 51002 69982 70361 82040 333 +10074 +21016 31005 55300 0//// 20000 3//// 60007 91001 91102= +15200 01598 81007 10092 20066 39784 49924 58038 69912 78081 883// 333 +10117 +20060 31014 55300 0//// 20000 3//// 69917 91011 91111= +15230 02997 71702 10086 20038 39690 49978 58010 60012 87070 333 10122 +20027 +31014 4/000 55300 10110 20000 30000 60007 91004 91105= +15260 05598 30802 10088 20031 39460 49990 58005 60002 7000/ 83500 333 +10109 +20016 31009 55300 0//// 20000 3//// 60007 91004 91105= +15280 01/90 92012 11041 21047 37349 47889 50004 60022 74143 333 11041 +21059 +3//// 49075 55300 0//// 20000 3//// 60007 91020 911// 92818 92946= +15292 02597 81505 10078 20052 39664 49953 58027 60022 883// 333 10112 +20045 +31014 55300 0//// 20000 3//// 60007 91010 91112= +15310 02997 11801 10090 20072 39953 40039 52011 60002 80001 333 10132 +20065 +31015 4/000 55300 10160 20000 30000 60007 91002 91105= +15335 02997 21804 10096 20079 30040 40047 52008 60002 82030 333 10134 +20072 +31016 4/000 55300 0//// 20000 3//// 60007 91006 91107= +15346 01597 83603 10064 20048 39737 40024 58005 60022 70262 8557/ 333 +10079 +20034 32010 4/000 55300 0//// 20000 3//// 60007 91005 91105= +15350 02997 01001 10058 20044 39916 40037 50006 60002 333 10121 20009 +31008 +4/000 55300 0//// 20000 3//// 60007 91003 91103= +15360 02998 01908 10092 20084 30042 40059 53009 60002 222// 06068 +20402 333 +10100 20087 3//// 55300 ///// 20000 3//// 60007 91009 91112 92438= +15410 02597 80302 10072 20047 39909 40004 58013 60032 8457/ 333 10080 +20035 +31009 4/000 55300 0//// 20000 3//// 60007 91004 91106= +15420 02997 60701 10057 20056 39919 40030 52002 69912 80006 333 10157 +20028 +31013 55300 0//// 20000 3//// 60007 91002 91102= +15450 02597 70706 10075 20060 39768 40001 58013 69912 83570 333 10093 +20038 +31012 55300 10119 20000 30000 60007 91009 91109= +15460 02997 41702 10126 20072 30011 40035 50003 69912 80008 333 10180 +20074 +31017 4/000 55300 0//// 20000 3//// 60007 91006 91106= +15470 02597 60503 10078 20064 39890 40015 58006 60002 82370 333 10143 +20037 +31016 55300 0//// 20000 3//// 60007 91003 91104= +15480 05997 41901 10114 20089 30035 40052 51009 60002 7000/ 80005 +222// 06074 +2//// 333 10150 20084 31015 4/000 55300 10188 20000 30000 60007 91004 +91106 +92427= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117181403_51669400.txt b/tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117181403_51669400.txt new file mode 100644 index 00000000..21e07a08 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK171800_C_EDZW_20230117181403_51669400.txt @@ -0,0 +1,53 @@ +SMRO01 YRBK 171800 +AAXX 17181 +15015 01598 82700 10039 20026 39397 42633 58006 60022 70282 8657/ 333 +10066 20029 31010 4/000 55300 0//// 20000 3//// 69917 91002 91104= +15020 02997 01503 10071 20018 39811 40008 51023 60002 333 10113 20024 +31012 4/000 55300 0//// 20000 3//// 60007 91006 91106= +15090 02997 12101 10084 20048 39928 40020 51019 60002 81030 333 10122 +20059 3/010 4/000 55300 10143 20000 30000 60007 91004 91104= +15108 01299 22710 11024 21040 37925 48339 52022 69972 73674 81540 333 +11021 21036 3//// 48014 55300 0//// 20000 3//// 60007 91016 911// +92727 92813 92911= +15120 02997 71303 10079 20027 39497 49984 58006 60002 87070 333 10117 +20049 31016 55300 10080 20000 30000 60007 91006 91106= +15150 02997 01902 10058 20046 39811 40024 51021 60002 333 10117 20028 +31013 4/000 55300 0//// 20000 3//// 60007 91004 91104= +15170 01998 23100 10001 21001 39242 42656 51002 69982 70361 82040 333 +10074 21016 31005 55300 0//// 20000 3//// 60007 91001 91102= +15200 01598 81007 10092 20066 39784 49924 58038 69912 78081 883// 333 +10117 20060 31014 55300 0//// 20000 3//// 69917 91011 91111= +15230 02997 71702 10086 20038 39690 49978 58010 60012 87070 333 10122 +20027 31014 4/000 55300 10110 20000 30000 60007 91004 91105= +15260 05598 30802 10088 20031 39460 49990 58005 60002 7000/ 83500 333 +10109 20016 31009 55300 0//// 20000 3//// 60007 91004 91105= +15280 01/90 92012 11041 21047 37349 47889 50004 60022 74143 333 11041 +21059 3//// 49075 55300 0//// 20000 3//// 60007 91020 911// 92818 +92946= +15292 02597 81505 10078 20052 39664 49953 58027 60022 883// 333 10112 +20045 31014 55300 0//// 20000 3//// 60007 91010 91112= +15310 02997 11801 10090 20072 39953 40039 52011 60002 80001 333 10132 +20065 31015 4/000 55300 10160 20000 30000 60007 91002 91105= +15335 02997 21804 10096 20079 30040 40047 52008 60002 82030 333 10134 +20072 31016 4/000 55300 0//// 20000 3//// 60007 91006 91107= +15346 01597 83603 10064 20048 39737 40024 58005 60022 70262 8557/ 333 +10079 20034 32010 4/000 55300 0//// 20000 3//// 60007 91005 91105= +15350 02997 01001 10058 20044 39916 40037 50006 60002 333 10121 20009 +31008 4/000 55300 0//// 20000 3//// 60007 91003 91103= +15360 02998 01908 10092 20084 30042 40059 53009 60002 222// 06068 +20402 333 10100 20087 3//// 55300 20000 3//// 60007 91009 91112 92438 += +15410 02597 80302 10072 20047 39909 40004 58013 60032 8457/ 333 10080 +20035 31009 4/000 55300 0//// 20000 3//// 60007 91004 91106= +15420 02997 60701 10057 20056 39919 40030 52002 69912 80006 333 10157 +20028 31013 55300 0//// 20000 3//// 60007 91002 91102= +15450 02597 70706 10075 20060 39768 40001 58013 69912 83570 333 10093 +20038 31012 55300 10119 20000 30000 60007 91009 91109= +15460 02997 41702 10126 20072 30011 40035 50003 69912 80008 333 10180 +20074 31017 4/000 55300 0//// 20000 3//// 60007 91006 91106= +15470 02597 60503 10078 20064 39890 40015 58006 60002 82370 333 10143 +20037 31016 55300 0//// 20000 3//// 60007 91003 91104= +15480 05997 41901 10114 20089 30035 40052 51009 60002 7000/ 80005 +222// 06074 2//// 333 10150 20084 31015 4/000 55300 10188 20000 30000 +60007 91004 91106 92427= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK180000CCA_C_EDZW_20230118004301_51967254.txt b/tests/data/observations/romania/A_SMRO01YRBK180000CCA_C_EDZW_20230118004301_51967254.txt new file mode 100644 index 00000000..eb6d235b --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK180000CCA_C_EDZW_20230118004301_51967254.txt @@ -0,0 +1,6 @@ +SMRO01 YRBK 180000 CCA +AAXX 18001 +15280 01/90 92034 11034 21040 37301 47838 53008 60001 74143 333 49080 +55300 0//// 20000 3//// 55000 0//// 20003 3//// 60007 91040 911// +92956= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118000502_51936144.txt b/tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118000502_51936144.txt new file mode 100644 index 00000000..dce0558c --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118000502_51936144.txt @@ -0,0 +1,76 @@ +SMRO01 YRBK 180000 +AAXX 18001 +15015 01597 83201 10072 20053 39345 42589 56019 60051 76186 885// 333 +4/000 +55300 0//// 20000 3//// 55008 0//// 20214 3//// 60057 91004 91107= +15020 02597 80804 10060 20035 39775 49971 58032 60001 8457/ 333 4/000 +55300 +0//// 20000 3//// 55032 0//// 20465 3//// 60007 91006 91110= +15090 02997 80804 10060 20044 39910 40002 57022 60001 80007 333 4/000 +55300 +10144 20000 30000 55011 10119 20331 30296 60007 91007 91107= +15108 01298 52018 10011 21018 37894 48299 58027 60001 73632 83540 333 +48013 +55300 0//// 20000 3//// 55022 0//// 20331 3//// 60007 91024 911// +92727 92912= +15120 01597 81103 10063 20042 39459 49947 56016 69981 78082 8437/ 333 +55300 +10138 20000 30000 55044 00077 20610 30225 69987 91005 91106= +15150 02997 41606 10060 20047 39786 49998 58018 60001 84070 333 4/000 +55300 +0//// 20000 3//// 55024 0//// 20470 3//// 60007 91010 91110= +15170 05598 81801 10021 20018 39215 42631 58021 60001 7000/ 8287/ 333 +55300 +0//// 20000 3//// 55020 0//// 20356 3//// 60007 91005 91106= +15200 01598 81613 10111 20073 39769 49909 53006 60011 70282 8437/ 333 +55300 +0//// 20000 3//// 55034 0//// 20552 3//// 69947 91018 91118= +15230 01597 82003 10091 20075 39662 49949 55008 60041 76162 885// 333 +4/000 +55300 10179 20000 30000 55022 10042 20387 30240 60047 91005 91107= +15260 05598 71404 10118 20032 39419 49941 58009 60001 7000/ 84570 333 +55300 +0//// 20000 3//// 55035 0//// 20456 3//// 60007 91007 91115= +15280 01/90 92032 11034 21040 37301 47838 53008 60001 74143 333 49080 +55300 +0//// 20000 3//// 55000 0//// 20003 3//// 60007 91040 911// 92956= +15292 01597 81506 10095 20062 39656 49943 50005 60011 72582 883// 333 +55300 +0//// 20000 3//// 55024 0//// 20432 3//// 69977 91011 91111= +15310 02997 01604 10105 20082 39933 40018 57014 60001 333 4/000 55300 +10138 +20000 30000 55036 10029 20545 30351 60007 91008 91109= +15335 02597 31807 10107 20087 30027 40034 58013 60001 82530 333 4/000 +55300 +0//// 20000 3//// 55027 0//// 20540 3//// 60007 91009 91109= +15346 01597 83603 10063 20055 39712 40000 57007 60011 76162 8657/ 333 +4/000 +55300 0//// 20000 3//// 55006 0//// 20225 3//// 60017 91004 91106= +15350 02997 03302 10044 20037 39897 40017 57017 60001 333 4/000 55300 +0//// +20000 3//// 55032 0//// 20419 3//// 60007 91003 91103= +15360 02997 01909 10097 20086 30031 40048 57012 60001 222// 0//// +2//// 333 +55300 ///// 20000 3//// 55030 ///// 20520 3//// 60007 91011 91111 +92437= +15410 02597 70202 10088 20055 39883 49978 56010 60001 83570 333 4/000 +55300 +0//// 20000 3//// 55020 0//// 20398 3//// 60007 91005 91105= +15420 02997 50101 10053 20053 39888 49999 57019 60001 85050 333 55300 +0//// +20000 3//// 55018 0//// 20402 3//// 60007 91001 91102= +15450 02997 11105 10080 20055 39747 49979 56010 60001 81030 333 55300 +10148 +20000 30000 55030 00055 20549 30334 60007 91007 91108= +15460 02997 61803 10132 20087 39987 40012 58016 60001 86050 333 4/000 +55300 +0//// 20000 3//// 55039 0//// 20592 3//// 60007 91006 91106= +15470 02997 71704 10116 20061 39866 49989 56012 60001 87070 333 55300 +0//// +20000 3//// 55025 0//// 20506 3//// 60007 91007 91107= +15480 05597 51703 10113 20096 30021 40038 58012 60001 7000/ 85500 +222// 0//// +2//// 333 4/000 55300 10136 20000 30000 55035 10042 20593 30480 60007 +91007 +91108 92427= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118001801_51945941.txt b/tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118001801_51945941.txt new file mode 100644 index 00000000..568c959b --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK180000_C_EDZW_20230118001801_51945941.txt @@ -0,0 +1,56 @@ +SMRO01 YRBK 180000 +AAXX 18001 +15015 01597 83201 10072 20053 39345 42589 56019 60051 76186 885// 333 +4/000 55300 0//// 20000 3//// 55008 0//// 20214 3//// 60057 91004 +91107= +15020 02597 80804 10060 20035 39775 49971 58032 60001 8457/ 333 4/000 +55300 0//// 20000 3//// 55032 0//// 20465 3//// 60007 91006 91110= +15090 02997 80804 10060 20044 39910 40002 57022 60001 80007 333 4/000 +55300 10144 20000 30000 55011 10119 20331 30296 60007 91007 91107= +15108 01298 52018 10011 21018 37894 48299 58027 60001 73632 83540 333 +48013 55300 0//// 20000 3//// 55022 0//// 20331 3//// 60007 91024 +911// 92727 92912= +15120 01597 81103 10063 20042 39459 49947 56016 69981 78082 8437/ 333 +55300 10138 20000 30000 55044 00077 20610 30225 69987 91005 91106= +15150 02997 41606 10060 20047 39786 49998 58018 60001 84070 333 4/000 +55300 0//// 20000 3//// 55024 0//// 20470 3//// 60007 91010 91110= +15170 05598 81801 10021 20018 39215 42631 58021 60001 7000/ 8287/ 333 +55300 0//// 20000 3//// 55020 0//// 20356 3//// 60007 91005 91106= +15200 01598 81613 10111 20073 39769 49909 53006 60011 70282 8437/ 333 +55300 0//// 20000 3//// 55034 0//// 20552 3//// 69947 91018 91118= +15230 01597 82003 10091 20075 39662 49949 55008 60041 76162 885// 333 +4/000 55300 10179 20000 30000 55022 10042 20387 30240 60047 91005 +91107= +15260 05598 71404 10118 20032 39419 49941 58009 60001 7000/ 84570 333 +55300 0//// 20000 3//// 55035 0//// 20456 3//// 60007 91007 91115= +15280 01/90 92032 11034 21040 37301 47838 53008 60001 74143 333 49080 +55300 0//// 20000 3//// 55000 0//// 20003 3//// 60007 91040 911// +92956= +15292 01597 81506 10095 20062 39656 49943 50005 60011 72582 883// 333 +55300 0//// 20000 3//// 55024 0//// 20432 3//// 69977 91011 91111= +15310 02997 01604 10105 20082 39933 40018 57014 60001 333 4/000 55300 +10138 20000 30000 55036 10029 20545 30351 60007 91008 91109= +15335 02597 31807 10107 20087 30027 40034 58013 60001 82530 333 4/000 +55300 0//// 20000 3//// 55027 0//// 20540 3//// 60007 91009 91109= +15346 01597 83603 10063 20055 39712 40000 57007 60011 76162 8657/ 333 +4/000 55300 0//// 20000 3//// 55006 0//// 20225 3//// 60017 91004 +91106= +15350 02997 03302 10044 20037 39897 40017 57017 60001 333 4/000 55300 +0//// 20000 3//// 55032 0//// 20419 3//// 60007 91003 91103= +15360 02997 01909 10097 20086 30031 40048 57012 60001 222// 0//// +2//// 333 55300 20000 3//// 55030 20520 3//// 60007 91011 91111 92437 += +15410 02597 70202 10088 20055 39883 49978 56010 60001 83570 333 4/000 +55300 0//// 20000 3//// 55020 0//// 20398 3//// 60007 91005 91105= +15420 02997 50101 10053 20053 39888 49999 57019 60001 85050 333 55300 +0//// 20000 3//// 55018 0//// 20402 3//// 60007 91001 91102= +15450 02997 11105 10080 20055 39747 49979 56010 60001 81030 333 55300 +10148 20000 30000 55030 00055 20549 30334 60007 91007 91108= +15460 02997 61803 10132 20087 39987 40012 58016 60001 86050 333 4/000 +55300 0//// 20000 3//// 55039 0//// 20592 3//// 60007 91006 91106= +15470 02997 71704 10116 20061 39866 49989 56012 60001 87070 333 55300 +0//// 20000 3//// 55025 0//// 20506 3//// 60007 91007 91107= +15480 05597 51703 10113 20096 30021 40038 58012 60001 7000/ 85500 +222// 0//// 2//// 333 4/000 55300 10136 20000 30000 55035 10042 20593 +30480 60007 91007 91108 92427= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK180600_C_EDZW_20230118060404_52242453.txt b/tests/data/observations/romania/A_SMRO01YRBK180600_C_EDZW_20230118060404_52242453.txt new file mode 100644 index 00000000..f3846275 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK180600_C_EDZW_20230118060404_52242453.txt @@ -0,0 +1,77 @@ +SMRO01 YRBK 180600 +AAXX 18061 +15015 01598 80903 10090 20050 39352 42595 52007 60132 78086 8537/ 333 +10090 +20041 32001 4/000 55300 0//// 20000 3//// 60027 70144 91005 91105= +15020 02597 71103 10077 20055 39764 49959 53003 60002 83570 333 10094 +20054 +31101 4/000 55300 0//// 20000 3//// 60007 70000 91004 91106= +15090 02497 51103 10088 20069 39889 49981 55000 60002 84630 333 10101 +20050 +31101 4/000 55300 10131 20000 30000 60007 70000 91005 91106= +15108 02298 52016 10016 21013 37892 48296 52005 60002 83540 333 10016 +21024 +3//// 44013 55300 0//// 20000 3//// 60007 70007 91024 911// 92727 +92823 92910= +15120 01997 71504 10094 20051 39458 49941 55001 69992 70282 87070 333 +10094 +20054 31004 4/000 55300 10121 20000 30000 60007 70009 91006 91106= +15150 01596 51606 10064 20057 39769 49980 55002 60002 71011 82670 333 +10068 +20032 31002 4/000 55300 0//// 20000 3//// 60007 70000 91009 91110= +15170 05598 71404 10075 20035 39210 42627 53004 60002 7000/ 83370 333 +10079 +21019 31000 55300 0//// 20000 3//// 60007 70008 91007 91107= +15200 01597 81610 10113 20078 39773 49913 55001 60012 78082 8637/ 333 +10121 +20089 31008 55300 0//// 20000 3//// 69917 70015 91015 91116= +15230 01597 82203 10091 20068 39665 49952 53006 60052 7616/ 885// 333 +10122 +20078 31007 55300 10147 20000 30000 69967 70057 91005 91106= +15260 05598 61203 10105 20041 39427 49952 50004 60002 7000/ 84870 333 +10124 +20087 31005 55300 0//// 20000 3//// 60007 70000 91007 91110= +15280 01/90 92020 11030 21035 37323 47863 53005 60002 74143 333 11030 +21041 +3//// 49080 55300 0//// 20000 3//// 60007 70000 91030 911// 92828 +92956= +15292 01597 81606 10100 20068 39658 49945 50002 60012 72582 883// 333 +10114 +20079 31008 55300 0//// 20000 3//// 69937 70034 91009 91109= +15310 02997 21704 10108 20086 39925 40010 55001 60002 82030 333 10109 +20072 +31002 4/000 55300 10201 20000 30000 60007 70000 91008 91109= +15335 02597 41809 10108 20089 30017 40024 55003 60002 83530 333 10111 +20075 +31007 55300 0//// 20000 3//// 60007 70000 91012 91113= +15346 01597 83503 10072 20060 39712 49998 52005 60012 70262 8557/ 333 +10077 +20059 32002 4/000 55300 0//// 20000 3//// 60007 70030 91004 91105= +15350 02997 33102 10043 20035 39894 40015 55000 60002 83040 333 10061 +20034 +31002 4/000 55300 0//// 20000 3//// 60007 70000 91003 91104= +15360 02997 61913 10106 20085 30020 40037 55001 60002 86070 222// +06062 20503 +333 10107 20091 3//// 55300 ///// 20001 3//// 60007 70000 91017 91117 +92447= +15410 02997 70202 10096 20066 39884 49978 53003 60002 87070 333 10096 +20072 +31007 4/000 55300 0//// 20000 3//// 60007 70034 91004 91107= +15420 02997 70601 10039 20039 39895 40006 53006 60002 87070 333 10066 +20008 +31000 55300 0//// 20000 3//// 60007 70001 91001 91102= +15450 02597 71105 10079 20053 39749 49981 53004 60002 84370 333 10096 +20073 +31007 55300 10073 20000 30000 60007 70001 91006 91107= +15460 02997 72005 10156 20078 39986 40010 53004 60002 87070 333 10158 +20118 +31006 4/000 55300 0//// 20000 3//// 60007 70001 91009 91113= +15470 02997 51505 10118 20067 39870 49993 53006 60002 85070 333 10123 +20073 +31005 55300 0//// 20000 3//// 60007 70000 91007 91110= +15480 05497 71801 10118 20095 30013 40030 53002 60002 7000/ 87600 +222// 06070 +2//// 333 10119 20110 31007 4/000 55300 10091 20000 30000 60007 70000 +91006 +91107 92437= + diff --git a/tests/data/observations/romania/A_SMRO01YRBK181200_C_EDZW_20230118120404_52514693.txt b/tests/data/observations/romania/A_SMRO01YRBK181200_C_EDZW_20230118120404_52514693.txt new file mode 100644 index 00000000..2d93e792 --- /dev/null +++ b/tests/data/observations/romania/A_SMRO01YRBK181200_C_EDZW_20230118120404_52514693.txt @@ -0,0 +1,72 @@ +SMRO01 YRBK 181200 +AAXX 18121 +15015 01597 82208 10074 20047 39376 42616 53024 60031 78082 883// 333 +4/000 +55300 0//// 20337 3//// 60037 91017 91117= +15020 02597 71403 10133 20073 39764 49955 58003 60001 83270 333 4/000 +55305 +0//// 20786 3//// 60007 91005 91106= +15090 02597 82103 10163 20068 39887 49976 58006 60001 83577 333 4/000 +55301 +00287 20566 30459 60007 91005 91108= +15108 02298 62520 10021 21005 37912 48315 53007 60001 83571 333 44012 +55302 +0//// 20562 3//// 60007 91028 911// 95100= +15120 01598 72904 10106 20052 39493 49975 53021 69951 72582 84370 333 +55304 +00318 20775 30462 69907 91008 91112= +15150 02597 41605 10134 20088 39774 49981 53002 60001 84800 333 4/000 +55308 +0//// 21282 3//// 60007 91009 91111= +15170 01597 72107 10114 20055 39228 42643 53016 69901 70382 87300 333 +55305 +0//// 20879 3//// 60007 91011 91113= +15200 01598 32107 10107 20071 39848 49989 51037 60041 70181 83200 333 +55305 +0//// 20902 3//// 60027 91010 91111= +15230 01597 72807 10089 20053 39720 40009 53044 60051 78086 85370 333 +55300 +10286 20169 30169 60027 91014 91116= +15260 06598 50904 10141 20053 39454 49974 50015 60001 7000/ 83370 333 +55308 +0//// 21330 3//// 60007 91007 91110 95010= +15280 01/90 92028 11024 21028 37343 47885 51025 60001 74143 333 49085 +55300 +0//// 20000 3//// 60007 91034 911// 92956= +15292 01597 70102 10099 20076 39726 40015 51052 60011 72998 87900 333 +55303 +0//// 20633 3//// 69987 91004 91111 95090= +15310 02997 11905 10181 20098 39928 40010 57002 60001 81040 333 4/000 +55309 +00670 21275 30228 60007 91012 91113= +15335 02597 21909 10186 20111 30018 40025 57004 60001 81530 333 55310 +0//// +21429 3//// 60007 91014 91114= +15346 01597 83502 10099 20084 39734 40019 51009 69951 76162 8657/ 333 +4/000 +55300 0//// 20242 3//// 69957 91003 91105 95090= +15350 02997 02204 10157 20086 39901 40017 54000 60001 333 4/000 55310 +0//// +20941 3//// 60007 91005 91106= +15360 02997 31915 10116 20087 30018 40035 56013 60001 83030 222// +06070 20502 +333 55309 ///// 21270 3//// 60007 91018 91118 92457= +15410 02597 40101 10154 20079 39908 40001 50009 60001 83330 333 4/000 +55307 +0//// 21254 3//// 60007 91003 91105= +15420 02597 41904 10197 20093 39901 40007 52004 60001 81240 333 55310 +0//// +21253 3//// 60007 91009 91112= +15450 02598 62105 10194 20065 39774 49998 51011 60001 83330 333 4/000 +55308 +00784 21510 30681 60007 91008 91110= +15460 02597 32108 10205 20076 39999 40023 53002 60001 81509 333 4/000 +55310 +0//// 21494 3//// 60007 91013 91114= +15470 02997 11604 10196 20097 39887 40007 52007 60001 81040 333 55310 +0//// +21499 3//// 60007 91007 91111= +15480 05997 22003 10178 20105 30020 40037 57002 60001 7000/ 80001 +222// 06088 +2//// 333 4/000 55310 00753 21508 30886 60007 91009 91113 92437= + diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 7f8011a8..c908ca4b 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -64,15 +64,15 @@ def test_metadata_station_publish(): stations = r.json() - assert stations['numberReturned'] == 56 - assert stations['numberMatched'] == 56 + assert stations['numberReturned'] == 79 + assert stations['numberMatched'] == 79 def test_metadata_discovery_publish(): """Test discovery metadata publishing""" r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items').json() - assert r['numberMatched'] == 3 + assert r['numberMatched'] == 4 r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items/{ID}').json() # noqa @@ -101,7 +101,7 @@ def test_metadata_discovery_publish(): r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items', params=params).json() - assert r['numberMatched'] == 3 + assert r['numberMatched'] == 4 def test_data_ingest(): @@ -185,7 +185,7 @@ def test_message_api(): url = f'{API_URL}/collections/messages/items?sortby=wigos_station_identifier' # noqa r = SESSION.get(url).json() - assert r['numberMatched'] == 82 + assert r['numberMatched'] == 197 msg = r['features'][0] assert msg['geometry'] is not None diff --git a/wis2box/data/base.py b/wis2box/data/base.py index 6fd5e211..a27bb9e3 100644 --- a/wis2box/data/base.py +++ b/wis2box/data/base.py @@ -225,7 +225,8 @@ def publish(self) -> bool: return True - def validate_filename_pattern(self, filename: str) -> bool: + def validate_filename_pattern( + self, filename: str) -> Union[re.Match, None]: """ Validate a filename pattern against a configured file_filter @@ -234,14 +235,8 @@ def validate_filename_pattern(self, filename: str) -> bool: :returns: `bool` of vadidation result """ - try: - LOGGER.debug(f'Validating {filename} against {self.file_filter}') - _ = re.match(self.file_filter, filename).group(1) - return True - except AttributeError: - msg = 'Validation failed' - LOGGER.error(msg) - return False + LOGGER.debug(f'Validating {filename} against {self.file_filter}') + return re.match(self.file_filter, filename) def files(self) -> Iterator[str]: """ diff --git a/wis2box/data/csv2bufr.py b/wis2box/data/csv2bufr.py index ae2a36cd..d98b8b96 100644 --- a/wis2box/data/csv2bufr.py +++ b/wis2box/data/csv2bufr.py @@ -62,7 +62,7 @@ def transform(self, input_data: Union[Path, bytes], LOGGER.debug('input_data is a Path') filename = input_data.name - if not self.validate_filename_pattern(filename): + if self.validate_filename_pattern(filename) is None: msg = f'Invalid filename format: {filename} ({self.file_filter})' LOGGER.error(msg) raise ValueError(msg) diff --git a/wis2box/data/synop2bufr.py b/wis2box/data/synop2bufr.py new file mode 100644 index 00000000..b274785c --- /dev/null +++ b/wis2box/data/synop2bufr.py @@ -0,0 +1,95 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### + +import logging +from pathlib import Path +from typing import Union + +from synop2bufr import transform as transform_synop + +from wis2box.data.base import BaseAbstractData +from wis2box.env import DATADIR + +LOGGER = logging.getLogger(__name__) + +STATION_METADATA = DATADIR / 'metadata' / 'station' / 'station_list.csv' + + +class ObservationDataSYNOP2BUFR(BaseAbstractData): + """Synoptic observation data""" + def __init__(self, defs: dict) -> None: + """ + ObservationDataSYNOP2BUFR data initializer + + :param def: `dict` object of resource mappings + + :returns: `None` + """ + + super().__init__(defs) + + self.mappings = {} + + with STATION_METADATA.open() as fh: + self.station_metadata = fh.read() + + def transform(self, input_data: Union[Path, bytes], + filename: str = '') -> bool: + + LOGGER.debug('Processing SYNOP ASCII data') + + if isinstance(input_data, Path): + LOGGER.debug('input_data is a Path') + filename = input_data.name + + file_match = self.validate_filename_pattern(filename) + + if file_match is None: + msg = f'Invalid filename format: {filename} ({self.file_filter})' + LOGGER.error(msg) + raise ValueError(msg) + + LOGGER.debug('Generating BUFR4') + input_bytes = self.as_bytes(input_data) + + year = int(file_match.group(1)) + month = int(file_match.group(2)) + + LOGGER.debug('Transforming data') + results = transform_synop(input_bytes.decode(), self.station_metadata, + year, month) + + # convert to list + LOGGER.debug('Iterating over BUFR messages') + for item in results: + LOGGER.debug('Setting obs date for filepath creation') + identifier = item['_meta']['id'] + data_date = item['_meta']['properties']['datetime'] + + self.output_data[identifier] = item + self.output_data[identifier]['_meta']['relative_filepath'] = \ + self.get_local_filepath(data_date) + + return True + + def get_local_filepath(self, date_): + yyyymmdd = date_.strftime('%Y-%m-%d') + return (Path(yyyymmdd) / 'wis' / self.topic_hierarchy.dirpath) diff --git a/wis2box/handler.py b/wis2box/handler.py index 4f6323ba..37825b96 100644 --- a/wis2box/handler.py +++ b/wis2box/handler.py @@ -106,7 +106,7 @@ def handle(self) -> bool: plugin.transform(self.filepath) except Exception as err: msg = f'Failed to transform file {self.filepath} : {err}' - LOGGER.warning(msg) + LOGGER.error(msg, exc_info=True) self.publish_failure_message( description="failed to transform file", plugin=plugin) @@ -115,7 +115,7 @@ def handle(self) -> bool: plugin.publish() except Exception as err: msg = f'Failed to publish file {self.filepath}: {err}' - LOGGER.warning(msg) + LOGGER.error(msg, exc_info=True) self.publish_failure_message( description="Failed to publish file to api-backend", plugin=plugin) diff --git a/wis2box/resources/data-mappings.yml b/wis2box/resources/data-mappings.yml index 9ddaf79e..23c1066d 100644 --- a/wis2box/resources/data-mappings.yml +++ b/wis2box/resources/data-mappings.yml @@ -1,4 +1,13 @@ data: + rou.rnimh.data.core.weather.surface-based-observations.synop: + plugins: + txt: + - plugin: wis2box.data.synop2bufr.ObservationDataSYNOP2BUFR + notify: true + file-pattern: '^A_SMR.*EDZW_(\d{4})(\d{2}).*.txt$' + bufr4: + - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON + file-pattern: '^A_SMR.*EDZW_(\d{4})(\d{2}).*.bufr4$' mwi.mwi_met_centre.data.core.weather.surface-based-observations.synop: plugins: csv: