From 74608387a9abfa7b224b6255b21dff4fdee35d25 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 15 Mar 2024 09:12:50 +0000 Subject: [PATCH] Fixing notebook errors. --- example/conf/collection-generator.yaml | 6 +- example/conf/item-generator.yaml | 8 +- example/example_notebook.ipynb | 75 +++++++++++++++++-- .../{output/asset.txt => input/assets.txt} | 0 example/input/collections.txt | 1 + example/output/collections/cmip6.json | 42 +++++++++++ ...P.MOHC.UKESM1-0-LL.Amon.tas.v20190502.json | 33 ++++++++ ...P.MOHC.UKESM1-0-LL.Amon.tas.v20191210.json | 33 ++++++++ .../CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml | 7 +- .../item/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml | 3 +- .../plugins/outputs/standard_out.py | 4 +- stac_generator/plugins/outputs/text_file.py | 3 +- 12 files changed, 194 insertions(+), 21 deletions(-) rename example/{output/asset.txt => input/assets.txt} (100%) create mode 100644 example/input/collections.txt create mode 100644 example/output/collections/cmip6.json create mode 100644 example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502.json create mode 100644 example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210.json diff --git a/example/conf/collection-generator.yaml b/example/conf/collection-generator.yaml index 2128951..10b21ce 100644 --- a/example/conf/collection-generator.yaml +++ b/example/conf/collection-generator.yaml @@ -2,12 +2,12 @@ generator: collection # The root directory of the recipes -recipes_root: example/recipes/ +recipes_root: recipes/ # The input plugins to be run for the generator inputs: - name: text_file - filepath: example/output/asset.txt + filepath: input/collections.txt # The output plugins to be run for the generator outputs: @@ -18,7 +18,7 @@ outputs: stac_version: '1.0.0' stac_extensions: [] - name: json_file - dirpath: example/output/collections + dirpath: output/collections filename_term: id mappings: - name: stac diff --git a/example/conf/item-generator.yaml b/example/conf/item-generator.yaml index 30566be..83f8f19 100644 --- a/example/conf/item-generator.yaml +++ b/example/conf/item-generator.yaml @@ -2,12 +2,12 @@ generator: item # The root directory of the recipes -recipes_root: example/recipes/ +recipes_root: recipes/ # The input plugins to be run for the generator inputs: - name: text_file - filepath: example/output/collection.txt + filepath: input/assets.txt # The output plugins to be run for the generator outputs: @@ -18,11 +18,11 @@ outputs: stac_version: '1.0.0' stac_extensions: [] - name: json_file - dirpath: example/output/items + dirpath: output/items filename_term: id mappings: - name: stac stac_version: '1.0.0' stac_extensions: [] - name: text_file - filepath: example/output/collection.txt + filepath: input/collections.txt diff --git a/example/example_notebook.ipynb b/example/example_notebook.ipynb index f119fcb..7eb2292 100644 --- a/example/example_notebook.ipynb +++ b/example/example_notebook.ipynb @@ -18,11 +18,61 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{ 'assets': { 'data0001': { 'href': 'https://cmip6-zarr-o.s3-ext.jc.rl.ac.uk/CMIP6.CMIP.MOHC.UKESM1-0-LL/historical.r12i1p1f2.Amon.tas.gn.v20191210.zarr',\n", + " 'roles': ['data']}},\n", + " 'collection': 'cmip6',\n", + " 'geometry': None,\n", + " 'id': 'CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210',\n", + " 'member_of_recipes': {'cmip6': '73c90e423f9cbc6bcc5bc446b5d21e19'},\n", + " 'properties': { 'activity_id': 'CMIP',\n", + " 'datetime': None,\n", + " 'experiment_id': 'historical',\n", + " 'grid_label': 'gn',\n", + " 'instance_id': 'CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210',\n", + " 'institution_id': 'MOHC',\n", + " 'member_id': 'r12i1p1f2',\n", + " 'mip_era': 'CMIP6',\n", + " 'source_id': 'UKESM1-0-LL',\n", + " 'table_id': 'Amon',\n", + " 'var_id': 'tas',\n", + " 'version': 'v20191210'},\n", + " 'stac_extensions': [],\n", + " 'stac_version': '1.0.0',\n", + " 'type': 'Feature'}\n", + "{ 'assets': { 'data0001': { 'href': 'https://cmip6-zarr-o.s3-ext.jc.rl.ac.uk/CMIP6.CMIP.MOHC.UKESM1-0-LL/historical.r12i1p1f2.Amon.tas.gn.v20191210.zarr',\n", + " 'roles': ['data']}},\n", + " 'collection': 'cmip6',\n", + " 'geometry': None,\n", + " 'id': 'CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502',\n", + " 'member_of_recipes': {'cmip6': '73c90e423f9cbc6bcc5bc446b5d21e19'},\n", + " 'properties': { 'activity_id': 'CMIP',\n", + " 'datetime': None,\n", + " 'experiment_id': 'historical',\n", + " 'grid_label': 'gn',\n", + " 'instance_id': 'CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502',\n", + " 'institution_id': 'MOHC',\n", + " 'member_id': 'r4i1p1f2',\n", + " 'mip_era': 'CMIP6',\n", + " 'source_id': 'UKESM1-0-LL',\n", + " 'table_id': 'Amon',\n", + " 'var_id': 'tas',\n", + " 'version': 'v20190502'},\n", + " 'stac_extensions': [],\n", + " 'stac_version': '1.0.0',\n", + " 'type': 'Feature'}\n", + "Processed 2 elasticsearch records in 0:00:27.579150\n" + ] + } + ], "source": [ - "!stac_generator -c conf/item-generator.yaml" + "!stac_generator -c conf/item-generator.yaml\n" ] }, { @@ -36,11 +86,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Usage: stac_generator [OPTIONS]\n", + "Try 'stac_generator --help' for help.\n", + "\n", + "Error: Missing option '--conf' / '-c'.\n" + ] + } + ], "source": [ - "!stac_generator conf/collection-generator.yaml" + "!stac_generator -c conf/collection-generator.yaml" ] } ], @@ -60,7 +121,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.0" + "version": "3.9.18" }, "vscode": { "interpreter": { diff --git a/example/output/asset.txt b/example/input/assets.txt similarity index 100% rename from example/output/asset.txt rename to example/input/assets.txt diff --git a/example/input/collections.txt b/example/input/collections.txt new file mode 100644 index 0000000..b817dd7 --- /dev/null +++ b/example/input/collections.txt @@ -0,0 +1 @@ +{"uri": "cmip6", "recipe_path": "cce91d867703049b5f46f227d8d71bad"} diff --git a/example/output/collections/cmip6.json b/example/output/collections/cmip6.json new file mode 100644 index 0000000..4161e7c --- /dev/null +++ b/example/output/collections/cmip6.json @@ -0,0 +1,42 @@ +{ + "type": "Collection", + "stac_version": "1.0.0", + "stac_extensions": [], + "id": "cmip6", + "extent": { + "temporal": { + "interval": null + }, + "spatial": { + "bbox": null + } + }, + "summaries": { + "mip_era": [ + "CMIP6" + ], + "activity_id": [ + "CMIP" + ], + "institution_id": [ + "MOHC" + ], + "source_id": [ + "UKESM1-0-LL" + ], + "table_id": [ + "Amon" + ], + "var_id": [ + "tas" + ], + "version": [ + "v20191210", + "v20190502" + ] + }, + "assets": {}, + "providers": [], + "license": "", + "member_of_recipes": {} +} \ No newline at end of file diff --git a/example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502.json b/example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502.json new file mode 100644 index 0000000..8fb96e5 --- /dev/null +++ b/example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502.json @@ -0,0 +1,33 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "stac_extensions": [], + "id": "CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502", + "geometry": null, + "assets": { + "data0001": { + "href": "https://cmip6-zarr-o.s3-ext.jc.rl.ac.uk/CMIP6.CMIP.MOHC.UKESM1-0-LL/historical.r12i1p1f2.Amon.tas.gn.v20191210.zarr", + "roles": [ + "data" + ] + } + }, + "properties": { + "datetime": null, + "mip_era": "CMIP6", + "activity_id": "CMIP", + "institution_id": "MOHC", + "source_id": "UKESM1-0-LL", + "experiment_id": "historical", + "member_id": "r4i1p1f2", + "table_id": "Amon", + "var_id": "tas", + "grid_label": "gn", + "version": "v20190502", + "instance_id": "CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20190502" + }, + "member_of_recipes": { + "cmip6": "cce91d867703049b5f46f227d8d71bad" + }, + "collection": "cmip6" +} \ No newline at end of file diff --git a/example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210.json b/example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210.json new file mode 100644 index 0000000..99de5d9 --- /dev/null +++ b/example/output/items/CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210.json @@ -0,0 +1,33 @@ +{ + "type": "Feature", + "stac_version": "1.0.0", + "stac_extensions": [], + "id": "CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210", + "geometry": null, + "assets": { + "data0001": { + "href": "https://cmip6-zarr-o.s3-ext.jc.rl.ac.uk/CMIP6.CMIP.MOHC.UKESM1-0-LL/historical.r12i1p1f2.Amon.tas.gn.v20191210.zarr", + "roles": [ + "data" + ] + } + }, + "properties": { + "datetime": null, + "mip_era": "CMIP6", + "activity_id": "CMIP", + "institution_id": "MOHC", + "source_id": "UKESM1-0-LL", + "experiment_id": "historical", + "member_id": "r12i1p1f2", + "table_id": "Amon", + "var_id": "tas", + "grid_label": "gn", + "version": "v20191210", + "instance_id": "CMIP6.CMIP.MOHC.UKESM1-0-LL.Amon.tas.v20191210" + }, + "member_of_recipes": { + "cmip6": "cce91d867703049b5f46f227d8d71bad" + }, + "collection": "cmip6" +} \ No newline at end of file diff --git a/example/recipes/collection/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml b/example/recipes/collection/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml index 2bc5b66..16de5c6 100644 --- a/example/recipes/collection/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml +++ b/example/recipes/collection/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml @@ -13,7 +13,7 @@ id: extraction_methods: - method: json_file inputs: - filepath: example/output/items/ + dirpath: output/items/ terms: - mip_era - activity_id @@ -22,3 +22,8 @@ extraction_methods: - table_id - var_id - version + + - method: remove + inputs: + keys: + - uri \ No newline at end of file diff --git a/example/recipes/item/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml b/example/recipes/item/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml index 1208c54..0a3aa06 100644 --- a/example/recipes/item/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml +++ b/example/recipes/item/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml @@ -24,6 +24,7 @@ extraction_methods: inputs: template: '{mip_era}.{activity_id}.{institution_id}.{source_id}.{table_id}.{var_id}.{version}' output_key: instance_id + # Some extraction methods generate assets which can also include their own list of extration methods to be run on the assets - method: intake_assets inputs: @@ -57,4 +58,4 @@ extraction_methods: # member of defines the other recipes that define a parent of this record member_of: - - example/recipes/collection/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml \ No newline at end of file + - recipes/collection/CMIP6.CMIP.MOHC.UKESM1-0-LL.yaml \ No newline at end of file diff --git a/stac_generator/plugins/outputs/standard_out.py b/stac_generator/plugins/outputs/standard_out.py index b2521ce..1cee516 100644 --- a/stac_generator/plugins/outputs/standard_out.py +++ b/stac_generator/plugins/outputs/standard_out.py @@ -46,6 +46,4 @@ def export(self, data: dict, **kwargs) -> None: :param kwargs: Not used """ pp = pprint.PrettyPrinter(indent=4) - # pp.pprint(data) - pp.pprint(data["instance_id"]) - # pp.pprint(data["dataset_id"]) + pp.pprint(data) diff --git a/stac_generator/plugins/outputs/text_file.py b/stac_generator/plugins/outputs/text_file.py index 2ff7a0f..645da29 100644 --- a/stac_generator/plugins/outputs/text_file.py +++ b/stac_generator/plugins/outputs/text_file.py @@ -25,6 +25,7 @@ filepath: location_to_destination_file """ + __author__ = "Mahir Rahman" __date__ = "23 Mar 2022" __copyright__ = "Copyright 2022 United Kingdom Research and Innovation" @@ -48,8 +49,6 @@ def __init__(self, **kwargs): self.filepath = self.filepath.rstrip("/") def export(self, data: dict, **kwargs) -> None: - if hasattr(self, "deduplicate"): - return if os.path.isdir(self.filepath): filepath = f"{self.filepath}/file_out.txt"