Skip to content

Commit

Permalink
Removing member of functionality.
Browse files Browse the repository at this point in the history
  • Loading branch information
@rhysrevans3
rhysrevans3 committed Feb 12, 2025
1 parent f8f7aa3 commit 5acf7d9
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 141 deletions.
35 changes: 2 additions & 33 deletions stac_generator/core/generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ def __init__(self, conf: dict):

self.recipes = Recipes(recipes_root)

self.default_id_methods = conf.pop("default_id_methods", {})

self.inputs = load_plugins(conf.pop("inputs", []), "stac_generator.inputs")

self.outputs = load_plugins(conf.pop("outputs", []), "stac_generator.outputs")
Expand Down Expand Up @@ -71,6 +69,7 @@ def _load_extraction_method(self, extraction_method_conf: dict, **kwargs) -> Ext
| kwargs
)

# Collect "sub" extraction methods
if "extraction_methods" in inputs:
extraction_methods = []

Expand Down Expand Up @@ -104,7 +103,7 @@ def _run_extraction_method(self, body: dict, extraction_method_conf: dict, **kwa

return extraction_method._run(body)

def run_extraction_methods(self, body: dict, extraction_methods: list, **kwargs: dict) -> dict:
def run_extraction_methods(self, body: dict, extraction_methods: list, **kwargs) -> dict:
"""
Extract facets from the listed extraction methods
Expand All @@ -120,34 +119,6 @@ def run_extraction_methods(self, body: dict, extraction_methods: list, **kwargs:

return body

def run_member_of_methods(self, body: dict, member_of: list, **kwargs: dict) -> dict:
"""
Extract the raw facets from the listed extraction methods
:param body: Dict of current extracted data
:param member_of: list of membership
:param kwargs:
:return: updated body
"""

update = defaultdict(list)

update["member_of_recipes"] = {}

for link in member_of:
member_body = self.run_extraction_methods({}, link.id, **kwargs)

link_id = member_body.pop("id")

update[f"{link.type}_id"].append(link_id)

update["member_of_recipes"][link_id] = link.key

body.update(update)

return body

def output(self, body: dict, recipe: Recipe, **kwargs) -> None:
"""
Run all configured outputs export methods.
Expand Down Expand Up @@ -181,8 +152,6 @@ def process(self, body: dict, **kwargs) -> None:

body = self.run_extraction_methods(body, recipe.extraction_methods, **kwargs)

body = self.run_member_of_methods(body, recipe.member_of, **kwargs)

self.output(body, recipe, **kwargs)

def run(self) -> None:
Expand Down
37 changes: 13 additions & 24 deletions stac_generator/plugins/mappings/stac.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ class STACMapping(BaseMapping):

config_class = STACConf

def datetime_field(self, body: dict, key: str) -> str:
dt = parser.parse(body.pop(key))
def datetime_field(self, date_str: str) -> str:
dt = parser.parse(date_str)
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")

def item(self, body: dict) -> dict:
Expand All @@ -61,40 +61,27 @@ def item(self, body: dict) -> dict:
"stac_version": self.conf.stac_version,
"stac_extensions": body.pop("stac_extensions", []) + self.conf.stac_extensions,
"id": body.pop("id"),
"geometry": None,
"assets": {},
"collection": body.pop("collection"),
"geometry": body.pop("geometry", None),
"assets": body.pop("assets", {}),
"properties": {
"datetime": None,
},
}

if "datetime" in body:
output["properties"]["datetime"] = self.datetime_field(body, "datetime")
output["properties"]["datetime"] = self.datetime_field(body.pop("datetime"))

if "start_datetime" in body:
output["properties"]["start_datetime"] = self.datetime_field(body, "start_datetime")
output["properties"]["start_datetime"] = self.datetime_field(body.pop("start_datetime"))

if "end_datetime" in body:
output["properties"]["end_datetime"] = self.datetime_field(body, "end_datetime")
output["properties"]["end_datetime"] = self.datetime_field(body.pop("end_datetime"))

if "bbox" in body:
output["bbox"] = body.pop("bbox")

if "geometry" in body:
output["geometry"] = body.pop("geometry")

if "assets" in body:
output["assets"] = body.pop("assets")

if "member_of_recipes" in body:
output["member_of_recipes"] = body.pop("member_of_recipes")

if "collection_id" in body:
output["collection"] = body.pop("collection_id")[0]

output["properties"] |= body

output["links"] = [
output["links"] = body.pop("links") + [
{
"rel": "self",
"type": "application/geo+json",
Expand All @@ -113,10 +100,12 @@ def item(self, body: dict) -> dict:
{
"rel": "root",
"type": "application/json",
"href": "{self.conf.stac_root_url}/",
"href": self.conf.stac_root_url,
},
]

output["properties"] |= body

return output

def collection(self, body: dict) -> dict:
Expand Down Expand Up @@ -183,7 +172,7 @@ def collection(self, body: dict) -> dict:
{
"rel": "root",
"type": "application/json",
"href": "{self.conf.stac_root_url}/",
"href": self.conf.stac_root_url,
},
]

Expand Down
171 changes: 87 additions & 84 deletions stac_generator/plugins/outputs/stac_fastapi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,10 @@ class STACFastAPIConf(BaseModel):
default=None,
description="Authentication for STAC API.",
)
headers: dict = Field(
default={},
description="Headers for API request.",
)
verify: bool = Field(
default=False,
description="API certificate verifcation.",
Expand All @@ -86,99 +90,98 @@ class STACFastAPIOutput(Output):
config_class = STACFastAPIConf

def item(self, data: dict, client: Client, auth: OAuth2ClientCredentials | None) -> None:
collections = data["collection"]

if isinstance(data["collection"], str):
collections = [collections]
collection = data["collection"]

for collection in collections:
collection = data["collection"] = collection.lower()
response = client.post(
urljoin(self.conf.api_url, f"collections/{collection}/items"),
json=data,
auth=auth,
headers=self.conf.headers,
)

response = client.post(
urljoin(self.conf.api_url, f"collections/{collection}/items"),
json=data,
auth=auth,
)
if response.status_code == 404:

response_json = response.json()

if response.status_code == 404:
response_json = response.json()

if response_json["description"] == f"Collection {collection} does not exist":
collection_data = {
"type": "Collection",
"id": collection,
"description": collection,
"stac_version": "0.1.0",
"stac_extensions": [],
"license": data.get("license", "No License"),
"extent": {
"spatial": {"bbox": [[-180, -90, 180, 90]]},
"temporal": {
"interval": [["1992-01-01T00:00:00Z", "2015-12-31T00:00:00Z"]]
},
if response_json["description"] == f"Collection {collection} does not exist":
collection_data = {
"type": "Collection",
"id": collection,
"description": collection,
"stac_version": "0.1.0",
"stac_extensions": [],
"license": data.get("license", "other"),
"extent": {
"spatial": {"bbox": [[-180, -90, 180, 90]]},
"temporal": {
"interval": [["1992-01-01T00:00:00Z", "2015-12-31T00:00:00Z"]]
},
"links": [
{
"rel": "self",
"type": "application/geo+json",
"href": f"{self.conf.api_url}/collections/{collection}",
},
{
"rel": "parent",
"type": "application/json",
"href": f"{self.conf.api_url}/",
},
{
"rel": "queryables",
"type": "application/json",
"href": f"{self.conf.api_url}/collections/{collection}/queryables",
},
{
"rel": "items",
"type": "application/geo+json",
"href": f"{self.conf.api_url}/collections/cmip6/{collection}",
},
{
"rel": "root",
"type": "application/json",
"href": f"{self.conf.api_url}/",
},
],
}

response = client.post(
urljoin(self.conf.api_url, "collections"),
json=collection_data,
auth=auth,
)
},
"links": data.get("links", []) + [
{
"rel": "self",
"type": "application/geo+json",
"href": f"{self.conf.api_url}/collections/{collection}",
},
{
"rel": "parent",
"type": "application/json",
"href": f"{self.conf.api_url}/",
},
{
"rel": "queryables",
"type": "application/json",
"href": f"{self.conf.api_url}/collections/{collection}/queryables",
},
{
"rel": "items",
"type": "application/geo+json",
"href": f"{self.conf.api_url}/collections/cmip6/{collection}",
},
{
"rel": "root",
"type": "application/json",
"href": self.conf.api_url,
},
],
}

response = client.post(
urljoin(self.conf.api_url, f"collections/{collection}/items"),
json=data,
auth=auth,
)
response = client.post(
urljoin(self.conf.api_url, "collections"),
json=collection_data,
auth=auth,
headers=headers,
)

if response.status_code == 409:
response_json = response.json()

if (
response_json["description"]
== f"Item {data['id']} in collection {collection} already exists"
):
response = client.put(
urljoin(self.conf.api_url, f"collections/{collection}/items/{data['id']}"),
json=data,
auth=auth,
)
response = client.post(
urljoin(self.conf.api_url, f"collections/{collection}/items"),
json=data,
auth=auth,
headers=headers,
)

if response.status_code == 409:
response_json = response.json()

if (
response_json["description"]
== f"Item {data['id']} in collection {collection} already exists"
):
response = client.put(
urljoin(self.conf.api_url, f"collections/{collection}/items/{data['id']}"),
json=data,
auth=auth,
)

if response.status_code != 200:
LOGGER.warning(
"FastAPI Output Item update failed with status code: %s and response text: %s",
response.status_code,
response.text,
)
if response.status_code != 200:
LOGGER.warning(
"FastAPI Output Item update failed with status code: %s and response text: %s",
response.status_code,
response.text,
)

elif response.status_code != 200:
elif response.status_code != 200:
LOGGER.warning(
"FastAPI Output failed to post to STAC Fastapi items endpoint returned status code: %s and response text: %s request data: %s",
response.status_code,
Expand Down

0 comments on commit 5acf7d9

Please sign in to comment.