feat(DCF-450): Add DRS Fields

Adds description, content_updated_time and content_created_time to IndexRecord.

Maps content_updated_time and content_created_time to updated_time and
created_time in the DRS responses.

Maps IndexRecord updated_time and created_time to index_updated_time and
index_created_time in DRS responses.

.gitignore

@@ -67,3 +67,5 @@ docs/_build/
 local_settings.py
 settings.yaml
 *.sq3
+#IDEs
+.idea

.secrets.baseline

@@ -224,6 +224,15 @@
         "line_number": 14
       }
     ],
+    "migrations/versions/a72f117515c5_add_description_created_time_and_.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "migrations/versions/a72f117515c5_add_description_created_time_and_.py",
+        "hashed_secret": "7c52f05d2823f0becb8196c04678c822ac71bcdf",
+        "is_verified": false,
+        "line_number": 14
+      }
+    ],
     "tests/default_test_settings.py": [
       {
         "type": "Basic Auth Credentials",
@@ -242,6 +251,15 @@
         "line_number": 18
       }
     ],
+    "tests/postgres/migrations/test_a72f117515c5_add_description_created_time_and_.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/postgres/migrations/test_a72f117515c5_add_description_created_time_and_.py",
+        "hashed_secret": "7c52f05d2823f0becb8196c04678c822ac71bcdf",
+        "is_verified": false,
+        "line_number": 31
+      }
+    ],
     "tests/postgres/migrations/test_legacy_schema_migration.py": [
       {
         "type": "Hex High Entropy String",
@@ -382,7 +400,7 @@
         "filename": "tests/test_deprecated_aliases_endpoints.py",
         "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
         "is_verified": false,
-        "line_number": 12
+        "line_number": 13
       }
     ],
     "tests/test_drs.py": [
@@ -391,9 +409,9 @@
         "filename": "tests/test_drs.py",
         "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
         "is_verified": false,
-        "line_number": 32
+        "line_number": 38
       }
     ]
   },
-  "generated_at": "2023-04-13T17:00:13Z"
+  "generated_at": "2023-04-19T18:14:25Z"
 }

indexd/drs/blueprint.py

@@ -150,7 +150,7 @@ def indexd_to_drs(record, expand=False):
 
     name = record["file_name"] if "file_name" in record else record["name"]
 
-    created_time = (
+    index_created_time = (
         record["created_date"] if "created_date" in record else record["created_time"]
     )
 
@@ -162,10 +162,14 @@ def indexd_to_drs(record, expand=False):
         else ""
     )
 
-    updated_date = (
+    index_updated_time = (
         record["updated_date"] if "updated_date" in record else record["updated_time"]
     )
 
+    created_time = record.get("content_created_date", "")
+
+    updated_time = record.get("content_updated_date", "")
+
     form = record["form"] if "form" in record else "bundle"
 
     description = record["description"] if "description" in record else None
@@ -180,11 +184,12 @@ def indexd_to_drs(record, expand=False):
 
     drs_object = {
         "id": did,
-        "description": "",
         "mime_type": "application/json",
         "name": name,
+        "index_created_time": index_created_time,
+        "index_updated_time": index_updated_time,
         "created_time": created_time,
-        "updated_time": updated_date,
+        "updated_time": updated_time,
         "size": record["size"],
         "aliases": alias,
         "self_uri": self_uri,

indexd/index/blueprint.py

@@ -407,6 +407,19 @@ def post_index_record():
     version = flask.request.json.get("version")
     baseid = flask.request.json.get("baseid")
     uploader = flask.request.json.get("uploader")
+    description = flask.request.json.get("description")
+    content_created_date = flask.request.json.get("created_time")
+    content_updated_date = flask.request.json.get("updated_time")
+
+    if content_updated_date is None:
+        content_updated_date = content_created_date
+
+    if content_updated_date is not None and content_created_date is None:
+        raise UserError("Cannot set updated_time without created_time")
+
+    if content_updated_date is not None and content_created_date is not None:
+        if content_updated_date < content_created_date:
+            raise UserError("updated_time cannot come before created_date")
 
     did, rev, baseid = blueprint.index_driver.add(
         form,
@@ -422,6 +435,9 @@ def post_index_record():
         hashes=hashes,
         baseid=baseid,
         uploader=uploader,
+        description=description,
+        content_created_date=content_created_date,
+        content_updated_date=content_updated_date,
     )
 
     ret = {"did": did, "rev": rev, "baseid": baseid}
@@ -508,9 +524,13 @@ def put_index_record(record):
         raise UserError(err)
 
     rev = flask.request.args.get("rev")
+    json = flask.request.json
+    if "updated_time" in json and "created_time" in json:
+        if json["updated_time"] < json["created_time"]:
+            raise UserError("updated_time cannot come before created_date")
 
     # authorize done in update
-    did, baseid, rev = blueprint.index_driver.update(record, rev, flask.request.json)
+    did, baseid, rev = blueprint.index_driver.update(record, rev, json)
 
     ret = {"did": did, "baseid": baseid, "rev": rev}
 
@@ -553,6 +573,16 @@ def add_index_record_version(record):
     metadata = flask.request.json.get("metadata")
     urls_metadata = flask.request.json.get("urls_metadata")
     version = flask.request.json.get("version")
+    description = flask.request.json.get("description")
+    content_created_date = flask.request.json.get("created_time")
+    content_updated_date = flask.request.json.get("updated_time")
+
+    if content_updated_date is None:
+        content_updated_date = content_created_date
+
+    if content_updated_date is not None and content_created_date is not None:
+        if content_updated_date < content_created_date:
+            raise UserError("updated_time cannot come before created_date")
 
     # authorize done in add_version for both the old and new authz
     did, baseid, rev = blueprint.index_driver.add_version(
@@ -568,6 +598,9 @@ def add_index_record_version(record):
         urls_metadata=urls_metadata,
         version=version,
         hashes=hashes,
+        description=description,
+        content_created_date=content_created_date,
+        content_updated_date=content_updated_date,
     )
 
     ret = {"did": did, "baseid": baseid, "rev": rev}

indexd/index/driver.py

@@ -58,6 +58,9 @@ def add(
         hashes=None,
         baseid=None,
         uploader=None,
+        description=None,
+        content_created_date=None,
+        content_updated_date=None,
     ):
         """
         Creates record for given data.
@@ -100,6 +103,9 @@ def add_version(
         acl=None,
         authz=None,
         hashes=None,
+        description=None,
+        created_time=None,
+        updated_time=None,
     ):
         """
         Add a record version given did

indexd/index/drivers/alchemy.py

@@ -80,6 +80,9 @@ class IndexRecord(Base):
     file_name = Column(String, index=True)
     version = Column(String, index=True)
     uploader = Column(String, index=True)
+    description = Column(String)
+    content_created_date = Column(DateTime)
+    content_updated_date = Column(DateTime)
 
     urls = relationship(
         "IndexRecordUrl", backref="index_record", cascade="all, delete-orphan"
@@ -120,6 +123,16 @@ def to_document_dict(self):
         }
         created_date = self.created_date.isoformat()
         updated_date = self.updated_date.isoformat()
+        content_created_date = (
+            self.content_created_date.isoformat()
+            if self.content_created_date is not None
+            else ""
+        )
+        content_updated_date = (
+            self.content_updated_date.isoformat()
+            if self.content_created_date is not None
+            else ""
+        )
 
         return {
             "did": self.did,
@@ -138,6 +151,9 @@ def to_document_dict(self):
             "form": self.form,
             "created_date": created_date,
             "updated_date": updated_date,
+            "description": self.description,
+            "content_created_date": content_created_date,
+            "content_updated_date": content_updated_date,
         }
 
 
@@ -675,6 +691,9 @@ def add(
         hashes=None,
         baseid=None,
         uploader=None,
+        description=None,
+        content_created_date=None,
+        content_updated_date=None,
     ):
         """
         Creates a new record given size, urls, acl, authz, hashes, metadata,
@@ -733,6 +752,20 @@ def add(
                 IndexRecordMetadata(did=record.did, key=m_key, value=m_value)
                 for m_key, m_value in metadata.items()
             ]
+
+            record.description = description
+
+            if content_created_date is not None:
+                record.content_created_date = datetime.datetime.fromisoformat(
+                    content_created_date
+                )
+                # Users cannot set content_updated_date without a content_created_date
+                record.content_updated_date = (
+                    datetime.datetime.fromisoformat(content_updated_date)
+                    if content_updated_date is not None
+                    else record.content_created_date  # Set updated to created if no updated is provided
+                )
+
             session.merge(base_version)
 
             try:
@@ -1137,7 +1170,15 @@ def update(self, did, rev, changing_fields):
         """
         authz_err_msg = "Auth error when attempting to update a record. User must have '{}' access on '{}' for service 'indexd'."
 
-        composite_fields = ["urls", "acl", "authz", "metadata", "urls_metadata"]
+        composite_fields = [
+            "urls",
+            "acl",
+            "authz",
+            "metadata",
+            "urls_metadata",
+            "created_time",
+            "updated_time",
+        ]
 
         with self.session as session:
             query = session.query(IndexRecord).filter(IndexRecord.did == did)
@@ -1209,6 +1250,19 @@ def update(self, did, rev, changing_fields):
 
                 create_urls_metadata(changing_fields["urls_metadata"], record, session)
 
+            if "created_time" in changing_fields:
+                record.content_created_date = datetime.datetime.fromisoformat(
+                    changing_fields["created_time"]
+                )
+            if "updated_time" in changing_fields:
+                if record.content_created_date is None:
+                    raise UserError(
+                        "Cannot set updated_time on record that does not have a created_time"
+                    )
+                record.content_updated_date = datetime.datetime.fromisoformat(
+                    changing_fields["updated_time"]
+                )
+
             for key, value in changing_fields.items():
                 if key not in composite_fields:
                     # No special logic needed for other updates.
@@ -1259,6 +1313,9 @@ def add_version(
         acl=None,
         authz=None,
         hashes=None,
+        description=None,
+        content_created_date=None,
+        content_updated_date=None,
     ):
         """
         Add a record version given did
@@ -1297,6 +1354,9 @@ def add_version(
             record.size = size
             record.file_name = file_name
             record.version = version
+            record.description = description
+            record.content_created_date = content_created_date
+            record.content_updated_date = content_updated_date
 
             record.urls = [IndexRecordUrl(did=record.did, url=url) for url in urls]
 

indexd/index/schema.py

@@ -31,6 +31,10 @@
             "description": "optional version string of the object",
             "type": "string",
         },
+        "description": {
+            "description": "optional description string of the object",
+            "type": "string",
+        },
         "uploader": {
             "description": "optional uploader of the object",
             "type": "string",
@@ -46,6 +50,16 @@
             "type": "string",
             "pattern": "^.*[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$",
         },
+        "created_time": {
+            "description": "Timestamp of content creation. Refers to the underyling content, not the JSON object.",
+            "type": "string",
+            "format": "date-time",
+        },
+        "updated_time": {
+            "description": "Timestamp of content update, identical to created_time in systems that do not support updates. Refers to the underyling content, not the JSON object.",
+            "type": "string",
+            "format": "date-time",
+        },
         "hashes": {
             "type": "object",
             "properties": {
@@ -82,6 +96,9 @@
         "uploader": {"type": ["string", "null"]},
         "metadata": {"type": "object"},
         "urls_metadata": {"type": "object"},
+        "description": {"type": ["string", "null"]},
+        "created_time": {"type": ["string", "null"], "format": "date-time"},
+        "updated_time": {"type": ["string", "null"], "format": "date-time"},
     },
 }
 

migrations/versions/a72f117515c5_add_description_created_time_and_.py

@@ -0,0 +1,32 @@
+"""Add description, created_time and updated_time columns to IndexRecord
+
+Revision ID: a72f117515c5
+Revises: 15f2e9345ade
+Create Date: 2023-04-11 10:00:59.250768
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a72f117515c5"
+down_revision = "15f2e9345ade"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "index_record", sa.Column("content_created_date", sa.DateTime, nullable=True)
+    )
+    op.add_column(
+        "index_record", sa.Column("content_updated_date", sa.DateTime, nullable=True)
+    )
+    op.add_column("index_record", sa.Column("description", sa.VARCHAR(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("index_record", "content_created_date")
+    op.drop_column("index_record", "content_updated_date")
+    op.drop_column("index_record", "description")