fix(core): update template checksum when updating Dockerfile (#3351)

renku/command/clone.py

@@ -62,8 +62,8 @@ def _project_clone(
         Tuple of cloned ``Repository`` and whether it's a Renku project or not.
     """
     from renku.command.mergetool import setup_mergetool
-    from renku.core.migration.migrate import is_renku_project
     from renku.core.util.git import clone_renku_repository
+    from renku.core.util.metadata import is_renku_project
 
     install_lfs = project_context.external_storage_requested
 

renku/command/migrate.py

@@ -177,13 +177,9 @@ def check_project():
 
 
 def _check_project():
-    from renku.core.migration.migrate import (
-        is_docker_update_possible,
-        is_migration_required,
-        is_project_unsupported,
-        is_renku_project,
-    )
+    from renku.core.migration.migrate import is_docker_update_possible, is_migration_required, is_project_unsupported
     from renku.core.template.usecase import check_for_template_update
+    from renku.core.util.metadata import is_renku_project
 
     if not is_renku_project():
         return NON_RENKU_REPOSITORY

renku/command/rollback.py

@@ -149,10 +149,10 @@ def _get_modifications_from_diff(diff):
             continue
 
         # normal file
-        if diff_index.change_type == "A":
+        if diff_index.added:
             modifications["files"]["removed"].append(entry)
 
-        elif diff_index.change_type == "D":
+        elif diff_index.deleted:
             modifications["files"]["restored"].append(entry)
         else:
             modifications["files"]["modified"].append(entry)

renku/command/schema/activity.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Activity JSON-LD schema."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema, SoftwareAgentSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -124,6 +124,28 @@ class Meta:
     started_at_time = fields.DateTime(prov.startedAtTime, add_value_types=True)
     usages = Nested(prov.qualifiedUsage, UsageSchema, many=True)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.started_at_time = obj.started_at_time.replace(microsecond=0)
+            obj.ended_at_time = obj.ended_at_time.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class WorkflowFileActivityCollectionSchema(JsonLDSchema):
     """WorkflowFileActivityCollection schema."""

renku/command/schema/composite_plan.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Represent a group of run templates."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.calamus import JsonLDSchema, Nested, fields, prov, renku, schema
@@ -49,3 +49,27 @@ class Meta:
     project_id = fields.IRI(renku.hasPlan, reverse=True)
     plans = Nested(renku.hasSubprocess, [PlanSchema, "CompositePlanSchema"], many=True)
     links = Nested(renku.workflowLinks, [ParameterLinkSchema], many=True, load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/dataset.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Datasets JSON-LD schemes."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -69,6 +69,27 @@ class Meta:
     id = fields.Id()
     name = fields.String(schema.name)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class LanguageSchema(JsonLDSchema):
     """Language schema."""
@@ -134,6 +155,27 @@ class Meta:
     is_external = fields.Boolean(renku.external, load_default=False)
     source = fields.String(renku.source, load_default=None)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.date_added = obj.date_added.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class DatasetSchema(JsonLDSchema):
     """Dataset schema."""
@@ -168,3 +210,30 @@ class Meta:
     same_as = Nested(schema.sameAs, UrlSchema, load_default=None)
     title = fields.String(schema.name)
     version = fields.String(schema.version, load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            if obj.date_created:
+                obj.date_created = obj.date_created.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            if obj.date_published:
+                obj.date_published = obj.date_published.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/plan.py

@@ -54,3 +54,27 @@ class Meta:
     parameters = Nested(renku.hasArguments, CommandParameterSchema, many=True, load_default=None)
     success_codes = fields.List(renku.successCodes, fields.Integer(), load_default=[0])
     annotations = Nested(oa.hasTarget, AnnotationSchema, reverse=True, many=True)
+
+    @marshmallow.pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/project.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Project JSON-LD schema."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -59,3 +59,22 @@ class Meta:
     )
     version = StringList(schema.schemaVersion, load_default="1")
     keywords = fields.List(schema.keywords, fields.String(), load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.date_created = obj.date_created.replace(microsecond=0)
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/core/git.py

@@ -207,6 +207,6 @@ def ensure_clean(ignore_std_streams=False):
         if dirty_paths - set(mapped_streams.values()):
             _clean_streams(repository, mapped_streams)
             raise errors.DirtyRepository(repository)
-    elif repository.is_dirty():
+    elif repository.is_dirty(untracked_files=False):
         _clean_streams(repository, mapped_streams)
         raise errors.DirtyRepository(repository)

renku/core/migration/m_0005__2_cwl.py

@@ -107,7 +107,7 @@ def sort_cwl_commits(e1, e2):
 
                 repository.add(cwl_file, path)
 
-                if repository.is_dirty():
+                if repository.is_dirty(untracked_files=False):
                     commit_msg = "renku migrate: committing migrated workflow"
                     committer = Actor(name=f"renku {__version__}", email=version_url)
                     repository.commit(commit_msg + project_context.transaction_id, committer=committer, no_verify=True)

renku/core/migration/m_0009__new_metadata_storage.py

@@ -96,7 +96,7 @@ def migrate(migration_context: MigrationContext):
 
 def _commit_previous_changes():
     repository = project_context.repository
-    if repository.is_dirty():
+    if repository.is_dirty(untracked_files=False):
         project_path = project_context.metadata_path.joinpath(OLD_METADATA_PATH)
         project = old_schema.Project.from_yaml(project_path)
         project.version = "8"
@@ -349,7 +349,7 @@ def _process_workflows(
     migration_context: MigrationContext, activity_gateway: IActivityGateway, commit: "Commit", remove: bool
 ):
 
-    for file in commit.get_changes(paths=f"{project_context.metadata_path}/workflow/*.yaml"):
+    for file in commit.get_changes(f"{project_context.metadata_path}/workflow/*.yaml"):
         if file.deleted:
             continue
 
@@ -640,7 +640,7 @@ def _process_datasets(
     is_last_commit,
     preserve_identifiers,
 ):
-    changes = commit.get_changes(paths=".renku/datasets/*/*.yml")
+    changes = commit.get_changes(".renku/datasets/*/*.yml")
     changed_paths = [c.b_path for c in changes if not c.deleted]
     paths = [p for p in changed_paths if len(Path(p).parents) == 4]  # Exclude files that are not in the right place
     deleted_paths = [c.a_path for c in changes if c.deleted]

renku/core/migration/m_0010__metadata_fixes.py

@@ -254,7 +254,7 @@ def fix_plan_times(plan_gateway: IPlanGateway):
             if plan.date_removed.tzinfo is None:
                 # NOTE: There was a bug that caused date_removed to be set without timezone (as UTC time)
                 # so we patch in the timezone here
-                plan.date_removed = plan.date_removed.replace(microsecond=0).astimezone(timezone.utc)
+                plan.date_removed = plan.date_removed.astimezone(timezone.utc)
             if plan.date_removed < plan.date_created:
                 # NOTE: Fix invalidation times set before creation date on plans
                 plan.date_removed = plan.date_created