chore: release v2.3.1 (#3357)

.github/workflows/test_deploy.yml

@@ -5,13 +5,16 @@ on:
     branches:
       - master
       - develop
-      - "release/**" # otherwise PRs created by the PR action don't execute tests
+      - "release/*" # otherwise PRs created by the PR action don't execute tests
     tags:
       - "v*.*.*"
   pull_request:
     types: [opened, reopened, synchronize]
     branches:
-      - "!release/**"
+      - "**"
+      - "!release/*"
+      - "!master"
+      - "!develop"
 
 env:
   DEPENDENCY_CACHE_PREFIX: "v2"

CHANGES.rst

@@ -18,6 +18,22 @@
 Changes
 =======
 
+`2.3.1 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.3.0...v2.3.1>`__ (2023-03-10)
+-------------------------------------------------------------------------------------------------------
+
+Bug Fixes
+~~~~~~~~~
+
+-  **cli:** set git credentials helper correctly on git clone
+   (`#3348 <https://github.com/SwissDataScienceCenter/renku-python/issues/3348>`__)
+   (`ac21c32 <https://github.com/SwissDataScienceCenter/renku-python/commit/ac21c3266246f011ec0bfd785e53fd03a553eee4>`__)
+-  **core:** update template checksum when updating Dockerfile
+   (`#3351 <https://github.com/SwissDataScienceCenter/renku-python/issues/3351>`__)
+   (`24407e5 <https://github.com/SwissDataScienceCenter/renku-python/commit/24407e5857faab0fe4d20883039f2a831977b15c>`__)
+-  **core:** parallelize dataset add to improve performance
+   (`#3338 <https://github.com/SwissDataScienceCenter/renku-python/issues/3338>`__)
+   (`85ab141 <https://github.com/SwissDataScienceCenter/renku-python/commit/85ab14199c83815f6d3a63fd6dc721638b3b4c95>`__)
+
 `2.3.0 <https://github.com/SwissDataScienceCenter/renku-python/compare/v2.2.0...v2.3.0>`__ (2023-02-27)
 -------------------------------------------------------------------------------------------------------
 

helm-chart/renku-core/Chart.yaml

@@ -3,4 +3,4 @@ appVersion: "1.0"
 description: A Helm chart for Kubernetes
 name: renku-core
 icon: https://avatars0.githubusercontent.com/u/53332360?s=400&u=a4311d22842343604ef61a8c8a1e5793209a67e9&v=4
-version: 2.3.0
+version: 2.3.1

helm-chart/renku-core/templates/deployment-nginx.yaml

@@ -11,6 +11,8 @@ spec:
   replicas: 1
   template:
     metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
       labels:
         app.kubernetes.io/name: {{ include "renku-core.fullname" $ }}-nginx
     spec:

helm-chart/renku-core/values.yaml

@@ -97,7 +97,7 @@ versions:
     fullnameOverride: ""
     image:
       repository: renku/renku-core
-      tag: "v2.3.0"
+      tag: "v2.3.1"
       pullPolicy: IfNotPresent
   v9:
     name: v9
@@ -106,7 +106,7 @@ versions:
     fullnameOverride: ""
     image:
       repository: renku/renku-core
-      tag: "v1.11.2"
+      tag: "v1.11.3"
       pullPolicy: IfNotPresent
 podSecurityContext:
   runAsUser: 1000

pyproject.toml

@@ -132,7 +132,7 @@ Flake8-pyproject = [
 ]
 isort = "<5.10.2,>=5.3.2"
 mypy = ">=0.942,<1.0"
-poetry-lock-package = "^0.4.3"
+poetry-lock-package = "^0.5.0"
 pre-commit = "^2.20.0"
 types-PyYAML = "<6.1.0,>=5.4"
 types-python-dateutil = "^2.8.10"

renku/command/clone.py

@@ -62,8 +62,8 @@ def _project_clone(
         Tuple of cloned ``Repository`` and whether it's a Renku project or not.
     """
     from renku.command.mergetool import setup_mergetool
-    from renku.core.migration.migrate import is_renku_project
     from renku.core.util.git import clone_renku_repository
+    from renku.core.util.metadata import is_renku_project
 
     install_lfs = project_context.external_storage_requested
 

renku/command/migrate.py

@@ -177,13 +177,9 @@ def check_project():
 
 
 def _check_project():
-    from renku.core.migration.migrate import (
-        is_docker_update_possible,
-        is_migration_required,
-        is_project_unsupported,
-        is_renku_project,
-    )
+    from renku.core.migration.migrate import is_docker_update_possible, is_migration_required, is_project_unsupported
     from renku.core.template.usecase import check_for_template_update
+    from renku.core.util.metadata import is_renku_project
 
     if not is_renku_project():
         return NON_RENKU_REPOSITORY

renku/command/rollback.py

@@ -149,10 +149,10 @@ def _get_modifications_from_diff(diff):
             continue
 
         # normal file
-        if diff_index.change_type == "A":
+        if diff_index.added:
             modifications["files"]["removed"].append(entry)
 
-        elif diff_index.change_type == "D":
+        elif diff_index.deleted:
             modifications["files"]["restored"].append(entry)
         else:
             modifications["files"]["modified"].append(entry)

renku/command/schema/activity.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Activity JSON-LD schema."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema, SoftwareAgentSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -124,6 +124,28 @@ class Meta:
     started_at_time = fields.DateTime(prov.startedAtTime, add_value_types=True)
     usages = Nested(prov.qualifiedUsage, UsageSchema, many=True)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.started_at_time = obj.started_at_time.replace(microsecond=0)
+            obj.ended_at_time = obj.ended_at_time.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class WorkflowFileActivityCollectionSchema(JsonLDSchema):
     """WorkflowFileActivityCollection schema."""

renku/command/schema/composite_plan.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Represent a group of run templates."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.calamus import JsonLDSchema, Nested, fields, prov, renku, schema
@@ -49,3 +49,27 @@ class Meta:
     project_id = fields.IRI(renku.hasPlan, reverse=True)
     plans = Nested(renku.hasSubprocess, [PlanSchema, "CompositePlanSchema"], many=True)
     links = Nested(renku.workflowLinks, [ParameterLinkSchema], many=True, load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/dataset.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Datasets JSON-LD schemes."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -69,6 +69,27 @@ class Meta:
     id = fields.Id()
     name = fields.String(schema.name)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class LanguageSchema(JsonLDSchema):
     """Language schema."""
@@ -134,6 +155,27 @@ class Meta:
     is_external = fields.Boolean(renku.external, load_default=False)
     source = fields.String(renku.source, load_default=None)
 
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.date_added = obj.date_added.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs
+
 
 class DatasetSchema(JsonLDSchema):
     """Dataset schema."""
@@ -168,3 +210,30 @@ class Meta:
     same_as = Nested(schema.sameAs, UrlSchema, load_default=None)
     title = fields.String(schema.name)
     version = fields.String(schema.version, load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            if obj.date_created:
+                obj.date_created = obj.date_created.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            if obj.date_published:
+                obj.date_published = obj.date_published.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/plan.py

@@ -54,3 +54,27 @@ class Meta:
     parameters = Nested(renku.hasArguments, CommandParameterSchema, many=True, load_default=None)
     success_codes = fields.List(renku.successCodes, fields.Integer(), load_default=[0])
     annotations = Nested(oa.hasTarget, AnnotationSchema, reverse=True, many=True)
+
+    @marshmallow.pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.unfreeze()
+            obj.date_created = obj.date_created.replace(microsecond=0)
+            obj.date_modified = obj.date_modified.replace(microsecond=0)
+            if obj.date_removed:
+                obj.date_removed = obj.date_removed.replace(microsecond=0)
+            obj.freeze()
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/schema/project.py

@@ -17,7 +17,7 @@
 # limitations under the License.
 """Project JSON-LD schema."""
 
-from marshmallow import EXCLUDE
+from marshmallow import EXCLUDE, pre_dump
 
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.annotation import AnnotationSchema
@@ -59,3 +59,22 @@ class Meta:
     )
     version = StringList(schema.schemaVersion, load_default="1")
     keywords = fields.List(schema.keywords, fields.String(), load_default=None)
+
+    @pre_dump(pass_many=True)
+    def removes_ms(self, objs, many, **kwargs):
+        """Remove milliseconds from datetimes.
+
+        Note: since DateField uses `strftime` as format, which only supports timezone info without a colon
+        e.g. `+0100` instead of `+01:00`, we have to deal with milliseconds manually instead of using a format string.
+        """
+
+        def _replace_times(obj):
+            obj.date_created = obj.date_created.replace(microsecond=0)
+
+        if many:
+            for obj in objs:
+                _replace_times(obj)
+            return objs
+
+        _replace_times(objs)
+        return objs

renku/command/session.py

@@ -29,7 +29,7 @@ def session_list_command():
 
 def session_start_command():
     """Start an interactive session."""
-    return Command().command(session_start).with_database()
+    return Command().command(session_start).with_database().require_migration()
 
 
 def session_stop_command():