save the repr of template variables in database (#4864)

template variables: save repr strings to the database

* This allows non-string template variables to be retrieved from the DB on restart.

CHANGES.md

@@ -48,6 +48,10 @@ workflow source argument), and rename the `--flow-name` option to
 
 ### Fixes
 
+[#4864](https://github.com/cylc/cylc-flow/pull/4864) - Allow strings
+and more complex data type template variables to be stored correctly
+in the workflow database.
+
 [#4863](https://github.com/cylc/cylc-flow/pull/4863) - Execution timeout is no
 longer set based on execution time limit. Fixes bug where execution timeout
 would get overridden.

cylc/flow/scheduler.py

@@ -95,6 +95,7 @@
 from cylc.flow.profiler import Profiler
 from cylc.flow.resources import get_resources
 from cylc.flow.subprocpool import SubProcPool
+from cylc.flow.templatevars import eval_var
 from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
 from cylc.flow.workflow_events import WorkflowEventHandler
 from cylc.flow.workflow_status import StopMode, AutoRestartMode
@@ -1236,7 +1237,7 @@ def _load_template_vars(self, _, row):
         key, value = row
         # Command line argument takes precedence
         if key not in self.template_vars:
-            self.template_vars[key] = value
+            self.template_vars[key] = eval_var(value)
 
     def run_event_handlers(self, event, reason=""):
         """Run a workflow event handler.

cylc/flow/workflow_db_mgr.py

@@ -395,7 +395,7 @@ def put_workflow_template_vars(self, template_vars):
         """
         for key, value in template_vars.items():
             self.db_inserts_map[self.TABLE_WORKFLOW_TEMPLATE_VARS].append(
-                {"key": key, "value": value})
+                {"key": key, "value": repr(value)})
 
     def put_task_event_timers(self, task_events_mgr):
         """Put statements to update the task_action_timers table."""

tests/functional/restart/56-db-saves-type.t

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test restarting a simple workflow with a task still running (orphaned)
+. "$(dirname "$0")/test_header"
+set_test_number 3
+init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__'
+#!Jinja2
+
+{{ assert(a_str == 'foo', 'variable "a_str" was not a string') }}
+{{ assert(an_int == 24, 'variable "an_int" was not an int') }}
+{{ assert(a_float ==  1.1111, 'variable "a_float" was not a float') }}
+{{ assert(complex == {'foo': [True, 42, 'string']}, 'Complex variable did not validate')}}
+
+
+[scheduler]
+    allow implicit tasks = True
+[scheduling]
+   [[graph]]
+        R1 = foo
+__FLOW_CONFIG__
+
+
+#-------------------------------------------------------------------------------
+run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" \
+    --set "a_str='foo'" --set "an_int=24" --set "a_float=1.1111" --set "complex={'foo': [True, 42, 'string']}"
+
+workflow_run_ok "${TEST_NAME_BASE}-run" \
+    cylc play --debug "${WORKFLOW_NAME}" --pause\
+    --set "a_str='foo'" --set "an_int=24" --set "a_float=1.1111" --set "complex={'foo': [True, 42, 'string']}"
+
+sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT * FROM workflow_template_vars' >'sqlite3.out'
+
+cylc stop "${WORKFLOW_NAME}"
+
+workflow_run_ok "${TEST_NAME_BASE}-restart" \
+    cylc play --debug --no-detach "${WORKFLOW_NAME}"
+#-------------------------------------------------------------------------------
+purge
+exit