feat(core): deprecate template

close #1907
kestra-io · Aug 21, 2023 · 9489a49 · 9489a49
1 parent 304d921
commit 9489a49
Show file tree

Hide file tree

Showing 41 changed files with 280 additions and 56 deletions.
diff --git a/cli/src/main/java/io/kestra/cli/commands/sys/RestoreQueueCommand.java b/cli/src/main/java/io/kestra/cli/commands/sys/RestoreQueueCommand.java
@@ -2,6 +2,7 @@
 
 import io.kestra.cli.AbstractCommand;
 import io.kestra.cli.services.RestoreQueueService;
+import io.kestra.core.repositories.TemplateRepositoryInterface;
 import io.micronaut.context.ApplicationContext;
 import jakarta.inject.Inject;
 import lombok.extern.slf4j.Slf4j;
@@ -46,7 +47,7 @@ public Integer call() throws Exception {
         }
 
         // templates
-        if (!this.noTemplates) {
+        if (!this.noTemplates && applicationContext.containsBean(TemplateRepositoryInterface.class)) {
             int size = restoreQueueService.templates(noRecreate);
             stdOut("Successfully send {0} templates", size);
         }

diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateCommand.java
@@ -3,6 +3,7 @@
 import io.kestra.cli.AbstractCommand;
 import io.kestra.cli.App;
 import io.kestra.cli.commands.templates.namespaces.TemplateNamespaceCommand;
+import io.kestra.core.models.templates.TemplateEnabled;
 import io.micronaut.configuration.picocli.PicocliRunner;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -19,6 +20,7 @@
     }
 )
 @Slf4j
+@TemplateEnabled
 public class TemplateCommand extends AbstractCommand {
     @SneakyThrows
     @Override

diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateExportCommand.java
@@ -1,6 +1,7 @@
 package io.kestra.cli.commands.templates;
 
 import io.kestra.cli.AbstractApiCommand;
+import io.kestra.core.models.templates.TemplateEnabled;
 import io.micronaut.http.HttpRequest;
 import io.micronaut.http.HttpResponse;
 import io.micronaut.http.MediaType;
@@ -19,6 +20,7 @@
     mixinStandardHelpOptions = true
 )
 @Slf4j
+@TemplateEnabled
 public class TemplateExportCommand extends AbstractApiCommand {
     private static final String DEFAULT_FILE_NAME = "templates.zip";
 

diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/TemplateValidateCommand.java
@@ -2,6 +2,7 @@
 
 import io.kestra.cli.AbstractValidateCommand;
 import io.kestra.core.models.templates.Template;
+import io.kestra.core.models.templates.TemplateEnabled;
 import io.kestra.core.models.validations.ModelValidator;
 import io.kestra.core.serializers.YamlFlowParser;
 import jakarta.inject.Inject;
@@ -11,6 +12,7 @@
     name = "validate",
     description = "validate a template"
 )
+@TemplateEnabled
 public class TemplateValidateCommand extends AbstractValidateCommand {
     @Inject
     private YamlFlowParser yamlFlowParser;

diff --git a/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommand.java b/cli/src/main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceCommand.java
@@ -1,6 +1,7 @@
 package io.kestra.cli.commands.templates.namespaces;
 
 import io.kestra.cli.AbstractCommand;
+import io.kestra.core.models.templates.TemplateEnabled;
 import lombok.extern.slf4j.Slf4j;
 import picocli.CommandLine;
 
@@ -13,6 +14,7 @@
     }
 )
 @Slf4j
+@TemplateEnabled
 public class TemplateNamespaceCommand extends AbstractCommand {
 
 }
diff --git a/...main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java b/...main/java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommand.java
@@ -3,6 +3,7 @@
 import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
 import io.kestra.cli.commands.templates.TemplateValidateCommand;
 import io.kestra.core.models.templates.Template;
+import io.kestra.core.models.templates.TemplateEnabled;
 import io.kestra.core.serializers.YamlFlowParser;
 import io.micronaut.core.type.Argument;
 import io.micronaut.http.HttpRequest;
@@ -13,17 +14,18 @@
 import lombok.extern.slf4j.Slf4j;
 import picocli.CommandLine;
 
-import javax.validation.ConstraintViolationException;
 import java.nio.file.Files;
 import java.util.List;
 import java.util.stream.Collectors;
+import javax.validation.ConstraintViolationException;
 
 @CommandLine.Command(
     name = "update",
     description = "handle namespace templates",
     mixinStandardHelpOptions = true
 )
 @Slf4j
+@TemplateEnabled
 public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCommand {
     @Inject
     public YamlFlowParser yamlFlowParser;

diff --git a/cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/templates/TemplateExportCommandTest.java
@@ -12,6 +12,7 @@
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.URL;
+import java.util.Map;
 import java.util.zip.ZipFile;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -25,7 +26,7 @@ void run() throws IOException {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         System.setOut(new PrintStream(out));
 
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
 
             EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
             embeddedServer.start();

diff --git a/cli/src/test/java/io/kestra/cli/commands/templates/TemplateValidateCommandTest.java b/cli/src/test/java/io/kestra/cli/commands/templates/TemplateValidateCommandTest.java
@@ -9,6 +9,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.net.URL;
+import java.util.Map;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.Is.is;
@@ -21,7 +22,7 @@ void runLocal()  {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         System.setErr(new PrintStream(out));
 
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
             String[] args = {
                 "--local",
                 directory.getPath()
@@ -40,7 +41,7 @@ void runServer()  {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         System.setErr(new PrintStream(out));
 
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
 
             EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
             embeddedServer.start();

diff --git a/.../java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommandTest.java b/.../java/io/kestra/cli/commands/templates/namespaces/TemplateNamespaceUpdateCommandTest.java
@@ -9,6 +9,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.net.URL;
+import java.util.Map;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.core.StringContains.containsString;
@@ -20,7 +21,7 @@ void run() {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         System.setOut(new PrintStream(out));
 
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
 
             EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
             embeddedServer.start();
@@ -46,7 +47,7 @@ void invalid() {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         System.setErr(new PrintStream(out));
 
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
 
             EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
             embeddedServer.start();
@@ -76,7 +77,7 @@ void runNoDelete() {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         System.setOut(new PrintStream(out));
 
-        try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
+        try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
 
             EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
             embeddedServer.start();

diff --git a/core/src/main/java/io/kestra/core/models/templates/TemplateEnabled.java b/core/src/main/java/io/kestra/core/models/templates/TemplateEnabled.java
@@ -0,0 +1,15 @@
+package io.kestra.core.models.templates;
+
+import io.micronaut.context.annotation.Requires;
+import io.micronaut.core.util.StringUtils;
+
+import java.lang.annotation.*;
+
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.PACKAGE, ElementType.TYPE})
+@Requires(property = "kestra.templates.enabled", value = StringUtils.TRUE, defaultValue = StringUtils.FALSE)
+@Inherited
+public @interface TemplateEnabled {
+
+}
diff --git a/core/src/main/java/io/kestra/core/tasks/flows/Template.java b/core/src/main/java/io/kestra/core/tasks/flows/Template.java
@@ -1,5 +1,6 @@
 package io.kestra.core.tasks.flows;
 
+import io.kestra.core.exceptions.DeserializationException;
 import io.kestra.core.exceptions.IllegalVariableEvaluationException;
 import io.kestra.core.exceptions.InternalException;
 import io.kestra.core.models.annotations.Example;
@@ -14,6 +15,7 @@
 import io.kestra.core.models.tasks.FlowableTask;
 import io.kestra.core.models.tasks.ResolvedTask;
 import io.kestra.core.models.tasks.Task;
+import io.kestra.core.models.templates.TemplateEnabled;
 import io.kestra.core.repositories.TemplateRepositoryInterface;
 import io.kestra.core.runners.FlowableUtils;
 import io.kestra.core.runners.RunContext;
@@ -79,6 +81,7 @@
         )
     }
 )
+@TemplateEnabled
 public class Template extends Task implements FlowableTask<Template.Output> {
     @Valid
     @PluginProperty
@@ -187,6 +190,10 @@ public Template.Output outputs(RunContext runContext, Execution execution, TaskR
     }
 
     protected io.kestra.core.models.templates.Template findTemplate(ApplicationContext applicationContext) throws IllegalVariableEvaluationException {
+        if (!applicationContext.containsBean(TemplateExecutorInterface.class)) {
+            throw new DeserializationException(new Exception("Templates are disabled, please check your configuration"));
+        }
+
         TemplateExecutorInterface templateExecutor = applicationContext.getBean(TemplateExecutorInterface.class);
 
         return templateExecutor.findById(this.namespace, this.templateId)
@@ -280,6 +287,7 @@ public interface TemplateExecutorInterface {
         Optional<io.kestra.core.models.templates.Template> findById(String namespace, String templateId);
     }
 
+    @TemplateEnabled
     public static class MemoryTemplateExecutor implements io.kestra.core.tasks.flows.Template.TemplateExecutorInterface {
         @Inject
         private TemplateRepositoryInterface templateRepository;

diff --git a/core/src/test/java/io/kestra/core/repositories/AbstractFlowRepositoryTest.java b/core/src/test/java/io/kestra/core/repositories/AbstractFlowRepositoryTest.java
@@ -6,22 +6,26 @@
 import io.kestra.core.events.CrudEvent;
 import io.kestra.core.events.CrudEventType;
 import io.kestra.core.models.flows.Flow;
+import io.kestra.core.models.flows.FlowWithException;
 import io.kestra.core.models.flows.FlowWithSource;
 import io.kestra.core.models.flows.Input;
 import io.kestra.core.models.flows.input.StringInput;
 import io.kestra.core.models.triggers.Trigger;
 import io.kestra.core.queues.QueueFactoryInterface;
 import io.kestra.core.queues.QueueInterface;
+import io.kestra.core.runners.FlowListeners;
 import io.kestra.core.schedulers.AbstractSchedulerTest;
 import io.kestra.core.serializers.JacksonMapper;
 import io.kestra.core.services.TaskDefaultService;
 import io.kestra.core.tasks.debugs.Return;
+import io.kestra.core.tasks.flows.Template;
 import io.kestra.core.tasks.log.Log;
 import io.kestra.core.utils.Await;
 import io.kestra.core.utils.IdUtils;
 import io.kestra.core.utils.TestsUtils;
 import io.micronaut.context.event.ApplicationEventListener;
 import io.micronaut.data.model.Pageable;
+import io.micronaut.data.model.Sort;
 import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
@@ -44,6 +48,8 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
 
 @MicronautTest(transactional = false)
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
@@ -368,6 +374,41 @@ void findDistinctNamespace() {
         assertThat((long) distinctNamespace.size(), is(4L));
     }
 
+    @Test
+    void templateDisabled() {
+        Template template = Template.builder()
+            .id(IdUtils.create())
+            .type(Template.class.getName())
+            .namespace("test")
+            .templateId("testTemplate")
+            .build();
+
+        Template templateSpy = spy(template);
+
+        doReturn(Collections.emptyList())
+            .when(templateSpy)
+            .allChildTasks();
+
+        Flow flow = Flow.builder()
+            .id(IdUtils.create())
+            .namespace("io.kestra.unittest")
+            .tasks(Collections.singletonList(templateSpy))
+            .build();
+
+        flowRepository.create(
+            flow,
+            flow.generateSource(),
+            flow
+        );
+
+        Optional<Flow> found = flowRepository.findById(flow.getNamespace(), flow.getId());
+
+        assertThat(found.isPresent(), is(true));
+        assertThat(found.get() instanceof FlowWithException, is(true));
+        assertThat(((FlowWithException) found.get()).getException(), containsString("Templates are disabled"));
+    }
+
+
     @Singleton
     public static class FlowListener implements ApplicationEventListener<CrudEvent<Flow>> {
         private static List<CrudEvent<Flow>> emits = new ArrayList<>();

diff --git a/core/src/test/java/io/kestra/core/repositories/AbstractTemplateRepositoryTest.java b/core/src/test/java/io/kestra/core/repositories/AbstractTemplateRepositoryTest.java
@@ -1,14 +1,13 @@
 package io.kestra.core.repositories;
 
-import io.kestra.core.Helpers;
 import io.kestra.core.events.CrudEvent;
 import io.kestra.core.events.CrudEventType;
-import io.kestra.core.models.flows.Flow;
-import io.kestra.core.models.flows.FlowWithSource;
 import io.kestra.core.models.templates.Template;
 import io.kestra.core.tasks.debugs.Return;
 import io.kestra.core.utils.IdUtils;
+import io.micronaut.context.annotation.Property;
 import io.micronaut.context.event.ApplicationEventListener;
+import io.micronaut.core.util.StringUtils;
 import io.micronaut.data.model.Pageable;
 import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
 import jakarta.inject.Inject;

diff --git a/core/src/test/java/io/kestra/core/tasks/flows/TemplateTest.java b/core/src/test/java/io/kestra/core/tasks/flows/TemplateTest.java
@@ -13,6 +13,8 @@
 import io.kestra.core.runners.RunnerUtils;
 import io.kestra.core.tasks.log.Log;
 import io.kestra.core.utils.TestsUtils;
+import io.micronaut.context.annotation.Property;
+import io.micronaut.core.util.StringUtils;
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
 import org.junit.jupiter.api.Test;
@@ -30,6 +32,7 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.*;
 
+@Property(name = "kestra.templates.enabled", value = StringUtils.TRUE)
 public class TemplateTest extends AbstractMemoryRunnerTest {
     @Inject
     protected TemplateRepositoryInterface templateRepository;
@@ -46,7 +49,8 @@ public class TemplateTest extends AbstractMemoryRunnerTest {
 
     public static void withTemplate(RunnerUtils runnerUtils, TemplateRepositoryInterface templateRepository, LocalFlowRepositoryLoader repositoryLoader, QueueInterface<LogEntry> logQueue) throws TimeoutException, IOException, URISyntaxException {
         templateRepository.create(TEMPLATE_1);
-        repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource("flows/tests/with-template.yaml")));
+        repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource(
+            "flows/templates/with-template.yaml")));
 
         List<LogEntry> logs = new CopyOnWriteArrayList<>();
         logQueue.receive(logs::add);
@@ -76,7 +80,8 @@ void withTemplate() throws TimeoutException, IOException, URISyntaxException {
 
 
     public static void withFailedTemplate(RunnerUtils runnerUtils, TemplateRepositoryInterface templateRepository, LocalFlowRepositoryLoader repositoryLoader, QueueInterface<LogEntry> logQueue) throws TimeoutException, IOException, URISyntaxException {
-        repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource("flows/tests/with-failed-template.yaml")));
+        repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource(
+            "flows/templates/with-failed-template.yaml")));
 
         List<LogEntry> logs = new CopyOnWriteArrayList<>();
         logQueue.receive(logs::add);

diff --git a/...ces/flows/tests/with-failed-template.yaml → ...flows/templates/with-failed-template.yaml b/...ces/flows/tests/with-failed-template.yaml → ...flows/templates/with-failed-template.yaml
diff --git a/.../resources/flows/tests/with-template.yaml → ...ources/flows/templates/with-template.yaml b/.../resources/flows/tests/with-template.yaml → ...ources/flows/templates/with-template.yaml
diff --git a/jdbc-h2/src/main/java/io/kestra/repository/h2/H2TemplateRepository.java b/jdbc-h2/src/main/java/io/kestra/repository/h2/H2TemplateRepository.java
@@ -1,6 +1,7 @@
 package io.kestra.repository.h2;
 
 import io.kestra.core.models.templates.Template;
+import io.kestra.core.models.templates.TemplateEnabled;
 import io.kestra.jdbc.repository.AbstractJdbcTemplateRepository;
 import io.micronaut.context.ApplicationContext;
 import jakarta.inject.Inject;
@@ -11,6 +12,7 @@
 
 @Singleton
 @H2RepositoryEnabled
+@TemplateEnabled
 public class H2TemplateRepository extends AbstractJdbcTemplateRepository {
     @Inject
     public H2TemplateRepository(ApplicationContext applicationContext) {