Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core): deprecate template #1916

Merged
merged 1 commit into from
Aug 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.kestra.cli.AbstractCommand;
import io.kestra.cli.services.RestoreQueueService;
import io.kestra.core.repositories.TemplateRepositoryInterface;
import io.micronaut.context.ApplicationContext;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -46,7 +47,7 @@ public Integer call() throws Exception {
}

// templates
if (!this.noTemplates) {
if (!this.noTemplates && applicationContext.containsBean(TemplateRepositoryInterface.class)) {
int size = restoreQueueService.templates(noRecreate);
stdOut("Successfully send {0} templates", size);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io.kestra.cli.AbstractCommand;
import io.kestra.cli.App;
import io.kestra.cli.commands.templates.namespaces.TemplateNamespaceCommand;
import io.kestra.core.models.templates.TemplateEnabled;
import io.micronaut.configuration.picocli.PicocliRunner;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -19,6 +20,7 @@
}
)
@Slf4j
@TemplateEnabled
public class TemplateCommand extends AbstractCommand {
@SneakyThrows
@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.kestra.cli.commands.templates;

import io.kestra.cli.AbstractApiCommand;
import io.kestra.core.models.templates.TemplateEnabled;
import io.micronaut.http.HttpRequest;
import io.micronaut.http.HttpResponse;
import io.micronaut.http.MediaType;
Expand All @@ -19,6 +20,7 @@
mixinStandardHelpOptions = true
)
@Slf4j
@TemplateEnabled
public class TemplateExportCommand extends AbstractApiCommand {
private static final String DEFAULT_FILE_NAME = "templates.zip";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.kestra.cli.AbstractValidateCommand;
import io.kestra.core.models.templates.Template;
import io.kestra.core.models.templates.TemplateEnabled;
import io.kestra.core.models.validations.ModelValidator;
import io.kestra.core.serializers.YamlFlowParser;
import jakarta.inject.Inject;
Expand All @@ -11,6 +12,7 @@
name = "validate",
description = "validate a template"
)
@TemplateEnabled
public class TemplateValidateCommand extends AbstractValidateCommand {
@Inject
private YamlFlowParser yamlFlowParser;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.kestra.cli.commands.templates.namespaces;

import io.kestra.cli.AbstractCommand;
import io.kestra.core.models.templates.TemplateEnabled;
import lombok.extern.slf4j.Slf4j;
import picocli.CommandLine;

Expand All @@ -13,6 +14,7 @@
}
)
@Slf4j
@TemplateEnabled
public class TemplateNamespaceCommand extends AbstractCommand {

}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import io.kestra.cli.commands.AbstractServiceNamespaceUpdateCommand;
import io.kestra.cli.commands.templates.TemplateValidateCommand;
import io.kestra.core.models.templates.Template;
import io.kestra.core.models.templates.TemplateEnabled;
import io.kestra.core.serializers.YamlFlowParser;
import io.micronaut.core.type.Argument;
import io.micronaut.http.HttpRequest;
Expand All @@ -13,17 +14,18 @@
import lombok.extern.slf4j.Slf4j;
import picocli.CommandLine;

import javax.validation.ConstraintViolationException;
import java.nio.file.Files;
import java.util.List;
import java.util.stream.Collectors;
import javax.validation.ConstraintViolationException;

@CommandLine.Command(
name = "update",
description = "handle namespace templates",
mixinStandardHelpOptions = true
)
@Slf4j
@TemplateEnabled
public class TemplateNamespaceUpdateCommand extends AbstractServiceNamespaceUpdateCommand {
@Inject
public YamlFlowParser yamlFlowParser;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import java.io.IOException;
import java.io.PrintStream;
import java.net.URL;
import java.util.Map;
import java.util.zip.ZipFile;

import static org.hamcrest.MatcherAssert.assertThat;
Expand All @@ -25,7 +26,7 @@ void run() throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setOut(new PrintStream(out));

try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {

EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
embeddedServer.start();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.net.URL;
import java.util.Map;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.Is.is;
Expand All @@ -21,7 +22,7 @@ void runLocal() {
ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setErr(new PrintStream(out));

try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {
String[] args = {
"--local",
directory.getPath()
Expand All @@ -40,7 +41,7 @@ void runServer() {
ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setErr(new PrintStream(out));

try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {

EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
embeddedServer.start();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.net.URL;
import java.util.Map;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.StringContains.containsString;
Expand All @@ -20,7 +21,7 @@ void run() {
ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setOut(new PrintStream(out));

try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {

EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
embeddedServer.start();
Expand All @@ -46,7 +47,7 @@ void invalid() {
ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setErr(new PrintStream(out));

try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {

EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
embeddedServer.start();
Expand Down Expand Up @@ -76,7 +77,7 @@ void runNoDelete() {
ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setOut(new PrintStream(out));

try (ApplicationContext ctx = ApplicationContext.run(Environment.CLI, Environment.TEST)) {
try (ApplicationContext ctx = ApplicationContext.run(Map.of("kestra.templates.enabled", "true"), Environment.CLI, Environment.TEST)) {

EmbeddedServer embeddedServer = ctx.getBean(EmbeddedServer.class);
embeddedServer.start();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package io.kestra.core.models.templates;

import io.micronaut.context.annotation.Requires;
import io.micronaut.core.util.StringUtils;

import java.lang.annotation.*;

@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.PACKAGE, ElementType.TYPE})
@Requires(property = "kestra.templates.enabled", value = StringUtils.TRUE, defaultValue = StringUtils.FALSE)
@Inherited
public @interface TemplateEnabled {

}
8 changes: 8 additions & 0 deletions core/src/main/java/io/kestra/core/tasks/flows/Template.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package io.kestra.core.tasks.flows;

import io.kestra.core.exceptions.DeserializationException;
import io.kestra.core.exceptions.IllegalVariableEvaluationException;
import io.kestra.core.exceptions.InternalException;
import io.kestra.core.models.annotations.Example;
Expand All @@ -14,6 +15,7 @@
import io.kestra.core.models.tasks.FlowableTask;
import io.kestra.core.models.tasks.ResolvedTask;
import io.kestra.core.models.tasks.Task;
import io.kestra.core.models.templates.TemplateEnabled;
import io.kestra.core.repositories.TemplateRepositoryInterface;
import io.kestra.core.runners.FlowableUtils;
import io.kestra.core.runners.RunContext;
Expand Down Expand Up @@ -79,6 +81,7 @@
)
}
)
@TemplateEnabled
public class Template extends Task implements FlowableTask<Template.Output> {
@Valid
@PluginProperty
Expand Down Expand Up @@ -187,6 +190,10 @@ public Template.Output outputs(RunContext runContext, Execution execution, TaskR
}

protected io.kestra.core.models.templates.Template findTemplate(ApplicationContext applicationContext) throws IllegalVariableEvaluationException {
if (!applicationContext.containsBean(TemplateExecutorInterface.class)) {
throw new DeserializationException(new Exception("Templates are disabled, please check your configuration"));
}

TemplateExecutorInterface templateExecutor = applicationContext.getBean(TemplateExecutorInterface.class);

return templateExecutor.findById(this.namespace, this.templateId)
Expand Down Expand Up @@ -280,6 +287,7 @@ public interface TemplateExecutorInterface {
Optional<io.kestra.core.models.templates.Template> findById(String namespace, String templateId);
}

@TemplateEnabled
public static class MemoryTemplateExecutor implements io.kestra.core.tasks.flows.Template.TemplateExecutorInterface {
@Inject
private TemplateRepositoryInterface templateRepository;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,26 @@
import io.kestra.core.events.CrudEvent;
import io.kestra.core.events.CrudEventType;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.FlowWithException;
import io.kestra.core.models.flows.FlowWithSource;
import io.kestra.core.models.flows.Input;
import io.kestra.core.models.flows.input.StringInput;
import io.kestra.core.models.triggers.Trigger;
import io.kestra.core.queues.QueueFactoryInterface;
import io.kestra.core.queues.QueueInterface;
import io.kestra.core.runners.FlowListeners;
import io.kestra.core.schedulers.AbstractSchedulerTest;
import io.kestra.core.serializers.JacksonMapper;
import io.kestra.core.services.TaskDefaultService;
import io.kestra.core.tasks.debugs.Return;
import io.kestra.core.tasks.flows.Template;
import io.kestra.core.tasks.log.Log;
import io.kestra.core.utils.Await;
import io.kestra.core.utils.IdUtils;
import io.kestra.core.utils.TestsUtils;
import io.micronaut.context.event.ApplicationEventListener;
import io.micronaut.data.model.Pageable;
import io.micronaut.data.model.Sort;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import jakarta.inject.Inject;
import jakarta.inject.Named;
Expand All @@ -44,6 +48,8 @@
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;

@MicronautTest(transactional = false)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Expand Down Expand Up @@ -368,6 +374,41 @@ void findDistinctNamespace() {
assertThat((long) distinctNamespace.size(), is(4L));
}

@Test
void templateDisabled() {
Template template = Template.builder()
.id(IdUtils.create())
.type(Template.class.getName())
.namespace("test")
.templateId("testTemplate")
.build();

Template templateSpy = spy(template);

doReturn(Collections.emptyList())
.when(templateSpy)
.allChildTasks();

Flow flow = Flow.builder()
.id(IdUtils.create())
.namespace("io.kestra.unittest")
.tasks(Collections.singletonList(templateSpy))
.build();

flowRepository.create(
flow,
flow.generateSource(),
flow
);

Optional<Flow> found = flowRepository.findById(flow.getNamespace(), flow.getId());

assertThat(found.isPresent(), is(true));
assertThat(found.get() instanceof FlowWithException, is(true));
assertThat(((FlowWithException) found.get()).getException(), containsString("Templates are disabled"));
}


@Singleton
public static class FlowListener implements ApplicationEventListener<CrudEvent<Flow>> {
private static List<CrudEvent<Flow>> emits = new ArrayList<>();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
package io.kestra.core.repositories;

import io.kestra.core.Helpers;
import io.kestra.core.events.CrudEvent;
import io.kestra.core.events.CrudEventType;
import io.kestra.core.models.flows.Flow;
import io.kestra.core.models.flows.FlowWithSource;
import io.kestra.core.models.templates.Template;
import io.kestra.core.tasks.debugs.Return;
import io.kestra.core.utils.IdUtils;
import io.micronaut.context.annotation.Property;
import io.micronaut.context.event.ApplicationEventListener;
import io.micronaut.core.util.StringUtils;
import io.micronaut.data.model.Pageable;
import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
import jakarta.inject.Inject;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import io.kestra.core.runners.RunnerUtils;
import io.kestra.core.tasks.log.Log;
import io.kestra.core.utils.TestsUtils;
import io.micronaut.context.annotation.Property;
import io.micronaut.core.util.StringUtils;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import org.junit.jupiter.api.Test;
Expand All @@ -30,6 +32,7 @@
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;

@Property(name = "kestra.templates.enabled", value = StringUtils.TRUE)
public class TemplateTest extends AbstractMemoryRunnerTest {
@Inject
protected TemplateRepositoryInterface templateRepository;
Expand All @@ -46,7 +49,8 @@ public class TemplateTest extends AbstractMemoryRunnerTest {

public static void withTemplate(RunnerUtils runnerUtils, TemplateRepositoryInterface templateRepository, LocalFlowRepositoryLoader repositoryLoader, QueueInterface<LogEntry> logQueue) throws TimeoutException, IOException, URISyntaxException {
templateRepository.create(TEMPLATE_1);
repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource("flows/tests/with-template.yaml")));
repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource(
"flows/templates/with-template.yaml")));

List<LogEntry> logs = new CopyOnWriteArrayList<>();
logQueue.receive(logs::add);
Expand Down Expand Up @@ -76,7 +80,8 @@ void withTemplate() throws TimeoutException, IOException, URISyntaxException {


public static void withFailedTemplate(RunnerUtils runnerUtils, TemplateRepositoryInterface templateRepository, LocalFlowRepositoryLoader repositoryLoader, QueueInterface<LogEntry> logQueue) throws TimeoutException, IOException, URISyntaxException {
repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource("flows/tests/with-failed-template.yaml")));
repositoryLoader.load(Objects.requireNonNull(ListenersTest.class.getClassLoader().getResource(
"flows/templates/with-failed-template.yaml")));

List<LogEntry> logs = new CopyOnWriteArrayList<>();
logQueue.receive(logs::add);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.kestra.repository.h2;

import io.kestra.core.models.templates.Template;
import io.kestra.core.models.templates.TemplateEnabled;
import io.kestra.jdbc.repository.AbstractJdbcTemplateRepository;
import io.micronaut.context.ApplicationContext;
import jakarta.inject.Inject;
Expand All @@ -11,6 +12,7 @@

@Singleton
@H2RepositoryEnabled
@TemplateEnabled
public class H2TemplateRepository extends AbstractJdbcTemplateRepository {
@Inject
public H2TemplateRepository(ApplicationContext applicationContext) {
Expand Down
Loading