feat(core): enhance plugin management

Changes: * add new interface PluginManager * add new CLI for un-installing plugins * refactor service for downloading plugins
kestra-io · Dec 16, 2024 · eb9de27 · eb9de27
1 parent 7696bae
commit eb9de27
Show file tree

Hide file tree

Showing 18 changed files with 984 additions and 297 deletions.
diff --git a/cli/src/main/java/io/kestra/cli/AbstractCommand.java b/cli/src/main/java/io/kestra/cli/AbstractCommand.java
@@ -5,13 +5,15 @@
 import io.kestra.cli.commands.servers.ServerCommandInterface;
 import io.kestra.cli.services.StartupHookInterface;
 import io.kestra.core.contexts.KestraContext;
+import io.kestra.core.plugins.PluginManager;
 import io.kestra.core.plugins.PluginRegistry;
 import io.kestra.webserver.services.FlowAutoLoaderService;
 import io.micronaut.context.ApplicationContext;
 import io.micronaut.context.env.yaml.YamlPropertySourceLoader;
 import io.micronaut.core.annotation.Introspected;
 import io.micronaut.management.endpoint.EndpointDefaultConfiguration;
 import io.micronaut.runtime.server.EmbeddedServer;
+import jakarta.inject.Provider;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.utils.URIBuilder;
 import io.kestra.core.utils.Rethrow;
@@ -49,6 +51,12 @@ abstract public class AbstractCommand implements Callable<Integer> {
     @Inject
     private io.kestra.core.utils.VersionProvider versionProvider;
 
+    @Inject
+    protected Provider<PluginRegistry> pluginRegistryProvider;
+
+    @Inject
+    Provider<PluginManager> pluginManagerProvider;
+
     private PluginRegistry pluginRegistry;
 
     @CommandLine.Option(names = {"-v", "--verbose"}, description = "Change log level. Multiple -v options increase the verbosity.", showDefaultValue = CommandLine.Help.Visibility.NEVER)
@@ -84,8 +92,10 @@ public Integer call() throws Exception {
         }
 
         if (this.pluginsPath != null && loadExternalPlugins()) {
-            pluginRegistry = pluginRegistry();
+            pluginRegistry = pluginRegistryProvider.get();
             pluginRegistry.registerIfAbsent(pluginsPath);
+
+            pluginManagerProvider.get(); // This will trigger initialization of the plugin-manager
         }
 
         startWebserver();
@@ -102,10 +112,6 @@ protected boolean loadExternalPlugins() {
         return true;
     }
 
-    protected PluginRegistry pluginRegistry() {
-        return KestraContext.getContext().getPluginRegistry(); // Lazy init
-    }
-
     private static String message(String message, Object... format) {
         return CommandLine.Help.Ansi.AUTO.string(
             format.length == 0 ? message : MessageFormat.format(message, format)

diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginDocCommand.java
@@ -3,7 +3,6 @@
 import com.google.common.base.Charsets;
 import io.kestra.cli.AbstractCommand;
 import io.kestra.core.docs.DocumentationGenerator;
-import io.kestra.core.docs.JsonSchemaGenerator;
 import io.kestra.core.plugins.PluginRegistry;
 import io.kestra.core.plugins.RegisteredPlugin;
 import io.kestra.core.serializers.JacksonMapper;
@@ -44,7 +43,8 @@ public Integer call() throws Exception {
         super.call();
         DocumentationGenerator documentationGenerator = applicationContext.getBean(DocumentationGenerator.class);
 
-        List<RegisteredPlugin> plugins = core ?  pluginRegistry().plugins() : pluginRegistry().externalPlugins();
+        PluginRegistry registry = pluginRegistryProvider.get();
+        List<RegisteredPlugin> plugins = core ?  registry.plugins() : registry.externalPlugins();
         for (RegisteredPlugin registeredPlugin : plugins) {
             documentationGenerator
                 .generate(registeredPlugin)

diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginInstallCommand.java
@@ -1,28 +1,21 @@
 package io.kestra.cli.commands.plugins;
 
+import io.kestra.core.contexts.MavenPluginRepositoryConfig;
+import io.kestra.core.plugins.PluginArtifact;
+import io.kestra.core.plugins.PluginManager;
 import io.micronaut.http.uri.UriBuilder;
-import org.apache.commons.io.FilenameUtils;
 import io.kestra.cli.AbstractCommand;
-import io.kestra.cli.plugins.PluginDownloader;
-import io.kestra.cli.plugins.RepositoryConfig;
 import io.kestra.core.utils.IdUtils;
-import org.apache.http.client.utils.URIBuilder;
+import jakarta.inject.Provider;
 import picocli.CommandLine;
 
 import java.net.URI;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Objects;
 
 import jakarta.inject.Inject;
 
-import static io.kestra.core.utils.Rethrow.throwConsumer;
-
 @CommandLine.Command(
     name = "install",
     description = "install a plugin"
@@ -38,63 +31,51 @@ public class PluginInstallCommand extends AbstractCommand {
     CommandLine.Model.CommandSpec spec;
 
     @Inject
-    private PluginDownloader pluginDownloader;
+    private Provider<PluginManager> pluginManager;
 
     @Override
     public Integer call() throws Exception {
         super.call();
 
-        if (this.pluginsPath == null) {
-            throw new CommandLine.ParameterException(this.spec.commandLine(), "Missing required options '--plugins' " +
-                "or environment variable 'KESTRA_PLUGINS_PATH"
-            );
-        }
-
-        if (!pluginsPath.toFile().exists()) {
-            if (!pluginsPath.toFile().mkdir()) {
-                throw new RuntimeException("Cannot create directory: " + pluginsPath.toFile().getAbsolutePath());
-            }
-        }
-
+        List<MavenPluginRepositoryConfig> repositoryConfigs = List.of();
         if (repositories != null) {
-            Arrays.stream(repositories)
-                .forEach(throwConsumer(s -> {
-                    URIBuilder uriBuilder = new URIBuilder(s);
-
-                    RepositoryConfig.RepositoryConfigBuilder builder = RepositoryConfig.builder()
+            repositoryConfigs = Arrays.stream(repositories)
+                .map(uri -> {
+                    MavenPluginRepositoryConfig.MavenPluginRepositoryConfigBuilder builder = MavenPluginRepositoryConfig
+                        .builder()
                         .id(IdUtils.create());
 
-                    if (uriBuilder.getUserInfo() != null) {
-                        int index = uriBuilder.getUserInfo().indexOf(":");
-
-                        builder.basicAuth(new RepositoryConfig.BasicAuth(
-                            uriBuilder.getUserInfo().substring(0, index),
-                            uriBuilder.getUserInfo().substring(index + 1)
+                    String userInfo = uri.getUserInfo();
+                    if (userInfo != null) {
+                        String[] userInfoParts = userInfo.split(":");
+                        builder = builder.basicAuth(new MavenPluginRepositoryConfig.BasicAuth(
+                            userInfoParts[0],
+                            userInfoParts[1]
                         ));
-
-                        uriBuilder.setUserInfo(null);
                     }
-
-                    builder.url(uriBuilder.build().toString());
-
-                    pluginDownloader.addRepository(builder.build());
-                }));
+                    builder.url(UriBuilder.of(uri).userInfo(null).build().toString());
+                    return builder.build();
+                }).toList();
         }
 
-        List<URL> resolveUrl = pluginDownloader.resolve(dependencies);
-        stdOut("Resolved Plugin(s) with {0}", resolveUrl);
-
-        for (URL url: resolveUrl) {
-            Files.copy(
-                Paths.get(url.toURI()),
-                Paths.get(pluginsPath.toString(), FilenameUtils.getName(url.toString())),
-                StandardCopyOption.REPLACE_EXISTING
-            );
+        List<PluginArtifact> pluginArtifacts;
+        try {
+           pluginArtifacts = dependencies.stream().map(PluginArtifact::fromCoordinates).toList();
+        } catch (IllegalArgumentException e) {
+            stdErr(e.getMessage());
+            return CommandLine.ExitCode.USAGE;
         }
-
-        stdOut("Successfully installed plugins {0} into {1}", dependencies, pluginsPath);
-
-        return 0;
+        PluginManager pluginManager = this.pluginManager.get();
+        List<PluginArtifact> installed = pluginManager.install(
+            pluginArtifacts,
+            repositoryConfigs,
+            false,
+            pluginsPath
+        );
+
+        List<URI> uris = installed.stream().map(PluginArtifact::uri).toList();
+        stdOut("Successfully installed plugins {0} into {1}", dependencies, uris);
+        return CommandLine.ExitCode.OK;
     }
 
     @Override

diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginListCommand.java
@@ -1,6 +1,7 @@
 package io.kestra.cli.commands.plugins;
 
 import io.kestra.cli.AbstractCommand;
+import io.kestra.core.plugins.PluginRegistry;
 import io.kestra.core.plugins.RegisteredPlugin;
 import picocli.CommandLine;
 
@@ -16,7 +17,7 @@ public class PluginListCommand extends AbstractCommand {
 
     @CommandLine.Option(names = {"--core"}, description = "Also write core tasks plugins")
     private boolean core = false;
-    
+
     @Override
     public Integer call() throws Exception {
         super.call();
@@ -26,8 +27,9 @@ public Integer call() throws Exception {
                 "or environment variable 'KESTRA_PLUGINS_PATH"
             );
         }
-
-        List<RegisteredPlugin> plugins = core ?  pluginRegistry().plugins() :  pluginRegistry().externalPlugins();
+
+        PluginRegistry registry = pluginRegistryProvider.get();
+        List<RegisteredPlugin> plugins = core ?  registry.plugins() : registry.externalPlugins();
         plugins.forEach(registeredPlugin -> stdOut(registeredPlugin.toString()));
 
         return 0;

diff --git a/cli/src/main/java/io/kestra/cli/commands/plugins/PluginUninstallCommand.java b/cli/src/main/java/io/kestra/cli/commands/plugins/PluginUninstallCommand.java
@@ -0,0 +1,59 @@
+package io.kestra.cli.commands.plugins;
+
+import io.kestra.cli.AbstractCommand;
+import io.kestra.core.plugins.PluginArtifact;
+import io.kestra.core.plugins.PluginManager;
+import jakarta.inject.Inject;
+import jakarta.inject.Provider;
+import picocli.CommandLine;
+import picocli.CommandLine.Parameters;
+import picocli.CommandLine.Spec;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+@CommandLine.Command(
+    name = "uninstall",
+    description = "uninstall a plugin"
+)
+public class PluginUninstallCommand extends AbstractCommand {
+    @Parameters(index = "0..*", description = "the plugins to uninstall")
+    List<String> dependencies = new ArrayList<>();
+
+    @Spec
+    CommandLine.Model.CommandSpec spec;
+
+    @Inject
+    private Provider<PluginManager> pluginManager;
+
+    @Override
+    public Integer call() throws Exception {
+        super.call();
+
+        List<PluginArtifact> pluginArtifacts;
+        try {
+            pluginArtifacts = dependencies.stream().map(PluginArtifact::fromCoordinates).toList();
+        } catch (IllegalArgumentException e) {
+            stdErr(e.getMessage());
+            return CommandLine.ExitCode.USAGE;
+        }
+
+        PluginManager pluginManager = this.pluginManager.get();
+
+        List<PluginArtifact> uninstalled = pluginManager.uninstall(
+            pluginArtifacts,
+            false,
+            pluginsPath
+        );
+
+        List<URI> uris = uninstalled.stream().map(PluginArtifact::uri).toList();
+        stdOut("Successfully uninstalled plugins {0} from {1}", dependencies, uris);
+        return CommandLine.ExitCode.OK;
+    }
+
+    @Override
+    protected boolean loadExternalPlugins() {
+        return false;
+    }
+}