diff --git a/docs/tasks/toml-tasks.md b/docs/tasks/toml-tasks.md index 4200c15239..dde42848d4 100644 --- a/docs/tasks/toml-tasks.md +++ b/docs/tasks/toml-tasks.md @@ -334,13 +334,47 @@ file = 'scripts/release.sh' # execute an external script ### Remote tasks -Task files can be fetched via http: +Task files can be fetched remotely with multiple protocols: + +#### HTTP ```toml [tasks.build] file = "https://example.com/build.sh" ``` +Please note that the file will be downloaded and executed. Make sure you trust the source. + +#### Git + +::: code-group + +```toml [ssh] +[tasks.build] +file = "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0" +``` + +```toml [https] +[tasks.build] +file = "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0" +``` + +::: + +Url format must follow these patterns `git:::////?` + +Required fields: + +- `protocol`: The git repository URL. +- `url`: The git repository URL. +- `path`: The path to the file in the repository. + +Optional fields: + +- `ref`: The git reference (branch, tag, commit). + +#### Cache + Each task file is cached in the `MISE_CACHE_DIR` directory. If the file is updated, it will not be re-downloaded unless the cache is cleared. :::tip diff --git a/e2e/run_test b/e2e/run_test index 00299b6f7a..970982bfcb 100755 --- a/e2e/run_test +++ b/e2e/run_test @@ -85,6 +85,7 @@ within_isolated_env() { TEST_ROOT="$SCRIPT_DIR" \ TEST_SCRIPT="$TEST_SCRIPT" \ TMPDIR="$TEST_TMPDIR" \ + EXCLUDE_FROM_CI="${CI:-}" \ "$@" || return $? } diff --git a/e2e/tasks/test_task_remote_git_https b/e2e/tasks/test_task_remote_git_https new file mode 100755 index 0000000000..05d2ba840d --- /dev/null +++ b/e2e/tasks/test_task_remote_git_https @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +cargo init --name hello_cargo + +################################################################################# +# Test remote tasks with no ref +################################################################################# + +cat <mise.toml +[tasks.remote_lint_https_latest] +file = "git::https://github.com/jdx/mise.git//xtasks/lint/clippy" +EOF + +assert_contains "mise tasks" "remote_lint_https_latest" +assert_succeed "mise run remote_lint_https_latest" # Remote task should be downloaded + +mise cache clear # Clear cache to force redownload + +assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_https_latest" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_https_latest --no-cache" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_https_latest" # Cache should be used + +################################################################################# +# Test remote tasks with with ref +################################################################################# + +cat <mise.toml +[tasks.remote_lint_https_ref] +file = "git::https://github.com/jdx/mise.git//xtasks/lint/clippy?ref=v2025.1.17" +EOF + +assert_contains "mise tasks" "remote_lint_https_ref" +assert_succeed "mise run remote_lint_https_ref" # Remote task should be downloaded + +mise cache clear # Clear cache to force redownload + +assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_https_ref" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_https_ref --no-cache" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_https_ref" # Cache should be used diff --git a/e2e/tasks/test_task_remote_git_ssh b/e2e/tasks/test_task_remote_git_ssh new file mode 100755 index 0000000000..f7367e61aa --- /dev/null +++ b/e2e/tasks/test_task_remote_git_ssh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +if [ -n "$EXCLUDE_FROM_CI" ]; then + echo "This test is not supported in CI, because it requires a SSH key to be added to the GitHub account" + exit 0 +fi + +cargo init --name hello_cargo + +################################################################################# +# Test remote tasks with no ref +################################################################################# + +cat <mise.toml +[tasks.remote_lint_ssh_latest] +file = "git::ssh://git@github.com:jdx/mise.git//xtasks/lint/clippy" +EOF + +assert_contains "mise tasks" "remote_lint_ssh_latest" +assert_succeed "mise run remote_lint_ssh_latest" # Remote task should be downloaded + +mise cache clear # Clear cache to force redownload + +assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_ssh_latest" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_ssh_latest --no-cache" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_ssh_latest" # Cache should be used + +################################################################################# +# Test remote tasks with with ref +################################################################################# + +cat <mise.toml +[tasks.remote_lint_ssh_ref] +file = "git::ssh://git@github.com:jdx/mise.git//xtasks/lint/clippy?ref=v2025.1.17" +EOF + +assert_contains "mise tasks" "remote_lint_ssh_ref" +assert_succeed "mise run remote_lint_ssh_ref" # Remote task should be downloaded + +mise cache clear # Clear cache to force redownload + +assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_ssh_ref" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_ssh_ref --no-cache" # Remote task should be redownloaded + +assert_succeed "mise run remote_lint_ssh_ref" # Cache should be used diff --git a/mise.lock b/mise.lock index 1ab8ea7d17..412f50c0bb 100644 --- a/mise.lock +++ b/mise.lock @@ -5,6 +5,7 @@ backend = "aqua:rhysd/actionlint" [tools.actionlint.checksums] "actionlint_1.7.7_darwin_arm64.tar.gz" = "sha256:2693315b9093aeacb4ebd91a993fea54fc215057bf0da2659056b4bc033873db" "actionlint_1.7.7_linux_amd64.tar.gz" = "sha256:023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757" +"actionlint_1.7.7_linux_arm64.tar.gz" = "sha256:401942f9c24ed71e4fe71b76c7d638f66d8633575c4016efd2977ce7c28317d0" [tools.bun] version = "1.2.2" @@ -12,6 +13,7 @@ backend = "core:bun" [tools.bun.checksums] "bun-darwin-aarch64.zip" = "sha256:c4d58e06c5c33885b526f4d91a38ca9ebdb9fc3fb4cd547f7d3302055c98e41c" +"bun-linux-aarch64.zip" = "sha256:d1dbaa3e9af24549fad92bdbe4fb21fa53302cd048a8f004e85a240984c93d4d" "bun-linux-x64-baseline.zip" = "sha256:cad7756a6ee16f3432a328f8023fc5cd431106822eacfa6d6d3afbad6fdc24db" [tools.cargo-binstall] @@ -20,6 +22,7 @@ backend = "aqua:cargo-bins/cargo-binstall" [tools.cargo-binstall.checksums] "cargo-binstall-aarch64-apple-darwin.zip" = "sha256:97ce4a2f18181f052dda266b042d8bb220e48ffe40ca75e796ae4c5e418b9e01" +"cargo-binstall-aarch64-unknown-linux-musl.tgz" = "sha256:b8c32b1b007482f42f6c4b5f8cfeb168f9674ec6448bfa29ae0c4ba01b7a370b" "cargo-binstall-x86_64-unknown-linux-musl.tgz" = "sha256:74d7c647c7e60bb8464fa551702fdd38a7241f5cedb2c4edc3b11639cd1dae47" [tools."cargo:cargo-edit"] @@ -115,6 +118,7 @@ version = "2.6.0" backend = "ubi:slsa-framework/slsa-verifier" [tools.slsa-verifier.checksums] +slsa-verifier-linux-aarch64 = "sha256:92b28eb2db998f9a6a048336928b29a38cb100076cd587e443ca0a2543d7c93d" slsa-verifier-linux-x86_64 = "sha256:1c9c0d6a272063f3def6d233fa3372adbaff1f5a3480611a07c744e73246b62d" slsa-verifier-macos-aarch64 = "sha256:8740e66832fd48bbaa479acd5310986b876ff545460add0cb4a087aec056189c" "slsa-verifier.exe-windows-x86_64" = "sha256:37ca29ad748e8ea7be76d3ae766e8fa505362240431f6ea7f0648c727e2f2507" diff --git a/src/aqua/aqua_registry.rs b/src/aqua/aqua_registry.rs index 91f2644062..5f1a209f68 100644 --- a/src/aqua/aqua_registry.rs +++ b/src/aqua/aqua_registry.rs @@ -3,7 +3,7 @@ use crate::backend::aqua; use crate::backend::aqua::{arch, os}; use crate::config::SETTINGS; use crate::duration::{DAILY, WEEKLY}; -use crate::git::Git; +use crate::git::{CloneOptions, Git}; use crate::{dirs, file, hashmap, http}; use expr::{Context, Parser, Program, Value}; use eyre::{eyre, ContextCompat, Result}; @@ -175,7 +175,7 @@ impl AquaRegistry { fetch_latest_repo(&repo)?; } else if let Some(aqua_registry_url) = &SETTINGS.aqua.registry_url { info!("cloning aqua registry to {path:?}"); - repo.clone(aqua_registry_url, None)?; + repo.clone(aqua_registry_url, CloneOptions::default())?; repo_exists = true; } Ok(Self { path, repo_exists }) diff --git a/src/backend/spm.rs b/src/backend/spm.rs index f8408b76f3..c65de6d2ac 100644 --- a/src/backend/spm.rs +++ b/src/backend/spm.rs @@ -3,7 +3,7 @@ use crate::backend::Backend; use crate::cli::args::BackendArg; use crate::cmd::CmdLineRunner; use crate::config::Settings; -use crate::git::Git; +use crate::git::{CloneOptions, Git}; use crate::install_context::InstallContext; use crate::toolset::ToolVersion; use crate::{dirs, file, github}; @@ -95,7 +95,10 @@ impl SPMBackend { package_repo.url.as_str(), repo.dir.display(), ); - repo.clone(package_repo.url.as_str(), Some(&ctx.pr))?; + repo.clone( + package_repo.url.as_str(), + CloneOptions::default().pr(&ctx.pr), + )?; } debug!("Checking out revision: {revision}"); repo.update_tag(revision.to_string())?; diff --git a/src/git.rs b/src/git.rs index c6358da33c..907b14b783 100644 --- a/src/git.rs +++ b/src/git.rs @@ -116,7 +116,7 @@ impl Git { Ok((prev_rev, post_rev)) } - pub fn clone(&self, url: &str, pr: Option<&Box>) -> Result<()> { + pub fn clone(&self, url: &str, options: CloneOptions) -> Result<()> { debug!("cloning {} to {}", url, self.dir.display()); if let Some(parent) = self.dir.parent() { file::mkdirp(parent)?; @@ -139,18 +139,30 @@ impl Git { err ), } - if let Some(pr) = pr { + if let Some(pr) = &options.pr { // in order to prevent hiding potential password prompt, just disable the progress bar pr.abandon(); } - CmdLineRunner::new("git") + + let mut cmd = CmdLineRunner::new("git") .arg("clone") .arg("-q") .arg("--depth") .arg("1") .arg(url) - .arg(&self.dir) - .execute()?; + .arg(&self.dir); + + if let Some(branch) = &options.branch { + cmd = cmd.args([ + "-b", + branch, + "--single-branch", + "-c", + "advice.detachedHead=false", + ]); + } + + cmd.execute()?; Ok(()) } @@ -265,3 +277,21 @@ impl Debug for Git { f.debug_struct("Git").field("dir", &self.dir).finish() } } + +#[derive(Default)] +pub struct CloneOptions<'a> { + pr: Option<&'a Box>, + branch: Option, +} + +impl<'a> CloneOptions<'a> { + pub fn pr(mut self, pr: &'a Box) -> Self { + self.pr = Some(pr); + self + } + + pub fn branch(mut self, branch: &str) -> Self { + self.branch = Some(branch.to_string()); + self + } +} diff --git a/src/plugins/asdf_plugin.rs b/src/plugins/asdf_plugin.rs index 56aa6ec33b..3c2c6bac44 100644 --- a/src/plugins/asdf_plugin.rs +++ b/src/plugins/asdf_plugin.rs @@ -1,7 +1,7 @@ use crate::config::{Config, Settings, SETTINGS}; use crate::errors::Error::PluginNotInstalled; use crate::file::{display_path, remove_all}; -use crate::git::Git; +use crate::git::{CloneOptions, Git}; use crate::plugins::{Plugin, PluginType, Script, ScriptManager}; use crate::result::Result; use crate::timeout::run_with_timeout; @@ -333,7 +333,7 @@ Plugins could support local directories in the future but for now a symlink is r } let git = Git::new(&self.plugin_path); pr.set_message(format!("clone {repo_url}")); - git.clone(&repo_url, Some(pr))?; + git.clone(&repo_url, CloneOptions::default().pr(pr))?; if let Some(ref_) = &repo_ref { pr.set_message(format!("check out {ref_}")); git.update(Some(ref_.to_string()))?; diff --git a/src/plugins/core/python.rs b/src/plugins/core/python.rs index 1ac097d19e..3bbaaa16fc 100644 --- a/src/plugins/core/python.rs +++ b/src/plugins/core/python.rs @@ -5,7 +5,7 @@ use crate::cli::args::BackendArg; use crate::cmd::CmdLineRunner; use crate::config::{Config, SETTINGS}; use crate::file::{display_path, TarOptions}; -use crate::git::Git; +use crate::git::{CloneOptions, Git}; use crate::http::{HTTP, HTTP_FETCH}; use crate::install_context::InstallContext; use crate::toolset::{ToolRequest, ToolVersion, Toolset}; @@ -66,7 +66,11 @@ impl PythonPlugin { file::create_dir_all(self.python_build_path().parent().unwrap())?; let git = Git::new(self.python_build_path()); let pr = ctx.map(|ctx| &ctx.pr); - git.clone(&SETTINGS.python.pyenv_repo, pr)?; + let mut clone_options = CloneOptions::default(); + if let Some(pr) = pr { + clone_options = clone_options.pr(pr); + } + git.clone(&SETTINGS.python.pyenv_repo, clone_options)?; Ok(()) } fn update_python_build(&self) -> eyre::Result<()> { diff --git a/src/plugins/core/ruby.rs b/src/plugins/core/ruby.rs index 5764b673d9..830db766f4 100644 --- a/src/plugins/core/ruby.rs +++ b/src/plugins/core/ruby.rs @@ -8,7 +8,7 @@ use crate::cmd::CmdLineRunner; use crate::config::{Config, Settings, SETTINGS}; use crate::duration::DAILY; use crate::env::PATH_KEY; -use crate::git::Git; +use crate::git::{CloneOptions, Git}; use crate::github::GithubRelease; use crate::http::{HTTP, HTTP_FETCH}; use crate::install_context::InstallContext; @@ -81,7 +81,11 @@ impl RubyPlugin { file::remove_all(&tmp)?; file::create_dir_all(tmp.parent().unwrap())?; let git = Git::new(tmp.clone()); - git.clone(&SETTINGS.ruby.ruby_build_repo, pr)?; + let mut clone_options = CloneOptions::default(); + if let Some(pr) = pr { + clone_options = clone_options.pr(pr); + } + git.clone(&SETTINGS.ruby.ruby_build_repo, clone_options)?; cmd!("sh", "install.sh") .env("PREFIX", self.ruby_build_path()) @@ -123,7 +127,11 @@ impl RubyPlugin { file::remove_all(&tmp)?; file::create_dir_all(tmp.parent().unwrap())?; let git = Git::new(tmp.clone()); - git.clone(&settings.ruby.ruby_install_repo, pr)?; + let mut clone_options = CloneOptions::default(); + if let Some(pr) = pr { + clone_options = clone_options.pr(pr); + } + git.clone(&settings.ruby.ruby_install_repo, clone_options)?; cmd!("make", "install") .env("PREFIX", self.ruby_install_path()) diff --git a/src/plugins/vfox_plugin.rs b/src/plugins/vfox_plugin.rs index 3422b66c12..ad7a34b13e 100644 --- a/src/plugins/vfox_plugin.rs +++ b/src/plugins/vfox_plugin.rs @@ -1,5 +1,5 @@ use crate::file::{display_path, remove_all}; -use crate::git::Git; +use crate::git::{CloneOptions, Git}; use crate::plugins::{Plugin, PluginType}; use crate::result::Result; use crate::tokio::RUNTIME; @@ -133,7 +133,8 @@ impl Plugin for VfoxPlugin { let url = self.get_repo_url()?; trace!("Cloning vfox plugin: {url}"); let pr = mpr.add(&format!("clone vfox plugin {}", url)); - self.repo().clone(url.as_str(), Some(&pr))?; + self.repo() + .clone(url.as_str(), CloneOptions::default().pr(&pr))?; } Ok(()) } @@ -208,7 +209,7 @@ Plugins could support local directories in the future but for now a symlink is r } let git = Git::new(&self.plugin_path); pr.set_message(format!("clone {repo_url}")); - git.clone(&repo_url, Some(pr))?; + git.clone(&repo_url, CloneOptions::default().pr(pr))?; if let Some(ref_) = &repo_ref { pr.set_message(format!("git update {ref_}")); git.update(Some(ref_.to_string()))?; diff --git a/src/task/task_file_providers/mod.rs b/src/task/task_file_providers/mod.rs index 37a364bd26..4e39ff0380 100644 --- a/src/task/task_file_providers/mod.rs +++ b/src/task/task_file_providers/mod.rs @@ -1,9 +1,11 @@ use std::{fmt::Debug, path::PathBuf}; mod local_task; +mod remote_task_git; mod remote_task_http; -pub use local_task::LocalTask; +use local_task::LocalTask; +use remote_task_git::RemoteTaskGitBuilder; use remote_task_http::RemoteTaskHttpBuilder; pub trait TaskFileProvider: Debug { @@ -41,6 +43,11 @@ impl TaskFileProviders { fn get_providers(&self) -> Vec> { vec![ + Box::new( + RemoteTaskGitBuilder::new() + .with_cache(self.use_cache) + .build(), + ), Box::new( RemoteTaskHttpBuilder::new() .with_cache(self.use_cache) @@ -64,7 +71,7 @@ mod tests { fn test_get_providers() { let task_file_providers = TaskFileProvidersBuilder::new().build(); let providers = task_file_providers.get_providers(); - assert_eq!(providers.len(), 2); + assert_eq!(providers.len(), 3); } #[test] @@ -75,7 +82,8 @@ mod tests { for file in cases { let provider = task_file_providers.get_provider(file); assert!(provider.is_some()); - assert!(format!("{:?}", provider.unwrap()).contains("LocalTask")); + let provider_name = format!("{:?}", provider.unwrap()); + assert!(provider_name.contains("LocalTask")); } } @@ -91,7 +99,26 @@ mod tests { for file in cases { let provider = task_file_providers.get_provider(file); assert!(provider.is_some()); - assert!(format!("{:?}", provider.unwrap()).contains("RemoteTaskHttp")); + let provider_name = format!("{:?}", provider.unwrap()); + assert!(provider_name.contains("RemoteTaskHttp")); + } + } + + #[test] + fn test_git_file_match_git_remote_task_provider() { + let task_file_providers = TaskFileProvidersBuilder::new().build(); + let cases = vec![ + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://user@myserver.com/example.git//subfolder/myfile.py", + "git::https://myserver.com/example.git//subfolder/myfile.sh", + ]; + + for file in cases { + let provider = task_file_providers.get_provider(file); + assert!(provider.is_some()); + let provider_name = format!("{:?}", provider.unwrap()); + assert!(provider_name.contains("RemoteTaskGit")); } } } diff --git a/src/task/task_file_providers/remote_task_git.rs b/src/task/task_file_providers/remote_task_git.rs new file mode 100644 index 0000000000..1e30799ca7 --- /dev/null +++ b/src/task/task_file_providers/remote_task_git.rs @@ -0,0 +1,383 @@ +use std::path::PathBuf; + +use regex::Regex; + +use crate::{ + dirs, env, + git::{self, CloneOptions}, + hash, +}; + +use super::TaskFileProvider; + +#[derive(Debug)] +pub struct RemoteTaskGitBuilder { + store_path: PathBuf, + use_cache: bool, +} + +impl RemoteTaskGitBuilder { + pub fn new() -> Self { + Self { + store_path: env::temp_dir(), + use_cache: false, + } + } + + pub fn with_cache(mut self, use_cache: bool) -> Self { + if use_cache { + self.store_path = dirs::CACHE.join("remote-git-tasks-cache"); + self.use_cache = true; + } + self + } + + pub fn build(self) -> RemoteTaskGit { + RemoteTaskGit { + storage_path: self.store_path, + is_cached: self.use_cache, + } + } +} + +#[derive(Debug)] +pub struct RemoteTaskGit { + storage_path: PathBuf, + is_cached: bool, +} + +#[derive(Debug, Clone)] +struct GitRepoStructure { + url_without_path: String, + path: String, + branch: Option, +} + +impl GitRepoStructure { + pub fn new(url_without_path: &str, path: &str, branch: Option) -> Self { + Self { + url_without_path: url_without_path.to_string(), + path: path.to_string(), + branch, + } + } +} + +impl RemoteTaskGit { + fn get_cache_key(&self, repo_structure: &GitRepoStructure) -> String { + let key = format!( + "{}{}", + &repo_structure.url_without_path, + &repo_structure.branch.to_owned().unwrap_or("".to_string()) + ); + hash::hash_sha256_to_str(&key) + } + + fn get_repo_structure(&self, file: &str) -> GitRepoStructure { + if self.detect_ssh(file).is_ok() { + return self.detect_ssh(file).unwrap(); + } + self.detect_https(file).unwrap() + } + + fn detect_ssh(&self, file: &str) -> Result> { + let re = Regex::new(r"^git::ssh://(?P((?P[^@]+)@)(?P[^/]+)/(?P[^/]+)\.git)//(?P[^?]+)(\?ref=(?P[^?]+))?$").unwrap(); + + if !re.is_match(file) { + return Err("Invalid SSH URL".into()); + } + + let captures = re.captures(file).unwrap(); + + let url_without_path = captures.name("url").unwrap().as_str(); + + let path = captures.name("path").unwrap().as_str(); + + let branch: Option = captures.name("branch").map(|m| m.as_str().to_string()); + + Ok(GitRepoStructure::new(url_without_path, path, branch)) + } + + fn detect_https(&self, file: &str) -> Result> { + let re = Regex::new(r"^git::(?Phttps://(?P[^/]+)/(?P[^/]+(?:/[^/]+)?)\.git)//(?P[^?]+)(\?ref=(?P[^?]+))?$").unwrap(); + + if !re.is_match(file) { + return Err("Invalid HTTPS URL".into()); + } + + let captures = re.captures(file).unwrap(); + + let url_without_path = captures.name("url").unwrap().as_str(); + + let path = captures.name("path").unwrap().as_str(); + + let branch: Option = captures.name("branch").map(|m| m.as_str().to_string()); + + Ok(GitRepoStructure::new(url_without_path, path, branch)) + } +} + +impl TaskFileProvider for RemoteTaskGit { + fn is_match(&self, file: &str) -> bool { + if self.detect_ssh(file).is_ok() { + return true; + } + + if self.detect_https(file).is_ok() { + return true; + } + + false + } + + fn get_local_path(&self, file: &str) -> Result> { + let repo_structure = self.get_repo_structure(file); + let cache_key = self.get_cache_key(&repo_structure); + let destination = self.storage_path.join(&cache_key); + let repo_file_path = repo_structure.path.clone(); + let full_path = destination.join(&repo_file_path); + + debug!("Repo structure: {:?}", repo_structure); + + match self.is_cached { + true => { + trace!("Cache mode enabled"); + + if full_path.exists() { + debug!("Using cached file: {:?}", full_path); + return Ok(full_path); + } + } + false => { + trace!("Cache mode disabled"); + + if full_path.exists() { + crate::file::remove_all(&destination)?; + } + } + } + + let git_repo = git::Git::new(destination); + + let mut clone_options = CloneOptions::default(); + + if let Some(branch) = &repo_structure.branch { + clone_options = clone_options.branch(branch); + } + + git_repo.clone(repo_structure.url_without_path.as_str(), clone_options)?; + + Ok(full_path) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_valid_detect_ssh() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com:myorg/example.git//terraform/myfile?ref=master", + "git::ssh://git@myserver.com/example.git//terraform/myfile", + "git::ssh://user@myserver.com/example.git//myfile?ref=master", + ]; + + for url in test_cases { + let result = remote_task_git.detect_ssh(url); + assert!(result.is_ok()); + } + } + + #[test] + fn test_invalid_detect_ssh() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::ssh://myserver.com/example.git//myfile?ref=master", + "git::ssh://user@myserver.com/example.git?ref=master", + "git::ssh://user@myserver.com/example.git", + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + ]; + + for url in test_cases { + let result = remote_task_git.detect_ssh(url); + assert!(result.is_err()); + } + } + + #[test] + fn test_valid_detect_https() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//terraform/myfile?ref=master", + "git::https://myserver.com/example.git//terraform/myfile", + "git::https://myserver.com/example.git//myfile?ref=master", + ]; + + for url in test_cases { + let result = remote_task_git.detect_https(url); + assert!(result.is_ok()); + } + } + + #[test] + fn test_invalid_detect_https() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::https://myserver.com/example.git?ref=master", + "git::https://user@myserver.com/example.git", + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + ]; + + for url in test_cases { + let result = remote_task_git.detect_https(url); + assert!(result.is_err()); + } + } + + #[test] + fn test_extract_ssh_url_information() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases: Vec<(&str, &str, &str, Option)> = vec![ + ( + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git@github.com:myorg/example.git", + "myfile", + Some("v1.0.0".to_string()), + ), + ( + "git::ssh://git@github.com:myorg/example.git//terraform/myfile?ref=master", + "git@github.com:myorg/example.git", + "terraform/myfile", + Some("master".to_string()), + ), + ( + "git::ssh://git@myserver.com/example.git//terraform/myfile", + "git@myserver.com/example.git", + "terraform/myfile", + None, + ), + ]; + + for (url, expected_repo, expected_path, expected_branch) in test_cases { + let repo = remote_task_git.detect_ssh(url).unwrap(); + assert_eq!(expected_repo, repo.url_without_path); + assert_eq!(expected_path, repo.path); + assert_eq!(expected_branch, repo.branch); + } + } + + #[test] + fn test_extract_https_url_information() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases: Vec<(&str, &str, &str, Option)> = vec![ + ( + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "https://github.com/myorg/example.git", + "myfile", + Some("v1.0.0".to_string()), + ), + ( + "git::https://github.com/myorg/example.git//terraform/myfile?ref=master", + "https://github.com/myorg/example.git", + "terraform/myfile", + Some("master".to_string()), + ), + ( + "git::https://myserver.com/example.git//terraform/myfile", + "https://myserver.com/example.git", + "terraform/myfile", + None, + ), + ]; + + for (url, expected_repo, expected_path, expected_branch) in test_cases { + let repo = remote_task_git.detect_https(url).unwrap(); + assert_eq!(expected_repo, repo.url_without_path); + assert_eq!(expected_path, repo.path); + assert_eq!(expected_branch, repo.branch); + } + } + + #[test] + fn test_compare_ssh_get_cache_key() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + ( + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v2.0.0", + false, + ), + ( + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://user@myserver.com/example.git//myfile?ref=master", + false, + ), + ( + "git::ssh://git@github.com/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com/example.git//subfolder/mysecondfile?ref=v1.0.0", + true, + ), + ( + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com:myorg/example.git//subfolder/mysecondfile?ref=v1.0.0", + true, + ), + ]; + + for (first_url, second_url, expected) in test_cases { + let first_repo = remote_task_git.detect_ssh(first_url).unwrap(); + let second_repo = remote_task_git.detect_ssh(second_url).unwrap(); + let first_cache_key = remote_task_git.get_cache_key(&first_repo); + let second_cache_key = remote_task_git.get_cache_key(&second_repo); + assert_eq!(expected, first_cache_key == second_cache_key); + } + } + + #[test] + fn test_compare_https_get_cache_key() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + ( + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//myfile?ref=v2.0.0", + false, + ), + ( + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://bitbucket.com/myorg/example.git//myfile?ref=v1.0.0", + false, + ), + ( + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//subfolder/myfile?ref=v1.0.0", + true, + ), + ( + "git::https://github.com/example.git//myfile?ref=v1.0.0", + "git::https://github.com/example.git//subfolder/myfile?ref=v1.0.0", + true, + ), + ]; + + for (first_url, second_url, expected) in test_cases { + let first_repo = remote_task_git.detect_https(first_url).unwrap(); + let second_repo = remote_task_git.detect_https(second_url).unwrap(); + let first_cache_key = remote_task_git.get_cache_key(&first_repo); + let second_cache_key = remote_task_git.get_cache_key(&second_repo); + assert_eq!(expected, first_cache_key == second_cache_key); + } + } +} diff --git a/src/task/task_file_providers/remote_task_http.rs b/src/task/task_file_providers/remote_task_http.rs index 18aecd0a7b..5c2394127d 100644 --- a/src/task/task_file_providers/remote_task_http.rs +++ b/src/task/task_file_providers/remote_task_http.rs @@ -50,6 +50,7 @@ impl RemoteTaskHttp { file: &str, destination: &PathBuf, ) -> Result<(), Box> { + trace!("Downloading file: {}", file); HTTP.download_file(file, destination, None)?; file::make_executable(destination)?; Ok(()) @@ -71,37 +72,29 @@ impl TaskFileProvider for RemoteTaskHttp { } fn get_local_path(&self, file: &str) -> Result> { + let cache_key = self.get_cache_key(file); + let destination = self.storage_path.join(&cache_key); + match self.is_cached { true => { trace!("Cache mode enabled"); - let cache_key = self.get_cache_key(file); - let destination = self.storage_path.join(&cache_key); if destination.exists() { debug!("Using cached file: {:?}", destination); return Ok(destination); } - - debug!("Downloading file: {}", file); - self.download_file(file, &destination)?; - Ok(destination) } false => { trace!("Cache mode disabled"); - let url = url::Url::parse(file)?; - let filename = url - .path_segments() - .and_then(|segments| segments.last()) - .unwrap(); - let destination = env::temp_dir().join(filename); if destination.exists() { file::remove_file(&destination)?; } - self.download_file(file, &destination)?; - Ok(destination) } } + + self.download_file(file, &destination)?; + Ok(destination) } } @@ -130,13 +123,13 @@ mod tests { #[test] fn test_http_remote_task_get_local_path_without_cache() { let paths = vec![ - ("/myfile.py", "myfile.py"), - ("/subpath/myfile.sh", "myfile.sh"), - ("/myfile.sh?query=1&sdfsdf=2", "myfile.sh"), + "/myfile.py", + "/subpath/myfile.sh", + "/myfile.sh?query=1&sdfsdf=2", ]; let mut server = mockito::Server::new(); - for (request_path, expected_file_name) in paths { + for request_path in paths { let mocked_server: mockito::Mock = server .mock("GET", request_path) .with_status(200) @@ -145,13 +138,14 @@ mod tests { .create(); let provider = RemoteTaskHttpBuilder::new().build(); - let mock = format!("{}{}", server.url(), request_path); + let request_url = format!("{}{}", server.url(), request_path); + let cache_key = provider.get_cache_key(&request_url); for _ in 0..2 { - let local_path = provider.get_local_path(&mock).unwrap(); + let local_path = provider.get_local_path(&request_url).unwrap(); assert!(local_path.exists()); assert!(local_path.is_file()); - assert!(local_path.ends_with(expected_file_name)); + assert!(local_path.ends_with(&cache_key)); } mocked_server.assert(); @@ -161,13 +155,13 @@ mod tests { #[test] fn test_http_remote_task_get_local_path_with_cache() { let paths = vec![ - ("/myfile.py", "myfile.py"), - ("/subpath/myfile.sh", "myfile.sh"), - ("/myfile.sh?query=1&sdfsdf=2", "myfile.sh"), + "/myfile.py", + "/subpath/myfile.sh", + "/myfile.sh?query=1&sdfsdf=2", ]; let mut server = mockito::Server::new(); - for (request_path, not_expected_file_name) in paths { + for request_path in paths { let mocked_server = server .mock("GET", request_path) .with_status(200) @@ -176,13 +170,14 @@ mod tests { .create(); let provider = RemoteTaskHttpBuilder::new().with_cache(true).build(); - let mock = format!("{}{}", server.url(), request_path); + let request_url = format!("{}{}", server.url(), request_path); + let cache_key = provider.get_cache_key(&request_url); for _ in 0..2 { - let path = provider.get_local_path(&mock).unwrap(); + let path = provider.get_local_path(&request_url).unwrap(); assert!(path.exists()); assert!(path.is_file()); - assert!(!path.ends_with(not_expected_file_name)); + assert!(path.ends_with(&cache_key)); } mocked_server.assert();