From 4a9a2a5f4616e2f7f5cdab0962535ef483ee37ef Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 12 Jul 2024 13:27:55 +0200 Subject: [PATCH 01/11] Move IJob implementation out of yagna directory --- Golem/{Yagna => }/Job.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Golem/{Yagna => }/Job.cs (100%) diff --git a/Golem/Yagna/Job.cs b/Golem/Job.cs similarity index 100% rename from Golem/Yagna/Job.cs rename to Golem/Job.cs From faf3b584c99b1a56b36fada8f39159ac6b251877 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 12 Jul 2024 13:32:04 +0200 Subject: [PATCH 02/11] Chnage status to Interrupted if modelserve reports chosen termination Reasons --- Golem/Job.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Golem/Job.cs b/Golem/Job.cs index 6165b9ce..3e2ede5e 100644 --- a/Golem/Job.cs +++ b/Golem/Job.cs @@ -195,6 +195,9 @@ public static JobStatus ResolveTerminationReason(string? code) "RequestorUnreachable" => JobStatus.Interrupted, "Shutdown" => JobStatus.Interrupted, "Interrupted" => JobStatus.Interrupted, + "HealthCheckFailed" => JobStatus.Interrupted, + "ConnectionTimedOut" => JobStatus.Interrupted, + "ProviderUnreachable" => JobStatus.Interrupted, "Expired" => JobStatus.Finished, "Cancelled" => JobStatus.Finished, _ => JobStatus.Finished, From 0d4e990392c8cae2fba22a9f0318ababe45e8e06 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 12 Jul 2024 17:27:41 +0200 Subject: [PATCH 03/11] Test scenarios with termination race conditions --- Golem.Tests/JobStatusTests.cs | 160 +++++++++++++++++++++++++++++++++- 1 file changed, 159 insertions(+), 1 deletion(-) diff --git a/Golem.Tests/JobStatusTests.cs b/Golem.Tests/JobStatusTests.cs index c3184e87..c184c5c1 100644 --- a/Golem.Tests/JobStatusTests.cs +++ b/Golem.Tests/JobStatusTests.cs @@ -3,6 +3,7 @@ using System.Text.RegularExpressions; using System.Threading.Channels; +using Golem.Model; using Golem.Tools; using Golem.Yagna.Types; @@ -137,7 +138,7 @@ public async Task RequestorBreaksAgreement_FastTerminatingScript() await Task.Delay(2 * 1000); _logger.LogInformation("=================== Terminating App ==================="); - var rest = _requestor.Rest != null ? _requestor.Rest : throw new Exception("Rest api on Requestor not initialized."); + var rest = _requestor.Rest ?? throw new Exception("Rest api on Requestor not initialized."); var activities = await rest.GetActivities(currentJob.Id); // Kill script so he doesn't have chance to handle tasks termination. @@ -341,5 +342,162 @@ public async Task RequestorBreaksAgreement_KillingYagna() // Stopping Golem await StopGolem(golem, golemPath, StatusChannel(golem)); } + + [Fact] + public async Task ProviderRestart_RequestorBreaksAgreement() + { + string golemPath = await PackageBuilder.BuildTestDirectory(); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + _logger.LogInformation($"Path: {golemPath}"); + + await StartGolem(golem, StatusChannel(golem)); + var jobChannel = JobChannel(golem); + + _logger.LogInformation("=================== Starting Sample App ==================="); + await using var app = _requestor?.CreateSampleApp() ?? throw new Exception("Requestor not started yet"); + Assert.True(app.Start()); + + // Wait for job. + Job? currentJob = await ReadChannel(jobChannel); + Assert.NotNull(currentJob); + + var jobStatusChannel = JobStatusChannel(currentJob); + var agreementId = currentJob.Id; + + // Wait until ExeUnit will be created. + // Workaround for situations, when status update was so fast, that we were not able to create + // channel yet, so waiting for update would be pointless. + if (currentJob.Status != JobStatus.Computing) + Assert.Equal(JobStatus.Computing, await ReadChannel(jobStatusChannel, + (JobStatus s) => s == JobStatus.DownloadingModel || s == JobStatus.Idle)); + // Let him compute for a while. + await Task.Delay(TimeSpan.FromSeconds(2)); + + _logger.LogInformation("=================== Killing Provider Agent ==================="); + // Provider is killed so he is not able to terminate Agreement. + // Yagna has chance to be closed gracefully and close net connection with Requestor. + // If we would double Stop here, re-establishing connection would last longer. + var golemStatusChannel = StatusChannel(golem); + var pid = golem.GetProviderPid(); + if (pid.HasValue) + Process.GetProcessById(pid.Value).Kill(); + + _logger.LogInformation("=================== Killing App ==================="); + // Note: We need to manually send termination from Requestor with correct Reason. + // It's better to avoid yapapi finding out that Provider stopped working, because it + // could take action otherwise. + await app.Stop(StopMethod.SigKill); + + await AwaitValue(jobStatusChannel, JobStatus.Interrupted, TimeSpan.FromSeconds(30)); + Assert.Equal(JobStatus.Interrupted, currentJob.Status); + await AwaitValue(jobChannel, null, TimeSpan.FromSeconds(1)); + Assert.Null(golem.CurrentJob); + + await AwaitValue(golemStatusChannel, GolemStatus.Error, TimeSpan.FromSeconds(3)); + + _logger.LogInformation("=================== Restart Provider Agent ==================="); + await golem.Start(); + + await Task.Delay(TimeSpan.FromSeconds(3)); + Assert.Equal(JobStatus.Interrupted, currentJob.Status); + + _logger.LogInformation("=================== Requestor terminates Agreement ==================="); + var rest = _requestor.Rest ?? throw new Exception("Rest api on Requestor not initialized."); + var reason = new Reason(null!, "Healthcheck failed"); + reason.RequestorCode = "HealthCheckFailed"; + await rest.TerminateAgreement(agreementId, reason); + + // Wait for propagation of termination event + await Task.Delay(TimeSpan.FromSeconds(2)); + + Assert.Equal(JobStatus.Interrupted, currentJob.Status); + } + + [Fact] + public async Task ProviderRestart_ProviderBreaksAgreement() + { + string golemPath = await PackageBuilder.BuildTestDirectory(); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + _logger.LogInformation($"Path: {golemPath}"); + + await StartGolem(golem, StatusChannel(golem)); + var jobChannel = JobChannel(golem); + + _logger.LogInformation("=================== Starting Sample App ==================="); + await using var app = _requestor?.CreateSampleApp() ?? throw new Exception("Requestor not started yet"); + Assert.True(app.Start()); + + // Wait for job. + Job? currentJob = await ReadChannel(jobChannel); + Assert.NotNull(currentJob); + + var jobStatusChannel = JobStatusChannel(currentJob); + var agreementId = currentJob.Id; + + // Wait until ExeUnit will be created. + // Workaround for situations, when status update was so fast, that we were not able to create + // channel yet, so waiting for update would be pointless. + if (currentJob.Status != JobStatus.Computing) + Assert.Equal(JobStatus.Computing, await ReadChannel(jobStatusChannel, + (JobStatus s) => s == JobStatus.DownloadingModel || s == JobStatus.Idle)); + // Let him compute for a while. + await Task.Delay(2 * 1000); + + _logger.LogInformation("=================== Killing Provider Agent ==================="); + // Provider is killed so he is not able to terminate Agreement. + // Yagna has chance to be closed gracefully and close net connection with Requestor. + // If we would double Stop here, re-establishing connection would last longer. + var golemStatusChannel = StatusChannel(golem); + var pid = golem.GetProviderPid(); + if (pid.HasValue) + Process.GetProcessById(pid.Value).Kill(); + + _logger.LogInformation("=================== Killing App ==================="); + // Note: Provider needs to send termination with correct Reason. + // It's better to avoid yapapi finding out that Provider stopped working, because it + // could take action otherwise. + await app.Stop(StopMethod.SigKill); + + await AwaitValue(jobStatusChannel, JobStatus.Interrupted, TimeSpan.FromSeconds(30)); + Assert.Equal(JobStatus.Interrupted, currentJob.Status); + await AwaitValue(jobChannel, null, TimeSpan.FromSeconds(1)); + Assert.Null(golem.CurrentJob); + + await AwaitValue(golemStatusChannel, GolemStatus.Error, TimeSpan.FromSeconds(3)); + + _logger.LogInformation("=================== Restart Provider Agent ==================="); + await golem.Start(); + + Assert.Null(golem.CurrentJob); + + // ListJobs is one of things that can trigger termination of interrupted job. + // This will happen as well, when new event from yagna API will be processed. + var jobs = await golem.ListJobs(DateTime.Now - TimeSpan.FromMinutes(1)); + var prevJob = jobs.Find(job => job.Id == agreementId) as Job; + + Assert.NotNull(prevJob); + Assert.Equal(JobStatus.Interrupted, prevJob.Status); + + _logger.LogInformation("=================== Requestor terminates Agreement ==================="); + // Try to terminate Agreement with Reason which in normal situation would be treated as correct termination + // resulting in Finished status. If Provider didn't terminate Agreement with Reason leading to Interrupted + // state, then JobStatus would be restored to Finished and it would reveal error. + var rest = _requestor.Rest ?? throw new Exception("Rest api on Requestor not initialized."); + var reason = new Reason(null!, "Agreement is no longer needed"); + reason.RequestorCode = "NoLongerNeeded"; + try + { + // Termination should fail, because Provider already terminated Agreement. + await rest.TerminateAgreement(agreementId, reason); + } + catch (Exception e) + { + _logger.LogInformation(e, "Failed to terminate agreement"); + } + + // Wait for propagation of termination event. Status should be still Interrupted. + await Task.Delay(TimeSpan.FromSeconds(2)); + Assert.Equal(JobStatus.Interrupted, prevJob.Status); + } } } From 1b18530165b6daf22506c5c236d3e083f9a6b275 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 15 Jul 2024 19:36:48 +0200 Subject: [PATCH 04/11] Fix statuses tests by using central net in tests --- Golem.Tests/JobStatusTests.cs | 21 ++--- Golem.Tests/JobTests.cs | 2 +- Golem.Tests/Utils.cs | 16 ++-- Golem.Tools/GolemCentralNet.cs | 41 ++++++++++ Golem.Tools/GolemRequestor.cs | 109 ++++++++++++++++--------- Golem.Tools/SampleApp.cs | 2 +- Golem.Tools/resources/test_key_2.plain | 1 + Golem/Yagna/EnvironmentBuilder.cs | 39 +++++---- Golem/Yagna/Net.cs | 9 ++ 9 files changed, 164 insertions(+), 76 deletions(-) create mode 100644 Golem.Tools/GolemCentralNet.cs create mode 100644 Golem.Tools/resources/test_key_2.plain diff --git a/Golem.Tests/JobStatusTests.cs b/Golem.Tests/JobStatusTests.cs index c184c5c1..f0f6bcfd 100644 --- a/Golem.Tests/JobStatusTests.cs +++ b/Golem.Tests/JobStatusTests.cs @@ -29,7 +29,7 @@ public JobStatusTests(ITestOutputHelper outputHelper, GolemFixture golemFixture) public async Task RequestorBreaksAgreement_KillingScript() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -72,7 +72,7 @@ public async Task RequestorBreaksAgreement_KillingScript() public async Task ProviderBreaksAgreement_Graceful() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -112,7 +112,7 @@ public async Task ProviderBreaksAgreement_Graceful() public async Task RequestorBreaksAgreement_FastTerminatingScript() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -164,7 +164,7 @@ public async Task RequestorBreaksAgreement_FastTerminatingScript() public async Task ProviderBreaksAgreement_KillingAgent() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -207,7 +207,7 @@ public async Task ProviderBreaksAgreement_KillingAgent() public async Task ProviderBreaksAgreement_KillingYagna() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -249,7 +249,7 @@ public async Task ProviderBreaksAgreement_KillingYagna() public async Task ProviderBreaksAgreement_KillingExeUnit() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -300,11 +300,12 @@ public async Task RequestorBreaksAgreement_KillingYagna() // Run Requestor yagna before starting Provider to speed up Offers propagation. await using var requestor = await GolemRequestor.Build("RequestorBreaksAgreement_KillingYagna", _loggerFactory.CreateLogger("Requestor2")); requestor.AutoSetUrls(11000); + requestor.SetSecret("test_key_2.plain"); Assert.True(requestor.Start()); - requestor.InitPayment(); + await requestor.InitPayment(); string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -347,7 +348,7 @@ public async Task RequestorBreaksAgreement_KillingYagna() public async Task ProviderRestart_RequestorBreaksAgreement() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); @@ -417,7 +418,7 @@ public async Task ProviderRestart_RequestorBreaksAgreement() public async Task ProviderRestart_ProviderBreaksAgreement() { string golemPath = await PackageBuilder.BuildTestDirectory(); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); _logger.LogInformation($"Path: {golemPath}"); await StartGolem(golem, StatusChannel(golem)); diff --git a/Golem.Tests/JobTests.cs b/Golem.Tests/JobTests.cs index 85b80f2e..ffa7ec9f 100644 --- a/Golem.Tests/JobTests.cs +++ b/Golem.Tests/JobTests.cs @@ -27,7 +27,7 @@ public async Task CompleteScenario() string golemPath = await PackageBuilder.BuildTestDirectory(); _logger.LogInformation($"Path: {golemPath}"); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); var golemStatusChannel = StatusChannel(golem); diff --git a/Golem.Tests/Utils.cs b/Golem.Tests/Utils.cs index 76a6102a..10992f81 100644 --- a/Golem.Tests/Utils.cs +++ b/Golem.Tests/Utils.cs @@ -212,7 +212,7 @@ public class JobsTestBase : WithAvailablePort, IDisposable, IAsyncLifetime, ICla { protected readonly ILoggerFactory _loggerFactory; protected readonly ILogger _logger; - protected GolemRelay? _relay; + protected GolemCentralNet? _router; protected GolemRequestor? _requestor; protected AppKey? _requestorAppKey; protected String _testClassName; @@ -240,15 +240,15 @@ public JobsTestBase(ITestOutputHelper outputHelper, GolemFixture golemFixture, s public async Task InitializeAsync() { - var testDir = PackageBuilder.TestDir($"{_testClassName}_relay"); - _relay = await GolemRelay.Build(testDir, _loggerFactory.CreateLogger("Relay")); - Assert.True(_relay.Start()); - NetConfig.SetEnv(RelayType.Local); + var testDir = PackageBuilder.TestDir($"{_testClassName}_router"); + _router = await GolemCentralNet.Build(testDir, _loggerFactory.CreateLogger("Router")); + Assert.True(_router.Start()); + NetConfig.SetEnv(RelayType.LocalCentral); System.Environment.SetEnvironmentVariable("RUST_LOG", "debug"); _requestor = await GolemRequestor.Build(_testClassName, _loggerFactory.CreateLogger("Requestor")); Assert.True(_requestor.Start()); - _requestor.InitPayment(); + await _requestor.InitPayment(); _requestorAppKey = _requestor.GetTestAppKey(); } @@ -319,8 +319,8 @@ public async Task DisposeAsync() if (_requestor != null) await _requestor.Stop(StopMethod.SigInt); - if (_relay != null) - await _relay.Stop(StopMethod.SigInt); + if (_router != null) + await _router.Stop(StopMethod.SigInt); } public void Dispose() diff --git a/Golem.Tools/GolemCentralNet.cs b/Golem.Tools/GolemCentralNet.cs new file mode 100644 index 00000000..9c984993 --- /dev/null +++ b/Golem.Tools/GolemCentralNet.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.Logging; + +namespace Golem.Tools +{ + public class GolemCentralNet : GolemRunnable + { + const string CURRENT_ROUTER_VERSION = "v0.7.2"; + + private GolemCentralNet(string dir, ILogger logger) : base(dir, logger) + { + } + + public async static Task Build(string testDir, ILogger logger) + { + var dir = await BuildCentralNetDir(testDir); + return new GolemCentralNet(dir, logger); + } + + public override bool Start() + { + var working_dir = Path.Combine(_dir, "modules", "golem-data", "central-net"); + Directory.CreateDirectory(working_dir); + return StartProcess("modules/golem/ya-sb-router", working_dir, "-l tcp://127.0.0.1:6464", new Dictionary()); + } + + protected static async Task BuildCentralNetDir(string test_dir) + { + var dir = PackageBuilder.PrepareTestDirectory(test_dir, true); + var binaries_dir = PackageBuilder.BinariesDir(dir); + + Directory.CreateDirectory(binaries_dir); + + var artifact = "ya-sb-router"; + var repo = "golemfactory/ya-service-bus"; + var tag = CURRENT_ROUTER_VERSION; + + await PackageBuilder.DownloadExtractPackage(binaries_dir, artifact, repo, tag); + return dir; + } + } +} diff --git a/Golem.Tools/GolemRequestor.cs b/Golem.Tools/GolemRequestor.cs index d982eb63..73e86682 100644 --- a/Golem.Tools/GolemRequestor.cs +++ b/Golem.Tools/GolemRequestor.cs @@ -26,9 +26,9 @@ public class AppKey public class GolemRequestor : GolemRunnable, IAsyncLifetime { - private Dictionary _env; + private readonly EnvironmentBuilder _env; - public string? AppKey; + public string AppKey; public string ApiUrl { get; set; } public string GsbUrl { get; set; } public string NetBindUrl { get; set; } @@ -40,6 +40,8 @@ public class GolemRequestor : GolemRunnable, IAsyncLifetime private GolemRequestor(string dir, bool mainnet, ILogger logger) : base(dir, logger) { + AppKey = ""; // Will be set in `SetAppKey` function. This line supresses warning. + ApiUrl = "http://127.0.0.1:7465"; GsbUrl = "tcp://127.0.0.1:7464"; NetBindUrl = "udp://0.0.0.0:11500"; @@ -53,7 +55,10 @@ private GolemRequestor(string dir, bool mainnet, ILogger logger) : base(dir, log envBuilder.WithGsbUrl(GsbUrl); envBuilder.WithYaNetBindUrl(NetBindUrl); envBuilder.WithMetricsGroup("Example-GamerHash"); - _env = envBuilder.Build(); + _env = envBuilder; + + SetSecret(_mainnet ? "main_key.plain" : "test_key.plain"); + SetAppKey(GenerateRandomAppkey()); } public async static Task Build(string test_name, ILogger logger, bool cleanupData = true, bool mainnet = false) @@ -70,14 +75,10 @@ public async static Task BuildRelative(string datadir, ILogger l public override bool Start() { - BuildEnv(); - var working_dir = Path.Combine(_dir, "modules", "golem-data", "yagna"); Directory.CreateDirectory(working_dir); - AppKey = generateRandomAppkey(); - var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value); - env["YAGNA_AUTOCONF_ID_SECRET"] = getRequestorAutoconfIdSecret(); - env["YAGNA_AUTOCONF_APPKEY"] = AppKey; + + var env = _env.Build(); var result = StartProcess("yagna", working_dir, "service run", env, false); Rest = CreateRestAPI(AppKey); @@ -93,47 +94,48 @@ public void AutoSetUrls(UInt16 portBase) ApiUrl = $"http://127.0.0.1:{apiPort}"; GsbUrl = $"tcp://127.0.0.1:{gsbPort}"; NetBindUrl = $"udp://0.0.0.0:{bindPort}"; + + _env.WithYagnaApiUrl(ApiUrl); + _env.WithGsbUrl(GsbUrl); + _env.WithYaNetBindUrl(NetBindUrl); } - private void BuildEnv() + public void SetSecret(string resourceFile) { - var envBuilder = new EnvironmentBuilder(); - envBuilder.WithYagnaDataDir(_dataDir); - envBuilder.WithYagnaApiUrl(ApiUrl); - envBuilder.WithGsbUrl(GsbUrl); - envBuilder.WithYaNetBindUrl(NetBindUrl); - envBuilder.WithMetricsGroup("Example-GamerHash"); - _env = envBuilder.Build(); + _env.WithPrivateKey(LoadSecret(resourceFile)); } - private string getRequestorAutoconfIdSecret() + private static string LoadSecret(string resourceFile) { - string? key = null; - if ((key = (string?)Environment.GetEnvironmentVariable("REQUESTOR_AUTOCONF_ID_SECRET")) != null) - { - return key; - } - var keyFilename = _mainnet ? "main_key.plain" : "test_key.plain"; - var keyReader = PackageBuilder.ReadResource(keyFilename); - return keyReader.ReadLine() ?? throw new Exception($"Failed to read key from file {keyFilename}"); + var keyReader = PackageBuilder.ReadResource(resourceFile); + return keyReader.ReadLine() ?? throw new Exception($"Failed to read key from file {resourceFile}"); } - private string generateRandomAppkey() + private static string GenerateRandomAppkey() { string? appKey = null; if ((appKey = (string?)Environment.GetEnvironmentVariable("REQUESTOR_AUTOCONF_APPKEY")) != null) { return appKey; } - byte[] data = RandomNumberGenerator.GetBytes(20); - return Convert.ToBase64String(data); + else + { + byte[] data = RandomNumberGenerator.GetBytes(20); + return Convert.ToBase64String(data); + } + + } + + private void SetAppKey(string appKey) + { + AppKey = appKey; + _env.WithAppKey(appKey); } public SampleApp CreateSampleApp(string? extraArgs = null) { - var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value); - env["YAGNA_APPKEY"] = AppKey ?? throw new Exception("Unable to create app process. No YAGNA_APPKEY."); - env["YAGNA_API_URL"] = ApiUrl; + var env = _env.Build(); + var pathEnvVar = Environment.GetEnvironmentVariable("PATH") ?? ""; var binariesDir = Path.GetFullPath(PackageBuilder.BinariesDir(_dir)); if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -149,11 +151,12 @@ public SampleApp CreateSampleApp(string? extraArgs = null) return new SampleApp(_dir, env, network, _logger, extraArgs); } - public void InitPayment(double minFundThreshold = 100.0) + public async Task InitPayment(double minFundThreshold = 100.0) { - Thread.Sleep(6000); - var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value); - env.Add("RUST_LOG", "none"); + await WaitForIdentityAsync(); + + var env = _env.Build(); + env["RUST_LOG"] = "none"; var network = Factory.Network(_mainnet); var payment_status_process = WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), $"payment status --json --network {network.Id}", env)); @@ -164,16 +167,42 @@ public void InitPayment(double minFundThreshold = 100.0) if (reserved > 0.0) { - WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), "payment release-allocations", _env)); + WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), "payment release-allocations", env)); } if (totalGlm < minFundThreshold && !_mainnet) { - WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), $"payment fund --network {network.Id}", _env)); + WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), $"payment fund --network {network.Id}", env)); } return; } + public async Task WaitForIdentityAsync(CancellationToken cancellationToken = default) + { + _logger.LogDebug("Waiting for yagna to start... Checking /me endpoint."); + + //yagna is starting and /me won't work until all services are running + for (int tries = 0; tries < 200; ++tries) + { + Thread.Sleep(300); + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var api = Rest ?? throw new Exception("REST API not initialized"); + MeInfo meInfo = await Rest.Me(cancellationToken); + + _logger.LogDebug("Yagna started; REST API is available."); + return meInfo.Identity; + } + catch (Exception) + { + // consciously swallow the exception... presumably REST call error... + } + } + return null; + } + private Command WaitAndPrintOnError(Command cmd) { try @@ -197,8 +226,8 @@ public AppKey GetTestAppKey() throw new Exception("No data dir"); } - var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value); - env.Add("RUST_LOG", "none"); + var env = _env.Build(); + env["RUST_LOG"] = "none"; var app_key_list_process = RunCommand("yagna", WorkingDir(), "app-key list --json", env); app_key_list_process.Wait(); diff --git a/Golem.Tools/SampleApp.cs b/Golem.Tools/SampleApp.cs index 4205ec00..1eda07a8 100644 --- a/Golem.Tools/SampleApp.cs +++ b/Golem.Tools/SampleApp.cs @@ -122,7 +122,7 @@ public async Task Run() Message = "Payment Initialization"; _logger.LogInformation("Initializing payment accounts for: " + Name); - await Task.Run(() => Requestor.InitPayment()); + await Task.Run(async () => await Requestor.InitPayment()); _logger.LogInformation("Creating requestor application: " + Name); Message = "Starting Application"; diff --git a/Golem.Tools/resources/test_key_2.plain b/Golem.Tools/resources/test_key_2.plain new file mode 100644 index 00000000..badb9022 --- /dev/null +++ b/Golem.Tools/resources/test_key_2.plain @@ -0,0 +1 @@ +b0f8813e42e0170c8a6c94e7c34b257df3a7a77322b61dc914f8b48b0d03d166 diff --git a/Golem/Yagna/EnvironmentBuilder.cs b/Golem/Yagna/EnvironmentBuilder.cs index a58799be..0d246762 100644 --- a/Golem/Yagna/EnvironmentBuilder.cs +++ b/Golem/Yagna/EnvironmentBuilder.cs @@ -21,77 +21,84 @@ public class EnvironmentBuilder { "BCAST_NODE_BAN_TIMEOUT", "5s" }, }; - private readonly Dictionary env = new Dictionary(); + private readonly Dictionary _env = new Dictionary(); + + + public string this[string key] + { + get => _env[key]; + set => _env[key] = value; + } public EnvironmentBuilder WithGsbUrl(string s) { - env["GSB_URL"] = s; + _env["GSB_URL"] = s; return this; } public EnvironmentBuilder WithMetricsGroup(string s) { - env["YAGNA_METRICS_GROUP"] = s; + _env["YAGNA_METRICS_GROUP"] = s; return this; } public EnvironmentBuilder WithYagnaApiUrl(string s) { - env["YAGNA_API_URL"] = s; + _env["YAGNA_API_URL"] = s; return this; } public EnvironmentBuilder WithYagnaAppKey(string s) { - env["YAGNA_APPKEY"] = s; + _env["YAGNA_APPKEY"] = s; return this; } public EnvironmentBuilder WithYaNetBindUrl(string s) { - env["YA_NET_BIND_URL"] = s; + _env["YA_NET_BIND_URL"] = s; return this; } public EnvironmentBuilder WithExeUnitPath(string s) { - env["EXE_UNIT_PATH"] = s; + _env["EXE_UNIT_PATH"] = s; return this; } public EnvironmentBuilder WithDataDir(string s) { - env["DATA_DIR"] = s; + _env["DATA_DIR"] = s; return this; } public EnvironmentBuilder WithYagnaDataDir(string s) { - env["YAGNA_DATADIR"] = s; + _env["YAGNA_DATADIR"] = s; return this; } public EnvironmentBuilder WithYaNetRelayHost(string s) { - env["YA_NET_RELAY_HOST"] = s; + _env["YA_NET_RELAY_HOST"] = s; return this; } public EnvironmentBuilder WithPrivateKey(string s) { - env["YAGNA_AUTOCONF_ID_SECRET"] = s; + _env["YAGNA_AUTOCONF_ID_SECRET"] = s; return this; } public EnvironmentBuilder WithAppKey(string s) { - env["YAGNA_AUTOCONF_APPKEY"] = s; + _env["YAGNA_AUTOCONF_APPKEY"] = s; return this; } public EnvironmentBuilder WithSslCertFile(string s) { - env["SSL_CERT_FILE"] = s; + _env["SSL_CERT_FILE"] = s; return this; } @@ -99,12 +106,12 @@ public Dictionary Build() { foreach (var kvp in DefaultEnv) { - if (!env.ContainsKey(kvp.Key) && Environment.GetEnvironmentVariable(kvp.Key) == null) + if (!_env.ContainsKey(kvp.Key) && Environment.GetEnvironmentVariable(kvp.Key) == null) { - env.Add(kvp.Key, kvp.Value); + _env.Add(kvp.Key, kvp.Value); } } - return env; + return _env; } } } diff --git a/Golem/Yagna/Net.cs b/Golem/Yagna/Net.cs index 7bdf6923..4c1b76d6 100644 --- a/Golem/Yagna/Net.cs +++ b/Golem/Yagna/Net.cs @@ -21,6 +21,10 @@ public enum RelayType /// Central, /// + /// Use central net local setup. + /// + LocalCentral, + /// /// Facade won't set any value. It must be set from outside using env variables. /// None, @@ -49,6 +53,11 @@ public static void SetEnv(RelayType relay) Environment.SetEnvironmentVariable("YA_NET_TYPE", "central"); // Will use default yagna central net configuration: resolving `_net._tcp.dev.golem.network` record. } + else if (relay == RelayType.LocalCentral) + { + Environment.SetEnvironmentVariable("YA_NET_TYPE", "central"); + Environment.SetEnvironmentVariable("CENTRAL_NET_HOST", "127.0.0.1:6464"); + } } } } From 751f48b71fdd989a47ae779f2fa957f8e38af421 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 15 Jul 2024 19:40:26 +0200 Subject: [PATCH 05/11] Fix passing AppKey to requestor script after chnages --- Golem.Tools/GolemRequestor.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Golem.Tools/GolemRequestor.cs b/Golem.Tools/GolemRequestor.cs index 73e86682..f65f4b0a 100644 --- a/Golem.Tools/GolemRequestor.cs +++ b/Golem.Tools/GolemRequestor.cs @@ -130,6 +130,7 @@ private void SetAppKey(string appKey) { AppKey = appKey; _env.WithAppKey(appKey); + _env.WithYagnaAppKey(appKey); } public SampleApp CreateSampleApp(string? extraArgs = null) From 7595577b33612a29a581ee7fce597406e812012f Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jul 2024 11:45:11 +0200 Subject: [PATCH 06/11] Fix paths to central net router --- Golem.Tests/ErrorHandlingTest.cs | 2 +- Golem.Tools/GolemCentralNet.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Golem.Tests/ErrorHandlingTest.cs b/Golem.Tests/ErrorHandlingTest.cs index bdc325ff..0758ff6b 100644 --- a/Golem.Tests/ErrorHandlingTest.cs +++ b/Golem.Tests/ErrorHandlingTest.cs @@ -29,7 +29,7 @@ public async Task StartTriggerErrorStop_VerifyStatusAsync() string golemPath = await PackageBuilder.BuildTestDirectory(); _logger.LogInformation("Path: " + golemPath); - await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local); + await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral); var statusChannel = StatusChannel(golem); diff --git a/Golem.Tools/GolemCentralNet.cs b/Golem.Tools/GolemCentralNet.cs index 9c984993..b34ce2c9 100644 --- a/Golem.Tools/GolemCentralNet.cs +++ b/Golem.Tools/GolemCentralNet.cs @@ -20,7 +20,7 @@ public override bool Start() { var working_dir = Path.Combine(_dir, "modules", "golem-data", "central-net"); Directory.CreateDirectory(working_dir); - return StartProcess("modules/golem/ya-sb-router", working_dir, "-l tcp://127.0.0.1:6464", new Dictionary()); + return StartProcess("ya-sb-router", working_dir, "-l tcp://127.0.0.1:6464", new Dictionary()); } protected static async Task BuildCentralNetDir(string test_dir) From 2f9d5e7d84f8dd1b2166363c422d88367203c2c8 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jul 2024 13:11:32 +0200 Subject: [PATCH 07/11] Disable requestor script to not terminate agreement prematurely --- Golem.Tests/JobStatusTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Golem.Tests/JobStatusTests.cs b/Golem.Tests/JobStatusTests.cs index f0f6bcfd..c3984b96 100644 --- a/Golem.Tests/JobStatusTests.cs +++ b/Golem.Tests/JobStatusTests.cs @@ -274,6 +274,10 @@ public async Task ProviderBreaksAgreement_KillingExeUnit() // Let him compute for a while. await Task.Delay(2 * 1000); + _logger.LogInformation("=================== Killing App ==================="); + // Killing app, so it won't terminate Agreement. + await app.Stop(StopMethod.SigKill); + _logger.LogInformation("=================== Killing Runtime ==================="); // TODO: How to avoid killing runtimes managed by non-test runs?? var runtimes = Process.GetProcessesByName("ya-runtime-ai").ToList(); @@ -283,10 +287,6 @@ public async Task ProviderBreaksAgreement_KillingExeUnit() await AwaitValue(jobStatusChannel, JobStatus.Idle, TimeSpan.FromSeconds(15)); - _logger.LogInformation("=================== Killing App ==================="); - // Killing app, so it won't terminate Agreement. - await app.Stop(StopMethod.SigKill); - // Task should be Interrupted after Provider won't get new Activity from Requestor. await AwaitValue(jobStatusChannel, JobStatus.Interrupted, TimeSpan.FromSeconds(100)); Assert.Equal(JobStatus.Interrupted, currentJob.Status); From c0ddbb95462ca7eb445817f237b2ccc282fe6c35 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jul 2024 13:35:05 +0200 Subject: [PATCH 08/11] Update Golem version --- Golem.Package/Args.cs | 2 +- Golem.Tools/GolemPackageBuilder.cs | 2 +- MockGUI/readme.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Golem.Package/Args.cs b/Golem.Package/Args.cs index 77970637..6c97f5d1 100644 --- a/Golem.Package/Args.cs +++ b/Golem.Package/Args.cs @@ -5,7 +5,7 @@ public class BuildArgs { [Option('t', "target", Default = "package", Required = false, HelpText = "Directory where binaries will be generated relative to working dir")] public required string Target { get; set; } - [Option('y', "yagna-version", Default = "pre-rel-v0.16.0-preview.ai.22", Required = false, HelpText = "Yagna version github tag")] + [Option('y', "yagna-version", Default = "pre-rel-v0.16.0-preview.ai.24", Required = false, HelpText = "Yagna version github tag")] public required string GolemVersion { get; set; } [Option('r', "runtime-version", Default = "v0.2.3", Required = false, HelpText = "Runtime version github tag")] public required string RuntimeVersion { get; set; } diff --git a/Golem.Tools/GolemPackageBuilder.cs b/Golem.Tools/GolemPackageBuilder.cs index 856d9099..bdda3502 100644 --- a/Golem.Tools/GolemPackageBuilder.cs +++ b/Golem.Tools/GolemPackageBuilder.cs @@ -18,7 +18,7 @@ namespace Golem.Tools { public class PackageBuilder { - public static string CURRENT_GOLEM_VERSION = "pre-rel-v0.16.0-preview.ai.22"; + public static string CURRENT_GOLEM_VERSION = "pre-rel-v0.16.0-preview.ai.24"; public static string CURRENT_RUNTIME_VERSION = "v0.2.3"; internal static string InitTestDirectory(string name, bool cleanupData = true) diff --git a/MockGUI/readme.md b/MockGUI/readme.md index ae3c3c1a..f6cdf60d 100644 --- a/MockGUI/readme.md +++ b/MockGUI/readme.md @@ -39,7 +39,7 @@ dotnet run --project Golem.Package -- download --target modules --version pre-re In case of building artifacts locally you can specify `yagna` and `runtimes` versions: ```sh -dotnet run --project Golem.Package -- build --target modules --yagna-version pre-rel-v0.16.0-preview.ai.22 --runtime-version v0.2.3 +dotnet run --project Golem.Package -- build --target modules --yagna-version pre-rel-v0.16.0-preview.ai.24 --runtime-version v0.2.3 ``` ## Running From 1d1ad24adc82a65f704c27a385ed49e1958b27db Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jul 2024 13:42:53 +0200 Subject: [PATCH 09/11] Kill process tree in tests to avoid leaving ya-runtime-ai running --- Golem.Tests/JobStatusTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Golem.Tests/JobStatusTests.cs b/Golem.Tests/JobStatusTests.cs index c3984b96..5d94589e 100644 --- a/Golem.Tests/JobStatusTests.cs +++ b/Golem.Tests/JobStatusTests.cs @@ -192,7 +192,7 @@ public async Task ProviderBreaksAgreement_KillingAgent() _logger.LogInformation("=================== Killing Provider Agent ==================="); var pid = golem.GetProviderPid(); if (pid.HasValue) - Process.GetProcessById(pid.Value).Kill(); + Process.GetProcessById(pid.Value).Kill(true); await AwaitValue(jobStatusChannel, JobStatus.Interrupted, TimeSpan.FromSeconds(30)); Assert.Equal(JobStatus.Interrupted, currentJob.Status); @@ -235,7 +235,7 @@ public async Task ProviderBreaksAgreement_KillingYagna() _logger.LogInformation("=================== Killing Provider Yagna ==================="); var pid = golem.GetYagnaPid(); if (pid.HasValue) - Process.GetProcessById(pid.Value).Kill(); + Process.GetProcessById(pid.Value).Kill(true); await AwaitValue(jobChannel, null, TimeSpan.FromSeconds(30)); Assert.Null(golem.CurrentJob); @@ -381,7 +381,7 @@ public async Task ProviderRestart_RequestorBreaksAgreement() var golemStatusChannel = StatusChannel(golem); var pid = golem.GetProviderPid(); if (pid.HasValue) - Process.GetProcessById(pid.Value).Kill(); + Process.GetProcessById(pid.Value).Kill(true); _logger.LogInformation("=================== Killing App ==================="); // Note: We need to manually send termination from Requestor with correct Reason. @@ -451,7 +451,7 @@ public async Task ProviderRestart_ProviderBreaksAgreement() var golemStatusChannel = StatusChannel(golem); var pid = golem.GetProviderPid(); if (pid.HasValue) - Process.GetProcessById(pid.Value).Kill(); + Process.GetProcessById(pid.Value).Kill(true); _logger.LogInformation("=================== Killing App ==================="); // Note: Provider needs to send termination with correct Reason. From 78e8cbab7b24f49e4029d9121503692c0051adae Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jul 2024 14:09:12 +0200 Subject: [PATCH 10/11] Inverse killing order in tests to prevent yapapi from terminating --- Golem.Tests/JobStatusTests.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Golem.Tests/JobStatusTests.cs b/Golem.Tests/JobStatusTests.cs index 5d94589e..61dcb053 100644 --- a/Golem.Tests/JobStatusTests.cs +++ b/Golem.Tests/JobStatusTests.cs @@ -374,6 +374,12 @@ public async Task ProviderRestart_RequestorBreaksAgreement() // Let him compute for a while. await Task.Delay(TimeSpan.FromSeconds(2)); + _logger.LogInformation("=================== Killing App ==================="); + // Note: We need to manually send termination from Requestor with correct Reason. + // It's better to avoid yapapi finding out that Provider stopped working, because it + // could take action otherwise. + await app.Stop(StopMethod.SigKill); + _logger.LogInformation("=================== Killing Provider Agent ==================="); // Provider is killed so he is not able to terminate Agreement. // Yagna has chance to be closed gracefully and close net connection with Requestor. @@ -383,12 +389,6 @@ public async Task ProviderRestart_RequestorBreaksAgreement() if (pid.HasValue) Process.GetProcessById(pid.Value).Kill(true); - _logger.LogInformation("=================== Killing App ==================="); - // Note: We need to manually send termination from Requestor with correct Reason. - // It's better to avoid yapapi finding out that Provider stopped working, because it - // could take action otherwise. - await app.Stop(StopMethod.SigKill); - await AwaitValue(jobStatusChannel, JobStatus.Interrupted, TimeSpan.FromSeconds(30)); Assert.Equal(JobStatus.Interrupted, currentJob.Status); await AwaitValue(jobChannel, null, TimeSpan.FromSeconds(1)); From 53d1c0b5e9c07beccba2d32b5290508c9f24bd8c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jul 2024 15:53:43 +0200 Subject: [PATCH 11/11] Review improvments --- Golem.Tests/JobStatusTests.cs | 6 +++--- Golem.Tools/GolemRequestor.cs | 13 ++++++------- Golem.Tools/SampleApp.cs | 2 +- Golem/Job.cs | 2 ++ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Golem.Tests/JobStatusTests.cs b/Golem.Tests/JobStatusTests.cs index 61dcb053..1efe31e1 100644 --- a/Golem.Tests/JobStatusTests.cs +++ b/Golem.Tests/JobStatusTests.cs @@ -480,9 +480,9 @@ public async Task ProviderRestart_ProviderBreaksAgreement() Assert.Equal(JobStatus.Interrupted, prevJob.Status); _logger.LogInformation("=================== Requestor terminates Agreement ==================="); - // Try to terminate Agreement with Reason which in normal situation would be treated as correct termination - // resulting in Finished status. If Provider didn't terminate Agreement with Reason leading to Interrupted - // state, then JobStatus would be restored to Finished and it would reveal error. + // Provider should have already noticed that Agreement was terminated and changed the status to Interrupted. + // Let's try to terminate Agreement from Requestor with Reason resulting in Finished status. + // If Provider didn't work correctly, status will be restored to Finished. var rest = _requestor.Rest ?? throw new Exception("Rest api on Requestor not initialized."); var reason = new Reason(null!, "Agreement is no longer needed"); reason.RequestorCode = "NoLongerNeeded"; diff --git a/Golem.Tools/GolemRequestor.cs b/Golem.Tools/GolemRequestor.cs index f65f4b0a..fee523e8 100644 --- a/Golem.Tools/GolemRequestor.cs +++ b/Golem.Tools/GolemRequestor.cs @@ -49,13 +49,12 @@ private GolemRequestor(string dir, bool mainnet, ILogger logger) : base(dir, log _mainnet = mainnet; _dataDir = Path.GetFullPath(Path.Combine(dir, "modules", "golem-data", "yagna")); - var envBuilder = new EnvironmentBuilder(); - envBuilder.WithYagnaDataDir(_dataDir); - envBuilder.WithYagnaApiUrl(ApiUrl); - envBuilder.WithGsbUrl(GsbUrl); - envBuilder.WithYaNetBindUrl(NetBindUrl); - envBuilder.WithMetricsGroup("Example-GamerHash"); - _env = envBuilder; + _env = new EnvironmentBuilder(); + _env.WithYagnaDataDir(_dataDir); + _env.WithYagnaApiUrl(ApiUrl); + _env.WithGsbUrl(GsbUrl); + _env.WithYaNetBindUrl(NetBindUrl); + _env.WithMetricsGroup("Example-GamerHash"); SetSecret(_mainnet ? "main_key.plain" : "test_key.plain"); SetAppKey(GenerateRandomAppkey()); diff --git a/Golem.Tools/SampleApp.cs b/Golem.Tools/SampleApp.cs index 1eda07a8..3994793a 100644 --- a/Golem.Tools/SampleApp.cs +++ b/Golem.Tools/SampleApp.cs @@ -122,7 +122,7 @@ public async Task Run() Message = "Payment Initialization"; _logger.LogInformation("Initializing payment accounts for: " + Name); - await Task.Run(async () => await Requestor.InitPayment()); + await Requestor.InitPayment(); _logger.LogInformation("Creating requestor application: " + Name); Message = "Starting Application"; diff --git a/Golem/Job.cs b/Golem/Job.cs index 3e2ede5e..0e37855b 100644 --- a/Golem/Job.cs +++ b/Golem/Job.cs @@ -199,6 +199,8 @@ public static JobStatus ResolveTerminationReason(string? code) "ConnectionTimedOut" => JobStatus.Interrupted, "ProviderUnreachable" => JobStatus.Interrupted, "Expired" => JobStatus.Finished, + "NotSpecified" => JobStatus.Finished, + "NoLongerNeeded" => JobStatus.Finished, "Cancelled" => JobStatus.Finished, _ => JobStatus.Finished, };