Merge pull request #174 from golemfactory/handle-new-termination-reasons

Fix JobStatus backward change from Interrupted to Finished
golemfactory · Jul 16, 2024 · 7c4e7e4 · 7c4e7e4
2 parents 15dab04 + 53d1c0b
commit 7c4e7e4
Show file tree

Hide file tree

Showing 14 changed files with 342 additions and 91 deletions.
diff --git a/Golem.Package/Args.cs b/Golem.Package/Args.cs
@@ -5,7 +5,7 @@ public class BuildArgs
 {
     [Option('t', "target", Default = "package", Required = false, HelpText = "Directory where binaries will be generated relative to working dir")]
     public required string Target { get; set; }
-    [Option('y', "yagna-version", Default = "pre-rel-v0.16.0-preview.ai.22", Required = false, HelpText = "Yagna version github tag")]
+    [Option('y', "yagna-version", Default = "pre-rel-v0.16.0-preview.ai.24", Required = false, HelpText = "Yagna version github tag")]
     public required string GolemVersion { get; set; }
     [Option('r', "runtime-version", Default = "v0.2.3", Required = false, HelpText = "Runtime version github tag")]
     public required string RuntimeVersion { get; set; }

diff --git a/Golem.Tests/ErrorHandlingTest.cs b/Golem.Tests/ErrorHandlingTest.cs
@@ -29,7 +29,7 @@ public async Task StartTriggerErrorStop_VerifyStatusAsync()
             string golemPath = await PackageBuilder.BuildTestDirectory();
             _logger.LogInformation("Path: " + golemPath);
 
-            await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local);
+            await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral);
 
             var statusChannel = StatusChannel(golem);
 

diff --git a/Golem.Tests/JobStatusTests.cs b/Golem.Tests/JobStatusTests.cs
diff --git a/Golem.Tests/JobTests.cs b/Golem.Tests/JobTests.cs
@@ -27,7 +27,7 @@ public async Task CompleteScenario()
 
             string golemPath = await PackageBuilder.BuildTestDirectory();
             _logger.LogInformation($"Path: {golemPath}");
-            await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.Local);
+            await using var golem = (Golem)await TestUtils.Golem(golemPath, _loggerFactory, null, RelayType.LocalCentral);
 
             var golemStatusChannel = StatusChannel(golem);
 

diff --git a/Golem.Tests/Utils.cs b/Golem.Tests/Utils.cs
@@ -212,7 +212,7 @@ public class JobsTestBase : WithAvailablePort, IDisposable, IAsyncLifetime, ICla
     {
         protected readonly ILoggerFactory _loggerFactory;
         protected readonly ILogger _logger;
-        protected GolemRelay? _relay;
+        protected GolemCentralNet? _router;
         protected GolemRequestor? _requestor;
         protected AppKey? _requestorAppKey;
         protected String _testClassName;
@@ -240,15 +240,15 @@ public JobsTestBase(ITestOutputHelper outputHelper, GolemFixture golemFixture, s
 
         public async Task InitializeAsync()
         {
-            var testDir = PackageBuilder.TestDir($"{_testClassName}_relay");
-            _relay = await GolemRelay.Build(testDir, _loggerFactory.CreateLogger("Relay"));
-            Assert.True(_relay.Start());
-            NetConfig.SetEnv(RelayType.Local);
+            var testDir = PackageBuilder.TestDir($"{_testClassName}_router");
+            _router = await GolemCentralNet.Build(testDir, _loggerFactory.CreateLogger("Router"));
+            Assert.True(_router.Start());
+            NetConfig.SetEnv(RelayType.LocalCentral);
             System.Environment.SetEnvironmentVariable("RUST_LOG", "debug");
 
             _requestor = await GolemRequestor.Build(_testClassName, _loggerFactory.CreateLogger("Requestor"));
             Assert.True(_requestor.Start());
-            _requestor.InitPayment();
+            await _requestor.InitPayment();
             _requestorAppKey = _requestor.GetTestAppKey();
         }
 
@@ -319,8 +319,8 @@ public async Task DisposeAsync()
             if (_requestor != null)
                 await _requestor.Stop(StopMethod.SigInt);
 
-            if (_relay != null)
-                await _relay.Stop(StopMethod.SigInt);
+            if (_router != null)
+                await _router.Stop(StopMethod.SigInt);
         }
 
         public void Dispose()

diff --git a/Golem.Tools/GolemCentralNet.cs b/Golem.Tools/GolemCentralNet.cs
@@ -0,0 +1,41 @@
+using Microsoft.Extensions.Logging;
+
+namespace Golem.Tools
+{
+    public class GolemCentralNet : GolemRunnable
+    {
+        const string CURRENT_ROUTER_VERSION = "v0.7.2";
+
+        private GolemCentralNet(string dir, ILogger logger) : base(dir, logger)
+        {
+        }
+
+        public async static Task<GolemCentralNet> Build(string testDir, ILogger logger)
+        {
+            var dir = await BuildCentralNetDir(testDir);
+            return new GolemCentralNet(dir, logger);
+        }
+
+        public override bool Start()
+        {
+            var working_dir = Path.Combine(_dir, "modules", "golem-data", "central-net");
+            Directory.CreateDirectory(working_dir);
+            return StartProcess("ya-sb-router", working_dir, "-l tcp://127.0.0.1:6464", new Dictionary<string, string>());
+        }
+
+        protected static async Task<string> BuildCentralNetDir(string test_dir)
+        {
+            var dir = PackageBuilder.PrepareTestDirectory(test_dir, true);
+            var binaries_dir = PackageBuilder.BinariesDir(dir);
+
+            Directory.CreateDirectory(binaries_dir);
+
+            var artifact = "ya-sb-router";
+            var repo = "golemfactory/ya-service-bus";
+            var tag = CURRENT_ROUTER_VERSION;
+
+            await PackageBuilder.DownloadExtractPackage(binaries_dir, artifact, repo, tag);
+            return dir;
+        }
+    }
+}
diff --git a/Golem.Tools/GolemPackageBuilder.cs b/Golem.Tools/GolemPackageBuilder.cs
@@ -18,7 +18,7 @@ namespace Golem.Tools
 {
     public class PackageBuilder
     {
-        public static string CURRENT_GOLEM_VERSION = "pre-rel-v0.16.0-preview.ai.22";
+        public static string CURRENT_GOLEM_VERSION = "pre-rel-v0.16.0-preview.ai.24";
         public static string CURRENT_RUNTIME_VERSION = "v0.2.3";
 
         internal static string InitTestDirectory(string name, bool cleanupData = true)

diff --git a/Golem.Tools/GolemRequestor.cs b/Golem.Tools/GolemRequestor.cs
@@ -26,9 +26,9 @@ public class AppKey
 
     public class GolemRequestor : GolemRunnable, IAsyncLifetime
     {
-        private Dictionary<string, string> _env;
+        private readonly EnvironmentBuilder _env;
 
-        public string? AppKey;
+        public string AppKey;
         public string ApiUrl { get; set; }
         public string GsbUrl { get; set; }
         public string NetBindUrl { get; set; }
@@ -40,20 +40,24 @@ public class GolemRequestor : GolemRunnable, IAsyncLifetime
 
         private GolemRequestor(string dir, bool mainnet, ILogger logger) : base(dir, logger)
         {
+            AppKey = "";    // Will be set in `SetAppKey` function. This line supresses warning.
+
             ApiUrl = "http://127.0.0.1:7465";
             GsbUrl = "tcp://127.0.0.1:7464";
             NetBindUrl = "udp://0.0.0.0:11500";
 
             _mainnet = mainnet;
             _dataDir = Path.GetFullPath(Path.Combine(dir, "modules", "golem-data", "yagna"));
 
-            var envBuilder = new EnvironmentBuilder();
-            envBuilder.WithYagnaDataDir(_dataDir);
-            envBuilder.WithYagnaApiUrl(ApiUrl);
-            envBuilder.WithGsbUrl(GsbUrl);
-            envBuilder.WithYaNetBindUrl(NetBindUrl);
-            envBuilder.WithMetricsGroup("Example-GamerHash");
-            _env = envBuilder.Build();
+            _env = new EnvironmentBuilder();
+            _env.WithYagnaDataDir(_dataDir);
+            _env.WithYagnaApiUrl(ApiUrl);
+            _env.WithGsbUrl(GsbUrl);
+            _env.WithYaNetBindUrl(NetBindUrl);
+            _env.WithMetricsGroup("Example-GamerHash");
+
+            SetSecret(_mainnet ? "main_key.plain" : "test_key.plain");
+            SetAppKey(GenerateRandomAppkey());
         }
 
         public async static Task<GolemRequestor> Build(string test_name, ILogger logger, bool cleanupData = true, bool mainnet = false)
@@ -70,14 +74,10 @@ public async static Task<GolemRequestor> BuildRelative(string datadir, ILogger l
 
         public override bool Start()
         {
-            BuildEnv();
-
             var working_dir = Path.Combine(_dir, "modules", "golem-data", "yagna");
             Directory.CreateDirectory(working_dir);
-            AppKey = generateRandomAppkey();
-            var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value);
-            env["YAGNA_AUTOCONF_ID_SECRET"] = getRequestorAutoconfIdSecret();
-            env["YAGNA_AUTOCONF_APPKEY"] = AppKey;
+
+            var env = _env.Build();
             var result = StartProcess("yagna", working_dir, "service run", env, false);
 
             Rest = CreateRestAPI(AppKey);
@@ -93,47 +93,49 @@ public void AutoSetUrls(UInt16 portBase)
             ApiUrl = $"http://127.0.0.1:{apiPort}";
             GsbUrl = $"tcp://127.0.0.1:{gsbPort}";
             NetBindUrl = $"udp://0.0.0.0:{bindPort}";
+
+            _env.WithYagnaApiUrl(ApiUrl);
+            _env.WithGsbUrl(GsbUrl);
+            _env.WithYaNetBindUrl(NetBindUrl);
         }
 
-        private void BuildEnv()
+        public void SetSecret(string resourceFile)
         {
-            var envBuilder = new EnvironmentBuilder();
-            envBuilder.WithYagnaDataDir(_dataDir);
-            envBuilder.WithYagnaApiUrl(ApiUrl);
-            envBuilder.WithGsbUrl(GsbUrl);
-            envBuilder.WithYaNetBindUrl(NetBindUrl);
-            envBuilder.WithMetricsGroup("Example-GamerHash");
-            _env = envBuilder.Build();
+            _env.WithPrivateKey(LoadSecret(resourceFile));
         }
 
-        private string getRequestorAutoconfIdSecret()
+        private static string LoadSecret(string resourceFile)
         {
-            string? key = null;
-            if ((key = (string?)Environment.GetEnvironmentVariable("REQUESTOR_AUTOCONF_ID_SECRET")) != null)
-            {
-                return key;
-            }
-            var keyFilename = _mainnet ? "main_key.plain" : "test_key.plain";
-            var keyReader = PackageBuilder.ReadResource(keyFilename);
-            return keyReader.ReadLine() ?? throw new Exception($"Failed to read key from file {keyFilename}");
+            var keyReader = PackageBuilder.ReadResource(resourceFile);
+            return keyReader.ReadLine() ?? throw new Exception($"Failed to read key from file {resourceFile}");
         }
 
-        private string generateRandomAppkey()
+        private static string GenerateRandomAppkey()
         {
             string? appKey = null;
             if ((appKey = (string?)Environment.GetEnvironmentVariable("REQUESTOR_AUTOCONF_APPKEY")) != null)
             {
                 return appKey;
             }
-            byte[] data = RandomNumberGenerator.GetBytes(20);
-            return Convert.ToBase64String(data);
+            else
+            {
+                byte[] data = RandomNumberGenerator.GetBytes(20);
+                return Convert.ToBase64String(data);
+            }
+
+        }
+
+        private void SetAppKey(string appKey)
+        {
+            AppKey = appKey;
+            _env.WithAppKey(appKey);
+            _env.WithYagnaAppKey(appKey);
         }
 
         public SampleApp CreateSampleApp(string? extraArgs = null)
         {
-            var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value);
-            env["YAGNA_APPKEY"] = AppKey ?? throw new Exception("Unable to create app process. No YAGNA_APPKEY.");
-            env["YAGNA_API_URL"] = ApiUrl;
+            var env = _env.Build();
+
             var pathEnvVar = Environment.GetEnvironmentVariable("PATH") ?? "";
             var binariesDir = Path.GetFullPath(PackageBuilder.BinariesDir(_dir));
             if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
@@ -149,11 +151,12 @@ public SampleApp CreateSampleApp(string? extraArgs = null)
             return new SampleApp(_dir, env, network, _logger, extraArgs);
         }
 
-        public void InitPayment(double minFundThreshold = 100.0)
+        public async Task InitPayment(double minFundThreshold = 100.0)
         {
-            Thread.Sleep(6000);
-            var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value);
-            env.Add("RUST_LOG", "none");
+            await WaitForIdentityAsync();
+
+            var env = _env.Build();
+            env["RUST_LOG"] = "none";
 
             var network = Factory.Network(_mainnet);
             var payment_status_process = WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), $"payment status --json --network {network.Id}", env));
@@ -164,16 +167,42 @@ public void InitPayment(double minFundThreshold = 100.0)
 
             if (reserved > 0.0)
             {
-                WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), "payment release-allocations", _env));
+                WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), "payment release-allocations", env));
             }
 
             if (totalGlm < minFundThreshold && !_mainnet)
             {
-                WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), $"payment fund --network {network.Id}", _env));
+                WaitAndPrintOnError(RunCommand("yagna", WorkingDir(), $"payment fund --network {network.Id}", env));
             }
             return;
         }
 
+        public async Task<string?> WaitForIdentityAsync(CancellationToken cancellationToken = default)
+        {
+            _logger.LogDebug("Waiting for yagna to start... Checking /me endpoint.");
+
+            //yagna is starting and /me won't work until all services are running
+            for (int tries = 0; tries < 200; ++tries)
+            {
+                Thread.Sleep(300);
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try
+                {
+                    var api = Rest ?? throw new Exception("REST API not initialized");
+                    MeInfo meInfo = await Rest.Me(cancellationToken);
+
+                    _logger.LogDebug("Yagna started; REST API is available.");
+                    return meInfo.Identity;
+                }
+                catch (Exception)
+                {
+                    // consciously swallow the exception... presumably REST call error...
+                }
+            }
+            return null;
+        }
+
         private Command WaitAndPrintOnError(Command cmd)
         {
             try
@@ -197,8 +226,8 @@ public AppKey GetTestAppKey()
                 throw new Exception("No data dir");
             }
 
-            var env = _env.ToDictionary(entry => entry.Key, entry => entry.Value);
-            env.Add("RUST_LOG", "none");
+            var env = _env.Build();
+            env["RUST_LOG"] = "none";
 
             var app_key_list_process = RunCommand("yagna", WorkingDir(), "app-key list --json", env);
             app_key_list_process.Wait();

diff --git a/Golem.Tools/SampleApp.cs b/Golem.Tools/SampleApp.cs
@@ -122,7 +122,7 @@ public async Task Run()
 
                 Message = "Payment Initialization";
                 _logger.LogInformation("Initializing payment accounts for: " + Name);
-                await Task.Run(() => Requestor.InitPayment());
+                await Requestor.InitPayment();
 
                 _logger.LogInformation("Creating requestor application: " + Name);
                 Message = "Starting Application";

diff --git a/Golem.Tools/resources/test_key_2.plain b/Golem.Tools/resources/test_key_2.plain
@@ -0,0 +1 @@
+b0f8813e42e0170c8a6c94e7c34b257df3a7a77322b61dc914f8b48b0d03d166
diff --git a/Golem/Yagna/Job.cs → Golem/Job.cs b/Golem/Yagna/Job.cs → Golem/Job.cs
@@ -195,7 +195,12 @@ public static JobStatus ResolveTerminationReason(string? code)
                 "RequestorUnreachable" => JobStatus.Interrupted,
                 "Shutdown" => JobStatus.Interrupted,
                 "Interrupted" => JobStatus.Interrupted,
+                "HealthCheckFailed" => JobStatus.Interrupted,
+                "ConnectionTimedOut" => JobStatus.Interrupted,
+                "ProviderUnreachable" => JobStatus.Interrupted,
                 "Expired" => JobStatus.Finished,
+                "NotSpecified" => JobStatus.Finished,
+                "NoLongerNeeded" => JobStatus.Finished,
                 "Cancelled" => JobStatus.Finished,
                 _ => JobStatus.Finished,
             };
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		b0f8813e42e0170c8a6c94e7c34b257df3a7a77322b61dc914f8b48b0d03d166