This is the behavior when having a running environment job and Poseidon and restarting the Nomad agents together.

On startup the environment and the runners are being successfully recovered:
time="2024-09-03T15:42:22.360344Z" level=info msg="Successfully recovered environment" jobID=template-8 package=environment
time="2024-09-03T15:42:22.465889Z" level=debug msg="Recovered Allocation" jobID=8-611db37e-69f5-11ef-ba1c-00155db34eb6 package=nomad status=running
time="2024-09-03T15:42:22.465965Z" level=debug msg="Recovered Allocation" jobID=8-0c672fe9-69f7-11ef-b5c6-00155db34eb6 package=nomad status=running
time="2024-09-03T15:42:22.466016Z" level=debug msg="Recovered Allocation" jobID=8-81e9646b-69f9-11ef-9835-00155db34eb6 package=nomad status=running
time="2024-09-03T15:42:22.466065Z" level=debug msg="Recovered Allocation" jobID=8-f10f73e7-69f8-11ef-bb47-00155db34eb6 package=nomad status=running
time="2024-09-03T15:42:22.466113Z" level=debug msg="Recovered Allocation" jobID=8-fe89989b-69f7-11ef-b5a7-00155db34eb6 package=nomad status=running
time="2024-09-03T15:42:22.601737Z" level=debug msg="Recovered Runner" isUsed=false package=runner runner_id=8-0c672fe9-69f7-11ef-b5c6-00155db34eb6
time="2024-09-03T15:42:22.640044Z" level=debug msg="Recovered Runner" isUsed=false package=runner runner_id=8-611db37e-69f5-11ef-ba1c-00155db34eb6
time="2024-09-03T15:42:22.677256Z" level=debug msg="Recovered Runner" isUsed=false package=runner runner_id=8-81e9646b-69f9-11ef-9835-00155db34eb6
time="2024-09-03T15:42:22.728976Z" level=debug msg="Recovered Runner" isUsed=false package=runner runner_id=8-f10f73e7-69f8-11ef-bb47-00155db34eb6
time="2024-09-03T15:42:22.768510Z" level=debug msg="Recovered Runner" isUsed=false package=runner runner_id=8-fe89989b-69f7-11ef-b5a7-00155db34eb6
time="2024-09-03T15:42:22.768606Z" level=info msg="Too many idle runner" environment_id=8 error="cannot scale down the environment" idle runners=5 package=environment pool size=1

When restarting the three Nomad agents:
{"Topic":"Allocation","Type":"PlanResult","Key":"16244930-acea-da7e-e410-ef583887ef86","FilterKeys":["8-0c672fe9-69f7-11ef-b5c6-00155db34eb6",""],"Index":323075,"Payload":{"Allocation":{"AllocModifyIndex":323075,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323017,"CreateTime":1725370832218942000,"DesiredDescription":"alloc not needed due to job update","DesiredStatus":"stop","EvalID":"d619fee4-a287-9f50-62e5-5977294c6851","ID":"16244930-acea-da7e-e410-ef583887ef86","JobID":"8-0c672fe9-69f7-11ef-b5c6-00155db34eb6","Metrics":{"AllocationTime":211934,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":3},"NodesEvaluated":3,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":3,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":0.9070296312712901,"Scores":{"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":-0.5752027071266841,"Scores":{"binpack":0.8495945857466318,"job-anti-affinity":-2,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323075,"ModifyTime":1725370967617845200,"Name":"8-0c672fe9-69f7-11ef-b5c6-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832244809500,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832257085400,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832628741600,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834453868000,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.453913016Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"PlanResult","Key":"39a19ef7-6980-35dd-5946-d7ccd892c840","FilterKeys":["template-8",""],"Index":323078,"Payload":{"Allocation":{"AllocModifyIndex":323078,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323019,"CreateTime":1725370832229150200,"DesiredDescription":"alloc not needed due to job update","DesiredStatus":"stop","EvalID":"28c4f16f-69da-00fd-b82e-010e5414b101","ID":"39a19ef7-6980-35dd-5946-d7ccd892c840","JobID":"template-8","Metrics":{"AllocationTime":200954,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":3},"NodesEvaluated":3,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":3,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.546485184364355,"Scores":{"binpack":0.9070296312712901,"job-anti-affinity":-2,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323078,"ModifyTime":1725370967625688300,"Name":"template-8.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NodeName":"nomad-agent-terraform-3","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832267342300,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832291844600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832452055000,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833721824000,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:33.721859474Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"PlanResult","Key":"f07a195e-f0c9-44fa-bbc7-9b80a73f7498","FilterKeys":["8-611db37e-69f5-11ef-ba1c-00155db34eb6",""],"Index":323081,"Payload":{"Allocation":{"AllocModifyIndex":323081,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323024,"CreateTime":1725370832238708000,"DesiredDescription":"alloc not needed due to job update","DesiredStatus":"stop","EvalID":"62c4ec31-946d-1805-44aa-d5e5f6fbb8b2","ID":"f07a195e-f0c9-44fa-bbc7-9b80a73f7498","JobID":"8-611db37e-69f5-11ef-ba1c-00155db34eb6","Metrics":{"AllocationTime":267761,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":3},"NodesEvaluated":3,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":3,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":0.9070296312712901,"Scores":{"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":-0.5821140876671933,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":-2,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323081,"ModifyTime":1725370967642174000,"Name":"8-611db37e-69f5-11ef-ba1c-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832265937400,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832310252800,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832602787800,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834390512600,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.39054909Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"39a19ef7-6980-35dd-5946-d7ccd892c840","FilterKeys":["template-8",""],"Index":323112,"Payload":{"Allocation":{"AllocModifyIndex":323078,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323019,"CreateTime":1725370832229150200,"DesiredDescription":"alloc not needed due to job update","DesiredStatus":"stop","EvalID":"28c4f16f-69da-00fd-b82e-010e5414b101","ID":"39a19ef7-6980-35dd-5946-d7ccd892c840","JobID":"template-8","Metrics":{"AllocationTime":200954,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":3},"NodesEvaluated":3,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":3,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.546485184364355,"Scores":{"binpack":0.9070296312712901,"job-anti-affinity":-2,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323112,"ModifyTime":1725370968941773600,"Name":"template-8.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NodeName":"nomad-agent-terraform-3","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832267342300,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832291844600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832452055000,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833721824000,"Type":"Started","ValidationError":"","VaultError":""},{"Details":{"kill_timeout":"5s"},"DiskLimit":0,"DisplayMessage":"Sent interrupt. Waiting 5s before force killing","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":5000000000,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370967992327400,"Type":"Killing","ValidationError":"","VaultError":""},{"Details":{"exit_code":"137","exit_message":"Docker container exited with non-zero exit code: 137","oom_killed":"false","signal":"0"},"DiskLimit":0,"DisplayMessage":"Exit Code: 137, Exit Message: \"Docker container exited with non-zero exit code: 137\"","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":137,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Docker container exited with non-zero exit code: 137","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370968135800000,"Type":"Terminated","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:33.721859474Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"39a19ef7-6980-35dd-5946-d7ccd892c840","FilterKeys":["template-8",""],"Index":323113,"Payload":{"Allocation":{"AllocModifyIndex":323078,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"All tasks have completed","ClientStatus":"complete","CreateIndex":323019,"CreateTime":1725370832229150200,"DesiredDescription":"alloc not needed due to job update","DesiredStatus":"stop","EvalID":"28c4f16f-69da-00fd-b82e-010e5414b101","ID":"39a19ef7-6980-35dd-5946-d7ccd892c840","JobID":"template-8","Metrics":{"AllocationTime":200954,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":3},"NodesEvaluated":3,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":3,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.546485184364355,"Scores":{"binpack":0.9070296312712901,"job-anti-affinity":-2,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323113,"ModifyTime":1725370969204545800,"Name":"template-8.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NodeName":"nomad-agent-terraform-3","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832267342300,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832291844600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832452055000,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833721824000,"Type":"Started","ValidationError":"","VaultError":""},{"Details":{"kill_timeout":"5s"},"DiskLimit":0,"DisplayMessage":"Sent interrupt. Waiting 5s before force killing","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":5000000000,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370967992327400,"Type":"Killing","ValidationError":"","VaultError":""},{"Details":{"exit_code":"137","exit_message":"Docker container exited with non-zero exit code: 137","oom_killed":"false","signal":"0"},"DiskLimit":0,"DisplayMessage":"Exit Code: 137, Exit Message: \"Docker container exited with non-zero exit code: 137\"","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":137,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Docker container exited with non-zero exit code: 137","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370968135800000,"Type":"Terminated","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":"2024-09-03T13:42:48.149806891Z","LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:33.721859474Z","State":"dead","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"PlanResult","Key":"04ea14a0-a621-e30c-7523-fc470dfe9811","FilterKeys":["8-f10f73e7-69f8-11ef-bb47-00155db34eb6",""],"Index":323135,"Payload":{"Allocation":{"AllocModifyIndex":323135,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":322995,"CreateTime":1725370832123828700,"DesiredDescription":"alloc is being migrated","DesiredStatus":"stop","DesiredTransition":{"ForceReschedule":null,"Migrate":true,"NoShutdownDelay":null,"Reschedule":null},"EvalID":"f36524a5-51f3-181f-a395-22e74081990e","ID":"04ea14a0-a621-e30c-7523-fc470dfe9811","JobID":"8-f10f73e7-69f8-11ef-bb47-00155db34eb6","Metrics":{"AllocationTime":123425,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.03902921501197415,"Scores":{"allocation-spread":-1,"binpack":0.9219415699760517,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323135,"ModifyTime":1725370977608316200,"Name":"8-f10f73e7-69f8-11ef-bb47-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NodeName":"nomad-agent-terraform-1","PreviousAllocation":"10059ca1-4596-2aaf-5c21-01e396cdf3ba","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832199118600,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833347655400,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833476218400,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834965139200,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.965191649Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"PlanResult","Key":"d65b32d6-8609-e2c2-b15e-77e9047a139d","FilterKeys":["8-fe89989b-69f7-11ef-b5a7-00155db34eb6",""],"Index":323146,"Payload":{"Allocation":{"AllocModifyIndex":323146,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientStatus":"pending","CreateIndex":323146,"CreateTime":1725370977641349600,"DesiredStatus":"run","EvalID":"4bdc81f6-f027-19a3-4a57-9de80cef0e1e","ID":"d65b32d6-8609-e2c2-b15e-77e9047a139d","JobID":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6","Metrics":{"AllocationTime":105993,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.023772947766184382,"Scores":{"allocation-spread":-1,"binpack":0.9524541044676312,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323146,"ModifyTime":1725370977641349600,"Name":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6.default-group[0]","Namespace":"default","NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","PreviousAllocation":"dedd154d-90b8-58aa-fd65-40d4c1775a72","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}}}}}
{"Topic":"Allocation","Type":"PlanResult","Key":"dedd154d-90b8-58aa-fd65-40d4c1775a72","FilterKeys":["8-fe89989b-69f7-11ef-b5a7-00155db34eb6",""],"Index":323146,"Payload":{"Allocation":{"AllocModifyIndex":323146,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":322989,"CreateTime":1725370832094990600,"DesiredDescription":"alloc is being migrated","DesiredStatus":"stop","DesiredTransition":{"ForceReschedule":null,"Migrate":true,"NoShutdownDelay":null,"Reschedule":null},"EvalID":"35a0bc5b-6283-6691-eac5-2c3e1599ee00","ID":"dedd154d-90b8-58aa-fd65-40d4c1775a72","JobID":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6","Metrics":{"AllocationTime":118295,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.03902921501197415,"Scores":{"allocation-spread":-1,"binpack":0.9219415699760517,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323146,"ModifyTime":1725370977641349600,"Name":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NextAllocation":"d65b32d6-8609-e2c2-b15e-77e9047a139d","NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NodeName":"nomad-agent-terraform-1","PreviousAllocation":"1bfb623b-f8d0-4847-8267-7a0de6cf6759","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832131636700,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833371706600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833542353400,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834705649700,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.705716436Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"PlanResult","Key":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","FilterKeys":["8-81e9646b-69f9-11ef-9835-00155db34eb6",""],"Index":323160,"Payload":{"Allocation":{"AllocModifyIndex":323160,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientStatus":"pending","CreateIndex":323160,"CreateTime":1725370977689382000,"DesiredStatus":"run","EvalID":"e497a0dd-e2c6-c121-19c7-82161dffb58b","ID":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","JobID":"8-81e9646b-69f9-11ef-9835-00155db34eb6","Metrics":{"AllocationTime":130719,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":2},"NodesEvaluated":2,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":2,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.01596898343948694,"Scores":{"allocation-spread":-1,"binpack":0.9680620331210261,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.04648518436435495,"Scores":{"allocation-spread":-1,"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323160,"ModifyTime":1725370977689382000,"Name":"8-81e9646b-69f9-11ef-9835-00155db34eb6.default-group[0]","Namespace":"default","NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","PreviousAllocation":"443aa8f2-9c1a-d388-24ae-be6c0eced8f8","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}}}}}
{"Topic":"Allocation","Type":"PlanResult","Key":"443aa8f2-9c1a-d388-24ae-be6c0eced8f8","FilterKeys":["8-81e9646b-69f9-11ef-9835-00155db34eb6",""],"Index":323160,"Payload":{"Allocation":{"AllocModifyIndex":323160,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323002,"CreateTime":1725370832155809800,"DesiredDescription":"alloc is being migrated","DesiredStatus":"stop","DesiredTransition":{"ForceReschedule":null,"Migrate":true,"NoShutdownDelay":null,"Reschedule":null},"EvalID":"8ef7f0f0-e170-5b8c-968e-a0341cb7a130","ID":"443aa8f2-9c1a-d388-24ae-be6c0eced8f8","JobID":"8-81e9646b-69f9-11ef-9835-00155db34eb6","Metrics":{"AllocationTime":207976,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":2},"NodesEvaluated":2,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":2,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":-0.04648518436435495,"Scores":{"allocation-spread":-1,"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.0610617393130608,"Scores":{"allocation-spread":-1,"binpack":0.8778765213738784,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323160,"ModifyTime":1725370977689382000,"Name":"8-81e9646b-69f9-11ef-9835-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NextAllocation":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NodeName":"nomad-agent-terraform-3","PreviousAllocation":"53826bbd-4e2b-7084-4ee2-9cd627ce44ab","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832183388700,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833404918500,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833528156700,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834384515600,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.385535569Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"dedd154d-90b8-58aa-fd65-40d4c1775a72","FilterKeys":["8-fe89989b-69f7-11ef-b5a7-00155db34eb6",""],"Index":323195,"Payload":{"Allocation":{"AllocModifyIndex":323146,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":322989,"CreateTime":1725370832094990600,"DesiredDescription":"alloc is being migrated","DesiredStatus":"stop","DesiredTransition":{"ForceReschedule":null,"Migrate":true,"NoShutdownDelay":null,"Reschedule":null},"EvalID":"35a0bc5b-6283-6691-eac5-2c3e1599ee00","ID":"dedd154d-90b8-58aa-fd65-40d4c1775a72","JobID":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6","Metrics":{"AllocationTime":118295,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.03902921501197415,"Scores":{"allocation-spread":-1,"binpack":0.9219415699760517,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323195,"ModifyTime":1725370977954797600,"Name":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NextAllocation":"d65b32d6-8609-e2c2-b15e-77e9047a139d","NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NodeName":"nomad-agent-terraform-1","PreviousAllocation":"1bfb623b-f8d0-4847-8267-7a0de6cf6759","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832131636700,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833371706600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833542353400,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834705649700,"Type":"Started","ValidationError":"","VaultError":""},{"Details":{"kill_timeout":"5s"},"DiskLimit":0,"DisplayMessage":"Sent interrupt. Waiting 5s before force killing","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":5000000000,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977676250400,"Type":"Killing","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.705716436Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"443aa8f2-9c1a-d388-24ae-be6c0eced8f8","FilterKeys":["8-81e9646b-69f9-11ef-9835-00155db34eb6",""],"Index":323197,"Payload":{"Allocation":{"AllocModifyIndex":323160,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"All tasks have completed","ClientStatus":"complete","CreateIndex":323002,"CreateTime":1725370832155809800,"DesiredDescription":"alloc is being migrated","DesiredStatus":"stop","DesiredTransition":{"ForceReschedule":null,"Migrate":true,"NoShutdownDelay":null,"Reschedule":null},"EvalID":"8ef7f0f0-e170-5b8c-968e-a0341cb7a130","ID":"443aa8f2-9c1a-d388-24ae-be6c0eced8f8","JobID":"8-81e9646b-69f9-11ef-9835-00155db34eb6","Metrics":{"AllocationTime":207976,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":2},"NodesEvaluated":2,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":2,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":-0.04648518436435495,"Scores":{"allocation-spread":-1,"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.0610617393130608,"Scores":{"allocation-spread":-1,"binpack":0.8778765213738784,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323197,"ModifyTime":1725370978091611000,"Name":"8-81e9646b-69f9-11ef-9835-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NextAllocation":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NodeName":"nomad-agent-terraform-3","PreviousAllocation":"53826bbd-4e2b-7084-4ee2-9cd627ce44ab","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832183388700,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833404918500,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833528156700,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834384515600,"Type":"Started","ValidationError":"","VaultError":""},{"Details":{"kill_timeout":"5s"},"DiskLimit":0,"DisplayMessage":"Sent interrupt. Waiting 5s before force killing","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":5000000000,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977716064800,"Type":"Killing","ValidationError":"","VaultError":""},{"Details":{"exit_code":"137","exit_message":"Docker container exited with non-zero exit code: 137","oom_killed":"false","signal":"0"},"DiskLimit":0,"DisplayMessage":"Exit Code: 137, Exit Message: \"Docker container exited with non-zero exit code: 137\"","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":137,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Docker container exited with non-zero exit code: 137","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977862610400,"Type":"Terminated","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task successfully killed","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977877054700,"Type":"Killed","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":"2024-09-03T13:42:57.878023151Z","LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.385535569Z","State":"dead","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"dedd154d-90b8-58aa-fd65-40d4c1775a72","FilterKeys":["8-fe89989b-69f7-11ef-b5a7-00155db34eb6",""],"Index":323198,"Payload":{"Allocation":{"AllocModifyIndex":323146,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"All tasks have completed","ClientStatus":"complete","CreateIndex":322989,"CreateTime":1725370832094990600,"DesiredDescription":"alloc is being migrated","DesiredStatus":"stop","DesiredTransition":{"ForceReschedule":null,"Migrate":true,"NoShutdownDelay":null,"Reschedule":null},"EvalID":"35a0bc5b-6283-6691-eac5-2c3e1599ee00","ID":"dedd154d-90b8-58aa-fd65-40d4c1775a72","JobID":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6","Metrics":{"AllocationTime":118295,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.03902921501197415,"Scores":{"allocation-spread":-1,"binpack":0.9219415699760517,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323198,"ModifyTime":1725370979058858500,"Name":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NextAllocation":"d65b32d6-8609-e2c2-b15e-77e9047a139d","NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NodeName":"nomad-agent-terraform-1","PreviousAllocation":"1bfb623b-f8d0-4847-8267-7a0de6cf6759","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832131636700,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833371706600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833542353400,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834705649700,"Type":"Started","ValidationError":"","VaultError":""},{"Details":{"kill_timeout":"5s"},"DiskLimit":0,"DisplayMessage":"Sent interrupt. Waiting 5s before force killing","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":5000000000,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977676250400,"Type":"Killing","ValidationError":"","VaultError":""},{"Details":{"exit_code":"137","exit_message":"Docker container exited with non-zero exit code: 137","oom_killed":"false","signal":"0"},"DiskLimit":0,"DisplayMessage":"Exit Code: 137, Exit Message: \"Docker container exited with non-zero exit code: 137\"","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":137,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Docker container exited with non-zero exit code: 137","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370978056081700,"Type":"Terminated","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task successfully killed","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370978115643600,"Type":"Killed","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":"2024-09-03T13:42:58.122643908Z","LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.705716436Z","State":"dead","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","FilterKeys":["8-81e9646b-69f9-11ef-9835-00155db34eb6",""],"Index":323199,"Payload":{"Allocation":{"AllocModifyIndex":323160,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"No tasks have started","ClientStatus":"pending","CreateIndex":323160,"CreateTime":1725370977689382000,"DesiredStatus":"run","EvalID":"e497a0dd-e2c6-c121-19c7-82161dffb58b","ID":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","JobID":"8-81e9646b-69f9-11ef-9835-00155db34eb6","Metrics":{"AllocationTime":130719,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":2},"NodesEvaluated":2,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":2,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.01596898343948694,"Scores":{"allocation-spread":-1,"binpack":0.9680620331210261,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.04648518436435495,"Scores":{"allocation-spread":-1,"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323199,"ModifyTime":1725370979089813200,"Name":"8-81e9646b-69f9-11ef-9835-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","PreviousAllocation":"443aa8f2-9c1a-d388-24ae-be6c0eced8f8","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977721723600,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370978161261600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370978278209300,"Type":"Driver","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":null,"State":"pending","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","FilterKeys":["8-81e9646b-69f9-11ef-9835-00155db34eb6",""],"Index":323201,"Payload":{"Allocation":{"AllocModifyIndex":323160,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323160,"CreateTime":1725370977689382000,"DesiredStatus":"run","EvalID":"e497a0dd-e2c6-c121-19c7-82161dffb58b","ID":"5d39ae65-0baf-5104-f1a5-4c2f55b818fc","JobID":"8-81e9646b-69f9-11ef-9835-00155db34eb6","Metrics":{"AllocationTime":130719,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":2},"NodesEvaluated":2,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":2,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.01596898343948694,"Scores":{"allocation-spread":-1,"binpack":0.9680620331210261,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.04648518436435495,"Scores":{"allocation-spread":-1,"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323201,"ModifyTime":1725370979770055000,"Name":"8-81e9646b-69f9-11ef-9835-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","PreviousAllocation":"443aa8f2-9c1a-d388-24ae-be6c0eced8f8","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977721723600,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370978161261600,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370978278209300,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370979720460300,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:42:59.720495824Z","State":"running","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"d65b32d6-8609-e2c2-b15e-77e9047a139d","FilterKeys":["8-fe89989b-69f7-11ef-b5a7-00155db34eb6",""],"Index":323201,"Payload":{"Allocation":{"AllocModifyIndex":323146,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"No tasks have started","ClientStatus":"pending","CreateIndex":323146,"CreateTime":1725370977641349600,"DesiredStatus":"run","EvalID":"4bdc81f6-f027-19a3-4a57-9de80cef0e1e","ID":"d65b32d6-8609-e2c2-b15e-77e9047a139d","JobID":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6","Metrics":{"AllocationTime":105993,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.023772947766184382,"Scores":{"allocation-spread":-1,"binpack":0.9524541044676312,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323201,"ModifyTime":1725370979770055000,"Name":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","PreviousAllocation":"dedd154d-90b8-58aa-fd65-40d4c1775a72","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977673716500,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370979125201000,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370979322060000,"Type":"Driver","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":null,"State":"pending","TaskHandle":null}}}}}
{"Topic":"Allocation","Type":"AllocationUpdated","Key":"d65b32d6-8609-e2c2-b15e-77e9047a139d","FilterKeys":["8-fe89989b-69f7-11ef-b5a7-00155db34eb6",""],"Index":323203,"Payload":{"Allocation":{"AllocModifyIndex":323146,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323146,"CreateTime":1725370977641349600,"DesiredStatus":"run","EvalID":"4bdc81f6-f027-19a3-4a57-9de80cef0e1e","ID":"d65b32d6-8609-e2c2-b15e-77e9047a139d","JobID":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6","Metrics":{"AllocationTime":105993,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":-0.023772947766184382,"Scores":{"allocation-spread":-1,"binpack":0.9524541044676312,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323203,"ModifyTime":1725370980307002000,"Name":"8-fe89989b-69f7-11ef-b5a7-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","PreviousAllocation":"dedd154d-90b8-58aa-fd65-40d4c1775a72","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370977673716500,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370979125201000,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370979322060000,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370980192157000,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:43:00.192196777Z","State":"running","TaskHandle":null}}}}}

- The environment job `template-8` goes in the complete-but-existing-state
- Two out of five runners survive (`8-81e9646b-69f9-11ef-9835-00155db34eb6`, `8-fe89989b-69f7-11ef-b5a7-00155db34eb6`)
  - `8-81e9646b-69f9-11ef-9835-00155db34eb6` survives by being migrated from agent-3 to agent-2
  - `8-fe89989b-69f7-11ef-b5a7-00155db34eb6` survives by being migrated from agent-1 to agent-2
  - `8-f10f73e7-69f8-11ef-bb47-00155db34eb6` is completely removed with just a single event stating that it is stopping to be migrated.
{"Topic":"Allocation","Type":"PlanResult","Key":"04ea14a0-a621-e30c-7523-fc470dfe9811","FilterKeys":["8-f10f73e7-69f8-11ef-bb47-00155db34eb6",""],"Index":323135,"Payload":{"Allocation":{"AllocModifyIndex":323135,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":322995,"CreateTime":1725370832123828700,"DesiredDescription":"alloc is being migrated","DesiredStatus":"stop","DesiredTransition":{"ForceReschedule":null,"Migrate":true,"NoShutdownDelay":null,"Reschedule":null},"EvalID":"f36524a5-51f3-181f-a395-22e74081990e","ID":"04ea14a0-a621-e30c-7523-fc470dfe9811","JobID":"8-f10f73e7-69f8-11ef-bb47-00155db34eb6","Metrics":{"AllocationTime":123425,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":1},"NodesEvaluated":1,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":1,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":-0.03902921501197415,"Scores":{"allocation-spread":-1,"binpack":0.9219415699760517,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323135,"ModifyTime":1725370977608316200,"Name":"8-f10f73e7-69f8-11ef-bb47-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NodeName":"nomad-agent-terraform-1","PreviousAllocation":"10059ca1-4596-2aaf-5c21-01e396cdf3ba","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832199118600,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833347655400,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370833476218400,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834965139200,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.965191649Z","State":"running","TaskHandle":null}}}}}
  - `8-0c672fe9-69f7-11ef-b5c6-00155db34eb6` is completely removed with just a single event starting that it is stopped because the allocation is no longer needed due to a job update
{"Topic":"Allocation","Type":"PlanResult","Key":"16244930-acea-da7e-e410-ef583887ef86","FilterKeys":["8-0c672fe9-69f7-11ef-b5c6-00155db34eb6",""],"Index":323075,"Payload":{"Allocation":{"AllocModifyIndex":323075,"AllocatedResources":{"Shared":{"DiskMB":10,"Networks":null,"Ports":null},"TaskLifecycles":{"default-task":null},"Tasks":{"default-task":{"Cpu":{"CpuShares":20,"ReservedCores":null},"Devices":null,"Memory":{"MemoryMB":30,"MemoryMaxMB":0},"Networks":null}}},"ClientDescription":"Tasks are running","ClientStatus":"running","CreateIndex":323017,"CreateTime":1725370832218942000,"DesiredDescription":"alloc not needed due to job update","DesiredStatus":"stop","EvalID":"d619fee4-a287-9f50-62e5-5977294c6851","ID":"16244930-acea-da7e-e410-ef583887ef86","JobID":"8-0c672fe9-69f7-11ef-b5c6-00155db34eb6","Metrics":{"AllocationTime":211934,"ClassExhausted":null,"ClassFiltered":null,"CoalescedFailures":0,"ConstraintFiltered":null,"DimensionExhausted":null,"NodesAvailable":{"dc1":3},"NodesEvaluated":3,"NodesExhausted":0,"NodesFiltered":0,"NodesInPool":3,"QuotaExhausted":null,"ResourcesExhausted":null,"ScoreMetaData":[{"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NormScore":0.9070296312712901,"Scores":{"binpack":0.9070296312712901,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"23f36565-f23e-31fd-fb3f-4723794fb54f","NormScore":0.8357718246656134,"Scores":{"binpack":0.8357718246656134,"job-anti-affinity":0,"node-affinity":0,"node-reschedule-penalty":0}},{"NodeID":"61afb6fa-38e0-5295-122c-21b9300beb45","NormScore":-0.5752027071266841,"Scores":{"binpack":0.8495945857466318,"job-anti-affinity":-2,"node-affinity":0,"node-reschedule-penalty":0}}],"Scores":null},"ModifyIndex":323075,"ModifyTime":1725370967617845200,"Name":"8-0c672fe9-69f7-11ef-b5c6-00155db34eb6.default-group[0]","Namespace":"default","NetworkStatus":{"Address":"","DNS":null,"InterfaceName":""},"NodeID":"dc532ed9-cecc-4b99-0c4b-7d4f66179cb9","NodeName":"nomad-agent-terraform-2","Resources":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":30,"NUMA":null,"Networks":null,"SecretsMB":0},"SharedResources":{"CPU":0,"Cores":0,"Devices":null,"DiskMB":10,"IOPS":0,"MemoryMB":0,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0},","SigningKeyID":"0f0bee64-a0df-0a82-ff45-b514a7c89288","TaskGroup":"default-group","TaskResources":{"default-task":{"CPU":20,"Cores":0,"Devices":null,"DiskMB":0,"IOPS":0,"MemoryMB":30,"MemoryMaxMB":0,"NUMA":null,"Networks":null,"SecretsMB":0}},"TaskStates":{"default-task":{"Events":[{"Details":{},"DiskLimit":0,"DisplayMessage":"Task received by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832244809500,"Type":"Received","ValidationError":"","VaultError":""},{"Details":{"message":"Building Task Directory"},"DiskLimit":0,"DisplayMessage":"Building Task Directory","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"Building Task Directory","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832257085400,"Type":"Task Setup","ValidationError":"","VaultError":""},{"Details":{"image":"openhpi/co_execenv_java:17"},"DiskLimit":0,"DisplayMessage":"Downloading image","DownloadError":"","DriverError":"","DriverMessage":"Downloading image","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370832628741600,"Type":"Driver","ValidationError":"","VaultError":""},{"Details":{},"DiskLimit":0,"DisplayMessage":"Task started by client","DownloadError":"","DriverError":"","DriverMessage":"","ExitCode":0,"FailedSibling":"","FailsTask":false,"GenericSource":"","KillError":"","KillReason":"","KillTimeout":0,"Message":"","RestartReason":"","SetupError":"","Signal":0,"StartDelay":0,"TaskSignal":"","TaskSignalReason":"","Time":1725370834453868000,"Type":"Started","ValidationError":"","VaultError":""}],"Failed":false,"FinishedAt":null,"LastRestart":null,"Paused":"","Restarts":0,"StartedAt":"2024-09-03T13:40:34.453913016Z","State":"running","TaskHandle":null}}}}}
  - Same for `8-611db37e-69f5-11ef-ba1c-00155db34eb6`

## Questions
- Are we handling such lost runner jobs right?

## Learnings
- Job-Events do not reliably notify about stopped jobs (on failed migrations)
- Do not restart all agents at the same time