FIX: Integartion tests, improve disconnect

.github/workflows/test-molecule.yml

@@ -71,6 +71,7 @@ jobs:
             { role: "update-changes", test: "215" },
             { role: "update-changes", test: "221" },
             { role: "update-changes", test: "222" },
+            { role: "update-changes", test: "223" },
           ]
       fail-fast: false
     concurrency: molecule-test-${{ matrix.tests.role }}-${{ matrix.tests.test }}

controls/roles/update-changes/molecule/223/converge.yml

@@ -0,0 +1,10 @@
+---
+- name: Converge
+  hosts: all
+  vars_files:
+  - ../../../../defaults/stereum_defaults.yaml
+
+  tasks:
+    - name: "Include update-changes"
+      include_role:
+        name: "update-changes"

controls/roles/update-changes/molecule/223/molecule.yml

@@ -0,0 +1,27 @@
+---
+#dependency:
+#  name: galaxy
+driver:
+  name: docker
+platforms:
+  - name: 'update-changes--2.2.3--ubuntu-23.04'
+    image: ubuntu:lunar
+  # - name: "configure-updates--default--centos-8"
+  #   image: geerlingguy/docker-centos8-ansible
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_PIPELINING: 'True'
+lint: |
+  set -e
+  yamllint .
+  ansible-lint .
+scenario:
+  test_sequence:
+    - destroy
+    - create
+    - prepare
+    - converge
+    - idempotence
+    - verify
+    - destroy

controls/roles/update-changes/molecule/223/playbook.yml

@@ -0,0 +1,7 @@
+---
+- name: Converge
+  hosts: all
+  tasks:
+    - name: "Include update-changes"
+      include_role:
+        name: "update-changes"

controls/roles/update-changes/molecule/223/prepare.yml

@@ -0,0 +1,105 @@
+---
+- name: Prepare
+  hosts: all
+  tasks:
+    - name: Make sure Stereum's config path exists
+      file:
+        path: '/etc/stereum/services'
+        state: directory
+        owner: 'root'
+        group: 'root'
+        mode: 0644
+      become: yes
+
+    - name: Create ValidatorEjectorService config
+      copy:
+        dest: '/etc/stereum/services/15ce3684-4dbd-4657-8a21-b1243b72b1ca.yaml'
+        owner: 'root'
+        group: 'root'
+        mode: 0644
+        content: |
+          service: ValidatorEjectorService
+          id: 15ce3684-4dbd-4657-8a21-b1243b72b1ca
+          configVersion: 1
+          command: []
+          entrypoint: []
+          env:
+            EXECUTION_NODE: http://stereum-ce456ae0-64d6-4886-a53b-6b90984d4913:8545
+            CONSENSUS_NODE: http://stereum-e7bd77d1-4d20-4a2e-85a6-1915f105a4c6:5052
+            LOCATOR_ADDRESS: "0x28FAB2059C713A7F9D8c86Db49f9bb0e96Af1ef8"
+            STAKING_MODULE_ID: "123"
+            OPERATOR_ID: "123"
+            MESSAGES_LOCATION: /app/messages
+            ORACLE_ADDRESSES_ALLOWLIST: '["0x140Bd8FbDc884f48dA7cb1c09bE8A2fAdfea776E",
+              "0xA7410857ABbf75043d61ea54e07D57A6EB6EF186",
+              "0x404335BcE530400a5814375E7Ec1FB55fAff3eA2",
+              "0x946D3b081ed19173dC83Cd974fC69e1e760B7d78",
+              "0x007DE4a5F7bc37E2F26c0cb2E8A95006EE9B89b5",
+              "0xEC4BfbAF681eb505B94E4a7849877DC6c600Ca3A",
+              "0x61c91ECd902EB56e314bB2D5c5C07785444Ea1c8",
+              "0x1Ca0fEC59b86F549e1F1184d97cb47794C8Af58d",
+              "0xc79F702202E3A6B0B6310B537E786B9ACAA19BAf"]'
+            HTTP_PORT: "8989"
+            RUN_METRICS: "true"
+            RUN_HEALTH_CHECK: "true"
+            DRY_RUN: "false"
+          image: lidofinance/validator-ejector:1.6.0
+          ports: []
+          volumes:
+            - /opt/stereum/validatorejector-15ce3684-4dbd-4657-8a21-b1243b72b1ca/messages:/app/messages
+          user: "2000"
+          autoupdate: true
+          network: mainnet
+          dependencies:
+            executionClients:
+              - service: GethService
+                id: ce456ae0-64d6-4886-a53b-6b90984d4913
+            consensusClients:
+              - service: LighthouseBeaconService
+                id: e7bd77d1-4d20-4a2e-85a6-1915f105a4c6
+            mevboost: []
+            otherServices: []
+      become: yes
+
+    - name: Create ValidatorEjectorService config
+      copy:
+        dest: '/etc/stereum/services/41b15ea7-12d4-49f6-8448-8a6164edef36.yaml'
+        owner: 'root'
+        group: 'root'
+        mode: 0644
+        content: |
+          service: ValidatorEjectorService
+          id: 41b15ea7-12d4-49f6-8448-8a6164edef36
+          configVersion: 1
+          command: []
+          entrypoint: []
+          env:
+            EXECUTION_NODE: http://stereum-cc1323ce-90a1-440f-8b78-f930a3300b38:8545
+            CONSENSUS_NODE: http://stereum-62cb879b-b0f0-4ea2-bc13-f606b7f4cfb9:5052
+            LOCATOR_ADDRESS: "0x28FAB2059C713A7F9D8c86Db49f9bb0e96Af1ef8"
+            STAKING_MODULE_ID: "123"
+            OPERATOR_ID: "123"
+            MESSAGES_LOCATION: /app/messages
+            ORACLE_ADDRESSES_ALLOWLIST: '["0x140Bd8FbDc884f48dA7cb1c09bE8A2fAdfea776E","0xA7410857ABbf75043d61ea54e07D57A6EB6EF186","0x404335BcE530400a5814375E7Ec1FB55fAff3eA2","0x946D3b081ed19173dC83Cd974fC69e1e760B7d78","0x007DE4a5F7bc37E2F26c0cb2E8A95006EE9B89b5","0xEC4BfbAF681eb505B94E4a7849877DC6c600Ca3A","0x61c91ECd902EB56e314bB2D5c5C07785444Ea1c8","0x1Ca0fEC59b86F549e1F1184d97cb47794C8Af58d","0xc79F702202E3A6B0B6310B537E786B9ACAA19BAf"]'
+            HTTP_PORT: "8989"
+            RUN_METRICS: "true"
+            RUN_HEALTH_CHECK: "true"
+            DRY_RUN: "false"
+          image: lidofinance/validator-ejector:1.6.0
+          ports: []
+          volumes:
+            - /opt/stereum/validatorejector-41b15ea7-12d4-49f6-8448-8a6164edef36/messages:/app/messages
+          user: "2000"
+          autoupdate: true
+          network: mainnet
+          dependencies:
+            executionClients:
+              - service: GethService
+                id: cc1323ce-90a1-440f-8b78-f930a3300b38
+            consensusClients:
+              - service: LighthouseBeaconService
+                id: 62cb879b-b0f0-4ea2-bc13-f606b7f4cfb9
+            mevboost: []
+            otherServices: []
+      become: yes
+#EOF

controls/roles/update-changes/molecule/223/verify.yml

@@ -0,0 +1,39 @@
+---
+- name: Verify
+  hosts: all
+  gather_facts: false
+  tasks:
+    # ValidatorEjectorService
+    - name: Read ValidatorEjectorService file
+      slurp:
+        src: '/etc/stereum/services/15ce3684-4dbd-4657-8a21-b1243b72b1ca.yaml'
+      register: Ejector_service_configuration_raw
+
+    # ValidatorEjectorService2
+    - name: Read ValidatorEjectorService file
+      slurp:
+        src: '/etc/stereum/services/41b15ea7-12d4-49f6-8448-8a6164edef36.yaml'
+      register: Ejector_service_configuration2_raw
+
+    - name: Parse Service configurations
+      set_fact:
+        Ejector_service_configuration: "{{ Ejector_service_configuration_raw['content'] | b64decode | from_yaml }}"
+        Ejector_service_configuration2: "{{ Ejector_service_configuration2_raw['content'] | b64decode | from_yaml }}"
+
+    - debug:
+        msg: '{{ Ejector_service_configuration }}'
+    - debug:
+        msg: "{{ Ejector_service_configuration_raw['content'] | b64decode }}"
+
+    - debug:
+        msg: '{{ Ejector_service_configuration2 }}'
+    - debug:
+        msg: "{{ Ejector_service_configuration2_raw['content'] | b64decode }}"
+
+    - assert:
+        that:
+          - Ejector_service_configuration.env.ORACLE_ADDRESSES_ALLOWLIST | from_yaml | select('match', '0xe57B3792aDCc5da47EF4fF588883F0ee0c9835C9') | length == 1
+          - Ejector_service_configuration.env.ORACLE_ADDRESSES_ALLOWLIST | from_yaml | select('match', '0xEC4BfbAF681eb505B94E4a7849877DC6c600Ca3A') | length == 0
+          - Ejector_service_configuration2.env.ORACLE_ADDRESSES_ALLOWLIST | from_yaml | select('match', '0xe57B3792aDCc5da47EF4fF588883F0ee0c9835C9') | length == 1
+          - Ejector_service_configuration2.env.ORACLE_ADDRESSES_ALLOWLIST | from_yaml | select('match', '0xEC4BfbAF681eb505B94E4a7849877DC6c600Ca3A') | length == 0
+# EOF

controls/roles/update-changes/tasks/2.2.3/ejector_changes.yaml

@@ -0,0 +1,20 @@
+---
+- name: Read service file
+  slurp:
+    src: '{{ config_file.path }}'
+  register: service_configuration_raw
+
+- name: Parse service's configuration
+  set_fact:
+    service_configuration: "{{ service_configuration_raw['content'] | b64decode | from_yaml }}"
+    service_configuration_text: "{{ service_configuration_raw['content'] | b64decode }}"
+
+- name: Update Oracle Allow List
+  replace:
+    path: '{{ config_file.path }}'
+    regexp: '(?s)ORACLE_ADDRESSES_ALLOWLIST:.*?]["'']{1}'
+    replace: 'ORACLE_ADDRESSES_ALLOWLIST: ''["0x140Bd8FbDc884f48dA7cb1c09bE8A2fAdfea776E","0xA7410857ABbf75043d61ea54e07D57A6EB6EF186","0x404335BcE530400a5814375E7Ec1FB55fAff3eA2","0x946D3b081ed19173dC83Cd974fC69e1e760B7d78","0x007DE4a5F7bc37E2F26c0cb2E8A95006EE9B89b5","0xe57B3792aDCc5da47EF4fF588883F0ee0c9835C9","0x61c91ECd902EB56e314bB2D5c5C07785444Ea1c8","0x1Ca0fEC59b86F549e1F1184d97cb47794C8Af58d","0xc79F702202E3A6B0B6310B537E786B9ACAA19BAf"]'''
+  when:
+    - service_configuration.service == "ValidatorEjectorService"
+    - service_configuration.network == "mainnet"
+    - service_configuration.env.ORACLE_ADDRESSES_ALLOWLIST | from_yaml | select('match', '0xEC4BfbAF681eb505B94E4a7849877DC6c600Ca3A') | length == 1

controls/roles/update-changes/tasks/2.2.3/updates-223.yaml

@@ -0,0 +1,11 @@
+---
+- name: Find service configs
+  find:
+    paths: "/etc/stereum/services"
+  register: service_config_files
+
+- name: Include Prysm Changes
+  include_tasks: ejector_changes.yaml
+  loop: "{{ service_config_files.files }}"
+  loop_control:
+    loop_var: config_file

controls/roles/update-changes/tasks/main.yml

@@ -9,4 +9,8 @@
 - name: Include 2.2.2 Update Scripts
   include_tasks: "2.2.2/updates-222.yaml"
   ignore_errors: yes
+
+- name: Include 2.2.3 Update Scripts
+  include_tasks: "2.2.3/updates-223.yaml"
+  ignore_errors: yes
 # EOF

launcher/src/backend/HetznerServer.js

@@ -28,6 +28,7 @@ export class HetznerServer {
     let connected = false;
     while (!connected && tries < 300) {
       try {
+        await this.Sleep(2000);
         tries++;
         log.info(`Trying to connect (${tries})`);
         connected = await nodeConnection.sshService.checkSSHConnection(nodeConnection.nodeConnectionParams, 5000);

launcher/src/backend/SSHService.js

@@ -188,10 +188,11 @@ export class SSHService {
         this.connectionInfo = null;
       }
       let counter = 0;
-      while (this.connectionPool.some((conn) => conn._chanMgr?._count > 0 && counter < 30)) {
+      while (this.connectionPool.length > 0 && counter < 30) {
         this.connectionPool.forEach((conn) => {
-          if (conn._chanMgr?._count > 0) {
+          if (conn._chanMgr?._count == 0) {
             conn.end();
+            this.connectionPool = this.connectionPool.filter((c) => c !== conn);
           }
         });
         await new Promise((resolve) => setTimeout(resolve, 1000));

launcher/src/backend/ethereum-services/ValidatorEjectorService.js

@@ -18,7 +18,7 @@ export class ValidatorEjectorService extends NodeService {
       case "mainnet":
         locatorAddress = "0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb"; //https://docs.lido.fi/deployed-contracts/#core-protocol
         oracleAllowList =
-          '["0x140Bd8FbDc884f48dA7cb1c09bE8A2fAdfea776E","0xA7410857ABbf75043d61ea54e07D57A6EB6EF186","0x404335BcE530400a5814375E7Ec1FB55fAff3eA2","0x946D3b081ed19173dC83Cd974fC69e1e760B7d78","0x007DE4a5F7bc37E2F26c0cb2E8A95006EE9B89b5","0xEC4BfbAF681eb505B94E4a7849877DC6c600Ca3A","0x61c91ECd902EB56e314bB2D5c5C07785444Ea1c8","0x1Ca0fEC59b86F549e1F1184d97cb47794C8Af58d","0xc79F702202E3A6B0B6310B537E786B9ACAA19BAf"]';
+          '["0x140Bd8FbDc884f48dA7cb1c09bE8A2fAdfea776E","0xA7410857ABbf75043d61ea54e07D57A6EB6EF186","0x404335BcE530400a5814375E7Ec1FB55fAff3eA2","0x946D3b081ed19173dC83Cd974fC69e1e760B7d78","0x007DE4a5F7bc37E2F26c0cb2E8A95006EE9B89b5","0xe57B3792aDCc5da47EF4fF588883F0ee0c9835C9","0x61c91ECd902EB56e314bB2D5c5C07785444Ea1c8","0x1Ca0fEC59b86F549e1F1184d97cb47794C8Af58d","0xc79F702202E3A6B0B6310B537E786B9ACAA19BAf"]';
         break;
       case "holesky":
         locatorAddress = "0x28FAB2059C713A7F9D8c86Db49f9bb0e96Af1ef8"; //https://docs.lido.fi/deployed-contracts/holesky#core-protocol

launcher/src/backend/tests/integration/FlashbotsMevBoostService.int.js

@@ -73,7 +73,7 @@ test("mevboost installation", async () => {
     await testServer.Sleep(30000);
     status = await nodeConnection.sshService.exec(`docker logs stereum-${mevboost.id}`);
     if (
-      /listening on 0.0.0.0:18550/.test(status.stdout) &&
+      /Listening on 0.0.0.0:18550/.test(status.stdout) &&
       /using 1 relays/.test(status.stdout) &&
       !/Invalid relay URL/.test(status.stdout)
     ) {
@@ -94,7 +94,7 @@ test("mevboost installation", async () => {
   }
 
   // check if Mevboost service is running properly
-  expect(status.stdout).toMatch(/listening on 0.0.0.0:18550/);
+  expect(status.stdout).toMatch(/Listening on 0.0.0.0:18550/);
   expect(status.stdout).toMatch(/using 1 relays/);
   expect(status.stdout).not.toMatch(/Invalid relay URL/);
 });