Add core isolation and change cores distribution

Author: wmousa

env_var

@@ -0,0 +1,5 @@
+SIMPLY_BLOCK_COMMAND_NAME=sbcli-isolate
+SIMPLY_BLOCK_VERSION=1.0.6
+
+SIMPLY_BLOCK_DOCKER_IMAGE=simplyblock/simplyblock:isolate
+DOCKER_USER=hamdysimplyblock

simplyblock_cli/cli.py

@@ -31,6 +31,8 @@ def __init__(self):
         # Add storage node
         sub_command = self.add_sub_command(subparser, "deploy", 'Deploy local services for remote ops (local run)')
         sub_command.add_argument("--ifname", help='Management interface name, default: eth0')
+        sub_command.add_argument("--cpu-mask", help='SPDK app CPU mask, default is all cores found',
+                                 dest='spdk_cpu_mask')
 
         self.add_sub_command(subparser, "deploy-cleaner", 'clean local deploy (local run)')
 
@@ -847,7 +849,13 @@ def run(self):
             sub_command = args_dict['storage-node']
 
             if sub_command == "deploy":
-                ret = storage_ops.deploy(args.ifname)
+                spdk_cpu_mask = None
+                if args.spdk_cpu_mask:
+                    if self.validate_cpu_mask(args.spdk_cpu_mask):
+                        spdk_cpu_mask = args.spdk_cpu_mask
+                    else:
+                        return f"Invalid cpu mask value: {args.spdk_cpu_mask}"
+                ret = storage_ops.deploy(args.ifname, spdk_cpu_mask)
 
             elif sub_command == "deploy-cleaner":
                 ret = storage_ops.deploy_cleaner()

simplyblock_core/constants.py

@@ -72,6 +72,7 @@ def get_from_env_var_file(name, default=None):
 SIMPLY_BLOCK_CLI_NAME = get_from_env_var_file(
         "SIMPLY_BLOCK_COMMAND_NAME", "sbcli")
 TASK_EXEC_INTERVAL_SEC = 10
+
 TASK_EXEC_RETRY_COUNT = 8
 SIMPLY_BLOCK_SPDK_CORE_IMAGE = "simplyblock/spdk-core:v24.05-tag-latest"
 
@@ -128,3 +129,7 @@ def get_from_env_var_file(name, default=None):
 
 NVMF_MAX_SUBSYSTEMS=50000
 HA_JM_COUNT=3
+
+JSON_CORES_FILE="/etc/simplyblock/cores_distribution.ini"
+TEMP_CORES_FILE = "/etc/simplyblock/tmp_cores_config"
+

simplyblock_core/scripts/install_deps.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
-sudo yum install -y yum-utils xorg-x11-xauth nvme-cli fio
+sudo yum install -y https://repo.almalinux.org/almalinux/9/devel/aarch64/os/Packages/tuned-profiles-realtime-2.24.0-1.el9.noarch.rpm
+sudo yum install -y yum-utils xorg-x11-xauth nvme-cli fio tuned
 sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
 sudo yum install hostname pkg-config git wget python3-pip yum-utils docker-ce docker-ce-cli \
   containerd.io docker-buildx-plugin docker-compose-plugin iptables -y

simplyblock_core/storage_node_ops.py

@@ -992,43 +992,31 @@ def add_node(cluster_id, node_ip, iface_name, data_nics_list,
     pollers_mask = ""
     app_thread_mask = ""
     jm_cpu_mask = ""
-    alceml_cpu_cores = []
-    distrib_cpu_cores = []
-    alceml_worker_cpu_cores = []
-
     alceml_cpu_index = 0
     alceml_worker_cpu_index = 0
     distrib_cpu_index = 0
     jc_singleton_mask = ""
 
+    cores_config= node_info["cores_config"]
+    spdk_cpu_mask = cores_config["cpu_mask"]
+    app_thread_core = cores_config["cores_distribution"]["app_thread_core"]
+    jm_cpu_core = cores_config["cores_distribution"]["jm_cpu_core"]
+    poller_cpu_cores = cores_config["cores_distribution"]["poller_cpu_cores"]
+    alceml_cpu_cores = cores_config["cores_distribution"]["alceml_cpu_cores"]
+    alceml_worker_cpu_cores = cores_config["cores_distribution"]["alceml_worker_cpu_cores"]
+    distrib_cpu_cores = cores_config["cores_distribution"]["distrib_cpu_cores"]
+    jc_singleton_core = cores_config["cores_distribution"]["jc_singleton_core"]
+    if is_secondary_node:
+        distrib_cpu_cores = distrib_cpu_cores + alceml_cpu_cores
 
-    poller_cpu_cores = []
-
-    if not spdk_cpu_mask:
-        spdk_cpu_mask = hex(int(math.pow(2, cpu_count))-2)
-
-    spdk_cores = utils.hexa_to_cpu_list(spdk_cpu_mask)
-    if cpu_count < spdk_cores[-1]:
-        print(f"ERROR: The cpu mask {spdk_cpu_mask} is greater than the total cpus on the system {cpu_count}")
-        return False
-    if spdk_cores[-1] >= 64:
-        print(f"ERROR: The provided cpu mask {spdk_cpu_mask} has values greater than 63, which is not allowed")
-        return False
-    if len(spdk_cores) >= 4:
-        app_thread_core, jm_cpu_core, poller_cpu_cores, alceml_cpu_cores, alceml_worker_cpu_cores, distrib_cpu_cores, jc_singleton_core = utils.calculate_core_allocation(
-            spdk_cores)
-
-        if is_secondary_node:
-            distrib_cpu_cores = distrib_cpu_cores+alceml_cpu_cores
-
-        pollers_mask = utils.generate_mask(poller_cpu_cores)
+    if jc_singleton_core:
+        jc_singleton_mask = utils.decimal_to_hex_power_of_2(jc_singleton_core[0])
+    if app_thread_core:
         app_thread_mask = utils.generate_mask(app_thread_core)
-        if jc_singleton_core:
-            jc_singleton_mask = utils.decimal_to_hex_power_of_2(jc_singleton_core[0])
-        #spdk_cpu_mask = utils.generate_mask(spdk_cores)
+    if jm_cpu_core:
         jm_cpu_mask = utils.generate_mask(jm_cpu_core)
-        #distrib_cpu_mask = utils.generate_mask(distrib_cpu_cores)
-
+    if poller_cpu_cores:
+        pollers_mask = utils.generate_mask(poller_cpu_cores)
     # Calculate pool count
     if cloud_instance['type']:
         ins_type = cloud_instance['type']
@@ -2558,10 +2546,12 @@ def get_node_port_iostats(port_id, history=None, records_count=20):
     return utils.print_table(out)
 
 
-def deploy(ifname):
+def deploy(ifname, spdk_cpu_mask):
     if not ifname:
         ifname = "eth0"
-
+    cpu_count = os.cpu_count()
+    if not spdk_cpu_mask:
+        spdk_cpu_mask = hex(int(math.pow(2, cpu_count))-2)
     dev_ip = utils.get_iface_ip(ifname)
     if not dev_ip:
         logger.error(f"Error getting interface ip: {ifname}")
@@ -2579,6 +2569,10 @@ def deploy(ifname):
     ret = scripts.configure_docker(dev_ip)
 
     start_storage_node_api_container(dev_ip)
+    isolated_full = utils.prepare_cores(spdk_cpu_mask)
+    if isolated_full:
+        utils.generate_realtime_variables_file(isolated_full)
+        utils.run_tuned()
     return f"{dev_ip}:5000"
 
 def start_storage_node_api_container(node_ip):