Genetic algorithm to find optimisation passes.

.gitignore

@@ -1,3 +1,5 @@
+__pycache__
+genetic.log
 target
 Cargo.lock
 .gdb_history

ykrt/pass_finder/cargo_run.py

@@ -0,0 +1,64 @@
+import os
+import subprocess
+import time
+
+RED = '\033[91m'
+GREEN = '\033[92m'
+PURPLE = '\033[38;5;128m'
+YELLOW = '\033[93m'
+RESET = '\033[0m'
+
+def run_test(id, yk_path, yklua_path, env, n=1):
+    """
+    The function loads the environment variables and
+    runs YK's test and YKLUA build plus tests. If both succeeds
+    the function then runs the benchmark script to get
+    the execution time which is used by the genetic algorithm
+    for fitness function.
+    """
+    #TODO: directly send the path instead of checking env variable
+    if yk_path is None:
+        raise ValueError("YK_PATH environment variable is not set") 
+    if yklua_path is None:
+        raise ValueError("YKLUA_PATH environment variable is not set") 
+    # Check if pre and post-link flags are set
+
+    prelink_passes = env.get(f'PRELINK_PASSES_{id}', 'Not Set') 
+    print(f"\n{YELLOW}PRELINK_PASSES: {prelink_passes}{RESET}")
+    postlink_passes = env.get(f'POSTLINK_PASSES_{id}', 'Not Set')
+    print(f"\n{YELLOW}POSTLINK_PASSES: {postlink_passes}{RESET}")
+
+    curdir = os.getcwd()
+    os.chdir(yklua_path)
+    times = []
+    c_test = None 
+    for _ in range(n):
+        subprocess.run(["make clean"], shell=True, env=env)
+        c = subprocess.run(["make && timeout 30 sh test.sh"], shell=True, env=env or os.environ)
+        os.chdir(yk_path)
+        if c.returncode == 0:
+            r = subprocess.run(f"timeout 60 cargo test --test c_tests", shell=True, env=env or os.environ)
+        else:
+            break
+        os.chdir(yklua_path)
+        os.chdir(os.path.join(yklua_path, 'tests'))
+        lua_interpreter_path = os.path.join(yklua_path, 'src', 'lua')
+        cmd = f"timeout 5 {lua_interpreter_path} db.lua"
+        if c.returncode == 0 and r.returncode == 0:
+            before = time.time()
+            c_test = subprocess.run(cmd, shell=True, env=env or os.environ)
+            elapsed = time.time() - before
+            times.append(elapsed)
+
+    if len(times) != 0:
+        mean_time = sum(times) / len(times)
+    else:
+        mean_time = None
+
+    if c_test is None:
+        return 130, None
+    elif c_test.returncode != 0:
+        return c_test.returncode, None
+
+    os.chdir(curdir)
+    return 0, mean_time

ykrt/pass_finder/setup_genetic.py

@@ -0,0 +1,76 @@
+import os, shutil, subprocess, sys, queue
+import multiprocessing
+from  multiprocessing import Manager, Process, Queue
+
+def setup_worker(curr_dir, temp_directories, base_temp_dir, yk_path, yklua, tasks):
+    while True:    
+        try:
+            i = tasks.get(block=False)
+        except queue.Empty:
+            print(f"Closing setup worker")
+            break
+        else:
+            # Create a directory with the custom name
+            temp_dir_name = f"tmp_{i}"
+            temp_dir = os.path.join(base_temp_dir, temp_dir_name)
+            git_repo_path = os.path.join(temp_dir, "yk")
+            yklua_dest_path = os.path.join(temp_dir, "yklua")
+            yk_test_src = os.path.join(git_repo_path, "tests", "src", "lib.rs")
+            langtester = os.path.join(temp_dir, "lang_tester")
+            os.makedirs(temp_dir, exist_ok=True)
+
+            if not os.path.exists(langtester):
+                shutil.copytree("/home/shreei/research/lang_tester", langtester)
+            if not os.path.exists(git_repo_path):
+                shutil.copytree(yk_path, git_repo_path, ignore=shutil.ignore_patterns('target'))
+                os.chdir(git_repo_path)
+
+                if os.path.exists(yk_test_src):
+                    subprocess.run(f"sed -i -e 's/PRELINK_PASSES/PRELINK_PASSES_{i}/g' {yk_test_src}", shell=True)
+                    subprocess.run(f"sed -i -e 's/POSTLINK_PASSES/POSTLINK_PASSES_{i}/g' {yk_test_src}", shell=True)
+                #subprocess.run("git submodule init", shell=True, env=os.environ)
+                #subprocess.run("git submodule update", shell=True, env=os.environ)
+                subprocess.run("cargo test", shell=True, env=os.environ)
+            elif os.path.exists(git_repo_path):
+                os.chdir(git_repo_path)
+                # subprocess.run("cargo test --test c_tests", shell=True, env=os.environ)
+                print("..")
+            else:
+                print(f"Directory {git_repo_path} does not exist.")
+                sys.exit()
+
+            if not os.path.exists(yklua_dest_path):
+                shutil.copytree(yklua, yklua_dest_path)   
+                yklua_src = os.path.join(yklua_dest_path, "src")
+                os.chdir(yklua_src)
+                subprocess.run(f"sed -i -e 's/PRELINK_PASSES/PRELINK_PASSES_{i}/g' Makefile", shell=True) 
+                subprocess.run(f"sed -i -e 's/POSTLINK_PASSES/POSTLINK_PASSES_{i}/g' Makefile", shell=True)  
+
+            temp_directories.append(temp_dir)
+            os.chdir(curr_dir)
+
+def setup(curr_dir, base_temp_dir, yk_path, yklua):
+    num_cores = multiprocessing.cpu_count() - 1
+    directories = []
+
+    with Manager() as manager:
+        temp_directories = manager.list()
+        tasks = Queue()
+        processes = []
+
+        #TODO: on bencher9 change num_cores to num_cores * 2
+        for i in range(num_cores):
+            tasks.put(i)
+
+        for i in range(num_cores): 
+            p = Process(target=setup_worker, args=(curr_dir, temp_directories, base_temp_dir, yk_path, yklua, tasks))
+            processes.append(p)
+            os.system(f"taskset -p -c {i} {p.pid}")
+            p.start()
+
+        for p in processes:
+            p.join()
+
+        directories = [dir for dir in temp_directories]
+
+    return directories