add the result

rdagent/app/quant_factor_benchmark/eval.py

@@ -1,3 +1,7 @@
+import os
+from pathlib import Path
+import pickle
+import time
 from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
 from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import (
     FactorTestCaseLoaderFromJsonFile,
@@ -11,6 +15,7 @@
 from rdagent.core.scenario import Scenario
 from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario
 
+from pprint import pprint
 
 # 1.read the settings
 bs = BenchmarkSettings()
@@ -35,6 +40,25 @@
 # 5.run the eval
 res = eval_method.eval()
 
+# 6.save the result
+pprint(res)
+
+res_workspace = (Path().cwd() / "git_ignore_folder" / "eval_results").absolute()
+print(str(res_workspace))
+
+# Save results
+timestamp = time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time()))
+
+if not os.path.exists(str(res_workspace)):
+    os.makedirs(str(res_workspace))
+
+df_file_path = res_workspace / ("result_" + timestamp + ".csv")
+res_pkl_path = res_workspace / ("res_promptV2" + timestamp + ".pkl")
+res_pkl_path = res_workspace / ("res_promptV2" + timestamp + ".pkl")
+with open(str(res_pkl_path), "wb") as file:
+    # file.write(str(res))
+    pickle.dump(res, file)
+
 # TODO:
 # - Run it:
 # - factor input data generator;

rdagent/components/benchmark/conf.py

@@ -15,7 +15,7 @@ class BenchmarkSettings(BaseSettings):
 
     bench_data_path: Path = DIRNAME / "example.json"
 
-    bench_test_round: int = 10
+    bench_test_round: int = 1
     bench_test_case_n: Optional[int] = None  # how many test cases to run; If not given, all test cases will be run
 
     bench_method_cls: str = "rdagent.components.coder.factor_coder.CoSTEER.FactorCoSTEER"

rdagent/components/coder/factor_coder/CoSTEER/evolving_agent.py

@@ -14,6 +14,6 @@ def filter_evolvable_subjects_by_feedback(self, evo: EvolvableSubjects, feedback
         assert len(evo.sub_workspace_list) == len(feedback)
 
         for index in range(len(evo.sub_workspace_list)):
-            if not feedback[index].final_decision:
+            if feedback[index] and not feedback[index].final_decision:
                 evo.sub_workspace_list[index].clear()
         return evo

rdagent/components/coder/factor_coder/config.py

@@ -10,7 +10,7 @@ class FactorImplementSettings(BaseSettings):
     class Config:
         env_prefix = "FACTOR_CODER_"  # Use FACTOR_CODER_ as prefix for environment variables
 
-    coder_use_cache: bool = False
+    coder_use_cache: bool = True
     data_folder: str = str(
         (Path().cwd() / "git_ignore_folder" / "factor_implementation_source_data").absolute(),
     )