improve factor_runner

rdagent/scenarios/qlib/developer/factor_runner.py

@@ -37,6 +37,32 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
     - results in `mlflow`
     """
 
+    def calculate_information_coefficient(
+        self, concat_feature: pd.DataFrame, SOTA_feature_column_size: int, new_feature_columns_size: int
+    ) -> pd.DataFrame:
+        res = pd.Series(index=range(SOTA_feature_column_size * new_feature_columns_size))
+        for col1 in range(SOTA_feature_column_size):
+            for col2 in range(SOTA_feature_column_size, SOTA_feature_column_size + new_feature_columns_size):
+                res.loc[col1 * new_feature_columns_size + col2 - SOTA_feature_column_size] = concat_feature.iloc[
+                    :, col1
+                ].corr(concat_feature.iloc[:, col2])
+        return res
+
+    def deduplicate_new_factors(self, SOTA_feature: pd.DataFrame, new_feature: pd.DataFrame) -> pd.DataFrame:
+        # calculate the IC between each column of SOTA_feature and new_feature
+        # if the IC is larger than a threshold, remove the new_feature column
+        # return the new_feature
+
+        concat_feature = pd.concat([SOTA_feature, new_feature], axis=1)
+        IC_max = (
+            concat_feature.groupby("datetime")
+            .apply(lambda x: self.calculate_information_coefficient(x, SOTA_feature.shape[1], new_feature.shape[1]))
+            .mean()
+        )
+        IC_max.index = pd.MultiIndex.from_product([range(SOTA_feature.shape[1]), range(new_feature.shape[1])])
+        IC_max = IC_max.unstack().max(axis=0)
+        return new_feature.iloc[:, IC_max[IC_max < 0.99].index]
+
     def develop(self, exp: QlibFactorExperiment) -> QlibFactorExperiment:
         """
         Generate the experiment by processing and combining factor data,
@@ -66,11 +92,15 @@ def develop(self, exp: QlibFactorExperiment) -> QlibFactorExperiment:
 
             # Combine the SOTA factor and new factors if SOTA factor exists
             if SOTA_factor is not None and not SOTA_factor.empty:
+                new_factors = self.deduplicate_new_factors(SOTA_factor, new_factors)
+                if new_factors.empty:
+                    raise FactorEmptyError("No valid factor data found to merge.")
                 combined_factors = pd.concat([SOTA_factor, new_factors], axis=1).dropna()
             else:
                 combined_factors = new_factors
 
             # Sort and nest the combined factors under 'feature'
+            # print(combined_factors)
             combined_factors = combined_factors.sort_index()
             combined_factors = combined_factors.loc[:, ~combined_factors.columns.duplicated(keep='last')]
             new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns])