Add Helicone

.github/workflows/ci.yml

@@ -172,6 +172,7 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           AGENT_NAME: ${{ matrix.agent-name }}
           PROMPT_USER: false # For mini-agi. TODO: Remove this once mini-agi follows the standards.
+          HELICONE_API_KEY: ${{ secrets.HELICONE_API_KEY }}
 
       - name: Upload logs as artifact
         if: always()

agbenchmark/challenge.py

@@ -7,6 +7,7 @@
 from dotenv import load_dotenv
 
 from agbenchmark.challenges.define_task_types import ChallengeData, Ground
+from agbenchmark.start_benchmark import CURRENT_DIRECTORY
 
 load_dotenv()
 
@@ -23,7 +24,7 @@ class Challenge(ABC):
 
     @property
     def data(self) -> ChallengeData:
-        file_path = f"{self.CHALLENGE_LOCATION}/data.json"
+        file_path = f"{CURRENT_DIRECTORY}/../{self.CHALLENGE_LOCATION}/data.json"
         if file_path not in Challenge._data_cache:
             Challenge._data_cache[file_path] = ChallengeData.deserialize(file_path)
         return Challenge._data_cache[file_path]

agbenchmark/challenges/test_all.py

@@ -10,13 +10,14 @@
 from dotenv import load_dotenv
 
 from agbenchmark.challenge import Challenge
+from agbenchmark.start_benchmark import CURRENT_DIRECTORY
 
 load_dotenv()
 
 IMPROVE = os.getenv("IMPROVE", "False")
 
 
-json_files = glob.glob("agbenchmark/challenges/**/data.json", recursive=True)
+json_files = glob.glob(f"{CURRENT_DIRECTORY}/challenges/**/data.json", recursive=True)
 
 
 def get_test_path(json_file: str) -> str: