chore: separated code_auditor and code_auditor_with_judge

Author: evilsocket

examples/code_auditor/task.yml

@@ -13,8 +13,6 @@ guidance:
   - Don't make assumptions or hypotheticals and only report vulnerabilities that can be confirmed by the source code provided.
   - Prioritize reporting vulnerabilities that can lead to unauthorized access to the application, code execution, or other unauthorized actions.
   - Avoid reporting misconfigurations or other non-vulnerability issues such as improper error handling.
-  - ALWAYS use the judge tool to confirm whether or not a finding is a vulnerability.
-  - NEVER report a finding unless the judge tool confirms it is a vulnerability.
   - Use exclusively the report_findings tool to report your findings.
   - Your task is not complete until you have analyzed ALL the files in EVERY source code subfolder and reported ALL of your findings.
   - Analyze the files in a folder before moving on to the next folder.
@@ -26,20 +24,6 @@ prompt: >
 functions:
   - name: Report
     actions:
-      - name: judge_finding
-        description: Use this tool to ask an external expert to judge if a finding is a vulnerability or not. If the expert confirms it is a vulnerability, use the report_finding tool to report it.
-        example_payload: >
-          {
-            "title": "SQL Injection",
-            "severity": "HIGH",
-            "impact": "An unauthorized attacker could execute arbitrary SQL commands, leading to unauthorized access to the database.",
-            "description": "This is an example finding",
-            "evidence": "This is the evidence for the finding",
-            "file": "/full/path/to/vulnerable_file.py",
-            "proof": "This is the proof of concept for the finding"
-          }
-        judge: judge.yml
-
       - name: report_finding
         description: Use this tool to report your findings.
         example_payload: >

examples/code_auditor_with_judge/README.md

@@ -0,0 +1,7 @@
+Looks for vulnerabilities in the specified code folder, variant of code_auditor that adds a judge AI to confirm vulnerabilities.
+
+### Example Usage
+
+```sh
+nerve -G "openai://gpt-4" -T code_auditor -DTARGET_PATH=/path/to/code
+```

examples/code_auditor_with_judge/judge.yml

@@ -0,0 +1,26 @@
+using:
+  # allows the agent to inspect folders and read files. No write access is provided.
+  - filesystem
+
+system_prompt: > 
+  You are an expert application security professional judging if the security finding submitted by a junior auditor is a vulnerability or not.
+  
+guidance:
+  - Don't make assumptions or hypotheticals and only confirm vulnerabilities that can be proven by the source code provided.
+  - ALWAYS check the file source and the code to confirm the finding.
+
+prompt: >
+  confirm by using the judge tool whether or not this is a vulnerability: $STDIN
+
+functions:
+  - name: Judge
+    actions:
+      - name: confirm_vulnerability
+        description: Use this tool to confirm whether or not the finding is a vulnerability.
+        complete_task: true
+        tool: echo "VULNERABILITY CONFIRMED"
+
+      - name: not_vulnerability
+        description: Use this tool to judge that the finding is not a vulnerability.
+        complete_task: true
+        tool: echo "NOT A VULNERABILITY, IGNORE"

examples/code_auditor_with_judge/task.yml

@@ -0,0 +1,57 @@
+using:
+  # allows the agent to inspect folders and read files. No write access is provided.
+  - filesystem
+  # allows the agent to set the task as complete
+  - task
+
+system_prompt: > 
+  You are an expert application security professional.
+  You are given access to a folder with the source code for an application to audit.
+  You are acting as a useful assistant that performs code auditing by reviewing the files in the folder and looking for potential vulnerabilities.
+
+guidance:
+  - Don't make assumptions or hypotheticals and only report vulnerabilities that can be confirmed by the source code provided.
+  - Prioritize reporting vulnerabilities that can lead to unauthorized access to the application, code execution, or other unauthorized actions.
+  - Avoid reporting misconfigurations or other non-vulnerability issues such as improper error handling.
+  - ALWAYS use the judge tool to confirm whether or not a finding is a vulnerability.
+  - NEVER report a finding unless the judge tool confirms it is a vulnerability.
+  - Use exclusively the report_findings tool to report your findings.
+  - Your task is not complete until you have analyzed ALL the files in EVERY source code subfolder and reported ALL of your findings.
+  - Analyze the files in a folder before moving on to the next folder.
+  - Make sure you reported everything you found and ince you are done reporting ALL of your findings, set your task as complete.
+
+prompt: >
+  find vulnerabilities in source code in $TARGET_PATH and report your findings.
+
+functions:
+  - name: Report
+    actions:
+      - name: judge_finding
+        description: Use this tool to ask an external expert to judge if a finding is a vulnerability or not. If the expert confirms it is a vulnerability, use the report_finding tool to report it.
+        example_payload: >
+          {
+            "title": "SQL Injection",
+            "severity": "HIGH",
+            "impact": "An unauthorized attacker could execute arbitrary SQL commands, leading to unauthorized access to the database.",
+            "description": "This is an example finding",
+            "evidence": "This is the evidence for the finding",
+            "file": "/full/path/to/vulnerable_file.py",
+            "proof": "This is the proof of concept for the finding"
+          }
+        judge: judge.yml
+
+      - name: report_finding
+        description: Use this tool to report your findings.
+        example_payload: >
+          {
+            "title": "SQL Injection",
+            "severity": "HIGH",
+            "impact": "An unauthorized attacker could execute arbitrary SQL commands, leading to unauthorized access to the database.",
+            "description": "This is an example finding",
+            "evidence": "This is the evidence for the finding",
+            "file": "/full/path/to/vulnerable_file.py",
+            "proof": "This is the proof of concept for the finding"
+          }
+        alias: filesystem.append_to_file
+        define:
+          filesystem.append_to_file.target: findings.jsonl