Merge pull request #2451 from Agenta-AI/release/v0.33.0

Release/v0.33.0

.gitignore

@@ -41,8 +41,6 @@ todo.md
 agenta-web/cypress/videos/
 agenta-backend/agenta_backend/cloud/**
 agenta-web/src/ee/**
-agenta-web/src/config/frontendConfig.ts
-agenta-web/src/config/appInfo.ts
 agenta-web/src/pages/apps/index.tsx
 agenta-web/src/pages/auth/[[...path]].tsx
 docker-compose.cloud.dev.yml

.husky/pre-push

@@ -15,10 +15,10 @@ run_frontend_checks() {
     fi
 
     # Run TypeScript type check
-    if ! npm run types:check; then
-        echo '❌ TypeScript type check failed.'
-        exit 1
-    fi
+    # if ! npm run types:check; then
+    #     echo '❌ TypeScript type check failed.'
+    #     exit 1
+    # fi
 
     echo '🎉 Frontend checks passed!'
     cd "$ORIGINAL_DIR" || exit

README.md

@@ -13,7 +13,7 @@
     <a href="https://join.slack.com/t/agenta-hq/shared_invite/zt-2yewk6o2b-DmhyA4h_lkKwecDtIsj1AQ">Slack</a>
   </p>
 <div align="center">
-  <strong> <h2> The Open source LLMOps Platform </h2></strong>
+  <strong> <h1> The Open source LLMOps Platform </h1></strong>
   Prompt playground, prompt management, evaluation, and observability
 </div>
 </br>
@@ -84,14 +84,14 @@
 
 ---
 
-# What is Agenta?
+## What is Agenta?
 
 Agenta is a platform for building production-grade LLM applications. It helps **engineering and product teams** create reliable LLM apps faster.
 
 
 Agenta provides end-to-end tools for the entire LLMOps workflow:  building (**LLM playground**, **evaluation**), deploying (**prompt and configuration management**), and monitoring (**LLM observability and tracing**).
 
-# Features
+## Features
 - **Prompt Playground**: Experiment, iterate on prompts, and compare outputs from over 50 LLM models side by side ([docs](https://docs.agenta.ai/prompt-management/using-the-playground?utm_source=github&utm_medium=referral&utm_campaign=readme))
 - **Custom Workflows**: Build a playground for any custom LLM workflow, such as RAG or agents. Enable all the team to easily iterate on its parameters and evaluate it from the web UI.
 - **LLM evaluation**: Run evaluation suite from the webUI using predefined evaluators like LLM-as-a-judge, RAG evaluators, or custom code evaluators. ([docs](https://docs.agenta.ai/evaluation/overview?utm_source=github&utm_medium=referral&utm_campaign=readme))
@@ -101,8 +101,8 @@ Agenta provides end-to-end tools for the entire LLMOps workflow:  building (**LL
 - **LLM Monitoring**: Track cost and latency and compare different deployments.
 
 
-# Getting Started 
-## Agenta Cloud:
+## Getting Started 
+### Agenta Cloud:
 The easiest way to get started is through Agenta Cloud. It is free to signup, and comes with a generous free-tier.
 
   <a href="https://cloud.agenta.ai?utm_source=github&utm_medium=referral&utm_campaign=readme">
@@ -113,32 +113,37 @@ The easiest way to get started is through Agenta Cloud. It is free to signup, an
       </picture>
   </a>
 
-## Self-host:
-```
+
+### Self-hosting Agenta
+
+#### Quick Setup (Port 80)
+```bash
 mkdir agenta && cd agenta
 curl -L https://raw.githubusercontent.com/agenta-ai/agenta/main/docker-compose.gh.yml -o docker-compose.gh.yml
 docker compose -f docker-compose.gh.yml up -d
 ```
-- Agenta uses port 80, to use a different port set the env var AGENTA_PORT. For instance `export AGENTA_PORT=90`
-- Add `--pull always` to the last command to upgrade your version of Agenta.
-
-# Disabling Anonymized Tracking
+
+#### Deployment Options
+- To use a different port or customize your deployment, see our [deployment documentation](https://docs.agenta.ai/self-host/host-locally?utm_source=github&utm_medium=referral&utm_campaign=readme)
+- For deploying on cloud platforms, refer to our guides for [AWS](https://docs.agenta.ai/self-host/deploy_remotly/host-on-aws?utm_source=github&utm_medium=referral&utm_campaign=readme), [GCP](https://docs.agenta.ai/self-host/deploy_remotly/host-on-gcp?utm_source=github&utm_medium=referral&utm_campaign=readme), or [other remote hosting options](https://docs.agenta.ai/self-host/deploy_remotly/host-remotely?utm_source=github&utm_medium=referral&utm_campaign=readme)
+
+## Disabling Anonymized Tracking
 
 By default, Agenta automatically reports anonymized basic usage statistics. This helps us understand how Agenta is used and track its overall usage and growth. This data does not include any sensitive information. To disable anonymized telemetry, follow these steps:
 
 - For web: Set `TELEMETRY_TRACKING_ENABLED` to `false` in your `agenta-web/.env` file.
 - For CLI: Set `telemetry_tracking_enabled` to `false` in your `~/.agenta/config.toml` file.
 
 
-# Contributing
+## Contributing
 
 We warmly welcome contributions to Agenta. Feel free to submit issues, fork the repository, and send pull requests.
 
 We are usually hanging in our Slack. Feel free to [join our Slack and ask us anything](https://join.slack.com/t/agenta-hq/shared_invite/zt-2yewk6o2b-DmhyA4h_lkKwecDtIsj1AQ)
 
 Check out our [Contributing Guide](https://docs.agenta.ai/misc/contributing/getting-started?utm_source=github&utm_medium=referral&utm_campaign=readme) for more information.
 
-## Contributors ✨
+### Contributors ✨
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
 [![All Contributors](https://img.shields.io/badge/all_contributors-49-orange.svg?style=flat-square)](#contributors-)

agenta-backend/agenta_backend/migrations/postgres/data_migrations/projects.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 import traceback
 from typing import Sequence
 
@@ -8,6 +9,7 @@
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, Session
 
+from agenta_backend.services import db_manager, evaluator_manager
 from agenta_backend.models.deprecated_models import ProjectScopedAppDB as AppDB
 from agenta_backend.models.db_models import (
     ProjectDB,
@@ -84,6 +86,73 @@ def create_default_project():
             raise e
 
 
+def add_completion_testset_to_project(session: Session, project_id: str):
+    try:
+        json_path = os.path.join(
+            db_manager.PARENT_DIRECTORY,
+            "resources",
+            "default_testsets",
+            "completion_testset.json",
+        )
+        if os.path.exists(json_path):
+            csvdata = db_manager.get_json(json_path)
+            testset = {
+                "name": f"completion_testset",
+                "csvdata": csvdata,
+            }
+            testset_db = TestSetDB(
+                **testset,
+                project_id=uuid.UUID(project_id),
+            )
+
+            session.add(testset_db)
+            session.commit()
+
+        print("Added completion testset to project.")
+    except Exception as e:
+        print(f"An error occurred in adding the default testset: {e}")
+
+
+def add_default_evaluators_to_project(session: Session, project_id: str):
+    try:
+        direct_use_evaluators = [
+            evaluator
+            for evaluator in evaluator_manager.get_evaluators()
+            if evaluator.direct_use
+        ]
+
+        for evaluator in direct_use_evaluators:
+            settings_values = {
+                setting_name: setting.get("default")
+                for setting_name, setting in evaluator.settings_template.items()
+                if setting.get("ground_truth_key") is True
+                and setting.get("default", "")
+            }
+
+            for setting_name, default_value in settings_values.items():
+                assert (
+                    default_value != ""
+                ), f"Default value for ground truth key '{setting_name}' in Evaluator is empty"
+
+            assert hasattr(evaluator, "name") and hasattr(
+                evaluator, "key"
+            ), f"'name' and 'key' does not exist in the evaluator: {evaluator}"
+
+            evaluator_config = EvaluatorConfigDB(
+                project_id=uuid.UUID(project_id),
+                name=evaluator.name,
+                evaluator_key=evaluator.key,
+                settings_values=settings_values,
+            )
+            session.add(evaluator_config)
+
+        session.commit()
+
+        print("Added default evalutors to project.")
+    except Exception as e:
+        print(f"An error occurred in adding default evaluators: {e}")
+
+
 def remove_default_project():
     engine = create_engine(os.getenv("POSTGRES_URI"))
     sync_session = sessionmaker(engine, expire_on_commit=False)
@@ -137,6 +206,14 @@ def add_project_id_to_db_entities():
                     session.commit()
                     offset += BATCH_SIZE
 
+            # add default testset and evaluators
+            add_completion_testset_to_project(
+                session=session,
+                project_id=str(default_project.id),
+            )
+            add_default_evaluators_to_project(
+                session=session, project_id=str(default_project.id)
+            )
         except Exception as e:
             session.rollback()
             click.echo(

agenta-backend/agenta_backend/migrations/postgres/versions/6cfe239894fb_set_user_id_column_in_db_entities_to_be_.py

@@ -5,6 +5,7 @@
 Create Date: 2024-09-12 15:25:29.462793
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op

agenta-backend/agenta_backend/models/api/api_models.py

@@ -58,6 +58,7 @@ class VariantAction(BaseModel):
 
 class CreateApp(BaseModel):
     app_name: str
+    template_key: Optional[str] = None
     project_id: Optional[str] = None
     workspace_id: Optional[str] = None
 
@@ -199,6 +200,20 @@ class AddVariantFromImagePayload(BaseModel):
     config_name: Optional[str]
 
 
+class AddVariantFromURLPayload(BaseModel):
+    variant_name: str
+    url: str
+    base_name: Optional[str]
+    config_name: Optional[str]
+
+
+class AddVariantFromKeyPayload(BaseModel):
+    variant_name: str
+    key: str
+    base_name: Optional[str]
+    config_name: Optional[str]
+
+
 class ImageExtended(Image):
     # includes the mongodb image id
     id: str