Skip to content

Commit b038acb

Browse files
Parsing (#5)
* Infra for conversation support * Comment fixes * Merging migration * Conversation Drafts * Conversation Drafts * Conversation Drafts * Conversation Drafts * Conversation Drafts * Remove unused models * Basic Agent Setup And Integration in place * logic updates * core parsing * Refactoring for controllers * Dummy user and project setup * Update models.py * Add connection pooling to db * Sanity fixes * Naming refactors * General logic for conversation creation + message management is up * moved to uuid7 to have time ordered ids for better indexing * Agent Table Support * Support for agent table and router * System message integration for conversation starter * User Conversation Fetching Support * Update conversation_service.py * Update conversation_service.py * Streaming chat * Insertion support in database for message_stream * Support for regeneration * Update conversation_service.py * Update projects_service.py * Fixing agent setup * Support for Tool Calling Agent * removing useless comments * Update .env.template * schema fixes * Tool fixes * Fixing runnable with historuy * Memory seperation * PG intergation * Delete history_manager.py * Better naming imports * Naming fixes * Update intelligent_tool_using_orchestrator.py * Tool and chain fix * merge from master * merge from master * Parsing and project management * update migrations * structure changes, logging etc * delete test apis --------- Co-authored-by: Vineet <[email protected]> Co-authored-by: Vineet <[email protected]>
1 parent dbee40b commit b038acb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+2915
-44
lines changed

app/alembic/versions/20240820182032_d3f532773223_changes_for_implementation_of_.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -95,4 +95,4 @@ def downgrade() -> None:
9595

9696
# Drop the ENUM type if it is no longer used
9797
message_status_enum.drop(op.get_bind(), checkfirst=False)
98-
# ### end Alembic commands ###
98+
# ### end Alembic commands ###
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
"""project id to string anddelete col
2+
3+
Revision ID: 20240823164559_05069444feee
4+
Revises: 20240820182032_d3f532773223
5+
Create Date: 2024-08-23 16:45:59.991109
6+
7+
"""
8+
from typing import Sequence, Union
9+
10+
from alembic import op
11+
import sqlalchemy as sa
12+
from sqlalchemy.dialects import postgresql
13+
14+
# revision identifiers, used by Alembic.
15+
revision: str = '20240823164559_05069444feee'
16+
down_revision: Union[str, None] = '20240820182032_d3f532773223'
17+
branch_labels: Union[str, Sequence[str], None] = None
18+
depends_on: Union[str, Sequence[str], None] = None
19+
20+
21+
def upgrade() -> None:
22+
# ### commands auto generated by Alembic - please adjust! ###
23+
op.alter_column('projects', 'id',
24+
existing_type=sa.INTEGER(),
25+
type_=sa.Text(),
26+
existing_nullable=False)
27+
op.drop_constraint('projects_directory_key', 'projects', type_='unique')
28+
op.drop_column('projects', 'directory')
29+
op.drop_column('projects', 'is_default')
30+
op.drop_column('projects', 'project_name')
31+
op.drop_constraint('check_status', 'projects', type_='check')
32+
op.create_check_constraint('check_status', 'projects',
33+
"status IN ('submitted', 'cloned', 'parsed', 'ready', 'error')")
34+
# ### end Alembic commands ###
35+
36+
37+
def downgrade() -> None:
38+
# ### commands auto generated by Alembic - please adjust! ###
39+
op.add_column('projects', sa.Column('project_name', sa.TEXT(), autoincrement=False, nullable=True))
40+
op.add_column('projects', sa.Column('is_default', sa.BOOLEAN(), autoincrement=False, nullable=True))
41+
op.add_column('projects', sa.Column('directory', sa.TEXT(), autoincrement=False, nullable=True))
42+
op.create_unique_constraint('projects_directory_key', 'projects', ['directory'])
43+
op.alter_column('projects', 'id',
44+
existing_type=sa.Text(),
45+
type_=sa.INTEGER(),
46+
existing_nullable=False)
47+
op.drop_constraint('check_status', 'projects', type_='check')
48+
op.create_check_constraint('check_status', 'projects',
49+
"status IN ('created', 'ready', 'error')")
50+
# ### end Alembic commands ###

app/core/config.py

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
from dotenv import load_dotenv
2+
import os
3+
4+
load_dotenv()
5+
6+
class ConfigProvider:
7+
def __init__(self):
8+
self.neo4j_config = {
9+
"uri": os.getenv("NEO4J_URI"),
10+
"username": os.getenv("NEO4J_USERNAME"),
11+
"password": os.getenv("NEO4J_PASSWORD"),
12+
}
13+
self.github_key = os.getenv("GITHUB_PRIVATE_KEY")
14+
15+
def get_neo4j_config(self):
16+
return self.neo4j_config
17+
18+
def get_github_key(self):
19+
return self.github_key
20+
21+
config_provider = ConfigProvider()

app/core/mongo_manager.py

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import os
2+
import logging
3+
from pymongo import MongoClient
4+
from pymongo.errors import ConnectionFailure, OperationFailure
5+
from typing import Optional
6+
import certifi
7+
8+
class MongoManager:
9+
_instance = None
10+
_client: Optional[MongoClient] = None
11+
_db = None
12+
13+
@classmethod
14+
def get_instance(cls):
15+
if cls._instance is None:
16+
cls._instance = cls()
17+
return cls._instance
18+
19+
def __init__(self):
20+
if self._instance is not None:
21+
raise RuntimeError("Use get_instance() to get the MongoManager instance")
22+
self._connect()
23+
24+
def _connect(self):
25+
if self._client is None:
26+
try:
27+
mongodb_uri = os.environ.get("MONGO_URI")
28+
if not mongodb_uri:
29+
raise ValueError("MONGO_URI environment variable is not set")
30+
31+
self._client = MongoClient(
32+
mongodb_uri,
33+
maxPoolSize=50,
34+
waitQueueTimeoutMS=2500,
35+
tlsCAFile=certifi.where() # Use the certifi package to locate the CA bundle
36+
)
37+
38+
db_name = os.environ.get("MONGODB_DB_NAME")
39+
if not db_name:
40+
raise ValueError("MONGODB_DB_NAME environment variable is not set")
41+
42+
self._db = self._client[db_name]
43+
44+
# Verify the connection and database
45+
self.verify_connection()
46+
47+
except (ConnectionFailure, ValueError) as e:
48+
logging.error(f"Failed to connect to MongoDB: {str(e)}")
49+
raise
50+
51+
def verify_connection(self):
52+
try:
53+
# Ping the server to check the connection
54+
self._client.admin.command('ping')
55+
56+
# List all collections to verify database access
57+
self._db.list_collection_names()
58+
59+
logging.info("Successfully connected to MongoDB and verified database access")
60+
except OperationFailure as e:
61+
logging.error(f"Failed to verify MongoDB connection: {str(e)}")
62+
raise
63+
64+
def get_collection(self, collection_name: str):
65+
self._connect() # Ensure connection is established
66+
return self._db[collection_name]
67+
68+
def put(self, collection_name: str, document_id: str, data: dict):
69+
try:
70+
collection = self.get_collection(collection_name)
71+
result = collection.update_one(
72+
{"_id": document_id},
73+
{"$set": data},
74+
upsert=True
75+
)
76+
logging.info(f"Document {'updated' if result.modified_count else 'inserted'} in {collection_name}")
77+
return result
78+
except Exception as e:
79+
logging.error(f"Failed to put document in {collection_name}: {str(e)}")
80+
raise
81+
82+
def get(self, collection_name: str, document_id: str):
83+
try:
84+
collection = self.get_collection(collection_name)
85+
document = collection.find_one({"_id": document_id})
86+
if document:
87+
logging.info(f"Document retrieved from {collection_name}")
88+
else:
89+
logging.info(f"Document not found in {collection_name}")
90+
return document
91+
except Exception as e:
92+
logging.error(f"Failed to get document from {collection_name}: {str(e)}")
93+
raise
94+
95+
def delete(self, collection_name: str, document_id: str):
96+
try:
97+
collection = self.get_collection(collection_name)
98+
result = collection.delete_one({"_id": document_id})
99+
if result.deleted_count:
100+
logging.info(f"Document deleted from {collection_name}")
101+
else:
102+
logging.info(f"Document not found in {collection_name}")
103+
return result
104+
except Exception as e:
105+
logging.error(f"Failed to delete document from {collection_name}: {str(e)}")
106+
raise
107+
108+
def close(self):
109+
if self._client:
110+
self._client.close()
111+
self._client = None
112+
self._db = None
113+
logging.info("MongoDB connection closed")
114+
115+
def reconnect(self):
116+
self.close()
117+
self._connect()
118+
logging.info("Reconnected to MongoDB")
119+
120+
@classmethod
121+
def close_connection(cls):
122+
if cls._instance:
123+
cls._instance.close()
124+
cls._instance = None
125+
logging.info("MongoDB connection closed and instance reset")
126+
127+
def __enter__(self):
128+
return self
129+
130+
def __exit__(self, exc_type, exc_val, exc_tb):
131+
pass # Don't close the connection here

app/main.py

+31-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import os
22
import logging
33

4+
# Configure logging
5+
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
6+
47
from dotenv import load_dotenv
58
from fastapi import FastAPI
69
from fastapi.middleware.cors import CORSMiddleware
@@ -12,15 +15,35 @@
1215

1316
from app.modules.utils.dummy_setup import DummyDataSetup
1417

18+
from app.modules.utils.firebase_setup import FirebaseSetup
19+
from app.modules.parsing.graph_construction.parsing_router import router as parsing_router
20+
from app.modules.auth.auth_router import auth_router
21+
from app.modules.key_management.secret_manager import router as secret_manager_router
22+
23+
from app.core.mongo_manager import MongoManager
24+
1525
class MainApp:
1626
def __init__(self):
1727
load_dotenv(override=True)
1828
self.app = FastAPI()
1929
self.setup_cors()
2030
self.initialize_database()
2131
self.check_and_set_env_vars()
22-
self.setup_data()
32+
if os.getenv("isDevelopmentMode") == "enabled":
33+
self.setup_data()
34+
else:
35+
FirebaseSetup.firebase_init()
2336
self.include_routers()
37+
self.verify_mongodb_connection()
38+
39+
def verify_mongodb_connection(self):
40+
try:
41+
mongo_manager = MongoManager.get_instance()
42+
mongo_manager.verify_connection()
43+
logging.info("MongoDB connection verified successfully")
44+
except Exception as e:
45+
logging.error(f"Failed to verify MongoDB connection: {str(e)}")
46+
raise
2447

2548
def setup_cors(self):
2649
origins = ["*"]
@@ -56,6 +79,9 @@ def setup_data(self):
5679
def include_routers(self):
5780
self.app.include_router(user_router, prefix="/api/v1", tags=["User"])
5881
self.app.include_router(conversations_router, prefix="/api/v1", tags=["Conversations"])
82+
self.app.include_router(parsing_router, prefix="/api/v1", tags=["Parsing"])
83+
self.app.include_router(auth_router, prefix="/api/v1", tags=["Auth"])
84+
self.app.include_router(secret_manager_router, prefix="/api/v1", tags=["Secret Manager"])
5985

6086

6187
def add_health_check(self):
@@ -67,7 +93,10 @@ def run(self):
6793
self.add_health_check()
6894
return self.app
6995

70-
7196
# Create an instance of MainApp and run it
7297
main_app = MainApp()
7398
app = main_app.run()
99+
100+
@app.on_event("shutdown")
101+
def shutdown_event():
102+
MongoManager.close_connection()

app/modules/auth/auth_router.py

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import json
2+
import os
3+
4+
from datetime import datetime
5+
from dotenv import load_dotenv
6+
7+
from fastapi import Depends, Request
8+
from fastapi.responses import JSONResponse, Response
9+
from sqlalchemy.orm import Session
10+
from app.core.database import get_db
11+
from app.modules.auth.auth_service import auth_handler
12+
from app.modules.users.user_service import UserService
13+
14+
from app.modules.utils.APIRouter import APIRouter
15+
16+
from .auth_schema import LoginRequest
17+
from app.modules.users.user_schema import CreateUser
18+
19+
import logging
20+
21+
auth_router = APIRouter()
22+
load_dotenv(override=True)
23+
24+
class AuthAPI:
25+
@auth_router.post("/login")
26+
async def login(login_request: LoginRequest):
27+
email, password = login_request.email, login_request.password
28+
29+
try:
30+
res = auth_handler.login(email=email, password=password)
31+
id_token = res.get("idToken")
32+
return JSONResponse(content={"token": id_token}, status_code=200)
33+
except Exception as e:
34+
return JSONResponse(
35+
content={"error": f"ERROR: {str(e)}"}, status_code=400
36+
)
37+
38+
@auth_router.post("/signup")
39+
async def signup(request: Request, db: Session = Depends(get_db)):
40+
body = json.loads(await request.body())
41+
uid = body["uid"]
42+
user_service = UserService(db)
43+
user = user_service.get_user_by_uid(uid)
44+
if user:
45+
message, error = user_service.update_last_login(uid)
46+
if error:
47+
return Response(content=message, status_code=400)
48+
else:
49+
return Response(content=json.dumps({"uid": uid}), status_code=200)
50+
else:
51+
first_login = datetime.utcnow()
52+
user = CreateUser(
53+
uid=uid,
54+
email=body["email"],
55+
display_name=body["displayName"],
56+
email_verified=body["emailVerified"],
57+
created_at=first_login,
58+
last_login_at=first_login,
59+
provider_info=body["providerData"][0],
60+
provider_username=body["providerUsername"]
61+
)
62+
uid, message, error = user_service.create_user(user)
63+
if error:
64+
return Response(content=message, status_code=400)
65+
return Response(content=json.dumps({"uid": uid}), status_code=201)

app/modules/auth/auth_schema.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from pydantic import BaseModel
2+
3+
4+
class LoginRequest(BaseModel):
5+
email: str
6+
password: str

0 commit comments

Comments
 (0)