Initial swap from redis to postgres

backend/alembic/versions/71d60844180b_add_summary_and_appstream_jsonb_columns.py

@@ -0,0 +1,27 @@
+"""add_summary_and_appstream_jsonb_columns
+
+Revision ID: 71d60844180b
+Revises: d7e1d2130fdb
+Create Date: 2025-02-07 23:54:12.326725
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB
+
+
+# revision identifiers, used by Alembic.
+revision = '71d60844180b'
+down_revision = 'd7e1d2130fdb'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column('apps', sa.Column('summary', JSONB, nullable=True))
+    op.add_column('apps', sa.Column('appstream', JSONB, nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column('apps', 'appstream')
+    op.drop_column('apps', 'summary')

backend/app/app.py

@@ -416,20 +416,42 @@ def get_summary(
     ),
     branch: Optional[str] = None,
 ):
+    with get_db("replica") as db_session:
+        app = models.App.by_appid(db_session, app_id)
+        if app and app.summary:
+            value = app.summary
+            if (
+                "metadata" in value
+                and value["metadata"]
+                and "runtime" in value["metadata"]
+            ):
+                runtime_appid, _, runtime_branch = value["metadata"]["runtime"].split(
+                    "/"
+                )
+                value["metadata"]["runtimeIsEol"] = bool(
+                    db.get_json_key(f"eol_message:{runtime_appid}:{runtime_branch}")
+                )
+            return value
+
+    # Fall back to Redis
     if not branch:
         branch = db.get_json_key(f"summary:{app_id}")
 
-    key = f"summary:{app_id}:{branch}"
-
-    if value := db.get_json_key(key):
-        if "metadata" in value and value["metadata"] and "runtime" in value["metadata"]:
-            runtime_appid, _, runtime_branch = value["metadata"]["runtime"].split("/")
-
-            value["metadata"]["runtimeIsEol"] = bool(
-                db.get_json_key(f"eol_message:{runtime_appid}:{runtime_branch}")
-            )
-
-        return value
+    if branch:
+        key = f"summary:{app_id}:{branch}"
+        if value := db.get_json_key(key):
+            if (
+                "metadata" in value
+                and value["metadata"]
+                and "runtime" in value["metadata"]
+            ):
+                runtime_appid, _, runtime_branch = value["metadata"]["runtime"].split(
+                    "/"
+                )
+                value["metadata"]["runtimeIsEol"] = bool(
+                    db.get_json_key(f"eol_message:{runtime_appid}:{runtime_branch}")
+                )
+            return value
 
     response.status_code = 404
     return None

backend/app/apps.py

@@ -153,10 +153,20 @@ def load_appstream(sqldb) -> None:
                 }
             )
 
-            models.App.set_app(sqldb, app_id, type, apps[app_id]["locales"])
+            app_data = apps[app_id].copy()
+            locales = app_data.pop("locales")
 
-            del apps[app_id]["locales"]
-            p.set(redis_key, json.dumps(apps[app_id]))
+            try:
+                app = models.App.set_app(sqldb, app_id, type, locales)
+                if app:
+                    app.appstream = app_data
+                    sqldb.session.add(app)
+                    sqldb.session.commit()
+
+                p.set(redis_key, json.dumps(app_data))
+            except Exception as e:
+                sqldb.session.rollback()
+                print(f"Error updating app {app_id}: {str(e)}")
 
         search.create_or_update_apps(search_apps)
 
@@ -203,23 +213,23 @@ def get_appids(type: AppType = AppType.APPS) -> list[str]:
 
 def get_addons(app_id: str, branch: str = "stable") -> list[str]:
     result = []
-    summary = db.get_json_key(f"summary:{app_id}:{branch}")
-    if (
-        summary
-        and "metadata" in summary
-        and summary["metadata"]
-        and "extensions" in summary["metadata"]
-        and summary["metadata"]["extensions"]
-    ):
-        extension_ids = list(summary["metadata"]["extensions"].keys())
-
-        with database.get_db() as sqldb:
-            addons = set(
-                addon.app_id
-                for addon in sqldb.query(models.App.app_id)
-                .filter(models.App.type == "addon")
-                .all()
-            )
+
+    with database.get_db() as sqldb:
+        app = models.App.by_appid(sqldb, app_id)
+        if not app or not app.summary:
+            return result
+
+        metadata = app.summary.get("metadata", {})
+        if not metadata or "extensions" not in metadata:
+            return result
+
+        extension_ids = list(metadata["extensions"].keys())
+        addons = set(
+            addon.app_id
+            for addon in sqldb.query(models.App.app_id)
+            .filter(models.App.type == "addon")
+            .all()
+        )
 
         for addon in addons:
             for extension_id in extension_ids:

backend/app/models.py

@@ -24,6 +24,7 @@
     func,
     or_,
 )
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column, relationship
 
 from . import utils
@@ -2249,6 +2250,8 @@ class App(Base):
     )
     last_updated_at = mapped_column(DateTime, nullable=True)
     localization_json = mapped_column(JSON, nullable=True)
+    summary = mapped_column(JSONB, nullable=True)
+    appstream = mapped_column(JSONB, nullable=True)
 
     __table_args__ = (Index("apps_unique", app_id, unique=True),)
 

backend/app/moderation.py

@@ -389,13 +389,25 @@ def submit_review_request(
                     current_values = {"direct upload": False}
                     keys = {"direct upload": True}
 
-        if current_summary := get_json_key(f"summary:{app_id}:stable"):
-            sentry_context[f"summary:{app_id}:stable"] = current_summary
+        with get_db("replica") as db:
+            current_summary = None
+
+            app = models.App.by_appid(db, app_id)
+            if app and app.summary:
+                current_summary = app.summary
+                sentry_context[f"summary:{app_id}:stable"] = current_summary
+
+                if current_metadata := current_summary.get("metadata", {}):
+                    current_permissions = current_metadata.get("permissions")
+                    current_extradata = bool(current_metadata.get("extra-data"))
+            elif current_summary := get_json_key(f"summary:{app_id}:stable"):
+                sentry_context[f"summary:{app_id}:stable"] = current_summary
 
-            if current_metadata := current_summary.get("metadata", {}):
-                current_permissions = current_metadata.get("permissions")
-                current_extradata = bool(current_metadata.get("extra-data"))
+                if current_metadata := current_summary.get("metadata", {}):
+                    current_permissions = current_metadata.get("permissions")
+                    current_extradata = bool(current_metadata.get("extra-data"))
 
+            if current_summary:
                 build_summary_metadata = build_summary.get(app_id, {}).get(
                     "metadata", {}
                 )

backend/app/summary.py

@@ -233,8 +233,6 @@ def update(sqldb) -> None:
         ]
     )
 
-    # todo: it seems like we only ever have one branch, so this is fine
-    # but why do we have branches then?
     db.redis_conn.mset(
         {
             f"summary:{app_id}": json.dumps(
@@ -253,6 +251,39 @@ def update(sqldb) -> None:
         }
     )
 
+    # collect all app IDs to update
+    apps_to_update = {}
+    for app_id, data in summary_dict.items():
+        try:
+            summary_json = json.loads(json.dumps(data, cls=JSONSetEncoder))
+            apps_to_update[app_id] = summary_json
+        except Exception as e:
+            print(f"Error encoding summary data for {app_id}: {str(e)}")
+            continue
+
+    # update all apps in a single transaction
+    try:
+        for app_id, summary_json in apps_to_update.items():
+            app = models.App.by_appid(sqldb, app_id)
+            if app:
+                appstream_data = app.appstream
+                app.summary = summary_json
+                if appstream_data:
+                    app.appstream = appstream_data
+                sqldb.session.add(app)
+            else:
+                app = models.App(
+                    app_id=app_id,
+                    type="desktop-application",
+                    summary=summary_json,
+                )
+                sqldb.session.add(app)
+
+        sqldb.session.commit()
+    except Exception as e:
+        sqldb.session.rollback()
+        print(f"Error updating apps: {str(e)}")
+
     eol_rebase: dict[str, str] = {}
     eol_message: dict[str, str] = {}
     for app, eol_dict in metadata["xa.sparse-cache"].items():

backend/app/worker.py

@@ -87,10 +87,14 @@ def update():
         if created_at:
             apps_created_at[app_id] = created_at.timestamp()
         else:
-            if metadata := db.get_json_key(f"summary:{app_id}:stable"):
-                created_at = metadata.get("timestamp")
-            else:
-                created_at = int(datetime.utcnow().timestamp())
+            with WorkerDB() as sqldb:
+                app = models.App.by_appid(sqldb, app_id)
+                if app and app.summary and "timestamp" in app.summary:
+                    created_at = app.summary["timestamp"]
+                elif metadata := db.get_json_key(f"summary:{app_id}:stable"):
+                    created_at = metadata.get("timestamp")
+                else:
+                    created_at = int(datetime.utcnow().timestamp())
 
             apps_created_at[app_id] = float(created_at)
             with WorkerDB() as sqldb: