Create py_oso package (#3054)

* feat(py_oso): create py_oso python package

* feat(sql): small improvements on error handling for sql route

* fix(py_oso): test with no auth key should mock env vars

* refactor(pyoso): rename package to pyoso

* refactor(frontend): change clickhouse sql to return streamed result

.gitignore

@@ -62,6 +62,8 @@ dbt_packages/
 
 # Python
 *.pyc
+__pycache__/
+*.egg-info/
 
 supabase/.temp/
 **/supabase/.temp/

apps/frontend/app/api/v1/sql/route.ts

@@ -1,5 +1,6 @@
 import { NextResponse, type NextRequest } from "next/server";
 import { getClickhouseClient } from "../../../../lib/clients/clickhouse";
+import { ClickHouseError } from "@clickhouse/client";
 //import { logger } from "../../../lib/logger";
 
 // Next.js route control
@@ -11,20 +12,15 @@ export const revalidate = 0;
 const QUERY_PARAM = "query";
 
 // Helper functions for creating responses suitable for Hasura
-const makeError = (errorMsg: string) => ({
-  error: errorMsg,
-});
+const makeErrorResponse = (errorMsg: string, status: number) =>
+  NextResponse.json({ error: errorMsg }, { status });
 
 async function doQuery(rawQuery: string) {
   const query = decodeURIComponent(rawQuery);
   console.log(`Running query: ${query}`);
   const client = getClickhouseClient();
-  const rows = await client.query({ query });
-  const resultSet = await rows.json();
-  // resultSet includes query statistics and metadata
-  //console.log(JSON.stringify(resultSet, null, 2));
-  const data = resultSet.data;
-  return data;
+  const rows = await client.query({ query, format: "JSONEachRow" });
+  return rows;
 }
 
 /**
@@ -34,30 +30,40 @@ async function doQuery(rawQuery: string) {
  * @param request
  * @returns
  */
-export async function GET(request: NextRequest) {
-  const searchParams = request.nextUrl.searchParams;
-  const query = searchParams.get(QUERY_PARAM);
+export async function POST(request: NextRequest) {
+  const body = await request.json();
+  const query = body?.[QUERY_PARAM];
+  // TODO: add authentication
   //const auth = request.headers.get("authorization");
 
   // If no query provided, short-circuit
   if (!query) {
     console.log(`/api/sql: Missing query`);
-    return NextResponse.json(makeError("Please provide a 'query' parameter"));
+    return makeErrorResponse("Please provide a 'query' parameter", 400);
   }
-  const result = await doQuery(query);
-  return NextResponse.json(result);
-}
+  try {
+    const stream = (await doQuery(query)).stream();
 
-export async function POST(request: NextRequest) {
-  const body = await request.json();
-  const query = body[QUERY_PARAM];
-  //const auth = request.headers.get("authorization");
+    const readableStream = new ReadableStream({
+      start(controller) {
+        stream.on("data", (chunk) => {
+          console.log("sending", chunk.length);
+          controller.enqueue(JSON.stringify(chunk.map((r) => r.json())));
+        });
+        stream.on("end", () => {
+          controller.close();
+        });
+        stream.on("error", (error) => {
+          controller.error(error);
+        });
+      },
+    });
 
-  // If no query provided, short-circuit
-  if (!query) {
-    console.log(`/api/sql: Missing query`);
-    return NextResponse.json(makeError("Please provide a 'query' parameter"));
+    return new NextResponse(readableStream);
+  } catch (e) {
+    if (e instanceof ClickHouseError) {
+      return makeErrorResponse(e.message, 400);
+    }
+    return makeErrorResponse("Unknown error", 500);
   }
-  const result = await doQuery(query);
-  return NextResponse.json(result);
 }

apps/frontend/package.json

@@ -28,7 +28,7 @@
     "@apollo/server": "^4.10.5",
     "@apollo/subgraph": "^2.8.4",
     "@as-integrations/next": "^3.0.0",
-    "@clickhouse/client": "^1.4.0",
+    "@clickhouse/client": "^1.10.1",
     "@cubejs-client/core": "^0.35.23",
     "@cubejs-client/react": "^0.35.48",
     "@emotion/react": "^11.11.4",