fix: bypass vercel serverless functions when uploading datasets

.env.sample

@@ -1,5 +1,5 @@
 
-BACKEND_ALLOW_ORIGINS=
+BACKEND_ALLOW_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
 BACKEND_HOST=localhost
 BACKEND_PORT=8000
 BACKEND_RELOAD=True

README.md

@@ -76,7 +76,7 @@ The easiest way to get the DocWrangler playground running:
 Create `.env` in the root directory:
 ```bash
 OPENAI_API_KEY=your_api_key_here
-BACKEND_ALLOW_ORIGINS=
+BACKEND_ALLOW_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
 BACKEND_HOST=0.0.0.0
 BACKEND_PORT=8000
 BACKEND_RELOAD=True
@@ -122,7 +122,7 @@ cd docetl
 2. Set up environment variables in `.env` in the root/top-level directory:
 ```bash
 OPENAI_API_KEY=your_api_key_here
-BACKEND_ALLOW_ORIGINS=
+BACKEND_ALLOW_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
 BACKEND_HOST=localhost
 BACKEND_PORT=8000
 BACKEND_RELOAD=True

docs/playground/index.md

@@ -35,7 +35,7 @@ Create `.env` in the root directory (for the FastAPI backend):
 # Required: API key for your preferred LLM provider (OpenAI, Anthropic, etc)
 # The key format will depend on your chosen provider (sk-..., anthro-...)
 OPENAI_API_KEY=your_api_key_here 
-BACKEND_ALLOW_ORIGINS=
+BACKEND_ALLOW_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
 BACKEND_HOST=localhost
 BACKEND_PORT=8000
 BACKEND_RELOAD=True
@@ -85,7 +85,7 @@ cd docetl
 2. Set up environment variables in `.env` in the root directory:
 ```bash
 LLM_API_KEY=your_api_key_here
-BACKEND_ALLOW_ORIGINS=
+BACKEND_ALLOW_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
 BACKEND_HOST=localhost
 BACKEND_PORT=8000
 BACKEND_RELOAD=True

website/src/app/api/convertDocuments/route.ts

@@ -49,7 +49,6 @@ export async function POST(request: NextRequest) {
       }`;
     }
 
-
     // Forward the request to the appropriate backend
     const response = await fetch(targetUrl, {
       method: "POST",

website/src/components/FileExplorer.tsx

@@ -55,6 +55,7 @@ import {
 } from "./ui/tooltip";
 import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
 import { useDatasetUpload } from "@/hooks/useDatasetUpload";
+import { getBackendUrl } from "@/lib/api-config";
 
 interface FileExplorerProps {
   files: File[];
@@ -326,7 +327,7 @@ export const FileExplorer: React.FC<FileExplorerProps> = ({
     setIsConverting(true);
     const formData = new FormData();
 
-    // First, save all original documents and collect their paths
+    // First, save all original documents
     const originalDocsFormData = new FormData();
     Array.from(selectedFiles).forEach((file) => {
       formData.append("files", file);
@@ -336,24 +337,23 @@ export const FileExplorer: React.FC<FileExplorerProps> = ({
       );
     });
     originalDocsFormData.append("namespace", namespace);
-    formData.append("conversion_method", conversionMethod);
 
     try {
-      // First save the original documents
-      const saveDocsResponse = await fetch("/api/saveDocuments", {
-        method: "POST",
-        body: originalDocsFormData,
-      });
+      // Save original documents directly to FastAPI
+      const saveDocsResponse = await fetch(
+        `${getBackendUrl()}/fs/save-documents`,
+        {
+          method: "POST",
+          body: originalDocsFormData,
+        }
+      );
 
       if (!saveDocsResponse.ok) {
         throw new Error("Failed to save original documents");
       }
 
       const savedDocs = await saveDocsResponse.json();
 
-      // Choose endpoint based on conversion method
-      const conversionEndpoint = "/api/convertDocuments";
-
       // Prepare headers for Azure if needed
       const headers: HeadersInit = {};
       if (conversionMethod === "azure") {
@@ -363,21 +363,29 @@ export const FileExplorer: React.FC<FileExplorerProps> = ({
         headers["custom-docling-url"] = customDoclingUrl;
       }
 
-      // Then proceed with conversion
-      const response = await fetch(conversionEndpoint, {
+      // Determine conversion endpoint
+      let targetUrl = `${getBackendUrl()}/api/convert-documents`;
+      if (conversionMethod === "azure") {
+        targetUrl = `${getBackendUrl()}/api/azure-convert-documents`;
+      } else if (conversionMethod === "docetl") {
+        targetUrl = `${getBackendUrl()}/api/convert-documents?use_docetl_server=true`;
+      }
+
+      // Convert documents
+      const response = await fetch(targetUrl, {
         method: "POST",
         body: formData,
         headers,
       });
 
       if (!response.ok) {
-        const errorData = await response.json();
+        const errorData = await response.json().catch(() => ({}));
         throw new Error(errorData.error || "Internal Server Error");
       }
 
       const result = await response.json();
 
-      // Create a folder name based on timestamp
+      // Create folder and upload files
       const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
       const folderName = `converted_${timestamp}`;
 
@@ -402,7 +410,8 @@ export const FileExplorer: React.FC<FileExplorerProps> = ({
       const jsonFormData = new FormData();
       jsonFormData.append("file", jsonFile);
       jsonFormData.append("namespace", namespace);
-      const uploadResponse = await fetch("/api/uploadFile", {
+
+      const uploadResponse = await fetch(`${getBackendUrl()}/fs/upload-file`, {
         method: "POST",
         body: jsonFormData,
       });
@@ -444,7 +453,7 @@ export const FileExplorer: React.FC<FileExplorerProps> = ({
   const handleFileDownload = async (file: File) => {
     try {
       const response = await fetch(
-        `/api/readFile?path=${encodeURIComponent(file.path)}`
+        `${getBackendUrl()}/fs/read-file?path=${encodeURIComponent(file.path)}`
       );
       if (!response.ok) {
         throw new Error("Failed to download file");

website/src/hooks/useDatasetUpload.ts

@@ -1,6 +1,7 @@
 import { useState } from "react";
 import { useToast } from "@/hooks/use-toast";
 import type { File } from "@/app/types";
+import { getBackendUrl } from "@/lib/api-config";
 
 interface UseDatasetUploadOptions {
   namespace: string;
@@ -71,24 +72,21 @@ export function useDatasetUpload({
       return;
     }
 
-    // Add loading indicator immediately
     toast({
       title: "Uploading dataset...",
       description: "This may take a few seconds",
     });
 
-    // Add to uploading files set to show spinner in file list
     setUploadingFiles((prev) => new Set(prev).add(file.name));
 
     try {
-      // Validate JSON structure before uploading
       await validateJsonDataset(file.blob);
 
       const formData = new FormData();
       formData.append("file", file.blob);
       formData.append("namespace", namespace);
 
-      const response = await fetch("/api/uploadFile", {
+      const response = await fetch(`${getBackendUrl()}/fs/upload-file`, {
         method: "POST",
         body: formData,
       });

website/src/lib/api-config.ts

@@ -0,0 +1,6 @@
+export const getBackendUrl = () => {
+  const protocol = process.env.NEXT_PUBLIC_BACKEND_HTTPS ? "https" : "http";
+  const host = process.env.NEXT_PUBLIC_BACKEND_HOST;
+  const port = process.env.NEXT_PUBLIC_BACKEND_PORT;
+  return `${protocol}://${host}:${port}`;
+};