diff --git a/packages/common-react/src/toastify-components/callbacks.tsx b/packages/common-react/src/toastify-components/callbacks.tsx index 34ef3a74e..c298a1f60 100644 --- a/packages/common-react/src/toastify-components/callbacks.tsx +++ b/packages/common-react/src/toastify-components/callbacks.tsx @@ -8,10 +8,14 @@ interface ToastKernelCallbacks { onLoad: NonNullable; onError: NonNullable; } -export function makeToastKernelCallbacks(): ToastKernelCallbacks { +export function makeToastKernelCallbacks(disableProgressToasts = false, disableErrorToasts = false): ToastKernelCallbacks { let prevToastId: ToastId | null = null; const toastIds: ToastId[] = []; const onProgress: StliteKernelOptions["onProgress"] = (message) => { + if (disableProgressToasts) { + return; + } + const id = toast(message, { position: toast.POSITION.BOTTOM_RIGHT, transition: Slide, @@ -33,6 +37,10 @@ export function makeToastKernelCallbacks(): ToastKernelCallbacks { toastIds.forEach((id) => toast.dismiss(id)); }; const onError: StliteKernelOptions["onError"] = (error) => { + if (disableErrorToasts) { + return; + } + toast( , { diff --git a/packages/common-react/tsconfig.json b/packages/common-react/tsconfig.json index 1b5ab51cd..01d6d4a35 100644 --- a/packages/common-react/tsconfig.json +++ b/packages/common-react/tsconfig.json @@ -28,11 +28,7 @@ // "rootDir": "./", /* Specify the root folder within your source files. */ "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ - "paths": { - "src/theme": ["../../streamlit/frontend/src/theme"], - "src/theme/*": ["../../streamlit/frontend/src/theme/*"], - "src/lib/*": ["../../streamlit/frontend/src/lib/*"], - } /* Specify a set of entries that re-map imports to additional lookup locations. */, + // "paths": {} /* Specify a set of entries that re-map imports to additional lookup locations. */, // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ // "types": [], /* Specify type package names to be included without being referenced in a source file. */ diff --git a/packages/kernel/py/stlite-server/stlite_server/bootstrap.py b/packages/kernel/py/stlite-server/stlite_server/bootstrap.py index 6ecccd862..7db70ecb2 100644 --- a/packages/kernel/py/stlite-server/stlite_server/bootstrap.py +++ b/packages/kernel/py/stlite-server/stlite_server/bootstrap.py @@ -123,6 +123,30 @@ def _on_pages_changed(_path: str) -> None: allow_nonexistent=True, ) +def _fix_altair(): + """Fix an issue with Altair and the mocked pyarrow module of stlite.""" + try: + from altair.utils import _importers + + def _pyarrow_available(): + return False + + _importers.pyarrow_available = _pyarrow_available + + def _import_pyarrow_interchange(): + raise ImportError("Pyarrow is not available in stlite.") + + _importers.import_pyarrow_interchange = _import_pyarrow_interchange + except: + pass + +def _fix_requests(): + try: + import pyodide_http + pyodide_http.patch_all() # Patch all libraries + except ImportError: + # pyodide_http is not installed. No need to do anything. + pass def prepare( main_script_path: str, @@ -135,6 +159,8 @@ def prepare( """ _fix_sys_path(main_script_path) _fix_matplotlib_crash() + _fix_altair() + _fix_requests() _fix_sys_argv(main_script_path, args) _fix_pydeck_mapbox_api_warning() _install_pages_watcher(main_script_path) diff --git a/packages/kernel/py/stlite-server/stlite_server/handler.py b/packages/kernel/py/stlite-server/stlite_server/handler.py index 7fb35446a..5be975ce8 100644 --- a/packages/kernel/py/stlite-server/stlite_server/handler.py +++ b/packages/kernel/py/stlite-server/stlite_server/handler.py @@ -19,5 +19,8 @@ class RequestHandler(abc.ABC): def get(self, request: Request) -> Response | Awaitable[Response]: return Response(status_code=405, headers={}, body="") - def post(self, request: Request) -> Response | Awaitable[Response]: + def put(self, request: Request) -> Response | Awaitable[Response]: + return Response(status_code=405, headers={}, body="") + + def delete(self, request: Request) -> Response | Awaitable[Response]: return Response(status_code=405, headers={}, body="") diff --git a/packages/kernel/py/stlite-server/stlite_server/server.py b/packages/kernel/py/stlite-server/stlite_server/server.py index 473e63ec2..7991b7fea 100644 --- a/packages/kernel/py/stlite-server/stlite_server/server.py +++ b/packages/kernel/py/stlite-server/stlite_server/server.py @@ -11,6 +11,10 @@ from streamlit.runtime import Runtime, RuntimeConfig, SessionClient from streamlit.runtime.memory_media_file_storage import MemoryMediaFileStorage from streamlit.runtime.runtime_util import serialize_forward_msg +from streamlit.runtime.memory_uploaded_file_manager import MemoryUploadedFileManager +from streamlit.web.cache_storage_manager_config import ( + create_default_cache_storage_manager, +) from .component_request_handler import ComponentRequestHandler from .handler import RequestHandler @@ -21,9 +25,10 @@ LOGGER = logging.getLogger(__name__) -# These route definitions are copied from the original impl at https://github.com/streamlit/streamlit/blob/1.18.1/lib/streamlit/web/server/server.py#L81-L89 # noqa: E501 +# These route definitions are copied from the original impl at https://github.com/streamlit/streamlit/blob/1.27.0/lib/streamlit/web/server/server.py#L83-L92 # noqa: E501 +UPLOAD_FILE_ENDPOINT: Final = "/_stcore/upload_file" MEDIA_ENDPOINT: Final = "/media" -STREAM_ENDPOINT: Final = r"_stcore/stream" +STREAM_ENDPOINT: Final = "_stcore/stream" HEALTH_ENDPOINT: Final = r"(?:healthz|_stcore/health)" @@ -34,12 +39,15 @@ def __init__(self, main_script_path: str, command_line: str | None) -> None: self._main_script_path = main_script_path self._media_file_storage = MemoryMediaFileStorage(MEDIA_ENDPOINT) + self.uploaded_file_mgr = MemoryUploadedFileManager(UPLOAD_FILE_ENDPOINT) self._runtime = Runtime( RuntimeConfig( script_path=main_script_path, command_line=command_line, media_file_storage=self._media_file_storage, + uploaded_file_manager=self.uploaded_file_mgr, + cache_storage_manager=create_default_cache_storage_manager(), ), ) @@ -152,8 +160,8 @@ def receive_http( on_response(404, {}, b"No handler found") return method_name = method.lower() - if method_name not in ("get", "post"): - on_response(405, {}, b"Now allowed") + if method_name not in ("get", "put", "delete"): + on_response(405, {}, b"Not allowed") return handler_method = getattr(handler, method_name, None) if handler_method is None: diff --git a/packages/kernel/py/stlite-server/stlite_server/server_util.py b/packages/kernel/py/stlite-server/stlite_server/server_util.py index f25d03a23..a921781a8 100644 --- a/packages/kernel/py/stlite-server/stlite_server/server_util.py +++ b/packages/kernel/py/stlite-server/stlite_server/server_util.py @@ -1,4 +1,4 @@ -# Copied from https://github.com/streamlit/streamlit/blob/1.18.1/lib/streamlit/web/server/server_util.py#L73-L77 # noqa: E501 +# Copied from https://github.com/streamlit/streamlit/blob/1.27.1/lib/streamlit/web/server/server_util.py#L73-L77 # noqa: E501 def make_url_path_regex(*path: str, **kwargs) -> str: """Get a regex of the form ^/foo/bar/baz/?$ for a path (foo, bar, baz).""" valid_path = [x.strip("/") for x in path if x] # Filter out falsely components. diff --git a/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py b/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py index 09c6af9b4..fefa7b600 100644 --- a/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py +++ b/packages/kernel/py/stlite-server/stlite_server/upload_file_request_handler.py @@ -1,14 +1,15 @@ import logging from typing import Callable, Dict, List +import re from streamlit.runtime.uploaded_file_manager import UploadedFileManager, UploadedFileRec from .handler import Request, RequestHandler, Response from .httputil import HTTPFile, parse_body_arguments -# /_stcore/upload_file/(optional session id)/(optional widget id) +# /_stcore/upload_file/(optional session id)/(optional file id) UPLOAD_FILE_ROUTE = ( - r"/_stcore/upload_file/?(?P[^/]*)?/?(?P[^/]*)?" + r"/_stcore/upload_file/(?P[^/]+)/(?P[^/]+)" ) LOGGER = logging.getLogger(__name__) @@ -40,7 +41,7 @@ def _require_arg(args: Dict[str, List[bytes]], name: str) -> str: # Convert bytes to string return arg[0].decode("utf-8") - def post(self, request: Request, **kwargs) -> Response: + def put(self, request: Request, **kwargs) -> Response: # NOTE: The original implementation uses an async function, # but it didn't make use of any async features, # so we made it a regular function here for simplicity sake. @@ -61,8 +62,11 @@ def post(self, request: Request, **kwargs) -> Response: ) try: - session_id = self._require_arg(args, "sessionId") - widget_id = self._require_arg(args, "widgetId") + path_args = re.match(UPLOAD_FILE_ROUTE, request.path) + session_id = path_args.group('session_id') + file_id = path_args.group('file_id') + # session_id = self._require_arg(args, "sessionId") + # file_id = self._require_arg(args, "fileId") if not self._is_active_session(session_id): raise Exception(f"Invalid session_id: '{session_id}'") @@ -78,7 +82,7 @@ def post(self, request: Request, **kwargs) -> Response: for file in flist: uploaded_files.append( UploadedFileRec( - id=0, + file_id=file_id, name=file.filename, type=file.content_type, data=file.body, @@ -92,10 +96,17 @@ def post(self, request: Request, **kwargs) -> Response: body=f"Expected 1 file, but got {len(uploaded_files)}", ) - added_file = self._file_mgr.add_file( - session_id=session_id, widget_id=widget_id, file=uploaded_files[0] + self._file_mgr.add_file( + session_id=session_id, file=uploaded_files[0] ) + return Response(status_code=204, headers={}, body="") - # Return the file_id to the client. (The client will parse - # the string back to an int.) - return Response(status_code=200, headers={}, body=str(added_file.id)) + def delete(self, request: Request, **kwargs): + """Delete file request handler.""" + + path_args = re.match(UPLOAD_FILE_ROUTE, request.path) + session_id = path_args.group('session_id') + file_id = path_args.group('file_id') + + self._file_mgr.remove_file(session_id=session_id, file_id=file_id) + self.set_status(204) diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index a8ec3acf4..6472649aa 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -1,5 +1,4 @@ export * from "./kernel"; export * from "./streamlit-replacements/lib/ConnectionManager"; -export * from "./streamlit-replacements/lib/FileUploadClient"; export * from "./react-helpers"; export * from "./types"; diff --git a/packages/kernel/src/kernel.ts b/packages/kernel/src/kernel.ts index 066a2f196..b06736426 100644 --- a/packages/kernel/src/kernel.ts +++ b/packages/kernel/src/kernel.ts @@ -107,6 +107,11 @@ export interface StliteKernelOptions { */ streamlitConfig?: StreamlitConfig; + /** + * If true, no toasts will be shown on loading progress steps. + */ + disableProgressToasts?: boolean; + onProgress?: (message: string) => void; onLoad?: () => void; @@ -186,6 +191,7 @@ export class StliteKernel { archives: options.archives, requirements: options.requirements, pyodideUrl: options.pyodideUrl, + disableProgressToasts: options.disableProgressToasts, wheels, mountedSitePackagesSnapshotFilePath: options.mountedSitePackagesSnapshotFilePath, diff --git a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts index 523c8c641..878be9560 100644 --- a/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts +++ b/packages/kernel/src/streamlit-replacements/lib/ConnectionManager.ts @@ -1,17 +1,22 @@ -// Mimic https://github.com/streamlit/streamlit/blob/1.9.0/frontend/src/lib/ConnectionManager.ts +// Mimic https://github.com/streamlit/streamlit/blob/1.27.0/frontend/app/src/connection/ConnectionManager.ts // and WebsocketConnection. import type { ReactNode } from "react" -import { BackMsg, ForwardMsg } from "@streamlit/lib/src/proto" -import type { IAllowedMessageOriginsResponse } from "@streamlit/lib/src/hostComm/types" -import type { BaseUriParts } from "@streamlit/lib/src/util/UriUtil" +import { + IAllowedMessageOriginsResponse, + BaseUriParts, + SessionInfo, + StreamlitEndpoints, + ensureError, + BackMsg, + ForwardMsg, +} from "@streamlit/lib" -import type { StliteKernel } from "../../kernel" -import { ConnectionState } from "@streamlit/app/src/connection/ConnectionState" -import type { SessionInfo } from "@streamlit/lib/src/SessionInfo" -import { ensureError } from "@streamlit/lib/src/util/ErrorHandling" import { DUMMY_BASE_HOSTNAME, DUMMY_BASE_PORT } from "../../consts" +import { ConnectionState } from "./ConnectionState" + +import type { StliteKernel } from "@stlite/kernel" interface MessageQueue { [index: number]: any @@ -26,6 +31,9 @@ interface Props { /** The app's SessionInfo instance */ sessionInfo: SessionInfo + /** The app's StreamlitEndpoints instance */ + endpoints: StreamlitEndpoints + /** * Function to be called when we receive a message from the server. */ @@ -41,6 +49,19 @@ interface Props { */ connectionStateChanged: (connectionState: ConnectionState) => void + /** + * Function to get the auth token set by the host of this app (if in a + * relevant deployment scenario). + */ + claimHostAuthToken: () => Promise + + /** + * Function to clear the withHostCommunication hoc's auth token. This should + * be called after the promise returned by claimHostAuthToken successfully + * resolves. + */ + resetHostAuthToken: () => void + /** * Function to set the list of origins that this app should accept * cross-origin messages from (if in a relevant deployment scenario). @@ -48,6 +69,9 @@ interface Props { setAllowedOriginsResp: (resp: IAllowedMessageOriginsResponse) => void } +/** + * Manages our connection to the Server. + */ export class ConnectionManager { private readonly props: Props @@ -137,6 +161,13 @@ export class ConnectionManager { // Because caching is disabled in stlite. See https://github.com/whitphx/stlite/issues/495 } + /** + * No-op in stlite. + */ + disconnect(): void { + // no-op. + } + private async handleMessage(data: ArrayBuffer): Promise { // Assign this message an index. const messageIndex = this.nextMessageIndex diff --git a/packages/kernel/src/streamlit-replacements/lib/FileUploadClient.ts b/packages/kernel/src/streamlit-replacements/lib/FileUploadClient.ts deleted file mode 100644 index 555258d32..000000000 --- a/packages/kernel/src/streamlit-replacements/lib/FileUploadClient.ts +++ /dev/null @@ -1,162 +0,0 @@ -/** - * @license - * Copyright 2018-2022 Streamlit Inc. - * Copyright 2022 Yuichiro Tachibana - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { CancelToken } from "axios" -import type { SessionInfo } from "@streamlit/lib/src/SessionInfo" -import _ from "lodash" -import type { BaseUriParts } from "@streamlit/lib/src/util/UriUtil" -import { FormDataEncoder, FormDataLike } from "form-data-encoder" -import { isValidFormId } from "@streamlit/lib/src/util/utils" -import type { StliteKernel } from "../../kernel" - -/** Common widget protobuf fields that are used by the FileUploadClient. */ -interface WidgetInfo { - id: string - formId: string -} - -interface Props { - /** The app's SessionInfo instance. */ - sessionInfo: SessionInfo - getServerUri: () => BaseUriParts | undefined - csrfEnabled: boolean - formsWithPendingRequestsChanged: (formIds: Set) => void -} - -/** - * Handles uploading files to the server. - */ -export class FileUploadClient { - private readonly sessionInfo: SessionInfo - - /** - * Map of . Updated whenever - * a widget in a form creates are completes a request. - */ - private readonly formsWithPendingRequests = new Map() - - /** - * Called when the set of forms that have pending file requests changes. - */ - private readonly pendingFormUploadsChanged: (formIds: Set) => void - - public constructor(props: Props) { - this.pendingFormUploadsChanged = props.formsWithPendingRequestsChanged - this.sessionInfo = props.sessionInfo - } - - private kernel: StliteKernel | undefined - public setKernel(kernel: StliteKernel) { - this.kernel = kernel - } - - /** - * Upload a file to the server. It will be associated with this browser's sessionID. - * - * @param widget: the FileUploader widget that's doing the upload. - * @param file: the files to upload. - * @param onUploadProgress: an optional function that will be called repeatedly with progress events during the upload. - * @param cancelToken: an optional axios CancelToken that can be used to cancel the in-progress upload. - * - * @return a Promise that resolves with the file's unique ID, as assigned by the server. - */ - public async uploadFile( - widget: WidgetInfo, - file: File, - onUploadProgress?: (progressEvent: any) => void, // TODO - cancelToken?: CancelToken // TODO - ): Promise { - const form = new FormData() - form.append("sessionId", this.sessionInfo.current.sessionId) - form.append("widgetId", widget.id) - form.append(file.name, file) - - this.offsetPendingRequestCount(widget.formId, 1) - - const encoder = new FormDataEncoder(form as unknown as FormDataLike) - const bodyBlob = new Blob(encoder as unknown as BufferSource[], { - type: encoder.contentType, - }) - - return bodyBlob.arrayBuffer().then((body) => { - if (this.kernel == null) { - throw new Error("Kernel not ready") - } - - return this.kernel - .sendHttpRequest({ - method: "POST", - path: "/_stcore/upload_file", - body, - headers: { ...encoder.headers }, - }) - .then((response) => { - const text = new TextDecoder().decode(response.body) - if (typeof text === "number") { - return text - } - if (typeof text === "string") { - const parsed = parseInt(text, 10) - if (!Number.isNaN(parsed)) { - return parsed - } - } - - throw new Error( - `Bad uploadFile response: expected a number but got '${text}'` - ) - }) - .finally(() => this.offsetPendingRequestCount(widget.formId, -1)) - }) - } - - private getFormIdSet(): Set { - return new Set(this.formsWithPendingRequests.keys()) - } - - private offsetPendingRequestCount(formId: string, offset: number): void { - if (offset === 0) { - return - } - - if (!isValidFormId(formId)) { - return - } - - const curCount = this.formsWithPendingRequests.get(formId) ?? 0 - const newCount = curCount + offset - if (newCount < 0) { - throw new Error( - `Can't offset pendingRequestCount below 0 (formId=${formId}, curCount=${curCount}, offset=${offset})` - ) - } - - const prevWidgetIds = this.getFormIdSet() - - if (newCount === 0) { - this.formsWithPendingRequests.delete(formId) - } else { - this.formsWithPendingRequests.set(formId, newCount) - } - - const newWidgetIds = this.getFormIdSet() - if (!_.isEqual(newWidgetIds, prevWidgetIds)) { - this.pendingFormUploadsChanged(newWidgetIds) - } - } -} diff --git a/packages/kernel/src/types.ts b/packages/kernel/src/types.ts index ca88cb69b..5959c5598 100644 --- a/packages/kernel/src/types.ts +++ b/packages/kernel/src/types.ts @@ -51,6 +51,7 @@ export interface WorkerInitialData { }; mountedSitePackagesSnapshotFilePath?: string; streamlitConfig?: StreamlitConfig; + disableProgressToasts?: boolean; } /** diff --git a/packages/kernel/src/worker.ts b/packages/kernel/src/worker.ts index fbf39ff05..c25b582ad 100644 --- a/packages/kernel/src/worker.ts +++ b/packages/kernel/src/worker.ts @@ -177,7 +177,6 @@ async function loadPyodideAndPackages() { console.debug("Loading stlite-server, and streamlit"); await pyodide.loadPackage("micropip"); const micropip = pyodide.pyimport("micropip"); - await micropip.install(["altair<5.2.0"]); // Altair>=5.2.0 checks PyArrow version and emits an error (https://github.com/altair-viz/altair/pull/3160) await micropip.install.callKwargs([wheels.stliteServer], { keep_going: true, }); @@ -280,8 +279,9 @@ async function loadPyodideAndPackages() { console.debug("Booting up the Streamlit server"); // The following Python code is based on streamlit.web.cli.main_run(). self.__streamlitFlagOptions__ = { + // gatherUsageStats is disabled as default, but can be enabled explicitly by setting it to true. + "browser.gatherUsageStats": false, ...streamlitConfig, - "browser.gatherUsageStats": false, "runner.fastReruns": false, // Fast reruns do not work well with the async script runner of stlite. See https://github.com/whitphx/stlite/pull/550#issuecomment-1505485865. }; await pyodide.runPythonAsync(` diff --git a/packages/mountable/src/index.tsx b/packages/mountable/src/index.tsx index 501bc57ff..2aac4485e 100644 --- a/packages/mountable/src/index.tsx +++ b/packages/mountable/src/index.tsx @@ -35,10 +35,11 @@ export function mount( options: MountOptions, container: HTMLElement = document.body ) { + const mountOptions = canonicalizeMountOptions(options); const kernel = new StliteKernel({ - ...canonicalizeMountOptions(options), + ...mountOptions, wheelBaseUrl, - ...makeToastKernelCallbacks(), + ...makeToastKernelCallbacks(mountOptions.disableProgressToasts === true), }); ReactDOM.render( diff --git a/packages/mountable/src/options.ts b/packages/mountable/src/options.ts index 16c337ac0..38249e567 100644 --- a/packages/mountable/src/options.ts +++ b/packages/mountable/src/options.ts @@ -15,6 +15,7 @@ export interface SimplifiedStliteKernelOptions { allowedOriginsResp?: StliteKernelOptions["allowedOriginsResp"]; pyodideUrl?: StliteKernelOptions["pyodideUrl"]; streamlitConfig?: StliteKernelOptions["streamlitConfig"]; + disableProgressToasts?: StliteKernelOptions["disableProgressToasts"]; } function canonicalizeFiles( @@ -106,5 +107,6 @@ export function canonicalizeMountOptions( allowedOriginsResp: options.allowedOriginsResp, pyodideUrl: options.pyodideUrl, streamlitConfig: options.streamlitConfig, + disableProgressToasts: options.disableProgressToasts, }; }