diff --git a/.cursorignore b/.cursorignore new file mode 100644 index 000000000000..183bcdfcad36 --- /dev/null +++ b/.cursorignore @@ -0,0 +1,2 @@ +*.old.* +_*/** diff --git a/.github/labeler.yml b/.github/labeler.yml index 07f6aac81766..58df859d7248 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -19,6 +19,6 @@ hamr: info: - changed-files: - any-glob-to-any-file: info/** -test-file-runner: +evals: - changed-files: - - any-glob-to-any-file: test-file-runner/** + - any-glob-to-any-file: evals/** diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53db95d9aea2..d35f8c8c23e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,15 +34,35 @@ jobs: run: deno task test - name: Generate lcov + if: matrix.os == 'ubuntu-latest' run: deno task cov:gen - name: Upload coverage + if: matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: name: ${{ matrix.os }} + - name: Run benchmark + if: matrix.os == 'ubuntu-latest' + run: deno task bench | tee output.txt + + - name: Store benchmark result + if: github.ref == 'refs/heads/main' && matrix.os == 'ubuntu-latest' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Artifact Benchmarks + tool: 'benchmarkjs' + output-file-path: output.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + lint: runs-on: ${{ matrix.os }} timeout-minutes: 30 diff --git a/.github/workflows/title.yml b/.github/workflows/title.yml index 5bb9e7c2f0fd..4f2a0a692e58 100644 --- a/.github/workflows/title.yml +++ b/.github/workflows/title.yml @@ -1,4 +1,4 @@ -name: "Lint PR title" +name: 'Lint PR title' on: pull_request_target: @@ -42,4 +42,4 @@ jobs: fixture(/unstable)? hamr(/unstable)? info(/unstable)? - test-file-runner(/unstable)? + evals(/unstable)? diff --git a/.gitignore b/.gitignore index c158ca8609b2..4dc843750069 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ http/testdata/file#2.txt http/testdata/test file.txt coverage/ docs/ -.env \ No newline at end of file +.env +concat.txt \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000000..db6610318717 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,43 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "request": "launch", + "name": "Circular", + "type": "node", + "cwd": "${workspaceFolder}", + "runtimeExecutable": "~/.deno/bin/deno", + "runtimeArgs": ["task", "lint:circular"], + + "attachSimplePort": 9229, + "internalConsoleOptions": "neverOpen", + "console": "integratedTerminal" + }, + { + "request": "launch", + "name": "Run Current", + "type": "node", + "cwd": "${workspaceFolder}", + "runtimeExecutable": "~/.deno/bin/deno", + "runtimeArgs": ["test", "--inspect-wait", "--watch", "-A", "${file}"], + + "attachSimplePort": 9229, + "internalConsoleOptions": "neverOpen", + "console": "integratedTerminal" + }, + { + "request": "launch", + "name": "Execute Current File", + "type": "node", + "cwd": "${fileDirname}", + "runtimeExecutable": "~/.deno/bin/deno", + "runtimeArgs": ["run", "-A", "--inspect-wait", "--watch", "${file}"], + "attachSimplePort": 9229, + "internalConsoleOptions": "neverOpen", + "console": "integratedTerminal" + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index e659d1421862..91ecff2e1d4f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,11 @@ { "deno.enable": true, - "deno.unstable": true, "editor.formatOnSave": true, - "editor.defaultFormatter": "denoland.vscode-deno" + "editor.defaultFormatter": "denoland.vscode-deno", + "[markdown]": { + "editor.defaultFormatter": "denoland.vscode-deno" + }, + "[typescript]": { + "editor.defaultFormatter": "denoland.vscode-deno" + } } diff --git a/README.md b/README.md index ac1872f8cfcd..a4582500e0ea 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,17 @@ for Artifact, the infrastructure of the The following list contains links to the Artifact Standard Library's packages and documentation: -| Package | Latest version | -| ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| [files](https://jsr.io/@artifact/files) | [](https://jsr.io/@artifact/files) | -| [openai](https://jsr.io/@artifact/openai) | [](https://jsr.io/@artifact/openai) | -| [dumb-bot](https://jsr.io/@artifact/dumb-bot) | [](https://jsr.io/@artifact/dumb-bot) | -| [reasoner](https://jsr.io/@artifact/reasoner) | [](https://jsr.io/@artifact/reasoner) | -| [fixture](https://jsr.io/@artifact/fixture) | [](https://jsr.io/@artifact/fixture) | -| [hamr](https://jsr.io/@artifact/hamr) | [](https://jsr.io/@artifact/hamr) | -| [info](https://jsr.io/@artifact/info) | [](https://jsr.io/@artifact/info) | -| [test-file-runner](https://jsr.io/@artifact/test-file-runner) | [](https://jsr.io/@artifact/test-file-runner) | +| Package | Latest version | +| ------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| [files](https://jsr.io/@artifact/files) | [](https://jsr.io/@artifact/files) | +| [openai](https://jsr.io/@artifact/openai) | [](https://jsr.io/@artifact/openai) | +| [dumb-bot](https://jsr.io/@artifact/dumb-bot) | [](https://jsr.io/@artifact/dumb-bot) | +| [reasoner](https://jsr.io/@artifact/reasoner) | [](https://jsr.io/@artifact/reasoner) | +| [fixture](https://jsr.io/@artifact/fixture) | [](https://jsr.io/@artifact/fixture) | +| [hamr](https://jsr.io/@artifact/hamr) | [](https://jsr.io/@artifact/hamr) | +| [info](https://jsr.io/@artifact/info) | [](https://jsr.io/@artifact/info) | +| [evals](https://jsr.io/@artifact/evals) | [](https://jsr.io/@artifact/evals) | +| [napp-tools](https://jsr.io/@artifact/napp-tools) | [](https://jsr.io/@artifact/napp-tools) | ## Architecture diff --git a/_import-artifact/.gitattributes b/_import-artifact/.gitattributes deleted file mode 100644 index 3c53eb39d62f..000000000000 --- a/_import-artifact/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -# Use Unix line endings in all text files. -* text=auto eol=lf diff --git a/_import-artifact/.github/codecov.yml b/_import-artifact/.github/codecov.yml deleted file mode 100644 index 8a0c37f164ad..000000000000 --- a/_import-artifact/.github/codecov.yml +++ /dev/null @@ -1,11 +0,0 @@ -comment: false -coverage: - status: - project: - default: - informational: true - patch: - default: - informational: true -ignore: - - '**/*.tsx' diff --git a/_import-artifact/.github/dependabot.yml b/_import-artifact/.github/dependabot.yml deleted file mode 100644 index af8120ff11ce..000000000000 --- a/_import-artifact/.github/dependabot.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 -updates: - - package-ecosystem: 'github-actions' - directory: '/' - schedule: - # Check for updates to GitHub Actions every week - interval: 'weekly' diff --git a/_import-artifact/.github/typos.toml b/_import-artifact/.github/typos.toml deleted file mode 100644 index 704e8ca37f82..000000000000 --- a/_import-artifact/.github/typos.toml +++ /dev/null @@ -1,2 +0,0 @@ -[files] -extend-exclude = ["__snapshots__"] \ No newline at end of file diff --git a/_import-artifact/.github/workflows/ci.yml b/_import-artifact/.github/workflows/ci.yml deleted file mode 100644 index 62dea1af077a..000000000000 --- a/_import-artifact/.github/workflows/ci.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: CI - -on: - push: - branches: [main, next] - pull_request: - branches: [main] - -jobs: - ci: - permissions: - id-token: write - # contents permission to update benchmark contents in gh-pages branch - # without benchmarks, would only need read permission - contents: write - # deployments permission to deploy GitHub pages website - deployments: write - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - - windows-latest - - macOS-latest - - steps: - - name: Setup repo - uses: actions/checkout@v4 - with: - submodules: true - - - name: Setup Deno - uses: denoland/setup-deno@v2 - with: - deno-version: v2.x - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: | - ~/.cache - ~/.cache/deno - key: ${{ runner.os }}-deno-${{ hashFiles('deno.json') }} - - - name: Check typos - if: matrix.os == 'ubuntu-latest' - uses: crate-ci/typos@master - with: - config: ./.github/typos.toml - - - name: Check formatting, linting, and types - run: deno task check - - - name: Test - run: deno task test - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - - - name: Create lcov file - if: github.ref == 'refs/heads/main' && matrix.os == 'ubuntu-latest' - run: deno task cov:gen - - - name: Upload coverage - if: github.ref == 'refs/heads/main' && matrix.os == 'ubuntu-latest' - uses: codecov/codecov-action@v4 - with: - name: ${{ matrix.os }} - files: cov.lcov - token: ${{ secrets.CODECOV_TOKEN }} - slug: dreamcatcher-tech/artifact - - - name: Run benchmark - if: matrix.os == 'ubuntu-latest' - run: deno task bench | tee output.txt - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - - name: Store benchmark result - if: github.ref == 'refs/heads/main' && matrix.os == 'ubuntu-latest' - uses: benchmark-action/github-action-benchmark@v1 - with: - name: Artifact Benchmarks - tool: 'benchmarkjs' - output-file-path: output.txt - github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - diff --git a/_import-artifact/.gitignore b/_import-artifact/.gitignore deleted file mode 100644 index 7eb667567ab0..000000000000 --- a/_import-artifact/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -.env -*.DS_Store -cov/ -cov.lcov -.idea -_fresh/ -backup.json -html_cov/ -coverage/ -node_modules/ -tmp/ \ No newline at end of file diff --git a/_import-artifact/.vscode/extensions.json b/_import-artifact/.vscode/extensions.json deleted file mode 100644 index c75f091bcc36..000000000000 --- a/_import-artifact/.vscode/extensions.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "recommendations": [ - "denoland.vscode-deno", - "ryanluker.vscode-coverage-gutters", - "dotenv.dotenv-vscode" - ] -} diff --git a/_import-artifact/.vscode/launch.json b/_import-artifact/.vscode/launch.json deleted file mode 100644 index 45e22d92d741..000000000000 --- a/_import-artifact/.vscode/launch.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "request": "launch", - "name": "Serve", - "type": "node", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "~/.deno/bin/deno", - "runtimeArgs": ["task", "api"], - "attachSimplePort": 9229, - "internalConsoleOptions": "neverOpen", - "console": "integratedTerminal" - }, - { - "request": "launch", - "name": "io", - "type": "node", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "~/.deno/bin/deno", - "runtimeArgs": ["task", "watch", "--inspect-wait", "io/io.test.ts"], - "attachSimplePort": 9229, - "internalConsoleOptions": "neverOpen", - "console": "integratedTerminal" - }, - { - "request": "launch", - "name": "longthread", - "type": "node", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "~/.deno/bin/deno", - "runtimeArgs": [ - "task", - "watch", - "--inspect-wait", - "isolates/longthread.test.ts" - ], - "attachSimplePort": 9229, - "internalConsoleOptions": "neverOpen", - "console": "integratedTerminal" - }, - { - "request": "launch", - "name": "FS", - "type": "node", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "~/.deno/bin/deno", - "runtimeArgs": ["task", "watch", "--inspect-wait", "git/fs.test.ts"], - "attachSimplePort": 9229, - "internalConsoleOptions": "neverOpen", - "console": "integratedTerminal" - }, - { - "request": "launch", - "name": "Engine", - "type": "node", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "~/.deno/bin/deno", - "runtimeArgs": ["task", "watch", "--inspect-wait", "engine.test.ts"], - "attachSimplePort": 9229, - "internalConsoleOptions": "neverOpen", - "console": "integratedTerminal" - }, - { - "request": "launch", - "name": "Cloud", - "type": "node", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "~/.deno/bin/deno", - "runtimeArgs": ["task", "tc", "--watch", "--inspect-wait"], - "attachSimplePort": 9229, - "internalConsoleOptions": "neverOpen", - "console": "integratedTerminal" - }, - { - "request": "launch", - "name": "Launch Program", - "type": "node", - "cwd": "${workspaceFolder}", - "runtimeExecutable": "~/.deno/bin/deno", - "runtimeArgs": ["task", "watch", "--inspect-wait", "${relativeFile}"], - "attachSimplePort": 9229, - "internalConsoleOptions": "neverOpen", - "console": "integratedTerminal" - } - ] -} diff --git a/_import-artifact/.vscode/settings.json b/_import-artifact/.vscode/settings.json deleted file mode 100644 index b16523d5fcd1..000000000000 --- a/_import-artifact/.vscode/settings.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "deno.enable": true, - "deno.lint": true, - "editor.formatOnSave": true, - "editor.defaultFormatter": "denoland.vscode-deno", - "deno.cacheOnSave": true, - "deno.codeLens.references": false, - "deno.codeLens.implementations": false, - "deno.codeLens.referencesAllFunctions": true, - "deno.codeLens.test": true, - "deno.testing.args": ["--allow-all", "--no-check", "--unstable-kv"], - "deno.codeLens.testArgs": ["--allow-all", "--no-check", "--unstable-kv"], - "coverage-gutters.coverageFileNames": [ - "cov.lcov", - "lcov.info", - "cov.xml", - "coverage.xml", - "jacoco.xml", - "coverage.cobertura.xml" - ], - "coverage-gutters.showLineCoverage": true, - "coverage-gutters.showRulerCoverage": false, - "coverage-gutters.showGutterCoverage": false, - "[markdown]": { - "editor.defaultFormatter": "esbenp.prettier-vscode" - } -} diff --git a/_import-artifact/LICENSE b/_import-artifact/LICENSE deleted file mode 100644 index 0ad25db4bd1d..000000000000 --- a/_import-artifact/LICENSE +++ /dev/null @@ -1,661 +0,0 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - <one line to give the program's name and a brief idea of what it does.> - Copyright (C) <year> <name of author> - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -<https://www.gnu.org/licenses/>. diff --git a/_import-artifact/api/isolates/files.ts b/_import-artifact/api/isolates/files.ts deleted file mode 100644 index 73a9d24369ad..000000000000 --- a/_import-artifact/api/isolates/files.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { reasoning, ToApiType } from '../types.ts' -import { z } from 'zod' - -export const parameters = { - write: z.object({ - reasoning, - path: z.string().describe('the relative path to the file'), - content: z.string().optional().describe( - 'the contents of the file to be written', - ), - }).describe( - 'Write to a file with optional contents. Will overwrite existing files. Will create all required parent directories. Returns the number of characters written in the operation', - ), - ls: z.object({ - reasoning, - path: z.string().optional().describe( - 'the relative path to the directory you want to list', - ), - count: z.boolean().optional().describe( - 'count the number of files and return this instead of the names of the files', - ), - all: z.boolean().optional().describe( - 'include all files including hidden files in the operation', - ), - }).describe( - 'List files for a given path. Returns file names with directory names ending in "/". The root ("/") is actually just ".". To count the number of files instead of list them, set "count" to true. To include hidden files in the list or count, set "all" to true.', - ), - read: z.object({ - reasoning, - path: z.string().describe( - 'the relative path to the file you want to read', - ), - }).describe('Read a file. The contents will be returned as a string.'), - update: z.object({ - reasoning, - expectedMatches: z.number().int().gt(0).describe( - 'the expected number of matches for the regex', - ), - path: z.string().describe( - 'the relative path to the file you want to update', - ), - regex: z.string().describe('a regular expression string'), - replacement: z.string().describe('the replacement string'), - }).refine(({ regex }) => { - try { - new RegExp(regex) - return true - } catch (error) { - return !error - } - }) - .describe( - 'Update a file using a regex and a replacement string. The number of occurrences replaced will be returned to you as an integer. If you want to append something to a file, you can use a regex to match the end of the file and replace it with the contents you want to append. To delete portions of a file, you can use a regex to match the contents you want to delete and replace it with an empty string. Path must be relative.', - ), - rm: z.object({ - reasoning, - path: z.string().describe( - 'the relative path to the file you want to remove', - ), - }).describe('Remove a file. This is recursive.'), - mv: z.object({ - reasoning, - from: z.string().describe( - 'the relative path to the file you want to move', - ), - to: z.string().describe( - 'the relative path to the new location of the file', - ), - }).describe('Move a file efficiently. This is a rename operation.'), - cp: z.object({ - reasoning, - from: z.string().describe( - 'the relative path to the file you want to copy', - ), - to: z.string().describe( - 'the relative path to the new location of the file', - ), - }).describe('Copy a file efficiently.'), - search: z.object({ - reasoning, - query: z.string().describe( - 'the relative path to the file or directory you want to find', - ), - }).describe( - 'Search for a file or directory. Returns the relative path to the first match.', - ), -} -export const returns = { - /** The number of bytes written */ - write: z.object({ - charactersWritten: z.number(), - }), - ls: z.union([z.array(z.string()), z.number()]), - read: z.string(), - /** The number of occurrences replaced */ - update: z.object({ matchesUpdated: z.number() }), - rm: z.void(), - mv: z.void(), - cp: z.void(), - search: z.array( - z.object({ path: z.string(), description: z.string() }), - ), -} - -export type Api = ToApiType<typeof parameters, typeof returns> diff --git a/_import-artifact/api/types.ts b/_import-artifact/api/types.ts deleted file mode 100644 index de0c4dc1ae9d..000000000000 --- a/_import-artifact/api/types.ts +++ /dev/null @@ -1,907 +0,0 @@ -// copied from the artifact project -import { Chalk } from 'chalk' -import { z, ZodSchema } from 'zod' -export type { AssistantMessage, CompletionMessage } from './zod.ts' -import { completionMessage } from './zod.ts' -import type { Backchat } from './client-backchat.ts' -import { assert } from '@sindresorhus/is' -import type OpenAI from 'openai' -import { randomness } from './randomness.ts' -export { randomness } -type CommitOid = string - -const sequenceInteger = z.number().int().gte(0) -const sequenceKey = z.string().refine((data) => { - try { - return sequenceInteger.safeParse(Number.parseInt(data)).success - } catch (error) { - return !error - } -}, 'sequence key must be an integer') - -export { type Backchat } -export const Proctype = z.enum(['SERIAL', 'BRANCH', 'DAEMON', 'EFFECT']) -// TODO FORGET = 'FORGET', // allow fire and forget actions -// BUT forget needs to be a separate option as we need DAEMON and FORGET -// together to allow for a fire and forget branches -// OR make DAEMON be the same as FORGET since no new info need be returned ? - -export type STATEBOARD_WIDGETS = z.infer<typeof STATEBOARD_WIDGETS> -export const STATEBOARD_WIDGETS = z.enum([ - 'TPS_REPORT', - 'FILE_EXPLORER', - 'MARKDOWN_EDITOR', - 'BRANCH_EXPLORER', - 'COMMIT_GRAPH', - 'COMMIT_INFO', - 'THREADS', - 'REPOS', -]) -export const md5 = z.string().regex(/^[a-f0-9]{40}$/, 'Invalid MD5 hash') -export const githubRegex = /^[a-zA-Z\d](?:[a-zA-Z\d]|[-.](?=[a-zA-Z\d])){0,38}$/ -export const repoIdRegex = /^rep_[0-9A-HJKMNP-TV-Z]{16}$/ -export const machineIdRegex = /^mac_[2-7a-z]{33}$/ -export const actorIdRegex = /^act_[0-9A-HJKMNP-TV-Z]{16}$/ -export const backchatIdRegex = /^bac_[0-9A-HJKMNP-TV-Z]{16}$/ -export const threadIdRegex = /^the_[0-9A-HJKMNP-TV-Z]{16}$/ -export const agentHashRegex = /^age_[0-9A-HJKMNP-TV-Z]{16}$/ - -export const SU_ACTOR = 'act_0000000000000000' -export const SU_BACKCHAT = 'bac_0000000000000000' -export const pidSchema = z.object({ - /** - * The hash of the genesis commit is used to identify this repo in a - * cryptographically secure way. This repoId is used to reference this repo - * unique with strong guarantees that this is the correct repo that - * communication was intended with. - */ - repoId: z.string().regex(repoIdRegex), - account: z.string().regex(githubRegex), - repository: z.string().regex(githubRegex), - branches: z.array(z.string()).min(1), -}) -export const triad = z.object({ - path: z.string(), - pid: pidSchema, - commit: z.string(), -}) -export type Triad = z.infer<typeof triad> -export type ApiFunction = { - (): unknown | Promise<unknown> - (...args: [{ [key: string]: unknown }]): unknown | Promise<unknown> -} -export type ApiFunctions = { - [key: string]: ApiFunction -} - -export type IsolateReturn = JsonValue | undefined | void -export type ProcessOptions = { - /** - * Any function called with this option will be executed in parallel - * in a new branch, with no guarantee of order of execution. A call to this - * function will cause 3 commits to occur, 2 of which may be pooled with other - * functions. The commits are: - * 1. The current branch, to declare the function invocation - may be pooled - * 2. The new branch, to conclude the function invocation - may be skippable - * if no fs changes were made - * 3. The current branch, to merge the result back in - may be pooled - * - * Without this option, the functions will be executed in the same - * branch as the caller, and will be executed in the order that any other - * similarly called functions were invoked. - * A call to this function will cause two commits to occur on the current - * branch - the first to store the function call, and the second to store the - * result. Both commits may be shared with other function calls. - */ - branch?: boolean - /** - * Enables Daemon mode, where the branch will not be closed after the process - * is done. - */ - noClose?: boolean - /** Set a prefix for the new branch name, which will be combined with a - * random id and separated by a "-". Implies branch = true - */ - prefix?: string - /** Set the name of the new branch. Will error if this exists already */ - branchName?: string - /** Provide file paths that will be deleted in the new branch */ - deletes?: string[] -} - -export type DispatchFunctions = { - [key: string]: (params?: Params) => Promise<unknown> | unknown -} - -export type IsolateApiSchema = { - [key: string]: object -} - -export const ENTRY_BRANCH = 'main' - -export type PartialPID = Omit<PID, 'repoId'> - -const int = z.number().int().gte(0) - -const usageSchema = z.object({ - /** - * Number of tokens in the generated completion. - */ - completion_tokens: int, - /** - * Number of tokens in the prompt. - */ - prompt_tokens: int, - /** - * Total number of tokens used in the request (prompt + completion). - */ - total_tokens: int, - /** - * Breakdown of tokens used in a completion. - */ - completion_tokens_details: z.object({ - /** - * Audio input tokens generated by the model. - */ - audio_tokens: int.optional(), - /** - * Tokens generated by the model for reasoning. - */ - reasoning_tokens: int.optional(), - }).optional(), - /** - * Breakdown of tokens used in the prompt. - */ - prompt_tokens_details: z.object({ - /** - * Audio input tokens present in the prompt. - */ - audio_tokens: int.optional(), - /** - * Cached tokens present in the prompt. - */ - cached_tokens: int.optional(), - }).optional(), -}) - -export const messageStatsSchema = z.object({ - /** The Unix timestamp (in seconds) of when the chat completion was created. */ - created: z.number().int().gte(0), - /** The model used for the completion. */ - model: z.string(), - /** The system fingerprint of the completion. */ - system_fingerprint: z.string(), - /** The duration of the completion in milliseconds from our side. */ - duration: z.number().int().gte(0), - /** The duration of the completion in milliseconds from OpenAI's side. - */ - openAiProcessingMs: z.number().int().gte(0), - /** The usage of the completion. */ - usage: usageSchema, -}) - -export const threadSchema = z.object({ - /** The current agent the conversation is with */ - agent: z.string(), - /** The remote thread the conversation is currently with */ - remote: pidSchema.optional(), - /** If the messages were truncated, this is the offset count */ - messageOffset: int, - messages: z.array(completionMessage), - toolCommits: z.record( - /** The tool call id */ - z.string(), - /** The commit hash the tool ended on */ - md5, - ), - messageStats: z.record( - /** The message index that this stat is for */ - z.string(), - messageStatsSchema, - ), - /** Have any files been changed in this threads branch */ - isDirty: z.boolean().optional(), - summaries: z.array( - z.object({ - title: z.string(), - summary: z.string(), - /** The message index that this summary starts with */ - start: z.number().int().gte(0), - /** The message index that this summary ends with */ - end: z.number().int().gte(0).optional(), - }).refine((data) => data.end === undefined || data.end >= data.start, { - message: "'end' must be greater than or equal to 'start'", - path: ['end'], - }), - ).optional(), - /** History of stateboard changes */ - stateboards: z.array(z.object({ - /** What message number set the stateboard change */ - - setter: int, - commit: z.string(), - })), - /** History of what the focus file path was set to (like the CWD). Allows - * statements like "the previous file", "that other file", and "three files - * ago" */ - focusedFiles: z.array(z.object({ - /** The message number that set the focus */ - setter: int, - focus: z.object({ - // Define the structure of PathTriad here - }), - })), -}) -export type Thread = z.infer<typeof threadSchema> - -export type AssistantsThread = Thread & { - externalId: string - messages: OpenAI.Beta.Threads.Message[] - additionalMessages: OpenAI.Beta.Threads.RunCreateParams.AdditionalMessage[] -} -export type RemoteThread = { - /** The location in the remote repo and the last known commit we have of it */ - triad: Triad -} - -export type PathTriad = { - path: string - pid?: PID - commit?: CommitOid -} - -export const isPierceRequest = (p: Request): p is Pierce => { - return 'ulid' in p -} -export type Params = { [key: string]: JsonValue } - -const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]) -type Literal = z.infer<typeof literalSchema> -export type JsonValue = Literal | { [key: string]: JsonValue } | JsonValue[] -export const jsonSchema: z.ZodType<JsonValue> = z.lazy(() => - z.union([literalSchema, z.array(jsonSchema), z.record(jsonSchema)]) -) - -const invocation = z.object({ - isolate: z.string(), - functionName: z.string(), - params: z.record(jsonSchema), - proctype: Proctype, - /** - * Allow a custom name for the new branch, if this is a branching request - */ - branch: z.string().optional(), - /** - * If the custom branch name might not be unique, a prefix can be given and - * the sequence number will be appended to the branch name, ensuring - * uniqueness. - */ - branchPrefix: z.string().optional(), - /** - * If the request is a branching request, this will be the name of the new - * branch. If the branch already exists, the request will fail. - */ - branchName: z.string().optional(), - /** Relative paths to delete in the branch */ - deletes: z.array(z.string()).optional(), - effect: z.union([ - z.boolean(), - z.object({ - /** does this side effect have access to the network ? */ - net: z.boolean().optional(), - /** does this side effect have access to the files of the repo ? */ - files: z.boolean().optional(), - /** can this side effect make execution requests in artifact ? */ - artifact: z.boolean().optional(), - /** Specify the maximum time to wait for this side effect to complete */ - timeout: z.number().optional(), - }), - ]).optional(), -}) -export type Invocation = z.infer<typeof invocation> -/** - * The Process Identifier used to address a specific process branch. - */ -export type PID = z.infer<typeof pidSchema> -/** - * A request that has been included in a commit, therefore has a sequence number - */ -export type SolidRequest = z.infer<typeof solidRequest> -const solidRequest = invocation.extend({ - target: pidSchema, - source: pidSchema, - sequence: sequenceInteger, -}) - -/** A request that travels between branches */ -export type RemoteRequest = z.infer<typeof remoteRequest> -export const remoteRequest = solidRequest.extend({ commit: md5 }) - -export type Pierce = z.infer<typeof pierceSchema> -export const pierceSchema = invocation.extend({ - target: pidSchema, - ulid: z.string(), -}) - -export type UnsequencedRequest = z.infer<typeof unsequencedRequest> -export const unsequencedRequest = invocation.extend({ target: pidSchema }) - -export type Request = z.infer<typeof requestSchema> -export const requestSchema = z.union([ - pierceSchema, - solidRequest, - remoteRequest, -]) -// TODO remove this by passing ProcessOptions in with the Request -export const getProcType = (procOpts?: ProcessOptions) => { - if (!procOpts) { - return Proctype.enum.SERIAL - } - if (procOpts.noClose) { - return Proctype.enum.DAEMON - } - if ( - procOpts.deletes || procOpts.branch || procOpts.branchName || - procOpts.prefix - ) { - return Proctype.enum.BRANCH - } - return Proctype.enum.SERIAL -} -/** Here is where additional AI models and runner techniques can be added */ -export enum AGENT_RUNNERS { - CHAT = 'ai-runner', -} - -export type Change = { - /** If present, represents the unified diff of the file at the given path, - * since the last commit. This is only provided if the file is a string. If - * the changes are too large, this will be missing and an oid will be provided. - */ - patch?: string - /** - * The oid of the object given by path, which can be used to fetch the object - * directly. If this file is binary, then patch will be missing, and the file - * should be directly retrieved. If oid is missing, then the change was - * fatal. - */ - oid?: string -} -export type Splice = { - pid: PID - /** - * The hash of the commit object - */ - oid: string - /** - * The commit this splice refers to - */ - commit: CommitObject - /** - * The timestamp of the commit, or if transient, the timestamp of the write - * that caused this update - */ - timestamp: number - changes: { [key: string]: Change } -} -export declare interface EventSourceMessage { - /** The data received for this message. */ - data: string - /** Event name sent from the server, or `undefined` if none is set for this message. */ - event?: string - /** ID of the message, if any was provided by the server. */ - id?: string -} -/** - * A git commit object. - */ -export type CommitObject = { - /** - * Commit message - */ - message: string - /** - * SHA-1 object id of corresponding file tree - */ - tree: string - /** - * an array of zero or more SHA-1 object ids - */ - parent: string[] - author: { - /** - * The author's name - */ - name: string - /** - * The author's email - */ - email: string - /** - * UTC Unix timestamp in seconds - */ - timestamp: number - /** - * Timezone difference from UTC in minutes - */ - timezoneOffset: number - } - committer: { - /** - * The committer's name - */ - name: string - /** - * The committer's email - */ - email: string - /** - * UTC Unix timestamp in seconds - */ - timestamp: number - /** - * Timezone difference from UTC in minutes - */ - timezoneOffset: number - } - /** - * PGP signature (if present) - */ - gpgsig?: string -} - -export interface EngineInterface { - /** - * The address in use as basis of identity for this engine. May be a repo - * hosted on external engines, or hosted in cooperation with other engines. - */ - homeAddress: PID - abortSignal: AbortSignal - upsertBackchat(machineId: string, resume?: string): Promise<PID> - stop(): Promise<void> | void - /** - * Send a ping to the edge isolate that will process requests, to establish - * basic network connectivity. To ping a chain, use an isolate. - * TODO ping should return some info about the deployment - * @param data Data that will be echoed back - */ - ping(data?: JsonValue): Promise<IsolateReturn> - apiSchema(isolate: string): Promise<Record<string, object>> - transcribe(audio: File): Promise<{ text: string }> - pierce(pierce: Pierce): Promise<void> - watch( - pid: PID, - path?: string, - after?: string, - signal?: AbortSignal, - ): AsyncIterable<Splice> - splice( - target: PID, - opts?: { commit?: string; path?: string; count?: number }, - ): Promise<Splice[]> - read(path: string, pid: PID, commit?: string): Promise<string> - readTree(path: string, pid: PID, commit?: string): Promise<TreeEntry[]> - readJSON<T>(path: string, pid: PID, commit?: string): Promise<T> - readBinary(path: string, pid?: PID, commit?: string): Promise<Uint8Array> - exists(path: string, pid: PID): Promise<boolean> -} -export const isPID = (value: unknown): value is PID => { - if (typeof value !== 'object' || value === null) { - return false - } - const pid = value as PID - return ( - typeof pid.account === 'string' && - typeof pid.repository === 'string' && - Array.isArray(pid.branches) && - pid.branches.every((branch) => typeof branch === 'string') - ) -} -const { black, red, green, blue, magenta, cyan, bold } = new Chalk({ level: 1 }) -const colors = [red, green, blue, magenta, cyan, black] -let colorIndex = 0 -const colorMap = new Map<string, number>() -export const colorize = ( - string: string, - noSubstring = false, - noColor = false, -) => { - let sub = string - if (!noSubstring) { - sub = string.substring(0, 7) - } - let index - if (noColor) { - return sub - } - if (colorMap.has(sub)) { - index = colorMap.get(sub)! - } else { - index = colorIndex++ - if (colorIndex === colors.length) { - colorIndex = 0 - } - colorMap.set(sub, index) - } - - return colors[index](bold(sub)) -} -export const print = (pid?: PID, noColor = false) => { - if (!pid) { - return '(no pid)' - } - const branches = pid.branches.map((segment) => { - const noSubstring = !segment.startsWith('mac_') && - !segment.startsWith('bac_') && - !segment.startsWith('act_') && - !segment.startsWith('rep_') && - !segment.startsWith('the_') - return colorize(segment, noSubstring, noColor) - }) - const noSubstring = false - const repoId = colorize(pid.repoId, noSubstring, noColor) - return `${repoId}/${pid.account}/${pid.repository}:${branches.join('/')}` -} -export const printPlain = (pid?: PID) => { - const noColor = true - return print(pid, noColor) -} -export const freezePid = (pid: PID) => { - if (!pid.repoId) { - throw new Error('repoId is required') - } - if (!repoIdRegex.test(pid.repoId)) { - throw new Error('Invalid repoId: ' + pid.repoId) - } - if (!pid.account) { - throw new Error('account is required') - } - if (!pid.repository) { - throw new Error('repository is required') - } - if (!pid.branches[0]) { - throw new Error('branch is required') - } - if (!githubRegex.test(pid.account) || !githubRegex.test(pid.repository)) { - const repo = `${pid.account}/${pid.repository}` - throw new Error('Invalid GitHub account or repository name: ' + repo) - } - Object.freeze(pid) - Object.freeze(pid.branches) - return pid -} -export const partialFromRepo = (repo: string) => { - const [account, repository] = repo.split('/') - const pid: PartialPID = { - account, - repository, - branches: [ENTRY_BRANCH], - } - return pid -} - -export const HAL: Omit<PID, 'repoId'> = { - account: 'dreamcatcher-tech', - repository: 'HAL', - branches: ['main'], -} -export type RpcOpts = { target?: PID } & ProcessOptions -export const toActions = <T = DispatchFunctions>( - target: PID, - isolate: string, - schema: IsolateApiSchema, - procOpts: ProcessOptions, - execute: (request: UnsequencedRequest) => unknown | Promise<unknown>, -) => { - procOpts = procOpts || {} - if (procOpts.prefix && procOpts.branchName) { - throw new Error('failed mutex: ' + print(target)) - } - const proctype = getProcType(procOpts) - const actions: DispatchFunctions = {} - for (const functionName of Object.keys(schema)) { - actions[functionName] = (arg1: Params = {}) => { - const params = safeParams(arg1) - const unsequencedRequest: UnsequencedRequest = { - target, - isolate, - functionName, - params, - proctype, - } - if (procOpts.prefix) { - unsequencedRequest.branchPrefix = procOpts.prefix - } - if (procOpts.branchName) { - unsequencedRequest.branchName = procOpts.branchName - } - if (procOpts.deletes) { - unsequencedRequest.deletes = procOpts.deletes - } - return execute(unsequencedRequest) - } - } - return actions as PromisifyFunctionReturnTypes<T> -} -type PromisifyFunctionReturnTypes<T> = { - [K in keyof T]: T[K] extends (...args: infer Args) => infer R - ? (...args: Args) => R extends Promise<unknown> ? R : Promise<R> - : T[K] -} -const safeParams = (params: Params) => { - const safe = { ...params } - for (const key in safe) { - if (safe[key] === undefined) { - delete safe[key] - } - } - checkUndefined(safe) - return safe -} -const checkUndefined = (params: Params) => { - for (const key in params) { - if (params[key] === undefined) { - throw new Error('undefined value: ' + key) - } - if (typeof params[key] === 'object') { - checkUndefined(params[key] as Params) - } - } -} - -export const generateActorId = () => { - return 'act_' + randomness() -} -export const generateBackchatId = () => { - return 'bac_' + randomness() -} - -export const getActorId = (source: PID) => { - const [base, actorId] = source.branches - const parent = { ...source, branches: [base] } - const fullHAL = { ...HAL, repoId: source.repoId } - if (!isPidEqual(parent, fullHAL)) { - throw new Error('source is not a child of HAL') - } - if (!actorIdRegex.test(actorId)) { - throw new Error('Invalid actor id: ' + actorId) - } - return actorId -} -export const getActorPid = (source: PID) => { - const actorId = getActorId(source) - const branches = [source.branches[0], actorId] - return { ...source, branches } -} -export const isActorBranch = (pid: PID) => { - if (pid.branches.length !== 2) { - return false - } - return !!getActorId(pid) -} - -export const isPidEqual = (pid1: PID, pid2: PID) => { - // TODO why not just use the fast-equals utility ? - if (pid1.repoId !== pid2.repoId) { - return false - } - if (pid1.account !== pid2.account) { - return false - } - if (pid1.repository !== pid2.repository) { - return false - } - if (pid1.branches.length !== pid2.branches.length) { - return false - } - for (let i = 0; i < pid1.branches.length; i++) { - if (pid1.branches[i] !== pid2.branches[i]) { - return false - } - } - return true -} -export const META_SYMBOL = Symbol.for('settling commit') -export type Meta = { - parent?: CommitOid - // TODO add the PID so we know what the id of the branch that returned was -} -export const withMeta = async <T>(promise: MetaPromise<T>) => { - const result = await promise - assert.truthy(META_SYMBOL in promise, 'missing commit symbol') - const meta = promise[META_SYMBOL] - assert.object(meta, 'missing meta on promise') - const { parent } = meta - if (parent) { - assert.string(parent, 'missing parent commit') - assert.truthy(sha1.test(parent), 'commit not sha1: ' + parent) - } - return { result, parent } -} -export const sha1 = /^[0-9a-f]{40}$/i -export type MetaPromise<T> = Promise<T> & { [META_SYMBOL]?: Meta } - -export const addBranches = (pid: PID, ...children: string[]) => { - const next = { ...pid, branches: [...pid.branches, ...children] } - return freezePid(next) -} -export const addPeer = (pid: PID, peer: string) => { - const branches = [...pid.branches] - branches.pop() - const next = { ...pid, branches: [...branches, peer] } - return freezePid(next) -} -export const getParent = (pid: PID) => { - const branches = [...pid.branches] - branches.pop() - return freezePid({ ...pid, branches }) -} -export const getRoot = (pid: PID) => { - const branches = [pid.branches[0]] - return freezePid({ ...pid, branches }) -} -export const getBaseName = (pid: PID) => { - return pid.branches[pid.branches.length - 1] -} - -export const getContent = (message: AssistantsThread['messages'][number]) => { - const { content } = message - if (content[0].type !== 'text') { - throw new Error('content not text') - } - return content[0].text.value -} -export const getThreadPath = (pid: PID) => { - const [, , ...actorChildBranches] = pid.branches - const threadPath = actorChildBranches.join('/') - const path = `threads/${threadPath}.json` - return path -} - -export const agentConfigSchema = z.object({ - model: z.enum([ - 'gpt-3.5-turbo', - 'gpt-4-turbo', - 'gpt-4o', - 'gpt-4o-mini', - 'o1-preview', - 'o1-mini', - ]), - temperature: z.number().gte(0).lte(2).optional(), - presence_penalty: z.number().optional(), - tool_choice: z.enum(['auto', 'none', 'required']).optional().describe( - 'control model behaviour to force it to call a tool or no tool', - ), - parallel_tool_calls: z.boolean().optional().describe( - 'Is the model permitted to call more than one function at a time. Must be false to use strict function calling', - ), -}) - -export const agentSchema = z.object({ - name: z.string().regex(/^[a-zA-Z0-9_-]+$/), - source: triad.describe('Where exactly did this agent come from'), - description: z.string().optional(), - config: agentConfigSchema, - runner: z.enum(['ai-runner']), - commands: z.array(z.string()), - napps: z.array(z.string()), - instructions: z.string().max(256000), -}) -export type Agent = z.infer<typeof agentSchema> - -export const chatParams = agentConfigSchema.extend({ - messages: z.array(completionMessage), - seed: z.literal(1337), - tools: z.array(z.object({ - type: z.literal('function'), - function: z.object({ - name: z.string(), - description: z.string().optional(), - parameters: z.object({}).passthrough().optional(), - strict: z.boolean().optional().nullable(), - }), - })).optional(), -}) -export type ChatParams = z.infer<typeof chatParams> - -export const backchatStateSchema = z.object({ - /** The base thread that this backchat session points to - the thread of last resort */ - target: pidSchema, - threadCount: z.number().int().gte(0), -}) - -export type Returns<T extends Record<string, ZodSchema>> = { - [K in keyof T]: ZodSchema -} -// TODO ensure that function return types are inferred from returns object -export type ToApiType< - P extends Record<string, ZodSchema>, - R extends Returns<P>, -> = { - [K in keyof P]: ( - params: z.infer<P[K]>, - ) => z.infer<R[K]> | Promise<z.infer<R[K]>> -} - -export const serializableError = z.object({ - name: z.string().optional(), - message: z.string(), - stack: z.string().optional(), -}) -export type SerializableError = z.infer<typeof serializableError> - -export const outcomeSchema = z.object({ - result: jsonSchema.optional(), - error: serializableError.optional(), -}).refine((data) => { - if (data.error !== undefined) { - return data.result === undefined - } - return true -}, 'result and error are mutually exclusive') - -export type Outcome = { result?: JsonValue; error?: SerializableError } - -export type IoStruct = z.infer<typeof ioStruct> -export const ioStruct = z.object({ - sequence: sequenceInteger, - /** The current sequence of the request being executed serially */ - executing: sequenceInteger.optional(), - /** The sequences of requests that have been executed serially */ - executed: z.record(sequenceKey, z.boolean()), - // TODO make the requests be a zod schema - requests: z.record(sequenceKey, requestSchema), - replies: z.record(sequenceKey, outcomeSchema), - /** If a reply is a merge reply, the commit that carried it is stored here */ - parents: z.record(sequenceKey, md5), - /** - * If a request generates child requests, they are tracked here. The commit - * in each entry is the commit that caused the child requests to be generated. - * This is used to replay by resetting the fs to that commit and doing a - * replay. - */ - pendings: z.record( - sequenceKey, - z.array(z.object({ - commit: md5, - sequences: z.array(sequenceInteger), - })), - ), - /** Active branches are stored here. A branch is a daemon if it is listed - * here but its request has been replied to or it is gone from the requests - * list */ - branches: z.record(sequenceKey, z.string()), - /** - * Isolates can store values here and know they will not leak into other - * branches, and will be quick to access since the io file is always loaded. - */ - state: z.record(jsonSchema), -}) -export const reasoning = z.array(z.string()).describe( - 'Step by step reasoning why this function was called and what it is trying to achieve. This is working space for clarifying thought and is not passed through to the function', -) -export type TreeEntry = { - /** - * - the 6 digit hexadecimal mode - */ - mode: string - /** - * - the name of the file or directory - */ - path: string - /** - * - the SHA-1 object id of the blob or tree - */ - oid: string - /** - * - the type of object - */ - type: 'blob' | 'tree' | 'commit' -} -export const pooledRef = z.object({ - commit: md5, - sequence: sequenceInteger, - source: pidSchema, - isReply: z.boolean(), -}) -export type PooledRef = z.infer<typeof pooledRef> diff --git a/_import-artifact/api/zod.ts b/_import-artifact/api/zod.ts deleted file mode 100644 index b26b81347e60..000000000000 --- a/_import-artifact/api/zod.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { z } from 'zod' - -// Define the schema for content parts (text and image) -const ChatCompletionContentPartTextSchema = z.object({ - type: z.literal('text'), - text: z.string(), -}) - -const ChatCompletionContentPartImageSchema = z.object({ - type: z.literal('image_url'), - image_url: z.object({ - url: z.string(), - detail: z.enum(['auto', 'low', 'high']).optional(), - }), -}) - -const ChatCompletionContentPartRefusalSchema = z.object({ - refusal: z.string(), - type: z.literal('refusal'), -}) - -const ChatCompletionContentSchema = z.union([ - z.string(), - z.array( - z.union([ - ChatCompletionContentPartTextSchema, - ChatCompletionContentPartImageSchema, - ]), - ), -]) - -const ChatCompletionContentResponseSchema = z.union([ - z.string(), - z.array(ChatCompletionContentPartTextSchema), -]) - -const systemMessage = z.object({ - content: ChatCompletionContentResponseSchema, - role: z.literal('system'), - name: z.string().optional(), -}) - -// Define the user message schema -const userMessage = z.object({ - content: ChatCompletionContentSchema, - role: z.literal('user'), - name: z.string().optional(), -}) - -// Define the assistant message schema -export type AssistantMessage = z.infer< - typeof assistantMessage -> -export const assistantMessage = z.object({ - role: z.literal('assistant'), - content: z.union([ - z.string(), - z.array(z.union([ - ChatCompletionContentPartTextSchema, - ChatCompletionContentPartRefusalSchema, - ])), - ]).optional().nullable(), - name: z.string().optional(), - refusal: z.string().optional().nullable(), - tool_calls: z - .array( - z.object({ - id: z.string(), - type: z.literal('function'), - function: z.object({ - name: z.string(), - arguments: z.string(), - }), - }), - ) - .optional(), -}) - -export type ToolMessage = z.infer<typeof toolMessage> -const toolMessage = z.object({ - role: z.literal('tool'), - content: ChatCompletionContentResponseSchema, - tool_call_id: z.string(), -}) - -export type CompletionMessage = z.infer<typeof completionMessage> -export const completionMessage = z.union([ - systemMessage, - userMessage, - assistantMessage, - toolMessage, -]) diff --git a/_import-artifact/constants.ts b/_import-artifact/constants.ts deleted file mode 100644 index 4662f775b509..000000000000 --- a/_import-artifact/constants.ts +++ /dev/null @@ -1,251 +0,0 @@ -import IA from './isolate-api.ts' -export type { IA } -export const IO_PATH = '.io.json' -import { - type Backchat, - Change, - CommitObject, - EngineInterface, - IsolateReturn, - MetaPromise, - Outcome, - Params, - PID, - Pierce, - RemoteRequest, - Request, - SolidRequest, - UnsequencedRequest, -} from './api/types.ts' -import FS from '@/git/fs.ts' -import type DB from '@/db.ts' -import type Executor from '@/exe/exe.ts' -import { assert, equal } from '@utils' -import { JsonSchema7ObjectType, zodToJsonSchema } from 'zod-to-json-schema' -import { ZodObject, ZodSchema, ZodUnknown } from 'zod' - -export const REPO_LOCK_TIMEOUT_MS = 5000 - -/** Artifact Context, including the db and executor */ -export type C = { - db: DB - exe: Executor - aesKey?: string - seed?: Deno.KvEntry<unknown>[] -} - -/** Extends the actions api to be the isolate api */ -export type Functions<Api> = { - [K in keyof Api]: Function<Api[K]> -} - -type Function<T> = T extends (...args: infer Args) => infer R - ? (...args: [...Args, IA]) => R - : never - -export type IsolateFunction = { - (): unknown | Promise<unknown> - (...args: [Params]): unknown | Promise<unknown> - (...args: [Params, IA]): unknown | Promise<unknown> -} - -export type IsolateFunctions = { - [key: string]: IsolateFunction -} -export type IsolateLifecycle = { - '@@mount'?: (api: IA) => Promise<IsolateReturn> | IsolateReturn - '@@unmount'?: (api: IA) => Promise<IsolateReturn> | IsolateReturn -} -export type Isolate = { - parameters: Record<string, ZodObject<Record<string, ZodUnknown>>> - returns: Record<string, ZodSchema> - functions: IsolateFunctions - lifecycles?: IsolateLifecycle -} - -export type Poolable = MergeReply | RemoteRequest -export type Reply = SolidReply | MergeReply -export type EffectRequest = { - target: PID - /** - * The hash of the function that was called, to ensure repeatability - */ - fingerprint: string - sequence: number -} -export type SolidReply = { - target: PID - sequence: number - outcome: Outcome -} -export type MergeReply = SolidReply & { - /** - * Where did this merge reply come from? - */ - source: PID - /** - * The commit that solidified this merge reply, which is used as a merge - * parent in the recipient branch, so that any changes to the fs can be - * accessed and so the provenance of the action is included. - */ - commit: string -} -export type IsolatePromise = - | BareIsolatePromise - | PromisedIsolatePromise - | SettledIsolatePromise -type BareIsolatePromise = { - request: UnsequencedRequest -} -export type PromisedIsolatePromise<T = unknown> = BareIsolatePromise & { - promise: MetaPromise<T> - resolve: (value: T) => void - reject: (error: Error) => void -} -export type SettledIsolatePromise<T = unknown> = - & (BareIsolatePromise | PromisedIsolatePromise<T>) - & { - outcome: Outcome - /** if an outcome is given, there must be a commit associated with it, so - * that the execution filesystem can be ticked forwards */ - commit: string - /** If the outcome was the result of a branch returning, then the parent - * commit of that branch is given here */ - parent?: string - } -export const isSettledIsolatePromise = ( - p: IsolatePromise, -): p is SettledIsolatePromise => { - return 'outcome' in p -} -export type Solids = { - oid: string - commit: CommitObject - /** Changed files in this commit. Empty change signals deletion. */ - changes: { [key: string]: Change } - exe?: { request: SolidRequest; sequence: number } - branches: number[] - poolables: (MergeReply | RemoteRequest)[] - deletes: { pid: PID; commit: string }[] -} -export type Branched = { - /** The first request in the new branch */ - origin: SolidRequest - /** The branch PID that needs to be created in /.git/refs */ - pid: PID - /** The head of the new branch that needs to be created in /.git/refs */ - head: string -} -export type ExeResult = ExeSettled | ExePending -type ExeResultBase = { - /** - * The last filesystem that was modified during the execution run. The FS - * might have been bumped forwards if accumulations occurred. - */ - fs: FS - /** If this is a side effect request, this is the lock held by for it */ - effectsLock?: Deno.KvEntry<string> -} -type ExeSettled = ExeResultBase & { - reply: SolidReply -} -type ExePending = ExeResultBase & { - pending: Pending -} -export type Pending = { - /** The commit that caused the requests to be generated */ - commit: string - /** The requests that were generated by the latest round of execution */ - requests: UnsequencedRequest[] - /** The sequence number to accumulate the pending requests against */ - sequence: number -} - -export const isMergeReply = ( - poolable: Poolable | SolidReply, -): poolable is MergeReply => { - return 'commit' in poolable && 'outcome' in poolable -} -export const isReply = ( - poolable: Poolable | Pierce | SolidReply, -): poolable is Reply => { - return 'outcome' in poolable -} -export const isRemoteRequest = ( - poolable: Request, -): poolable is RemoteRequest => { - return 'commit' in poolable && 'proctype' in poolable -} -/** - * Messages that go on the queue are one of three types. Each one is an - * operation that will result in a new commit, atomically. Each operation is - * able to detect when it is a duplicate task due to duplicate message delivery. - * Each task will continue to retry until it is successful, as long as its check - * for duplication reassures it to keep trying. - */ -export type QueueMessage = QueuePool | QueueExe | QueueBranch -export enum QueueMessageType { - POOL = 'pool', - EXECUTION = 'exe', - BRANCH = 'branch', -} -export type QueuePool = { - type: QueueMessageType.POOL - pid: PID -} -export type QueueExe = { - type: QueueMessageType.EXECUTION - pid: PID - commit: string - sequence: number -} -export type QueueBranch = { - type: QueueMessageType.BRANCH - parentCommit: string - parentPid: PID - sequence: number -} -export const isQueuePool = (m: QueueMessage): m is QueuePool => { - return m.type === QueueMessageType.POOL -} -export const isQueueExe = (m: QueueMessage): m is QueueExe => { - return m.type === QueueMessageType.EXECUTION -} -export const isQueueBranch = (m: QueueMessage): m is QueueBranch => { - return m.type === QueueMessageType.BRANCH -} -export const isChildOf = (child: PID, parent: PID) => { - const childParent = { ...child, branches: child.branches.slice(0, -1) } - return equal(childParent, parent) -} -export const isBaseRepo = (pid: PID) => pid.branches.length === 1 - -export type Provisioner = (superBackchat: Backchat) => Promise<void> - -export type CradleMaker = ( - t: Deno.TestContext, - /** The file url that the snapshots are associated with */ - snapshotsFor: string, - updateSnapshots?: 'updateSnapshots', - init?: Provisioner, -) => Promise< - { - backchat: Backchat - engine: EngineInterface - privateKey: string - [Symbol.asyncDispose](): Promise<void> - } -> - -export const toApi = (parameters: Record<string, ZodSchema>) => { - const api: Record<keyof typeof parameters, JsonSchema7ObjectType> = {} - for (const key of Object.keys(parameters)) { - const schema = zodToJsonSchema(parameters[key]) - delete schema.$schema - assert('properties' in schema, 'schema must have properties') - api[key] = schema - } - return api -} - -export * from './api/types.ts' diff --git a/_import-artifact/exe/accumulator.ts b/_import-artifact/exe/accumulator.ts deleted file mode 100644 index 4058f7637e63..000000000000 --- a/_import-artifact/exe/accumulator.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { - META_SYMBOL, - PID, - PromisedIsolatePromise, - SettledIsolatePromise, - UnsequencedRequest, -} from '@/constants.ts' -import { IsolatePromise } from '@/constants.ts' -import { assert, deserializeError, equal, expect } from '@utils' -import FS from '@/git/fs.ts' - -export default class Accumulator { - #index = 0 - #buffer: IsolatePromise[] - #fs: FS - #highestFs: FS - #new: PromisedIsolatePromise[] = [] - #isActive = false - #trigger: (() => void) | undefined - private constructor(highestFs: FS, buffer: IsolatePromise[]) { - this.#buffer = buffer - this.#highestFs = highestFs - this.#fs = highestFs - const [first] = buffer - if (first) { - assert('commit' in first, 'first accumulation must have a commit') - this.#fs = highestFs.tick(first.commit) - } - } - static create(highestFs: FS, buffer: IsolatePromise[] = []) { - const acc = new Accumulator(highestFs, buffer) - return acc - } - get accumulations() { - return [...this.#new] - } - get fs() { - return this.#fs - } - push(promised: PromisedIsolatePromise) { - assert(this.isActive, 'Activity is denied') - assert(typeof this.#trigger === 'function', 'Trigger is not set') - this.#trigger() - this.#new.push(promised) - this.#buffer.push(promised) - } - #tickFs() { - const next = this.#buffer[this.#index] - if (next && 'commit' in next) { - this.#fs = this.#fs.tick(next.commit) - } else if (this.fs.oid !== this.#highestFs.oid) { - // we are at the final layer, so use the latest fs - this.#fs = this.#fs.tick(this.#highestFs.oid) - } - } - recover(request: UnsequencedRequest) { - assert(this.isActive, 'Activity is denied') - const index = this.#index++ - if (this.#buffer[index]) { - const recovered = this.#buffer[index] - if (!equal(recovered.request, request)) { - expect(recovered.request, 'Requests are not equal').toEqual(request) - } - if ('outcome' in recovered) { - this.#tickFs() - } - return recovered - } - } - isParent(parent: string) { - assert(this.isActive, 'Activity is denied') - for (const promise of this.#buffer) { - if ('parent' in promise) { - if (promise.parent === parent) { - return true - } - } - } - return false - } - activate(symbol: symbol) { - assert(!this.isActive, 'Activity is already active') - assert(!this.#trigger, 'Trigger is already set') - this.#isActive = true - return new Promise<symbol>((resolve) => { - this.#trigger = () => resolve(symbol) - }) - } - deactivate() { - assert(this.isActive, 'Activity is not active') - this.#isActive = false - this.#trigger = undefined - } - get isActive() { - return this.#isActive - } - /** - * Used so that an execution can be paused, then receive replies from - * accumulated actions, then continue without restarting the execution. Makes - * it easier to debug these functions, but also can be faster to execute. - * This is a nice to have and the operation is equally capable of starting - * again, if we find ourselves replaying the operation with no existing cache. - * - * As new layers of the accumulation process occur, the filesystem object - * referenced by the isolate-api object will tick forwards. - * - * @param from The newer accumulator that should be copied in to the old one - */ - absorb(from: Accumulator) { - assert(!this.isActive, '"this" is already active') - assert(!from.isActive, '"from" is already active') - if (this === from) { - return - } - assert(this.#buffer.length <= from.#buffer.length, '"this" must be shorter') - this.#highestFs = from.#highestFs - - let index = 0 - for (const source of from.#buffer) { - const sink = this.#buffer[index++] - if (!sink) { - this.#buffer.push(source) - continue - } - if (!equal(source.request, sink.request)) { - expect(source.request).toEqual(sink.request) - } - if ('outcome' in sink) { - assert('outcome' in source, 'source has no outcome') - assert(equal(source.outcome, sink.outcome), 'outcomes are not equal') - assert(equal(source.commit, sink.commit), 'commits are not equal') - } else if ('outcome' in source) { - const settledSink = sink as SettledIsolatePromise - settledSink.outcome = source.outcome - settledSink.commit = source.commit - settledSink.parent = source.parent - } - if ('outcome' in sink && 'promise' in sink) { - sink.promise[META_SYMBOL] = { parent: sink.parent } - if (sink.outcome.error) { - sink.reject(deserializeError(sink.outcome.error)) - } else { - sink.resolve(sink.outcome.result) - } - } - } - this.#tickFs() - // TODO assert the #new matched the incoming updates exactly - this.#new = [] - } - isPidExists(pid: PID) { - // TODO log this action in the buffer and make it replayable - return this.fs.isPidExists(pid) - } -} diff --git a/_import-artifact/io/compartment.ts b/_import-artifact/io/compartment.ts deleted file mode 100644 index 08551b68377d..000000000000 --- a/_import-artifact/io/compartment.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { assert, Debug } from '@utils' -import { Isolate, Params, toApi } from '@/constants.ts' -import IA from '../isolate-api.ts' - -// deno has no dynamic runtime imports, so this is a workaround -import isolates from '../isolates/index.ts' -import { DispatchFunctions } from '@/constants.ts' -import { ZodSchema } from 'zod' - -const log = Debug('AI:compartment') -const cache = new Map<string, Compartment>() - -export default class Compartment { - #module: Isolate - #isolate: string - #check() { - assert(this.#module, 'code not loaded') - } - private constructor(isolate: string) { - log('load isolate:', isolate) - assert(isolates[isolate as keyof typeof isolates], `not found: ${isolate}`) - // TODO this is incredibly messy for types - this.#module = - isolates[isolate as keyof typeof isolates] as unknown as Isolate - this.#isolate = isolate - const { parameters, returns, functions } = this.#module - assert(typeof parameters === 'object', 'no parameters exported: ' + isolate) - assert(typeof returns === 'object', 'no returns exported: ' + isolate) - - zodCheck(parameters, returns, isolate) - assert(typeof functions === 'object', 'functions not exported: ' + isolate) - const missing = Object.keys(parameters).filter((key) => !functions[key]) - assert(!missing.length, `${isolate} Missing: ${missing.join(', ')}`) - } - static async create(isolate: string) { - if (!cache.has(isolate)) { - const compartment = new Compartment(isolate) - cache.set(isolate, compartment) - await Promise.resolve() // simulates loading from filesystem - } - const compartment = cache.get(isolate) - assert(compartment, 'compartment not found: ' + isolate) - return compartment - } - get api() { - this.#check() - return toApi(this.#module.parameters) - } - /** - * Mount the isolate as a side effect, and give it the chance to initialize - * some context that will get passed between different invocations on the - * same mount. - * @param api : IsolateApi - * @returns Promise<void> | void - */ - mount(api: IA) { - // TODO use exe to ensure that mount stops working arfter invocation - this.#check() - if (this.#module.lifecycles) { - if (typeof this.#module.lifecycles['@@mount'] === 'function') { - return this.#module.lifecycles['@@mount'](api) - } - } - } - /** - * Unmount the isolate as a side effect, and give it the chance to clean up - * @param api : IsolateApi - */ - unmount(api: IA) { - this.#check() - if (this.#module.lifecycles) { - if (typeof this.#module.lifecycles['@@unmount'] === 'function') { - return this.#module.lifecycles['@@unmount'](api) - } - } - } - functions<T = DispatchFunctions>(api: IA) { - this.#check() - const actions: DispatchFunctions = {} - for (const functionName in this.#module.parameters) { - actions[functionName] = this.#toFunction(functionName, api) - } - return actions as T - } - #toFunction(functionName: string, api: IA) { - return (parameters?: Params) => { - log('dispatch: %o', functionName) - if (parameters === undefined) { - parameters = {} - } - const schema = this.#module.parameters[functionName] - const path = this.#isolate + '/' + functionName - try { - schema.parse(parameters) - } catch (error) { - const message = error instanceof Error ? error.message : '' - throw new Error( - `Zod schema parameters validation error at path: ${path}\nError: ${message}`, - ) - } - return Promise.resolve( - this.#module.functions[functionName](parameters, api), - ) - .then((result) => { - const schema = this.#module.returns[functionName] - const parsed = schema.safeParse(result) - if (!parsed.success) { - throw new Error( - `Unrecoverable system error in ${path}. ${parsed.error.message}`, - ) - } - return result - }) - } - } -} -const zodCheck = ( - parameters: Record<string, ZodSchema>, - returns: Record<string, ZodSchema>, - isolate: string, -) => { - assert(Object.keys(parameters).length, 'no api exported: ' + isolate) - assert( - Object.keys(returns).length === Object.keys(parameters).length, - 'parameters do not match returns: ' + isolate, - ) - - for (const key in parameters) { - assert(parameters[key] instanceof ZodSchema, 'invalid schema: ' + key) - assert(returns[key] instanceof ZodSchema, 'invalid schema: ' + key) - } -} diff --git a/_import-artifact/isolate-api.ts b/_import-artifact/isolate-api.ts deleted file mode 100644 index 7429443659f9..000000000000 --- a/_import-artifact/isolate-api.ts +++ /dev/null @@ -1,326 +0,0 @@ -import { deserializeError } from 'serialize-error' -import Accumulator from './exe/accumulator.ts' -import Compartment from './io/compartment.ts' -import { assert, Debug, equal } from '@utils' -import micromatch from 'micromatch' -import { - ApiFunctions, - DispatchFunctions, - freezePid, - getThreadPath, - IoStruct, - isChildOf, - isSettledIsolatePromise, - META_SYMBOL, - MetaPromise, - PID, - print, - PromisedIsolatePromise, - RpcOpts, - SolidRequest, - threadSchema, - toActions, - UnsequencedRequest, -} from '@/constants.ts' -import { type Isolate } from '@/isolates/index.ts' -import * as files from '@/isolates/files.ts' -import IOChannel from '@io/io-channel.ts' -import { z, ZodTypeAny } from 'zod' -const log = Debug('AI:isolateApi') -interface Default { - [key: string]: unknown -} -type EffectOptions = { - isEffect: boolean - isEffectRecovered: boolean -} - -export default class IA<T extends object = Default> { - #accumulator: Accumulator - #origin: SolidRequest - #originCommit: string | undefined - // TODO assign a mount id for each side effect execution context ? - #context: Partial<T> = {} - #isEffect = false - #isEffectRecovered = false - #abort = new AbortController() - private constructor( - accumulator: Accumulator, - origin: SolidRequest, - originCommit?: string, - ) { - this.#accumulator = accumulator - this.#origin = origin - this.#originCommit = originCommit - } - static create( - accumulator: Accumulator, - origin: SolidRequest, - originCommit?: string, - opts?: EffectOptions, - ) { - const api = new IA(accumulator, origin, originCommit) - if (opts) { - api.#isEffect = opts.isEffect || false - api.#isEffectRecovered = opts.isEffectRecovered || false - } - return api - } - static createContext<T extends object = Default>() { - // TODO find a more graceful way to do this for cradle setup - return new IA<T>( - null as unknown as Accumulator, - null as unknown as SolidRequest, - ) - } - get #fs() { - return this.#accumulator.fs - } - get pid() { - return this.#fs.pid - } - get origin() { - return this.#origin - } - /** The commit from the origin action */ - get originCommit() { - return this.#originCommit - } - get commit() { - return this.#fs.oid - } - /** If this execution is side effect capable. May extend to get permissions - * information */ - get isEffect() { - return this.#isEffect - } - /** If the side effect lock was broken in order to start this instance. - * Implies the previous executing instance was aborted */ - get isEffectRecovered() { - return this.#isEffectRecovered - } - /** Side effects can listen to this signal to abort their activities */ - get signal() { - assert(this.#accumulator.isActive, 'Activity is denied') - // return an abort signal - assert(this.isEffect, 'signal only available for side effects') - return this.#abort.signal - } - // TODO make get and set config be synchronous - async state<T extends z.ZodObject<Record<string, ZodTypeAny>>>( - schema: T, - fallback: z.infer<T>, - ) { - assert(this.#accumulator.isActive, 'Activity is denied') - const io = await IOChannel.read(this.#fs) - assert(io, 'io not found') - if (equal(io.state, {})) { - return schema.parse(fallback) as z.infer<T> - } - return schema.parse(io.state) as z.infer<T> - } - async updateState<T extends z.ZodObject<Record<string, ZodTypeAny>>>( - updater: (state: z.infer<T>) => z.infer<T>, - schema: T, - ) { - assert(this.#accumulator.isActive, 'Activity is denied') - const io = await IOChannel.load(this.#fs) - assert(io, 'io not found') - const state = io.state as z.infer<T> - const result = updater(state) - if (!equal(state, result)) { - const next = schema.parse(result) as z.infer<T> - assert(io, 'io not found') - io.state = next - io.save() - } - } - - async actions<T = DispatchFunctions>(isolate: Isolate, opts: RpcOpts = {}) { - const { target = this.pid, ...procOpts } = opts - freezePid(target) - const schema = await this.apiSchema(isolate) - const execute = (request: UnsequencedRequest) => this.action(request) - return toActions<T>(target, isolate, schema, procOpts, execute) - } - async requests<T extends ApiFunctions>(isolate: Isolate, opts: RpcOpts = {}) { - const { target = this.pid, ...procOpts } = opts - freezePid(target) - - type Unseq = { - [K in keyof T]: (...args: Parameters<T[K]>) => UnsequencedRequest - } - - const schema = await this.apiSchema(isolate) - const execute = (request: UnsequencedRequest) => request - return toActions<Unseq>(target, isolate, schema, procOpts, execute) - } - action(request: UnsequencedRequest) { - const recovered = this.#accumulator.recover(request) - if (recovered) { - assert(isSettledIsolatePromise(recovered), 'recovered is not settled') - const { outcome } = recovered - let promise: MetaPromise<typeof outcome.result> - if (outcome.error) { - promise = Promise.reject(deserializeError(outcome.error)) - } else { - promise = Promise.resolve(outcome.result) - } - promise[META_SYMBOL] = { parent: recovered.parent } - return promise - } - let resolve, reject - const promise: MetaPromise<unknown> = new Promise((_resolve, _reject) => { - resolve = _resolve - reject = _reject - }) - assert(resolve) - assert(reject) - const promised: PromisedIsolatePromise = { - promise, - request, - resolve, - reject, - } - this.#accumulator.push(promised) - return promise - } - /** - * Used to call the functions of an isolate purely, without going thru the IO - * subsystem which would otherwise cost a commit to the chain. - * @param isolate The name of the isolate to load the functions for - * @returns An object keyed by API function name, with values being the - * function itself. - */ - async functions<T = DispatchFunctions>(isolate: Isolate): Promise<T> { - // TODO these need some kind of PID attached ? - const compartment = await Compartment.create(isolate) - // TODO but these need to be wrapped in a dispatch call somewhere - return compartment.functions<T>(this) - } - async apiSchema(isolate: Isolate) { - const compartment = await Compartment.create(isolate) - return compartment.api - } - writeJSON(path: string, json: unknown) { - assert(this.#accumulator.isActive, 'Activity is denied') - log('writeJSON', path) - this.#fs.writeJSON(path, json) - } - write(path: string, content: string | Uint8Array = '') { - assert(this.#accumulator.isActive, 'Activity is denied') - log('write', path) - this.#fs.write(path, content) - } - async readJSON<T>( - path: string, - opts?: { target?: PID; commit?: string }, - ): Promise<T> { - // TODO implement readJSON<type> for remote reads, and take a schema - assert(this.#accumulator.isActive, 'Activity is denied') - log('readJSON', path) - if (opts && opts.target) { - // TODO move to something native - const { read } = await this.actions<files.Api>('files', opts) - const params = { path, reasoning: [], commit: opts.commit } - const string = await read(params) - return JSON.parse(string) as T - } - return this.#fs.readJSON<T>(path, opts?.commit) - } - async readThread( - threadPath?: string, - opts?: { target?: PID; commit?: string }, - ) { - if (!threadPath) { - threadPath = getThreadPath(opts?.target || this.pid) - } - const thread = await this.readJSON(threadPath, opts) - return threadSchema.parse(thread) - } - async read(path: string, opts?: { target?: PID; commit?: string }) { - assert(this.#accumulator.isActive, 'Activity is denied') - if (opts?.target) { - // TODO move to something native - const { read } = await this.actions<files.Api>('files', opts) - const params = { path, reasoning: [], commit: opts.commit } - return read(params) - } - return this.#fs.read(path, opts?.commit) - } - readOid(path: string) { - assert(this.#accumulator.isActive, 'Activity is denied') - return this.#fs.readOid(path) - } - readBinary(path: string) { - assert(this.#accumulator.isActive, 'Activity is denied') - log('readBinary', path) - return this.#fs.readBinary(path) - } - exists(path: string) { - assert(this.#accumulator.isActive, 'Activity is denied') - log('exists', path) - return this.#fs.exists(path) - } - ls(path: string = '.') { - assert(this.#accumulator.isActive, 'Activity is denied') - log('ls', path) - return this.#fs.ls(path) - } - delete(filepath: string) { - assert(this.#accumulator.isActive, 'Activity is denied') - log('delete', filepath) - return this.#fs.delete(filepath) - } - - async isActiveChild(pid: PID) { - if (!isChildOf(pid, this.pid)) { - throw new Error('not child: ' + print(pid) + ' of ' + print(this.pid)) - // TODO allow recursive PID walking - // TODO allow walking parents and remote repos - } - // TODO use a direct db lookup, relying on the atomic guarantees - - const obj = await this.readJSON<IoStruct>('.io.json') - const child = pid.branches[pid.branches.length - 1] - log('readTip', child) - for (const branchName of Object.values(obj.branches)) { - if (branchName === child) { - return true - } - } - return false - } - async lsChildren(patterns: string[] = []) { - const obj = await this.readJSON<IoStruct>('.io.json') - const branches = Object.values(obj.branches) - return micromatch(branches, patterns) - } - get context() { - // TODO at creation, this should flag context capable and reject if not - return this.#context as T - } - set context(context: Partial<T>) { - // TODO reject if any fs operations or actions are attempted - assert(typeof context === 'object', 'context must be an object') - assert(context !== null, 'context must not be null') - this.#context = context - } - isPidExists(pid: PID) { - // TODO push a self responding action to the accumulator for repeatability - return this.#fs.isPidExists(pid) - } - merge(commit: string, ...excludes: string[]) { - if (commit !== this.originCommit) { - assert(this.#accumulator.isParent(commit), 'Parent is not in scope') - } - log('overwrite', commit, excludes) - return this.#fs.overwrite(commit, ...excludes) - } - mv(from: string, to: string) { - return this.#fs.mv(from, to) - } - cp(from: string, to: string) { - return this.#fs.cp(from, to) - } -} diff --git a/_import-artifact/isolates/ai-completions.test.ts b/_import-artifact/isolates/ai-completions.test.ts deleted file mode 100644 index 1481e7d0948c..000000000000 --- a/_import-artifact/isolates/ai-completions.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { loadAgent } from '@/isolates/utils/load-agent.ts' -import { complete, image } from '@/isolates/ai-completions.ts' -import { functions, safeAssistantName } from '@/isolates/ai-completions.ts' -import { expect } from '@utils' -import { createMockApi } from '@/tests/fixtures/mock-api.ts' - -Deno.test('test the regex for agent name sanitization', () => { - const result = safeAssistantName({ - role: 'assistant', - name: 'agents/o1.md', - }) - expect(result).toEqual({ role: 'assistant', name: 'agents_o1_md' }) -}) -Deno.test('generate images', async (t) => { - image.mock.useRecorder(t) - const { api, stop } = await createMockApi('test/images') - const path = 'images/test.jpg' - const result = await functions.image({ - path, - prompt: 'a dystopian robot overlord mecha', - lowQuality: true, - size: '1024x1024', - style: 'natural', - }, api) - const file = await api.readBinary(path) - expect(file.byteLength).toBeGreaterThan(0) - expect(file.byteLength).toEqual(result.size) - - image.mock.teardown() - stop() -}) - -Deno.test('inject empty', async (t) => { - complete.mock.useRecorder(t) - const { api, stop } = await createMockApi('test/inject-single') - const path = 'fake/agent.md' - api.write(path, '') - const agent = await loadAgent(path, api) - - const result = await complete(agent, [{ - role: 'system', - content: 'say cheese in emoji', - }], api) - expect(result).toHaveProperty('assistant') - expect(result).toHaveProperty('stats') - - complete.mock.teardown() - stop() -}) diff --git a/_import-artifact/isolates/ai-completions.ts b/_import-artifact/isolates/ai-completions.ts deleted file mode 100644 index 60391275bee3..000000000000 --- a/_import-artifact/isolates/ai-completions.ts +++ /dev/null @@ -1,321 +0,0 @@ -import { mockCreator } from './utils/mocker.ts' -import { assert, Debug } from '@utils' -import { base64 } from '@scure/base' -import '@std/dotenv/load' // load .env variables -import OpenAI from 'openai' -import { decode, Image } from 'imagescript' -import { - Agent, - AssistantMessage, - ChatParams, - chatParams, - CompletionMessage, - Functions, - getThreadPath, - type IA, - messageStatsSchema, - print, - printPlain, - Returns, - Thread, - ToApiType, -} from '@/constants.ts' -import { loadTools } from './utils/ai-load-tools.ts' -import { loadAgent } from './utils/load-agent.ts' -import { z } from 'zod' -import { assistantMessage } from '@/api/zod.ts' -import { expect } from '@std/expect/expect' - -const log = Debug('AI:completions') - -const apiKey = Deno.env.get('OPENAI_API_KEY') -if (!apiKey) { - throw new Error('missing openai api key: OPENAI_API_KEY') -} -const ai = new OpenAI({ apiKey, maxRetries: 5 }) - -export const transcribe = async (file: File) => { - const transcription = await ai.audio.transcriptions - .create({ - file, - model: 'whisper-1', - prompt: - 'Backchat, GPT4, GPT3, Dreamcatcher, CRM, HAL, Deno, Stucks, Redlid, Pijul', - }) - return transcription.text -} - -export const parameters = { - /** Complete the thread with the given agent */ - complete: z.object({ - path: z.string(), - }), - /** Gives slightly quicker feedback to users when waiting for completions */ - completionEffect: z.object({ - path: z.string(), - }), - image: z.object({ - path: z.string().regex(/\.jpg$/, { - message: 'The path must end with .jpg', - }).describe( - 'The path to the image to generate, which should be a meaningful name in the images/ directory, and should end in .jpg', - ), - prompt: z.string().max(4000), - lowQuality: z.boolean().optional().describe( - 'Generate a low quality image as opposed to the default high quality image', - ), - size: z.enum(['1024x1024', '1792x1024', '1024x1792']).optional().describe( - 'The size of the image to generate', - ), - style: z.enum(['vivid', 'natural']).optional().describe( - 'The style of the image to generate, which defaults to vivid', - ), - }).describe( - 'Generate an image using DALL-E-3 from the provided prompt. The image will be saved to the provided path. The revised prompt that the image generator used will be returned, as well as the size of the image in bytes.', - // TODO add how to show in stateboard - ), -} -export const returns: Returns<typeof parameters> = { - complete: z.void(), - completionEffect: z.void(), - image: z.object({ - /** The revised prompt that the image generation system used */ - revisedPrompt: z.string(), - /** The size in bytes of the written image */ - size: z.number().int().gte(0), - }), -} - -export type Api = ToApiType<typeof parameters, typeof returns> - -export const functions: Functions<Api> = { - async complete({ path }, api) { - const threadPath = getThreadPath(api.pid) - log('completing thread %o', threadPath, print(api.pid)) - - const agent = await loadAgent(path, api) - const thread = await api.readThread(threadPath) - thread.messages.push({ role: 'assistant', name: agent.source.path }) - api.writeJSON(threadPath, thread) - - const { completionEffect } = await api.actions<Api>('ai-completions') - await completionEffect({ path }) - }, - async completionEffect({ path }, api) { - const threadPath = getThreadPath(api.pid) - log('completing thread %o', threadPath, print(api.pid)) - - const agent = await loadAgent(path, api) - - const thread = await api.readThread(threadPath) - const last = thread.messages.pop() - assert(last, 'no messages in thread') - assert(last.role === 'assistant', 'last message must be assistant') - assert(last.name === agent.source.path, 'last message must be from agent') - assert(!last.content, 'last message must be empty') - - const { assistant, stats } = await complete(agent, thread.messages, api) - - const id = thread.messages.length + thread.messageOffset - thread.messageStats[id] = stats - thread.messages.push(assistant) - api.writeJSON(threadPath, thread) - log('completion complete', assistant.tool_calls?.[0], assistant.content) - }, - async image(params, api) { - const { path } = params - const { data, response } = await image(params, api) - log('headers', response.statusText) - const { b64_json, revised_prompt } = data.data[0] - if (!b64_json) { - throw new Error('no image data') - } - const imageData = base64.decode(b64_json) - log('image length', imageData.length) - - const png = await decode(imageData) - assert(png instanceof Image, 'image must be an instance of Image') - const jpg = await png.encodeJPEG() - log('length', jpg.length) - - api.write(path, jpg) - return { revisedPrompt: revised_prompt, size: jpg.length } - }, -} - -export const image = async ( - params: z.infer<typeof parameters['image']>, - api: IA, -) => { - const { prompt, lowQuality, size, style } = params - const id = printPlain(api.pid) - const recording = imageMock.next(id) - if (recording) { - expect(recording.request).toEqual(params) - return recording.reply as { - data: OpenAI.Images.ImagesResponse - response: Response - } - } - - const { data, response } = await ai.images - .generate({ - prompt, - model: 'dall-e-3', - quality: lowQuality ? 'standard' : 'hd', - response_format: 'b64_json', - size, - style, - }).withResponse() - - const squeezed = squeezeMockData(data) - imageMock.store(id, { request: params, reply: { data: squeezed, response } }) - return { data, response } -} - -const imageReplySchema = z.object({ - data: z.unknown(), - response: z.unknown(), -}) -const imagePairSchema = z.object({ - request: parameters['image'], - reply: imageReplySchema, -}) - -const imageMock = mockCreator(imagePairSchema) -image.mock = imageMock - -const completionsReplySchema = z.object({ - assistant: assistantMessage, - stats: messageStatsSchema, -}) - -const completionsPairSchema = z.object({ - request: chatParams, - reply: completionsReplySchema, -}) - -export const complete = async ( - agent: Agent, - messages: Thread['messages'], - api: IA, -) => { - const tools = await loadTools(agent, api) - const args = getChatParams(agent, messages, tools) - - const id = printPlain(api.pid) - const recording = completionsMock.next(id) - if (recording) { - expect(recording.request).toEqual(args) - return recording.reply - } - - log('completion started with model: %o', args.model, print(api.pid)) - let retries = 0 - const RETRY_LIMIT = 5 - let errorMessage = '' - while (retries++ < RETRY_LIMIT) { - try { - const start = Date.now() - const { data: completion, response: raw } = await ai.chat.completions - .create(args).withResponse() - const duration = Date.now() - start - const openAiProcessingMs = raw.headers.get('openai-processing-ms') - const { created, model, system_fingerprint, usage } = completion - - const { message } = completion.choices[0] - log('completion complete', agent.source.path, message) - const assistant: AssistantMessage = { - ...message, - name: agent.source.path, - } - const reply = { - assistant, - stats: messageStatsSchema.parse({ - created, - model, - system_fingerprint, - usage, - duration, - openAiProcessingMs: openAiProcessingMs ? +openAiProcessingMs : 0, - }), - } - completionsMock.store(id, { request: args, reply }) - - return reply - } catch (error) { - console.error('ai completion error', error) - if (error instanceof Error) { - errorMessage = error.message - } - } - } - throw new Error(`Failed after ${retries} attempts: ${errorMessage}`) -} -const completionsMock = mockCreator(completionsPairSchema) -complete.mock = completionsMock - -export const getChatParams = ( - agent: Agent, - messages: Thread['messages'], - tools: OpenAI.ChatCompletionTool[], -) => { - const { config } = agent - const { - model, - temperature, - tool_choice, - parallel_tool_calls, - presence_penalty, - } = config - - messages = [...messages] - const sysprompt: CompletionMessage = { - role: 'system', - content: agent.instructions, - name: agent.source.path, - } - if (agent.instructions || agent.commands.length || agent.napps.length) { - messages.push(sysprompt) - } - messages = messages.map(safeAssistantName) - - let args: ChatParams = { - model, - temperature, - messages, - seed: 1337, - tools: tools.length ? tools : undefined, - tool_choice: tools.length ? tool_choice : undefined, - presence_penalty, - parallel_tool_calls: tools.length ? parallel_tool_calls : undefined, - } - args = Object.fromEntries( - Object.entries(args).filter(([_, value]) => value !== undefined), - ) as ChatParams - return args -} -export const safeAssistantName = (message: CompletionMessage) => { - if (message.role !== 'assistant' && message.role !== 'system') { - return message - } - if (!message.name) { - return message - } - if (!/^[a-zA-Z0-9_-]+$/.test(message.name)) { - return { ...message, name: message.name.replaceAll(/[^a-zA-Z0-9_-]/g, '_') } - } - return message -} - -const squeezeMockData = (result: OpenAI.Images.ImagesResponse) => { - const fake = - 'iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAABQUlEQVR4nAE2Acn+AFWFasx5BD3kQfuYK9zNJJXzR81JXGsdYwxOP/ggyAQC+dlbzrmdUZ+Xgo9sBhbNytzXE2IPfxblFuNZ9vYCD9Qw+7UUN/xCErc8I0gk21S+uCU4EX33VN1CFz5pAHaODSC/a8ZsheMYxX37BIOoMcfnpS8k6KwY7zE9VgKU4CsTmvjyE8lur1yLUAo7SKUu07rY8zYwmNASNh0CvvWpIaH1bs1V01RZE2se73fmxRfts48N4bf03otfBDHQ3j52DzyUBguaBfEcXnDOsBuN6HeYeMbrXRzVtADOYHNpl5tMMNBP2PSqx3YI7MFiBfb2wXFXAxH0JSsA1ZsAv2N5fzoy1PrKBJ8201yyqOsPZKnyqJ10ME7oAfSuLGJaACH+38rm67hvIcxsB99FWZb3Ah6CMO1CEnIBlb6tLzIdAAAAAElFTkSuQmCC' - - assert(result.data.length, 'data length must be 1') - const { b64_json: _, ...rest } = result.data[0] - return { - ...result, - data: [{ b64_json: fake, ...rest }], - } -} diff --git a/_import-artifact/isolates/files.ts b/_import-artifact/isolates/files.ts deleted file mode 100644 index d83d81dd8273..000000000000 --- a/_import-artifact/isolates/files.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { Debug } from '@utils' -import { Functions, print } from '@/constants.ts' -import { type Api, parameters, returns } from '@/api/isolates/files.ts' -export { type Api, parameters, returns } - -const log = Debug('AI:files') - -export const functions: Functions<Api> = { - // TODO this should be a full mirror of the IsolateApi functions - write: ({ path, content = '' }, api) => { - log('add', path, content) - api.write(path, content) - return { charactersWritten: content.length } - }, - ls: async ({ path = '.', count, all }, api) => { - log('ls', path) - let result = await api.ls(path) - if (!all) { - result = result.filter((name) => !name.startsWith('.')) - } - if (count) { - return result.length - } - return result - }, - read: async ({ path }, api) => { - log('read', path, print(api.pid)) - const result = await api.read(path) - log('read result', result) - return result - }, - // TODO migrate this to be linecount updates, not regex - update: async ({ expectedMatches, path, regex, replacement }, api) => { - log('update', expectedMatches, path, regex, replacement) - const contents = await api.read(path) - const { matches, result } = replace(contents, regex, replacement) - if (matches.length !== expectedMatches) { - throw new Error( - `Expected ${expectedMatches} matches but found ${matches.length}`, - ) - } - api.write(path, result) - return { matchesUpdated: matches.length } - }, - // TODO make a json update tool that specifies a path in the object to change - // by using a standard json path specification - rm: ({ path }, api) => { - log('rm', path) - api.delete(path) - }, - mv: ({ from, to }, api) => { - log('mv', from, to) - return api.mv(from, to) - }, - cp: ({ from, to }, api) => { - log('cp', from, to) - return api.cp(from, to) - }, - search: async ({ query }, api) => { - log('search', query, print(api.pid)) - - // to start with, this function should just return all the file paths ? - // or, read everything in, and make a call based on the contents of all ? - - // read all files and then pump into context, who cares about the price ? - // or vector store them all based on dir hashes ? - const ls = await api.ls() - return ls.map((path) => ({ path, description: '' })) - }, -} -export const replace = ( - contents: string, - regex: string, - replacement: string, -) => { - const matches = contents.match(new RegExp(regex, 'g')) || [] - const result = contents.replace(new RegExp(regex, 'g'), replacement) - return { matches, result } -} diff --git a/_import-artifact/isolates/index.ts b/_import-artifact/isolates/index.ts deleted file mode 100644 index 5dbb6e2b02c6..000000000000 --- a/_import-artifact/isolates/index.ts +++ /dev/null @@ -1,74 +0,0 @@ -import * as system from './system.ts' - -import * as fetch from './fetch.ts' -import * as files from './files.ts' -import * as ioFixture from './io-fixture.ts' -import * as utils from './utils.ts' -import * as artifact from './artifact.ts' -import * as session from './session.ts' - -import * as actors from './actors.ts' -import * as actor from './actor.ts' -import * as machines from './machines.ts' -import * as backchat from './backchat.ts' -import * as isolates from './isolates.ts' -import * as branches from './branches.ts' -import * as agents from './agents.ts' -import * as youtube from './youtube.ts' -import * as github from './github.ts' -import * as threads from './threads.ts' - -import * as completions from './ai-completions.ts' -import * as loadTools from './utils/ai-load-tools.ts' - -import * as longthread from './longthread.ts' -import * as stateboard from './stateboard.ts' - -import * as testCaseRunner from './test-case-runner.ts' -import * as testController from './test-controller.ts' -import * as testRegistry from './test-registry.ts' -import * as tpsReport from './tps-report.ts' - -import * as napps from './napps.ts' - -const isolatesExport = { - system, - - fetch, - files, - 'io-fixture': ioFixture, - utils, - artifact, - session, - - github, - actors, - actor, - machines, - backchat, - isolates, - branches, - agents, - threads, - youtube, - - 'ai-completions': completions, - 'ai-load-tools': loadTools, - - longthread, - stateboard, - - 'test-case-runner': testCaseRunner, - 'test-controller': testController, - 'test-registry': testRegistry, - 'tps-report': tpsReport, - - napps, -} - -export default isolatesExport - -export type Isolate = keyof typeof isolatesExport -export const isIsolate = (isolate: string): isolate is Isolate => { - return isolate in isolatesExport -} diff --git a/_import-artifact/isolates/threads.ts b/_import-artifact/isolates/threads.ts deleted file mode 100644 index f2066010888a..000000000000 --- a/_import-artifact/isolates/threads.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Functions, print, ToApiType } from '@/constants.ts' -import { log } from '@utils' -import { z } from 'zod' - -export const parameters = { - search: z.object({ - query: z.string().describe('The search query'), - after: z.number().int().optional().describe( - 'The timestamp to search after', - ), - before: z.number().int().optional().describe( - 'The timestamp to search before', - ), - }).describe('Search for threads that can complete the job'), -} -export const returns = { - search: z.array( - z.object({ - threadId: z.string(), - description: z.string(), - timestamp: z.number().int(), - }), - ), -} - -export type Api = ToApiType<typeof parameters, typeof returns> - -export const functions: Functions<Api> = { - search: ({ query }, api) => { - log('search', query, print(api.pid)) - return [] - }, -} diff --git a/_import-artifact/isolates/youtube.test.ts b/_import-artifact/isolates/youtube.test.ts deleted file mode 100644 index 448c56333f06..000000000000 --- a/_import-artifact/isolates/youtube.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import Compartment from '@/io/compartment.ts' -import { Api, VideoDetails } from './youtube.ts' -import { expect } from '@utils' -import { createMockApi } from '@/tests/fixtures/mock-api.ts' -Deno.test('youtube', async () => { - const compartment = await Compartment.create('youtube') - const { api, stop } = await createMockApi('test/youtube') - const { fetch } = compartment.functions<Api>(api) - - const videoID = 'zIB7YsC34Tc' - const lang = 'en' - - const path = 'test/youtube/fetch.json' - const result = await fetch({ path, videoID, lang }) - expect(result.path).toEqual(path) - expect(result.success).toBeTruthy() - - type Result = { - details: VideoDetails - transcript: { start: string; text: string }[] - } - const data = await api.readJSON<Result>(path) - - expect(data.details).toBeDefined() - expect(data.transcript).toBeDefined() - stop() -}) diff --git a/_import-artifact/isolates/youtube.ts b/_import-artifact/isolates/youtube.ts deleted file mode 100644 index 07bd6d1fb464..000000000000 --- a/_import-artifact/isolates/youtube.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Functions, ToApiType } from '@/constants.ts' -import { - getSubtitles, - getVideoDetails, - type VideoDetails, -} from 'youtube-caption-extractor' -import { Debug } from '@utils' -import { z } from 'zod' - -const log = Debug('AI:youtube') - -export { type VideoDetails } - -export const parameters = { - fetch: z.object({ - path: z.string().regex(/.*\.json$/).describe( - 'Relative path to save the fetched transcription result. Must end in ".json" and must start with "./"', - ), - videoID: z - .string() - .describe( - 'The last part of a YouTube URL. For example, in the URL https://www.youtube.com/watch?v=zIB7YsC34Tc, the videoID is "zIB7YsC34Tc"', - ), - lang: z - .string() - .optional() - .describe('Language code for the subtitles - defaults to "en"'), - }).strict() - .describe( - 'Fetch video details and subtitles (subs). This will write the transcription json object to the given path, and return that path if everything was ok', - ), -} -export const returns = { - fetch: z.object({ - success: z.boolean().describe('Whether the fetch was successful'), - path: z.string().describe('The path to the saved transcription json file'), - }), -} - -export type Api = ToApiType<typeof parameters, typeof returns> - -export const functions: Functions<Api> = { - async fetch({ path, videoID, lang = 'en' }, api) { - const details = await fetchVideoDetails(videoID, lang) - const subs: VideoDetails['subtitles'] | string = await fetchSubtitles( - videoID, - lang, - ) - const transcript = subs.map(({ start, text }) => ({ start, text })) - api.writeJSON(path, { details, transcript }) - return { success: true, path } - }, -} - -const fetchSubtitles = async (videoID: string, lang = 'en') => { - const subtitles = await getSubtitles({ videoID, lang }) - log(subtitles) - return subtitles -} - -const fetchVideoDetails = async (videoID: string, lang = 'en') => { - const videoDetails = await getVideoDetails({ videoID, lang }) - const { subtitles, ...rest } = videoDetails - log('details had subtitles:', !!subtitles) - return rest -} diff --git a/_import-artifact/napps/example-napp/instructions.md b/_import-artifact/napps/example-napp/instructions.md deleted file mode 100644 index 4183df7d07f9..000000000000 --- a/_import-artifact/napps/example-napp/instructions.md +++ /dev/null @@ -1 +0,0 @@ -These are the system prompt instructions that get given to the entry agent to run. \ No newline at end of file diff --git a/_import-artifact/napps/example-napp/napp.jsonc b/_import-artifact/napps/example-napp/napp.jsonc deleted file mode 100644 index 6c24efc34ccc..000000000000 --- a/_import-artifact/napps/example-napp/napp.jsonc +++ /dev/null @@ -1,129 +0,0 @@ -{ - "name": "napp-name", - "version": "0.0.1", - "napp-format": "v1", - "description": "This is a description of the napp", - // what species of runtime needs to be used to execute this napp - "runtime": "deno", - // folder containing the branding assets of the napp - "branding": "path/to/branding/", - - "agent": { - // entry point that takes plain text in and returns text plus files - "napp": "napp-runner-name", - "tool": "some-exported-function", // if use a function other than main - "parameters": { - // parameters for the chat runner, based on what the runner is - "model": "gpt-4o", - "parallel_tool_calls": false, - "stop_on_tools": ["tool1", "tool3"], - // tools are reverse called back to the calling napp file - // if the tool list is not given, pass all the tools - // if the tool lists some things, use those - // can only be strings, else gets too complicated ? - // or allow a full json function definition here ? - "tools": [] - // content is the reserved name for the input text - - // if instructions was not named instructions, then can pass it in here - // instructions.md is preferred, or README.md, else error - } - }, - "main": "some-function", - "tools": { - "some-function": { - "description": "This function does something", - "parameters": { - "param1": { - "description": "This is the first parameter", - "type": "string", - "required": true - }, - "param2": { - "description": "This is the second parameter", - "type": "number", - "required": false - }, - "permissions": [ - // list of permissions required to run this function - ], - "gateways": [ - // list of gateways required to run this function - ] - }, - "returns": { - "description": "This is what the function returns", - "type": "string" - }, - "throws": { - "FileNotFoundError": { - // list of expected error types and what to do about it - } - } - }, - // if the function name is the same, as what is in a napp, - "some-other-function": "dependent-napp-name", - "modified-passthru": { - // passes thru to a tool of the same name - "parameters": {}, - "napp": { - "name": "dependent-napp-name", - "tool": "some-exported-function" - } - } - }, - "help": [ - // list of files that can be read to learn how to interface with this napp - // intended to be read by an AI - ], - "test": { - "napp": "napp-test-runner-name", - "parameters": { - // config for the test runner, based on what the runner is - "files": [ - // list of test files that can be run to verify the napp is working correctly - ] - } - }, - "dependencies": { - // list of napps that this napp depends on, including version numbers and - // commits so that every reference to the napp can be by name. - "dependent-napp-name": { - "version": "0.0.1", - "commit": "some-commit-hash" - } - }, - // list of the api gateway targets that this thing needs to do its job - "sideEffects": { - // default configurations for the production running of side effects - }, - "testSideEffects": { - // special services that can be used for testing - these are still live but - // are somehow test grade, as in might be cheaper, or no credentials - - }, - "mockSideEffects": { - // when running in the mock environment, these side effects are supplied - }, - "graphics": { - // a list of exported widgets that can be displayed on the stateboard - "googleMapsViewer": { - "type": "react", - "component": "./path/to/component", - "parameters": { - // mock react props that can be configured by the LLM - } - } - } -} - -// gateways can be just a fetch function with some urls -// can make a moneyworks connector, which could be a napp, running as a gateway -// host -// upstream dependencies ? -// tools being the same exported as what is in the agent -// need an agent config that is a generic way - -// gateways need to have backups and retry policies -// gateways need environments, like mock, test, prod - diff --git a/_import-artifact/tests/crm/initialize.test.ts b/_import-artifact/tests/crm/initialize.test.ts deleted file mode 100644 index 93cc1d6ba973..000000000000 --- a/_import-artifact/tests/crm/initialize.test.ts +++ /dev/null @@ -1,2 +0,0 @@ -// test.skip('start a new crm') -// test.skip('connect this up to moneyworks') diff --git a/_import-artifact/tests/git.test.ts b/_import-artifact/tests/git.test.ts deleted file mode 100644 index 61949cfbb339..000000000000 --- a/_import-artifact/tests/git.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -// import { debug, expect, goal, test } from '../src/test-context.js' - -// // if we could make a help that was aware of the git repo, and the io rules -// // within it, then we could get it to answer questions about how the flow of -// // control worked in the system. The user could then make modifications to the -// // pseudocode and see what changes happened to their now forked version of the -// // underlying code that would be running in a sandbox. - -// test.skip('what was the last action to this process ?') -// test.skip('what parameters were used to call the runner at this point ?') -// goal.todo('please commit everything', async ({ result }) => { -// // this should push whatever the user has done to their branch on github -// // then we can do whatever we wish from this point -// }) diff --git a/_import-artifact/tests/helps.test.ts b/_import-artifact/tests/helps.test.ts deleted file mode 100644 index a83a0ed7dfa8..000000000000 --- a/_import-artifact/tests/helps.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -// const isolate = 'engage-help' -// const help = 'help-fixture' - -Deno.test.ignore('chat', async () => { - // const { engageInBand } = await artifact.actions(isolate) - // const text = 'say a single "x" character and do not call any functions' - // const result = await engageInBand({ help, text }) - // expect(result).toEqual('x') -}) -// test('calling another help file', async ({ artifact }) => { -// // this is where one executing help file needs to call another on, in another -// // branch. -// const text = 'ask the calculator tool what five time five is' - -// // give it the command to find suitable helps -// // give it the command to execute the helps it finds in a new branch only -// // this would be 'spawn-help' vs 'self-help' -// }) -// test('multi step', async ({ artifact }) => { -// // when the help hasn't finished, but it sends back a message to the parent -// // that it can't continue on without the response. -// // like if adding a customer, and the request is missing name info. -// }) -// test.skip('add a file to the database') -// test.skip('editing the help and rerunning it') diff --git a/_import-artifact/utils.ts b/_import-artifact/utils.ts deleted file mode 100644 index c846bb4e65a4..000000000000 --- a/_import-artifact/utils.ts +++ /dev/null @@ -1,54 +0,0 @@ -export { default as equal } from 'fast-deep-equal/es6' -export { delay } from '@std/async' -export { expect } from '@std/expect' -export { assert, AssertionError } from '@std/assert' -export { default as merge } from 'lodash.merge' -import Debug from 'debug' -import { deserializeError, serializeError } from 'serialize-error' -export { deserializeError, serializeError } -export { Debug } -const _log = Debug('AI:tests') -export const log = (...args: unknown[]) => { - _log(...args) -} -log.enable = (...args: string[]) => { - Debug.enable(...args) -} -export * as posix from '@std/path/posix' - -const isDenoDeploy = Deno.env.get('DENO_DEPLOYMENT_ID') !== undefined -let _isTestMode = false -export const isKvTestMode = () => { - return _isTestMode -} -export const openKv = async () => { - if (isDenoDeploy) { - return Deno.openKv() - } - const KEY = 'DENO_KV_PATH' - let path = ':memory:' - const permission = await Deno.permissions.query({ - name: 'env', - variable: KEY, - }) - if (permission.state === 'granted') { - const env = Deno.env.get(KEY) - if (env) { - path = env - } - } - log('open kv', path) - _isTestMode = path === ':memory:' - return Deno.openKv(path) -} -const getDebug = () => { - if (isDenoDeploy) { - const string = Deno.env.get('DEBUG') - if (string) { - return string - } - } - return '' -} - -Debug.enable(getDebug()) diff --git a/_tools/check_circular_package_dependencies.ts b/_tools/check_circular_package_dependencies.ts index d96b813f95c4..6f6dd02d8753 100644 --- a/_tools/check_circular_package_dependencies.ts +++ b/_tools/check_circular_package_dependencies.ts @@ -42,7 +42,7 @@ type Mod = | "fixture" | "hamr" | "info" - | "test-file-runner"; + | "evals"; const ENTRYPOINTS: Record<Mod, string[]> = { files: ["mod.ts"], @@ -52,7 +52,7 @@ const ENTRYPOINTS: Record<Mod, string[]> = { fixture: ["mod.ts"], hamr: ["mod.ts"], info: ["mod.ts"], - "test-file-runner": ["mod.ts"], + "evals": ["mod.ts"], }; const root = new URL("../", import.meta.url).href; diff --git a/_tools/check_docs.ts b/_tools/check_docs.ts index 9a6bc7b7745f..78a5ba65c3ff 100644 --- a/_tools/check_docs.ts +++ b/_tools/check_docs.ts @@ -36,7 +36,7 @@ const ENTRY_POINTS = [ "../fixture/mod.ts", "../hamr/mod.ts", "../info/mod.ts", - "../test-file-runner/mod.ts", + "../evals/mod.ts", ] as const; const TS_SNIPPET = /```ts[\s\S]*?```/g; diff --git a/_import-artifact/isolates/actor.ts b/actors/actor.ts similarity index 96% rename from _import-artifact/isolates/actor.ts rename to actors/actor.ts index 868b635d97d4..06aa7c8c2b16 100644 --- a/_import-artifact/isolates/actor.ts +++ b/actors/actor.ts @@ -1,7 +1,7 @@ import { z } from 'zod' -import * as backchat from './backchat.ts' -import * as longthread from './longthread.ts' -import * as system from './system.ts' +import * as backchat from '../_import-artifact/isolates/backchat.ts' +import * as longthread from '../_import-artifact/isolates/longthread.ts' +import * as system from '../_import-artifact/isolates/system.ts' import { addBranches, Functions, diff --git a/_import-artifact/isolates/actors.ts b/actors/actors.ts similarity index 95% rename from _import-artifact/isolates/actors.ts rename to actors/actors.ts index f08739c4f5d8..73f6f94d999c 100644 --- a/_import-artifact/isolates/actors.ts +++ b/actors/actors.ts @@ -16,8 +16,8 @@ import { } from '@/constants.ts' import * as actor from '@/api/isolates/actor.ts' import { assert, Debug, expect } from '@utils' -import * as session from './session.ts' -import * as machines from './machines.ts' +import * as session from '../_import-artifact/isolates/session.ts' +import * as machines from '../_import-artifact/isolates/machines.ts' import { Crypto } from '../api/crypto.ts' import { z } from 'zod' @@ -118,7 +118,10 @@ export const functions: Functions<Api> = { }, async addAuthProvider({ provider, name }, api) { - assert(isBaseRepo(provider), 'addAuthProvider not base: ' + print(provider)) + assert( + isBaseRepo(provider), + 'addAuthProvider not base: ' + print(provider), + ) log('addAuthProvider provider', print(provider)) log('addAuthProvider in', print(api.pid)) await api.updateState((state) => { diff --git a/actors/deno.json b/actors/deno.json new file mode 100644 index 000000000000..fd622cac54ae --- /dev/null +++ b/actors/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/actors", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/test-file-runner/mod.ts b/actors/mod.ts similarity index 100% rename from test-file-runner/mod.ts rename to actors/mod.ts diff --git a/actors/napp.json b/actors/napp.json new file mode 100644 index 000000000000..2c63c0851048 --- /dev/null +++ b/actors/napp.json @@ -0,0 +1,2 @@ +{ +} diff --git a/_import-artifact/api/isolates/actor.ts b/actors/zod.ts similarity index 58% rename from _import-artifact/api/isolates/actor.ts rename to actors/zod.ts index d74361007321..f7be92a31b24 100644 --- a/_import-artifact/api/isolates/actor.ts +++ b/actors/zod.ts @@ -1,13 +1,4 @@ import { z } from 'zod' -import { - backchatIdRegex, - jsonSchema, - machineIdRegex, - md5, - pidSchema, - threadIdRegex, - ToApiType, -} from '../types.ts' const repoParams = z.object({ repo: z.string(), @@ -56,3 +47,38 @@ export const returns = { pull: repoResult, } export type Api = ToApiType<typeof parameters, typeof returns> + +export const generateActorId = () => { + return 'act_' + randomness() +} + +export const getActorId = (source: PID) => { + const [base, actorId] = source.branches + const parent = { ...source, branches: [base] } + const fullHAL = { ...HAL, repoId: source.repoId } + if (!isPidEqual(parent, fullHAL)) { + throw new Error('source is not a child of HAL') + } + if (!actorIdRegex.test(actorId)) { + throw new Error('Invalid actor id: ' + actorId) + } + return actorId +} +export const getActorPid = (source: PID) => { + const actorId = getActorId(source) + const branches = [source.branches[0], actorId] + return { ...source, branches } +} +export const isActorBranch = (pid: PID) => { + if (pid.branches.length !== 2) { + return false + } + return !!getActorId(pid) +} + +export const actorIdRegex = /^act_[0-9A-HJKMNP-TV-Z]{16}$/ +export const backchatIdRegex = /^bac_[0-9A-HJKMNP-TV-Z]{16}$/ +export const threadIdRegex = /^the_[0-9A-HJKMNP-TV-Z]{16}$/ + +export const SU_ACTOR = 'act_0000000000000000' +export const SU_BACKCHAT = 'bac_0000000000000000' diff --git a/api/README.md b/api/README.md new file mode 100644 index 000000000000..44b09b068d6f --- /dev/null +++ b/api/README.md @@ -0,0 +1,13 @@ +# NAPPS API Package + +This package provides a client-side API for interacting with the NAPPS project. +It includes TypeScript modules that facilitate client-server communication, +cryptographic operations, randomness generation, process identification, action +creators, and backchat interactions. Developers can use this API to build +applications that communicate with NAPPS services, manage processes and threads, +perform cryptographic tasks, and integrate with backchat systems within the +NAPPS environment. + +## Addressability + +dimenions of addressability are history, branch, process, repo diff --git a/api/actions.test.ts b/api/actions.test.ts new file mode 100644 index 000000000000..21dd80a267dc --- /dev/null +++ b/api/actions.test.ts @@ -0,0 +1,18 @@ +import functionCreator from './actions.ts' +import { expect } from '@std/expect' +import { assert } from '@std/assert' + +Deno.test('functions', async (t) => { + const functions = await functionCreator('@artifact/files') + expect(functions).toHaveProperty('write') + + assert(functions.write, 'write function not found') + expect(functions.write).toThrow('Invalid parameters') + const result = functions.write({ path: 'test.txt', content: 'hello world' }) + expect(result).toEqual({ + napp: '@artifact/files', + tool: 'write', + parameters: { path: 'test.txt', content: 'hello world' }, + files: [], + }) +}) diff --git a/api/actions.ts b/api/actions.ts new file mode 100644 index 000000000000..37c40319697d --- /dev/null +++ b/api/actions.ts @@ -0,0 +1,115 @@ +import { readNappConfig } from '@artifact/napp-tools' +import { assert } from '@std/assert' +import type { NappTypes } from './napps-list.ts' +import { z } from 'zod' +import { Ajv } from 'ajv' +import { betterAjvErrors } from '@apideck/better-ajv-errors' +const ajv = new Ajv({ allErrors: true }) + +const literalSchema = z.union([z.string(), z.number(), z.boolean(), z.null()]) +type Literal = z.infer<typeof literalSchema> +export type JsonValue = + | Literal + | { [key: string]: JsonValue | undefined } + | JsonValue[] +export const jsonSchema: z.ZodType<JsonValue> = z.lazy(() => + z.union([literalSchema, z.array(jsonSchema), z.record(jsonSchema.optional())]) +) + +const nappName = z.string() + +export const actionSchema = z.object({ + napp: nappName, + tool: z.string(), + parameters: z.record(jsonSchema), + /** An array of filepaths to files and directories that are attached to this + * action */ + files: z.array(z.string()).optional(), +}) +export type Action = z.infer<typeof actionSchema> + +/** + * Given a napp name, generate a set of action creators. + * @param name the name of the napp to generate functions for + * @example + * const functions = await actionCreators('@artifact/files') + */ +const actionCreators = async <T extends keyof NappTypes>(name: T) => { + const config = await readNappConfig(name) + type Tools = NappTypes[T] + + const actionCreators = {} as { + [K in keyof Tools]: ( + params: Parameters<Tools[K]>[0], + ) => Action + } + + if (!config.tools) { + return actionCreators + } + for (const [tool, schema] of Object.entries(config.tools)) { + if (typeof schema === 'string') { + console.log('string tool', tool, schema) + // const toolConfig = await readNappConfig(tool) + } else { + const { parameters, returns } = schema + if (parameters) { + assert(returns, 'tool must have returns: ' + tool) + type Params = Parameters<Tools[typeof tool]>[0] + actionCreators[tool as keyof Tools] = (params: Params = {}): Action => { + if (typeof params !== 'object') { + throw new Error('parameters must be an object') + } + if (params === null) { + throw new Error('parameters must not be null') + } + + const valid = ajv.validate(parameters, params) + + if (!valid) { + const betterErrors = betterAjvErrors({ + basePath: 'parameters', + schema: parameters, + data: params, + errors: ajv.errors, + }) + const message = betterErrors.map((e) => JSON.stringify(e, null, 2)) + .join('\n') + throw new Error('Invalid parameters: ' + message) + } + return { + napp: name, + tool, + parameters: params, + files: [], + } + } + } + } + } + + return actionCreators + + // need to include the types in the package, and sniff for them +} + +export default actionCreators + +export const serializableError = z.object({ + name: z.string().optional(), + message: z.string().optional(), + stack: z.string().optional(), +}) +export type SerializableError = z.infer<typeof serializableError> + +export const outcomeSchema = z.object({ + result: jsonSchema.optional(), + error: serializableError.optional(), +}).refine((data) => { + if (data.error !== undefined) { + return data.result === undefined + } + return true +}, 'result and error are mutually exclusive') + +export type Outcome = { result?: JsonValue; error?: SerializableError } diff --git a/_import-artifact/api/client-backchat.ts b/api/client-backchat.ts similarity index 99% rename from _import-artifact/api/client-backchat.ts rename to api/client-backchat.ts index 1a379cdfc968..b29c59f0df72 100644 --- a/_import-artifact/api/client-backchat.ts +++ b/api/client-backchat.ts @@ -20,7 +20,7 @@ import { toActions, UnsequencedRequest, } from './types.ts' -import * as actor from './isolates/actor.ts' +import * as actor from '../actors/zod.ts' import * as files from './isolates/files.ts' import { ulid } from 'ulid' import { PierceWatcher } from './watcher.ts' diff --git a/_import-artifact/api/client-engine.ts b/api/client-engine.ts similarity index 100% rename from _import-artifact/api/client-engine.ts rename to api/client-engine.ts diff --git a/_import-artifact/api/crypto.ts b/api/crypto.ts similarity index 95% rename from _import-artifact/api/crypto.ts rename to api/crypto.ts index 4e8bf0b017a7..805229a8eb55 100644 --- a/_import-artifact/api/crypto.ts +++ b/api/crypto.ts @@ -1,7 +1,6 @@ import * as secp from '@noble/secp256k1' import { ripemd160 } from '@noble/hashes/ripemd160' import { base32 } from 'multiformats/bases/base32' -import { machineIdRegex } from './types.ts' export class Crypto { readonly #privKey: Uint8Array @@ -36,3 +35,5 @@ export class Crypto { return this.#machineId } } + +export const machineIdRegex = /^mac_[2-7a-z]{33}$/ diff --git a/api/deno.json b/api/deno.json new file mode 100644 index 000000000000..64e89de97985 --- /dev/null +++ b/api/deno.json @@ -0,0 +1,11 @@ +{ + "name": "@artifact/api", + "version": "0.0.1", + "exports": { + ".": "./mod.ts", + "./napp.json": "./napp.json", + "./actions": "./actions.ts", + "./addressing": "./addressing.ts", + "./napp-api": "./napp-api.ts" + } +} diff --git a/api/mod.ts b/api/mod.ts new file mode 100644 index 000000000000..81fe2f71b2fe --- /dev/null +++ b/api/mod.ts @@ -0,0 +1 @@ +export * from './napps-list.ts' diff --git a/api/napp-api.ts b/api/napp-api.ts new file mode 100644 index 000000000000..116414d24aa3 --- /dev/null +++ b/api/napp-api.ts @@ -0,0 +1,224 @@ +import type { ZodRecord, ZodTypeAny } from 'zod' +import { z } from 'zod' +import type { NappTypes } from './napps-list.ts' +import { type Action, jsonSchema, type JsonValue } from './actions.ts' +import Debug from 'debug' +import type { Outcome } from './actions.ts' +import { deserializeError } from 'serialize-error' +const log = Debug('@artifact/api') + +export type TreeEntry = { + /** the 6 digit hexadecimal mode */ + readonly mode: string + /** the name of the file or directory */ + readonly path: string + /** the hash identifier of the blob or tree */ + readonly oid: string + /** the type of object */ + readonly type: 'blob' | 'tree' + /** the snapshot identifier, since lookup by oid is not permitted as there is + * no way to lookup permissions cheaply */ + readonly snapshot: string +} + +export type Upsert = + | { readonly meta: { readonly snapshot: string; readonly path: string } } + | { readonly json: JsonValue } // TODO implement object cache using structured clone + | { readonly text: string } + | { readonly data: Uint8Array } + +export const addressSchema = z + .object({ + /** Posix style path that locates what process thread we want to communicate + * with. In git, this would be branch names, for example: `exe/proc-1/child-2` */ + process: z.string(), + /** The cryptographic identifier of the whole repository. This would be the + * chainId in conventional blockchains, but could be a group of public keys, + * or some other root of trust */ + crypto: z.string(), + /** Whatever snapshot model is used, the branch concept represents an isolated + * line of changes. In git, this would be a branch */ + branch: z.string(), + /** Depending on the snapshot format being used, represents the state at a + * specific point in the history. For write commands, snapshot is used to + * guarantee the state being changed has not been altered since it was read */ + snapshot: z.string(), + }) + .partial() + +export type Address = z.infer<typeof addressSchema> + +export interface NappSnapshots<ReadOptions = Address> { + /** Posix style path that locates what process thread we want to communicate + * with. In git, this would be branch names, for example: `exe/proc-1/child-2` */ + readonly latest: ( + options?: Omit<ReadOptions, 'snapshot'>, + ) => Promise<string | undefined> + readonly parents: (options?: ReadOptions) => Promise<string[]> + readonly history: ( + options?: ReadOptions & { count?: number }, + ) => Promise<string[]> +} + +export interface NappRead<ReadOptions = Address> { + readonly meta: (path: string, options?: ReadOptions) => Promise<TreeEntry> + readonly json: <T extends ZodTypeAny = typeof jsonSchema>( + path: string, + options?: ReadOptions & { schema?: T }, + ) => Promise<z.infer<T>> + readonly text: (path: string, options?: ReadOptions) => Promise<string> + readonly binary: (path: string, options?: ReadOptions) => Promise<Uint8Array> + readonly exists: (path: string, options?: ReadOptions) => Promise<boolean> + readonly ls: (path?: string, options?: ReadOptions) => Promise<TreeEntry[]> +} + +export interface SnapshotsProvider<ReadOptions = Address> { + readonly snapshots: NappSnapshots<ReadOptions> + readonly read: NappRead<ReadOptions> + readonly commit: ( + upserts: Map<string, Upsert>, + deletes: Set<string>, + ) => Promise<void> +} + +export interface NappWrite<WriteOptions = Address> { + readonly json: ( + path: string, + content: JsonValue, + options?: WriteOptions, + ) => Promise<void> + readonly text: ( + path: string, + content: string, + options?: WriteOptions, + ) => Promise<void> + readonly binary: ( + path: string, + content: Uint8Array, + options?: WriteOptions, + ) => Promise<void> + readonly rm: (path: string, options?: WriteOptions) => Promise<void> + readonly mv: ( + from: WriteOptions & { path: string }, + to: WriteOptions & { path: string }, + ) => Promise<void> + readonly cp: ( + from: WriteOptions & { path: string }, + to: WriteOptions & { path: string }, + ) => Promise<void> +} + +type SpawnOptions = + & Address + & ( + | { readonly name: string; prefix?: never } + | { name?: never; readonly prefix: string } + ) + & { + readonly files?: string[] + /** If process exists, exit gracefully */ + readonly upsert?: boolean + /** Priority of the process */ + readonly nice?: number + } + +type MetaResult = { + readonly meta: Required<Address> + readonly outcome: Outcome +} + +interface NappProcesses { + /** start a new process and install the given napp. */ + readonly spawn: ( + napp: keyof NappTypes, + options: SpawnOptions, + ) => Promise<void> + /** tear down the specified process, and resturn the result of teardown */ + readonly kill: (options: Address) => Promise<JsonValue> + /** spawns a new process, installs the napp specified in the action, awaits + * the execution, and then returns, killing the process */ + readonly async: (action: Action, options: SpawnOptions) => Promise<JsonValue> + /** move a process to another parent. Can be used to daemonize a running + * process by moving it to be a child of init */ + readonly mv: (to: Address, from?: Address) => Promise<void> + /** change the priority of a process */ + readonly nice: (level: number, options: Address) => void + + readonly dispatch: ( + action: Action, + options: Address, + ) => Promise<JsonValue | void> + readonly dispatchWithMeta: ( + action: Action, + options: Address, + ) => Promise<MetaResult> +} + +export const stateSchema = z.record(jsonSchema) + +/** State is stored in the process json files */ +interface NappState<ReadOptions = Address> { + readonly get: <T extends ZodRecord = typeof stateSchema>( + options: ReadOptions & { schema?: T; fallback?: z.infer<T> }, + ) => Promise<z.infer<T>> + // TODO return metadata of the state so we know if a part remains unchanged + // TODO allow fetching paths within the state + readonly set: <T extends ZodRecord = typeof stateSchema>( + state: z.infer<T>, + options: ReadOptions & { schema?: T }, + ) => Promise<void> +} + +interface NappEffects { + /** Side effects can listen to this signal to abort their activities */ + readonly signal: AbortSignal + + /** If the side effect lock was broken in order to start this instance. + * Implies the previous executing instance was aborted */ + readonly isEffectRecovered: boolean + + /** The context of the current side effect, which acts like a React ref, and + * is a mutable store of any value at all */ + set context(value: unknown) + + get context(): unknown +} + +export interface NappApi { + readonly state: NappState + readonly read: NappRead + readonly write: NappWrite + readonly processes: NappProcesses + readonly effects: NappEffects +} + +/** + * Use this to unwrap the results of a dispatch that came back with metadata as + * well as an outcome. Calling this function makes it behave the same as + * calling `dispatch` directly. + * @param meta the result of calling `dispatchWithMeta` + * @returns a promise that resolves or rejects to the result of the action + */ +export const settle = (meta: MetaResult) => { + const { outcome } = meta + if (outcome.error) { + return Promise.reject(deserializeError(outcome.error)) + } else { + return Promise.resolve(outcome.result) + } +} + +export class FileNotFoundError extends Error { + code = 'ENOENT' + constructor(path: string) { + super('Could not find file or directory: ' + path) + this.name = 'FileNotFoundError' + } +} +export class LineageError extends Error { + code = 'EINVALID' + constructor(id: string) { + super('Lineage fault for: ' + id) + this.name = 'LineageError' + } +} diff --git a/api/napp.json b/api/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/api/napp.json @@ -0,0 +1 @@ +{} diff --git a/api/napps-list.ts b/api/napps-list.ts new file mode 100644 index 000000000000..f21c9b34e024 --- /dev/null +++ b/api/napps-list.ts @@ -0,0 +1,12 @@ +import type { NappTypes as Files } from '@artifact/files' +import type { NappTypes as Fixture } from '@artifact/fixture' +import type { JsonValue } from './actions.ts' + +type NappType = { + [key: string]: (params: Record<string, JsonValue>) => unknown +} + +export type NappTypes = Record<string, NappType> & { + '@artifact/files': Files + '@artifact/fixture': Fixture +} diff --git a/_import-artifact/api/randomness.ts b/api/randomness.ts similarity index 100% rename from _import-artifact/api/randomness.ts rename to api/randomness.ts diff --git a/api/types.ts b/api/types.ts new file mode 100644 index 000000000000..c70fbd3a71d7 --- /dev/null +++ b/api/types.ts @@ -0,0 +1,155 @@ +// copied from the artifact project +import { z, type ZodSchema } from 'zod' +export type { AssistantMessage, CompletionMessage } from '../openai/zod.ts' +import { completionMessage } from '../openai/zod.ts' +import type { Backchat } from './client-backchat.ts' +import { assert } from '@std/assert' +import type OpenAI from 'openai' +import { randomness } from './randomness.ts' +export { randomness } +type CommitOid = string + +export const md5 = z.string().regex(/^[a-f0-9]{40}$/, 'Invalid MD5 hash') + +export type Change = { + /** If present, represents the unified diff of the file at the given path, + * since the last commit. This is only provided if the file is a string. If + * the changes are too large, this will be missing and an oid will be provided. + */ + patch?: string + /** + * The oid of the object given by path, which can be used to fetch the object + * directly. If this file is binary, then patch will be missing, and the file + * should be directly retrieved. If oid is missing, then the change was + * fatal. + */ + oid?: string +} +export type Splice = { + pid: PID + /** + * The hash of the commit object + */ + oid: string + /** + * The commit this splice refers to + */ + commit: CommitObject + /** + * The timestamp of the commit, or if transient, the timestamp of the write + * that caused this update + */ + timestamp: number + changes: { [key: string]: Change } +} +export declare interface EventSourceMessage { + /** The data received for this message. */ + data: string + /** Event name sent from the server, or `undefined` if none is set for this message. */ + event?: string + /** ID of the message, if any was provided by the server. */ + id?: string +} +/** + * A git commit object. + */ +export type CommitObject = { + /** + * Commit message + */ + message: string + /** + * SHA-1 object id of corresponding file tree + */ + tree: string + /** + * an array of zero or more SHA-1 object ids + */ + parent: string[] + author: { + /** + * The author's name + */ + name: string + /** + * The author's email + */ + email: string + /** + * UTC Unix timestamp in seconds + */ + timestamp: number + /** + * Timezone difference from UTC in minutes + */ + timezoneOffset: number + } + committer: { + /** + * The committer's name + */ + name: string + /** + * The committer's email + */ + email: string + /** + * UTC Unix timestamp in seconds + */ + timestamp: number + /** + * Timezone difference from UTC in minutes + */ + timezoneOffset: number + } + /** + * PGP signature (if present) + */ + gpgsig?: string +} + +export interface EngineInterface { + /** + * The address in use as basis of identity for this engine. May be a repo + * hosted on external engines, or hosted in cooperation with other engines. + */ + homeAddress: PID + abortSignal: AbortSignal + upsertBackchat(machineId: string, resume?: string): Promise<PID> + stop(): Promise<void> | void + /** + * Send a ping to the edge isolate that will process requests, to establish + * basic network connectivity. To ping a chain, use an isolate. + * TODO ping should return some info about the deployment + * @param data Data that will be echoed back + */ + ping(data?: JsonValue): Promise<IsolateReturn> + apiSchema(isolate: string): Promise<Record<string, object>> + transcribe(audio: File): Promise<{ text: string }> + pierce(pierce: Pierce): Promise<void> + watch( + pid: PID, + path?: string, + after?: string, + signal?: AbortSignal, + ): AsyncIterable<Splice> + splice( + target: PID, + opts?: { commit?: string; path?: string; count?: number }, + ): Promise<Splice[]> + read(path: string, pid: PID, commit?: string): Promise<string> + readTree(path: string, pid: PID, commit?: string): Promise<TreeEntry[]> + readJSON<T>(path: string, pid: PID, commit?: string): Promise<T> + readBinary(path: string, pid?: PID, commit?: string): Promise<Uint8Array> + exists(path: string, pid: PID): Promise<boolean> +} + +export const generateBackchatId = () => { + return 'bac_' + randomness() +} + +export const backchatStateSchema = z.object({ + /** The base thread that this backchat session points to - the thread of last resort */ + target: pidSchema, + threadCount: z.number().int().gte(0), +}) diff --git a/_import-artifact/api/watcher.ts b/api/watcher.ts similarity index 100% rename from _import-artifact/api/watcher.ts rename to api/watcher.ts diff --git a/browser-compat.tsconfig.json b/browser-compat.tsconfig.json index 202cafb54986..701ef1586109 100644 --- a/browser-compat.tsconfig.json +++ b/browser-compat.tsconfig.json @@ -8,47 +8,20 @@ }, "importMap": "./import_map.json", "workspace": [ - "./archive", - "./assert", - "./async", - "./bytes", - "./cache", - "./cbor", - "./cli", - "./collections", - "./crypto", - "./csv", - "./data_structures", - "./datetime", - "./dotenv", - "./encoding", - "./expect", - "./fmt", - "./front_matter", - "./fs", - "./html", - "./http", - "./ini", - "./internal", - "./io", - "./json", - "./jsonc", - "./log", - "./media_types", - "./msgpack", - "./net", - "./path", - "./random", - "./regexp", - "./semver", - "./streams", - "./tar", - "./testing", - "./text", - "./toml", - "./ulid", - "./uuid", - "./webgpu", - "./yaml" + "./actors", + "./api", + "./dumb-bot", + "./engine", + "./evals", + "./execution", + "./files", + "./fixture", + "./hamr", + "./info", + "./napp-tools", + "./openai", + "./reasoner", + "./server", + "./stateboard" ] } diff --git a/compartment/chats/chat_2024-12-18_d5fb64e.md b/compartment/chats/chat_2024-12-18_d5fb64e.md new file mode 100644 index 000000000000..490f759f1b31 --- /dev/null +++ b/compartment/chats/chat_2024-12-18_d5fb64e.md @@ -0,0 +1,5096 @@ +# Conversation +URL: https://chatgpt.com/share/67622a65-26f8-800b-b280-033fed5fb64e + +### Message ID: f1e36b26-4af2-4af9-9572-f5c74281355b +**Author**: system + +**Content:** + + + +--- + +### Message ID: ec0d6b27-8088-4401-aefc-c0d4593923d4 +**Author**: user + +**Content:** + +Original custom instructions no longer available + +--- + +### Message ID: aaa26b34-3ede-4254-9e41-8d4abeae8376 +**Author**: user + +**Content:** + +can you make me a ts file to use endo, running on nodejs, where I can give it a url for a typescript module, and it will be able to download that code and everything it depends on, then run it in a ses compartment + +--- + +### Message ID: 9bbffdce-f1bb-40af-80fd-2ee3455fdb6d +**Author**: tool + +**Content:** + +**Crafting a secure environment** + +I’m focusing on creating a TypeScript file with Endo in a SES compartment on Node.js, using ES6 modules and arrow functions. + +**Building the process** + +I’m gathering info on using Endo to run TypeScript code from a URL in a SES compartment, importing necessary Endo packages, and ensuring the JavaScript environment is in SES mode. + +**Setting up the environment** + +I’m preparing an environment for isolated execution by importing necessary Endo packages, setting up a new compartment, and evaluating code from a URL within this compartment. + +**Gathering insights** + +I’m organizing the TypeScript file to load code from a URL and run it in a Node.js environment using Endo, ensuring it adheres to ES6 modules and arrow functions. + +**Fetching and running** + +I’m thinking through how to fetch code from a URL, utilize `@endo/lockdown`, and create a custom read function to ensure the code runs in a SES compartment. + +**Setting up a fetch mechanism** + +I’m configuring a `fetch` function to retrieve code from a URL, ensuring it’s integrated with `loadLocation` for loading modules in a SES compartment on Node.js. + +**Setting up a fetch mechanism** + +Thinking about how to set up a fetch function to retrieve code from a URL, ensuring it integrates smoothly with `loadLocation` for module loading in a SES compartment on Node.js. + +**Crafting a streamlined snippet** + +I'm mapping out a minimal TypeScript snippet to fetch code from a URL and execute it in a SES compartment on Node.js, ensuring it aligns with the user's preferences. + +**Fetching code** + +I’m creating a minimal TypeScript snippet to fetch code from a URL and run it in a SES compartment on Node.js, including error handling and isolation setup. + +--- + +### Message ID: b4ec97f0-2658-46e2-9a70-e5c57ad9da9e +**Author**: assistant + +**Content:** + +```typescript +// run-remote-module.ts +import 'ses'; +import fetch from 'node-fetch'; +import { makeImporter } from '@endo/compartment-mapper'; + +// Lock down the environment before anything else +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }); + +const loadAndRun = async (moduleURL: string) => { + const read = async (url: string): Promise<string> => { + const res = await fetch(url); + if (!res.ok) throw new Error(`Failed to fetch ${url}: ${res.statusText}`); + return res.text(); + }; + + const importer = makeImporter({ + read, + resolve: (spec: string, ref: string) => new URL(spec, ref).toString(), + makeImportHook: (srcSpec: string) => (specifier: string) => + importer.import(specifier, srcSpec), + }); + + const compartment = new Compartment({}); + const { namespace } = await importer.import(moduleURL, 'file:///'); + // namespace now holds the exported bindings from the remote module + console.log(namespace); +}; + +loadAndRun('https://example.com/path/to/yourModule.ts').catch(console.error); +``` + +--- + +### Message ID: aaa209ea-66e5-4695-9272-c28dc109f250 +**Author**: user + +**Content:** + +makeImporter isn't a valid export. Here are the docs for the @endo/compartment-mapper module: + +# Compartment mapper + +The compartment mapper builds _compartment maps_ for Node.js style +applications, finding their dependencies and describing how to create +[Compartments][] for each package in the application. + +Creating a compartment map for a Node.js application allows us to harness +the SES module loader to encapsulate each dependency and grant the least +necessary authority to each third-party package, mitigating prototype pollution +attacks and some supply chain attacks. +Since most Node.js packages do not modify objects in global scope, +many libraries and applications work in Compartments without modification. + +The `importLocation` function runs a compartmentalized application off the file +system. +The `globals` are properties to add to the `globalThis` in the global scope +of the application's main package compartment. +The `modules` are built-in modules to grant the application's main package +compartment. + +```js +import fs from "fs"; +import { importLocation } from "@endo/compartment-mapper"; + +// ... + +const modules = { fs }; +const globals = { console }; + +const read = async location => + fs.promises.readFile(new URL(location).pathname); + +const { namespace } = await importLocation( + read, + moduleLocation, + { + globals, + modules + } +); +``` + +The compartment mapper does nothing to arrange for the realm to be frozen. +The application using the compartment mapper is responsible for applying the +[SES] shim (if necessary) and calling `lockdown` to freeze the realm (if +necessary). +The compartment mapper is also not coupled specifically to Node.js IO and does +not import any powerful modules like `fs`. +The user must provide `read` and `write` functions from whatever IO powers they +have. These powers can be provided as individual functions or as objects +carrying functions. `ReadPowers` has optional functions which can be used to +unlock compatibility features. When `fileURLToPath` is available, `__dirname` +and `__filename` will be provided to CJS modules. If `requireResolve` is +available, it will be called whenever a CJS module calls `require.resolve()`. + +```js +type ReadPowers = { + read: (location: string) => Promise<Uint8Array>, + canonical: (location: string) => Promise<string>, + computeSha512: { (bytes: Uint8Array) => string }?, + fileURLToPath: { (url: string | URL) => string }?, + pathToFileURL: { (path: string) => URL }?, + requireResolve: { (from: string, request: string, options?: {}) => string }? +} +``` + +> TODO +> +> A future version will allow application authors to distribute their choices +> of globals and built-in modules to third-party packages within the +> application, as with [LavaMoat]. + +The `importLocation` function uses `loadLocation`. +Using `loadLocation` directly allows for deferred execution or multiple runs +with different globals or modules in the same process. +Calling `loadLocation` returns an `Application` object with an +`import({ globals?, modules? })` method. + +Use `writeArchive` to capture an application in an archival format. +Archives are `zip` files with a `compartment-map.json` manifest file. + +```js +import fs from "fs"; +import { writeArchive } from "@endo/compartment-mapper"; + +const read = async location => + fs.promises.readFile(new URL(location).pathname); +const write = async (location, content) => + fs.promises.writeFile(new URL(location).pathname, content); + +await writeArchive( + write, + read, + new URL('app.zip', import.meta.url).toString(), // the archive to write + new URL('app.js', import.meta.url).toString() // the application to capture +); +``` + +The `writeArchive` function uses `makeArchive`. +Using `makeArchive` directly gives you the archive bytes. + +Use `importArchive` to run an application from an archive. +Note the similarity to `importLocation`. + +```js +import fs from "fs"; +import { importArchive } from "@endo/compartment-mapper"; + +// ... + +const modules = { fs }; +const globals = { console }; + +const read = async location => + fs.promises.readFile(new URL(location).pathname); + +const { namespace } = await importArchive( + read, + archiveLocation, + { + globals, + modules + } +); +``` + +The `importArchive` function composes `loadArchive` and `parseArchive`. +Use `loadArchive` to defer execution or run multiple times with varying +globals. +Use `parseArchive` to construct a runner from the bytes of an archive. +The `loadArchive` and `parseArchive` functions return an `Application` +object with an `import({ globals?, modules? })` method. + +`loadArchive` and `parseArchive` do not run the archived program, +so they can be used to check the hash of a program without running it. + +# Package Descriptors + +The compartment mapper uses [Compartments], one for each Node.js package your +application needs. +The compartment mapper generates a compartment graph from Node.js packaged +module descriptors: the `package.json` files of the application and all its +dependencies. +Consequently, an application must have a `package.json`. + +Each package has its own descriptor, `package.json`. +Some standard properties of the descriptor are relevant and used by a +compartment map. + +* `name` +* `type` +* `main` +* `exports` +* `browser` +* `dependencies` +* `files` + +The compartment map will contain one compartment for each `package.json` +necessary to build the application. +Like Node.js, the compartment mapper trusts the package manager to arrange the +packages such that a satisfactory version of every package's dependencies rests +in a parent directory, under `node_modules`. + +The `main`, `browser`, and `exports` properties determine the modules each +package exports to other compartments. + +The `exports` property describes [package entry points][] and can be influenced +by build _conditions_. +Currently, the only conditions supported by the compartment mapper are +`import`, `browser`, and `endo`. +The `imports` condition indicates that the module map should use ESM modules +over CommonJS modules or other variants, and `endo`. +The `browser` condition also draws in the `browser` property from +`package.json` instead of `main`. +The `endo` condition only indicates that this tool is in use. + +If no `exports` apply to the root of the compartment namespace (`"."`), +the `main` property serves as a default. + +> TODO +> +> A future version may also respect the `imports` property. + +> TODO +> +> A future version may also respect wildcard patterns in `exports` and +> `imports`. + +The `files` property indicates all of the files in the package that +should be vended out to applications. +The file set implicitly includes all `**.js`, `**.mjs`, and `**.cjs` files. +The file set implicitly excludes anything under `node_modules`. + +With the compartment mapper, just as in Node.js, a module specifier that has no +extension may refer either to the file with the `js` extension, or if that file +does not exist, to the `index.js` file in the directory with the same name. + +> TODO +> +> The compartment mapper does not yet do anything with the `files` globs but a +> future version of the compartment mapper will collect these in archives. +> The compartment mapper should eventually provide the means for any +> compartment to access its own files using an attenuated `fs` module or +> `fetch` global, in conjunction with usable values for `import.meta.url` in +> ECMAScript modules or `__dirname` and `__filename` in CommonJS modules. + +## Language Extensions + +Officially beginning with Node.js 14, Node.js treats `.mjs` files as ECMAScript +modules and `.cjs` files as CommonJS modules. +The `.js` extension indicates a CommonJS module by default, to maintain +backward compatibility. +However, packages that have a `type` property that explicitly says `module` +will treat a `.js` file as an ECMAScript module. + +Many Node.js applications using CommonJS modules expect to be able to `require` +a JSON file like `package.json`. +The compartment mapper supports loading JSON modules from any type of module. +As of Node.js 14, Node does not support importing JSON using ECMAScript +`import` directives, so using this feature may limit compatibility with the +Node.js platform. + +The compartment mapper supports loading CommonJS modules from ECMAScript +modules as well as ECMAScript modules importing CommonJS modules. +This presumes that the CommonJS modules exclusively use `require` calls with a +single string argument, where `require` is not lexically bound, to declare +their shallow dependencies, so that these modules and their transitive +dependencies can be loaded before any module executes. +As of Node.js 14, Node does not support loading ECMAScript modules from +CommonJS modules, so using this feature may limit compatibility with the +Node.js platform. + +The compartment mapper supports language plugins. +The languages supported by default are: + +- `mjs` for ECMAScript modules, +- `cjs` for CommonJS modules, +- `json` for JSON modules, +- `text` for UTF-8 encoded text files, +- `bytes` for any file, exporting a `Uint8Array` as `default`, +- `pre-mjs-json` for pre-compiled ECMAScript modules captured as JSON in + archives, and +- `pre-cjs-json` for pre-compiled CommonJS modules captured as JSON in + archives. + +The compartment mapper accepts extensions to this set of languages with +the `parserForLanguage` option supported by many functions. +See `src/types/external.ts` for the type and expected behavior for +parsers. + +These language identifiers are keys for the `moduleTransforms` and +`syncModuleTransforms` options, which may map each language to a transform +function. +The language identifiers are also the values for a `languageForExtension`, +`moduleLanguageForExtension`, and `commonjsLanguageForExtension` options to +configure additional extension-to-language mappings for a module and its +transitive dependencies. + +For any package that has `type` set to `"module"` in its `package.json`, +`moduleLangaugeForExtension` will precede `languageForExtension`. +Packages with `type` set to `"commonjs"` or simply not set, +`commonjsLanguageForExtension` will precede `languageForExtension`. +This provides an hook for mapping TypeScript's `.ts` to either `.cts` or +`.mts`. + +The analogous `workspaceLanguageForExtension`, +`workspaceCommonjsLanguageForExtension`, and +`workspaceModuleLanguageForExtension` options apply more specifically for +packages that are not under a `node_modules` directory, indicating that they +are in the set of linked workspaces and have not been built or published to +npm. + +In the scope any given package, the `parsers` property in `package.json` may +override the extension-to-language mapping. + +```json +{ + "parsers": { "png": "bytes" } +} +``` + +> TODO +> +> The compartment mapper may elect to respect some properties specified for +> import maps. + +> TODO +> +> A future version of the compartment mapper may add support for +> source-to-source translation in the scope of a package or compartment. +> This would be expressed in `package.json` using a property like +> `translate` that would contain a map from file extension +> to a module that exports a suitable translator. +> +> For browser applications, the compartment mapper would use the translator +> modules in two modes. +> During development, the compartment mapper would be able to load the +> translator in the client, with the `browser` condition. +> The compartment mapper would also be able to run the translator in a separate +> non-browser compartment during bundling, so the translator can be excluded +> from the production application and archived applications. + +> TODO +> +> The compartment mapper may also add support for compartment map plugins that +> would recognize packages in `devDependencies` that need to introduce globals. +> For example, _packages_ that use JSX and a virtual DOM would be able to add a +> module-to-module translator and endow the compartment with the `h` the +> translated modules need. + +# Source Maps + +The `makeArchive`, `makeAndHashArchive`, and `writeArchive` tools can receive a +`sourceMapHook` as one of its options. +The `sourceMapHook` receives a source map `string` for every module it +archives, along with details `compartment`, `module`, `location`, and `sha512`. +The `compartment` is the fully-qualified file URL of the package root. +The `module` is the package-relative module specifier. +The `location` is the fully-qualified file URL of the module file. +The `sha512`, if present, was generated with the `computeSha512` power from the +generated module bytes. + +The functions `importArchive`, `loadArchive`, and `parseArchive` +tools can receive a `computeSourceMapLocation` option that recives the same +details as above and must return a URL. +These will be appended to each module from the archive, for debugging purposes. + +The `@endo/bundle-source` and `@endo/import-bundle` tools integrate source maps +for an end-to-end debugging experience. + +# XS (experimental) + +The Compartment Mapper can use native XS `Compartment` and `ModuleSource` under +certain conditions: + +1. The application must be an XS script that was compiled with the `xs` + package condition. + This causes `ses`, `@endo/module-source`, and `@endo/import-bundle` to + provide slightly different implementations that can fall through to native + behavior. +2. The application must opt-in with the `__native__: true` option on any + of the compartment mapper methods that import modules like `importLocation` + and `importArchive`. + +# Design + +Each of the workflows the compartment mapper executes a portion of one sequence +of underlying internals. + +* search (`search.js`): Scan the parent directories of a given `moduleLocation` + until successfully finding and reading a `package.json` for the containing + application. +* map compartments from Node.js packages (`node-modules.js`): Find and gather + all the `package.json` files for the application's transitive dependencies. + Use these to construct a compartment map describing how to construct a + `Compartment` for each application package and how to link the modules each + exports in the compartments that import them. +* load compartments (`archive.js`): Using `compartment.load`, or + implicitly through `compartment.import`, create a module graph for the + application's entire working set. + When creating an archive, this does not execute any of the modules. + The compartment mapper uses the compartments and a special `importHook` that + records the text of every module the main module needed. +* import modules (`import.js`, `import-archive.js`): Actually execute the + working set. + +Around this sequence, we can enter late or depart early to store or retrieve an +archive. +The compartment mapper provides workflows that use `read` and `write` hooks +when interacting with a filesystem or work with the archive bytes directly. + +This diagram represents the the workflows of each of the public methods like +`importLocation`. +Each column of pipes `|` is a workflow from top to bottom. +Each asterisk `*` denotes a step that is taken by that workflow. +The dotted lines `.'. : '.'` indicate carrying an archive file from the end of +one workflow to the beginning of another, either as bytes or a location. + +In the diagram, "powers" refer to globals and built-in modules that may provide +capabilities to a compartment graph. +For `writeArchive` and `makeArchive`, these may be provided but will be ignored +since the application does not execute. + +``` + loadLocation writeArchive + importLocation | | makeArchive + | | | | + | | | | parseArchive + | | | | | loadArchive + | | | | | | importArchive + | | | | | | |... + search -> * * * * | |'| . ' + map compartments -> * * * * .'.| | |' : : + read archive -> | | | ' | * * : : + unpack archive -> | | | : * * * : : +assemble compartments -> * * * : * : : <- powers + load compartments -> * * * : * : : + import modules -> * | | : * : : + pack archive -> * * ' : : + write archive -> * '.' <- data : : + '..............' : <- files + '...............' +``` + +# Compartment maps + +The compartment mapper works by generating a _compartment map_ from your +application workspace and all of the `node_modules` it needs. +A compartment map is similar to a lock file because it collects information +from all of the installed modules. +A compartment map describes how to construct compartments for each +package in your application and link their module namespaces. + +The compartment map shape: + +```ts +// CompartmentMap describes how to prepare compartments +// to run an application. +type CompartmentMap = { + tags: Conditions, + entry: Entry, + compartments: Record<CompartmentName, Compartment>, + realms: Record<RealmName, Realm>, // TODO +}; + +// Conditions influence which modules are selected +// to represent the implementation of various modules. +// These may include terms like "browser", meaning +// each compartment uses the implementation of each +// module suitable for use in a browser environment. +type Conditions = Array<Condition>; +type Condition = string; + +// Entry is a reference to the module that is the module to initially import. +type Entry = CompartmentModule; + +// CompartmentName is an arbitrary string to name +// a compartment for purposes of inter-compartment linkage. +type CompartmentName = string; + +// Compartment describes where to find the modules +// for a compartment and how to link the compartment +// to modules in other compartments, or to built-in modules. +type Compartment = { + location: Location, + modules: ModuleMap, + parsers: ParserMap, + types: ModuleParserMap, + scopes: ScopeMap, + // The name of the realm to run the compartment within. + // The default is a single frozen realm that has no name. + realm: RealmName? // TODO +}; + +// Location is the URL relative to the compartment-map.json's +// containing location to the compartment's files. +type Location = string; + +// ModuleMap describes modules available in the compartment +// that do not correspond to source files in the same compartment. +type ModuleMap = Record<InternalModuleSpecifier, Module>; + +// Module describes a module in a compartment. +type Module = CompartmentModule | FileModule | ExitModule; + +// CompartmentModule describes a module that isn't in the same +// compartment and how to introduce it to the compartment's +// module namespace. +type CompartmentModule = { + // The name of the foreign compartment: + // TODO an absent compartment name may imply either + // that the module is an internal alias of the + // same compartment, or given by the user. + compartment: CompartmentName?, + // The name of the module in the foreign compartment's + // module namespace: + module: ExternalModuleSpecifier?, +}; + +// FileLocation is a URL for a module's file relative to the location of the +// containing compartment. +type FileLocation = string + +// FileModule is a module from a file. +// When loading modules off a file system (src/import.js), the assembler +// does not need any explicit FileModules, and instead relies on the +// compartment to declare a ParserMap and optionally ModuleParserMap and +// ScopeMap. +// The compartment mapper provides a Compartment importHook and moduleMapHook +// that will search the filesystem for candidate module files and infer the +// type from the extension when necessary. +type FileModule = { + location: FileLocation, + parser: Parser, +}; + +// ExitName is the name of a built-in module, to be threaded in from the +// modules passed to the module executor. +type ExitName = string; + +// ExitModule refers to a module that comes from outside the compartment map. +type ExitModule = { + exit: ExitName +}; + +// InternalModuleSpecifier is the module specifier +// in the namespace of the native compartment. +type InternalModuleSpecifier = string; + +// ExternalModuleSpecifier is the module specifier +// in the namespace of the foreign compartment. +type ExternalModuleSpecifier = string; + +// ParserMap indicates which parser to use to construct module sources +// from sources, for each supported file extension. +// For parity with Node.js, a package with `"type": "module"` in its +// `package.json` would have a parser map of `{"js": "mjs", "cjs": "cjs", +// "mjs": "mjs"}`. +// If `"module"` is not defined in package.json, the legacy parser map // is +// `{"js": "cjs", "cjs": "cjs", "mjs": "mjs"}`. +// The compartment mapper adds `{"json": "json"}` for good measure in both +// cases, although Node.js (as of version 0.14.5) does not support importing +// JSON modules from ESM. +type ParserMap = Record<Extension, Parser>; + +// Extension is a file extension such as "js" for "main.js" or "" for "README". +type Extension = string; + +// Parser is a union of built-in parsers for module sources. +// "mjs" corresponds to ECMAScript modules. +// "cjs" corresponds to CommonJS modules. +// "json" corresponds to JSON. +type Parser = "mjs" | "cjs" | "json"; + +// ModuleParserMap is a table of internal module specifiers +// to the parser that should be used, regardless of that module's +// extension. +// Node.js allows the "module" property in package.json to denote +// a file that is an ECMAScript module, regardless of its extension. +// This is the mechanism that allows the compartment mapper to respect that +// behavior. +type ModuleParserMap = Record<InternalModuleSpecifier, Parser>; + +// ScopeMap is a map from internal module specifier prefixes +// like "dependency" or "@organization/dependency" to another +// compartment. +// The compartment mapper uses this to build a moduleMapHook that can dynamically +// generate entries for a compartment's moduleMap into Node.js packages that do +// not explicitly state their "exports". +// For these modules, any specifier under that prefix corresponds +// to a link into some internal module of the foreign compartment. +>> When the compartment mapper creates an archive, it captures all of the Modules +>> explicitly and erases the scopes entry. +type ScopeMap = Record<InternalModuleSpecifier, Scope>; + +// Scope describes the compartment to use for all ad-hoc +// entries in the compartment's module map. +type Scope = { + compartment: CompartmentName +}; + + +// TODO everything hereafter... + +// Realm describes another realm to contain one or more +// compartments. +// The default realm is frozen by lockdown with no +// powerful references. +type Realm = { + // TODO lockdown options +}; + +// RealmName is an arbitrary identifier for realms +// for reference from any Compartment description. +// No names are reserved; the default realm has no name. +type RealmName = string; + +// ModuleParameter indicates that the module does not come from +// another compartment but must be passed expressly into the +// application by the user. +// For example, the Node.js `fs` built-in module provides +// powers that must be expressly granted to an application +// and may be attenuated or limited by the compartment mapper on behalf of the +// user. +// The string value is the name of the module to be provided +// in the application's given module map. +type ModuleParameter = string; +``` + +# Compartment map policy + +The `policy` option accepted by the compartment-mapper API methods provides means to narrow down the endowments passed to each compartment independently. +The rules defined by policy get preserved in the compartment map and enforced in the application. To explore how policies work, see [Policy Demo]. + +The shape of the `policy` object is based on `policy.json` from LavaMoat. MetaMask's [LavaMoat] generates a `policy.json` file that serves the same purposes, using a tool called TOFU: _trust on first use_. + +> TODO +> +> Endo policy support is intended to reach parity with LavaMoat's policy.json. +> Policy generation may be ported to Endo. + + + [LavaMoat]: https://github.com/LavaMoat/lavamoat + [Compartments]: ../ses/README.md#compartment + [Policy Demo]: ./demo/policy/README.md + [package entry points]: https://nodejs.org/api/esm.html#esm_package_entry_points + +--- + +### Message ID: ebf6d6f3-209e-451c-b85a-06d3d6492b9b +**Author**: tool + +**Content:** + +**Piecing together remote imports** + +I’m working through a TypeScript snippet to fetch a remote module and run it in a SES compartment. The code needs `lockdown()` and `importLocation` from `@endo/compartment-mapper`. + +**Setting up a fetch function** + +I’m establishing a fetch-based `read` function for `importLocation`. The function needs to convert fetch results to `Uint8Array`, and globals and modules must be specified, possibly including `console` and an empty modules object. + +**Navigating module handling** + +Interestingly enough, `@endo/compartment-mapper` doesn't naturally handle TypeScript modules. To solve this, I'd use a transform or translator, assuming the given file is `.js` or `.mjs`. + +**Importing and running** + +The goal is to use `importLocation` with a custom fetch function to bring in a TypeScript file from a URL. This method will ensure smooth execution in a SES compartment. + +**Ensuring secure execution** + +I'm preparing to address `read` with `Promise<Uint8Array>`, utilizing `await fetch(...).arrayBuffer()` and `new Uint8Array(await res.arrayBuffer())`. It's worth noting the inclusion of `lockdown()` from `ses`. + +**Setting up the snippet** + +I’m defining a TypeScript script to fetch and execute a remote module in a SES environment using `@endo/compartment-mapper`. The `read` function employs `fetch` to handle data retrieval. + +**Adjusting language handling** + +I’m thinking through how to handle TypeScript files by configuring `` in the `package.json` file, ensuring they are processed correctly. This would typically involve setting up TypeScript to compile to JavaScript. + +**Fetch and translate** + +I’m showing how to fetch a TypeScript file, translate it to JavaScript with `mjs` extensions, and run it in a SES environment using `@endo/compartment-mapper`. + +--- + +### Message ID: 69448896-7cad-4321-b3f8-d3e061005700 +**Author**: assistant + +**Content:** + +```typescript +// run-remote-module.ts +import 'ses'; +import fetch from 'node-fetch'; +import { importLocation } from '@endo/compartment-mapper'; + +// Lock down the environment +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }); + +const loadAndRun = async (moduleURL: string) => { + // We'll treat the given module URL as part of a virtual package where: + // - A synthetic package.json is served at the same origin + // - The module itself is the "main" entry + const packageURL = new URL('./package.json', moduleURL).toString(); + + const read = async (location: string): Promise<Uint8Array> => { + if (location === packageURL) { + // Minimal package.json: treat all JS/TS as ESM + // We set main to the TS file and type=module so .ts is treated as ES modules. + const pkg = { + name: "remote-module", + type: "module", + main: new URL(moduleURL).pathname.replace(/^\/+/, '') + }; + return new TextEncoder().encode(JSON.stringify(pkg)); + } else { + // Fetch any module file (ts, js, etc.) from the remote source + const res = await fetch(location); + if (!res.ok) { + throw new Error(`Failed to fetch ${location}: ${res.statusText}`); + } + const content = await res.arrayBuffer(); + return new Uint8Array(content); + } + }; + + // Treat .ts as ESM (mjs) + const languageForExtension = { ts: 'mjs' }; + + // Provide limited globals/modules + const globals = { console }; + const modules = {}; + + // Import the remote module as if it were a local app entry point + const { namespace } = await importLocation( + read, + moduleURL, + { globals, modules, languageForExtension } + ); + + console.log(namespace); +}; + +loadAndRun('https://example.com/yourModule.ts').catch(err => { + console.error(err); + process.exit(1); +}); +``` + +--- + +### Message ID: aaa2022e-a366-421e-b547-d0f19a6721b7 +**Author**: user + +**Content:** + +additionally, here are all the docs from all the @endo packages. Please see if they can help you improve at all: + +-----BEGIN FILE base64/README.md----- +# base64 + +Base64 is a JavaScript package that encodes and decodes +[Base64](https://en.wikipedia.org/wiki/Base64) between strings and +`Uint8Array`s. + +Base64 exports TypeScript definitions and is suitable for both web and Node.js. + +## Install + +```sh +npm install @endo/base64 +``` + +## Usage + +```js +import { encodeBase64, decodeBase64 } from '@endo/base64'; + +const string = encodeBase64(bytes); +const bytes = decodeBase64(string); +``` + +-----END FILE base64/README.md----- +-----BEGIN FILE bundle-source/README.md----- +# Bundle Source + +This package creates source bundles from ES Modules, compatible with Endo +applications, Agoric contracts, and SwingSet vats. + +To bundle a program that enters at `program.js` from the command line, use the +`bundle-source` tool: + +```console +> yarn bundle-source --cache-json bundles program.js program +``` + +To do the same programmatically: + +```js +import 'ses'; +import bundleSource from '@endo/bundle-source'; +import url from 'url'; + +const sourceBundleURL = new URL('program.js', import.meta.url); +const sourceBundlePath = url.fileURLToPath(sourceBundleURL); +const sourceBundleP = bundleSource(sourceBundlePath); +``` + +…to get a promise for a source bundle, that resolves after reading the +named sources and bundling them into a form that vats can load, as indicated +by the `moduleFormat` below. + +The resulting bundle is suitable for use with `@endo/import-bundle`. +The default format is of a bundle is `"endoZipBase64"`. + +## Conditions + +Node.js introduced [conditions](https://nodejs.org/api/packages.html#conditional-exports). +The `--condition` and `-C` flags accordingly influence `bundle-source` module +resolution decisions. + +The `browser` condition additionally implies the selection of the `browser` +entry instead of `main` in `package.json`, if not overridden by explicit +`exports`. + +The `development` condition additionally implies that the bundle may import +`devDependencies` from the package containing the entry module. + +## Comment Elision + +The `--elide-comments (`-e`) flag with `--format` (`-f`) `endoScript` or +`endoZipBase64` (default) causes the bundler to blank out the interior of +comments, without compromising line or column number cursor advancement. +This can reduce bundle size without harming the debug experience any more than +other transforms. + +Comment elision preserves `/*! slashasterbang /` comments and JSDoc comments +with `@preserve`, `@copyright`, `@license` pragmas or the Internet Explorer +`@cc_on` pragma. + +Comment elision does not strip comments entirely. +The syntax to begin or end comments remains. + +## TypeScript type erasure + +TypeScript modules with the `.ts`, `.mts`, and `.cts` extensions in +packages that are not under a `node_modules` directory are automatically +converted to JavaScript through type erasure using +[`ts-blank-space`](https://bloomberg.github.io/ts-blank-space/). + +This will not function for packages that are published as their original +TypeScript sources, as is consistent with `node +--experimental-strip-types`. +This will also not function properly for TypeScript modules that have +[runtime impacting syntax](https://github.com/bloomberg/ts-blank-space/blob/main/docs/unsupported_syntax.md), +such as `enum`. + +This also does not support importing a `.ts` file using the corresponding +imaginary, generated module with a `.js` extension. +Use this feature in conjunction with +[`--allowImportingTsExtensions`](https://www.typescriptlang.org/tsconfig/#allowImportingTsExtensions). + +## Source maps + +With the `moduleFormat` of `endoZipBase64`, the bundler can generate source +maps but does not include them in the bundle itself. +Use the `cacheSourceMaps` option to render source maps into a per-user per-host +cache. + +The `@endo/import-bundle` utility can add references to these generated +source maps when it unpacks a bundle, provided a suitable +`computeSourceMapLocation` power, like the one provided by +`@endo/import-bundle/source-map-node.js`. + +```js +import 'ses'; +import { importBundle } from '@endo/import-bundle'; +import { computeSourceMapLocation } from '@endo/import-bundle/source-map-node.js'; +await importBundle( + bundle, + { endowments: { console } }, + { computeSourceMapLocation }, +); +``` + +Use the `@endo/cli` to find your cache. + +```console +> yarn add -D @endo/cli +> yarn endo where cache +``` + +Use the `XDG_CACHE_HOME` environment variable to override the default location +of caches in general. +The caches will be in `endo/source-map` and `endo/source-map-track`. +The former is a content-address-store keyed on the SHA-512 of each bundled +module file. +The latter is a location-address-store keyed on the SHA-512 of the fully +qualified path of the module source, indicating the last known bundle hash. +The bundler uses the tracker to ensure that the cache only contains one source +map for every physical module. +It is not yet quite clever enough to collect source maps for sources that do +not exist. + +## getExport moduleFormat + +The most primitive `moduleFormat` is the `"getExport"` format. +It generates source like: + +```js +function getExport() { + let exports = {}; + const module = { exports }; + // CommonJS source translated from the inputs. + ... + return module.exports; +} +``` + +To evaluate it and obtain the resulting module namespace, you need to endow +a `require` function to resolve external imports. + +## nestedEvaluate moduleFormat + +This is logically similar to the `getExport` format, except that the code +may additionally depend upon a `nestedEvaluate(src)` function to be used +to evaluate submodules in the same context as the parent function. + +The advantage of this format is that it helps preserve the filenames within +the bundle in the event of any stack traces. + +Also, the toplevel `getExport(filePrefix = "/bundled-source")` accepts an +optional `filePrefix` argument (which is prepended to relative paths for the +bundled files) in order to help give context to stack traces. + +## endoZipBase64 moduleFormat + +An Endo (zip, base64) bundle is an object with properties: + +- `moduleFormat` is `"endoZipBase64"` +- `endoZipBase64` is a base 64 encoded zip file. +- `endoZipBase64Sha512`, if present, is the SHA-512 of the + `compartment-map.json` file inside the `endoZipBase64` archive. + If the `compartment-map.json` includes the SHA-512 of every module, this is + sufficient as a hash of the bundled application for checking its integrity + and is consistent regardless of whether the program is extracted from the + archive. + +To inspect the contents of a bundle in a JSON file: + +``` +jq -r .endoZipBase64 | base64 -d | xxd | less +``` + +To extract the contents: + +``` +jq -r .endoZipBase64 | base64 -d > bundle.zip +unzip bundle.zip -d bundle +``` + +Inside the zip file, the `compartment-map.json` expresses the entire linkage of +the bundled program starting at its entry module, with explicitly marked "exit" +modules (host modules that must be endowed). + +The compartment map then names all of its compartments, and within each +compartment, specifies each module that will be evaluated in that compartment. +These indicate the path within the archive of the physical text of the module. +The `parser` indicates how `importBundle` or the equivalent Compartment Mapper +utilities will interpret the physical text of the module. + +To avoid entraining large dependencies and a slow precompilation step, modules +in a bundle are currently precompiled, so instead of finding source text, you +will find a JSON record describing the bindings and behavior of the module, +including code that is similar to the source but not identical. + +The bundle may have any of these `"parser"` properties: + +- `pre-mjs-json`: precompiled ESM +- `pre-cjs-json`: precompiled CommonJS +- `json`: raw JSON (exports the corresponding value as `default`) +- `text`: UTF-8 encoded text (exports the corresponding `string` as `default`) +- `bytes`: bytes (exports the corresponding `Uint8Array` as `default`) + +The JSON of a `pre-mjs-json` module will have all the properties of an object +generated with `StaticModuleRecord` from `@endo/static-module-record`, but +particularly: + +- `__syncModuleProgram__`: the code, which has been transformed from the ESM + source to a program that a compartment can evaluate and bind to other ESM + modules, and also had certain censorship-evasion transforms applied. + +So, to extract the source-similar program for visual inspection: + +``` +jq -r .__syncModuleProgram module.js > module.source.js +``` + + +-----END FILE bundle-source/README.md----- +-----BEGIN FILE captp/README.md----- +# `@endo/captp` + +A minimal CapTP implementation leveraging Agoric's published modules. + +## Usage + +NOTE: `myconn` below is not part of the CapTP library, it represents a connection +object that you have created where `makeCapTP` is called on both sides of the +connection, passing in the function to send a JSON-able object on the connection, and returning +a `dispatch` function to receive a decoded JSON object from the connection. + +```js +import { E, makeCapTP } from '@endo/captp'; + +// Create a message dispatcher and bootstrap. +// Messages on myconn are exchanged with JSON-able objects. +const { dispatch, getBootstrap, abort } = makeCapTP('myid', myconn.send, myBootstrap); +myconn.onReceive = obj => dispatch(obj); + +// Get the remote's bootstrap object and call a remote method. +E(getBootstrap()).method(args).then(res => console.log('got res', res)); + +// Tear down the CapTP connection if it fails (e.g. connection is closed). +abort(Error('Connection aborted by user.')); +``` + +## Loopback + +The `makeLoopback()` function creates an async barrier between "near" and "far" +objects. This is useful for testing and isolation within the same address +space. + +## TrapCaps + +In addition to the normal CapTP facilities, this library also has the notion of +"TrapCaps", which enable a "guest" endpoint to call a "host" object (which may +resolve an answer promise at its convenience), but the guest synchronously +blocks until it receives the resolved answer. + +This is a specialized and advanced use case, not for mutually-suspicious CapTP +parties, but instead for clear "guest"/"host" relationship, such as user-space +code and synchronous devices. + +1. Supply the `trapHost` and `trapGuest` protocol implementation (such as the + one based on `SharedArrayBuffers` in `src/atomics.js`) to the host and guest + `makeCapTP` calls. +2. On the host side, use the returned `makeTrapHandler(target)` to mark a target + as synchronous-enabled. +3. On the guest side, use the returned `Trap(target)` proxy maker much like + `E(target)`, but it will return a synchronous result. `Trap` will throw an + error if `target` was not marked as a TrapHandler by the host. + +To understand how `trapHost` and `trapGuest` relate, consider the `trapHost` as +a maker of AsyncIterators which don't return any useful value. These specific +iterators are used to drive the transfer of serialized data back to the guest. + +`trapGuest` receives arguments to describe the specific trap request, including +`startTrap()` which sends data to the host to perform the actual work of the +trap. The returned (synchronous) iterator from `startTrap()` drives the async +iterator of the host until it fully transfers the trap results to the guest, and +the guest unblocks. + +The Loopback implementation provides partial support for TrapCaps, except it +cannot unwrap promises. Loopback TrapHandlers must return synchronously, or an +exception will be thrown. + +-----END FILE captp/README.md----- +-----BEGIN FILE check-bundle/README.md----- +# check-bundle + +`checkBundle` verifies the integrity of a bundle, inspects all of its internal hashes and its one external hash. +`checkBundle` verifies the internal consistency, completeness, coherence, and conciseness (no extra files) of the bundle. +The function returns a rejected promise if the bundle fails the check. + +```js +const bundle = await bundleSource('path/to/bundle.js'); +// 'bundle' is JSON-serializable +await checkBundle(bundle); +``` + +This must be run in an Endo environment. To run on Node.js, import `@endo/init` before importing `@endo/import-bundle`. + +-----END FILE check-bundle/README.md----- +-----BEGIN FILE cjs-module-analyzer/README.md----- + +-----END FILE cjs-module-analyzer/README.md----- +-----BEGIN FILE cli/README.md----- +# Endo CLI + +The Endo command line is a user interface for managing the Endo application +runner (daemon). +This includes managing the lifecycle of the daemon process. + +-----END FILE cli/README.md----- +-----BEGIN FILE common/README.md----- +# `@endo/common` + +A collection of common low level utilities. + +Each of the utilities in this packages +- are low level in the sense of not depending on anything higher level than `ses`, `@endo/eventual-send`, and `@endo/promise-kit`. Many depend on nothing beyond plain old JavaScript. +- highly reusable, i.e., potentially useful many places. +- sufficiently general that it would be awkward to import from a more specialized package. +- can be explained and motivated without much external knowledge. + +Each utility is in its own top-level source file, named after the main export of that utility. (This is often that file's only export.) The `package.json` also lists each as a distinct `"export":`. There is no `index.js` file that rolls them together. Thus, each importer must do a deep import of exactly the export it needs. Some implementations (bundlers, packagers) can thus do tree-shaking, omitted code that isn't reachable by imports. + +Currently there are no `src/something.js` files. The only source files that would go in `src/` are those that do not represent separately exported utilities. + +Generally each utility also has its own test file. (An exception is that `make-iterator.js` is indirectly but adequately tested by `test-make-array-iterator.js`). + +See the doc-comments within the source file of each utility for documentation of that utility. Sometimes the associated test files also serve as informative examples. + +-----END FILE common/README.md----- +-----BEGIN FILE compartment-mapper/README.md----- +# Compartment mapper + +The compartment mapper builds _compartment maps_ for Node.js style +applications, finding their dependencies and describing how to create +[Compartments][] for each package in the application. + +Creating a compartment map for a Node.js application allows us to harness +the SES module loader to encapsulate each dependency and grant the least +necessary authority to each third-party package, mitigating prototype pollution +attacks and some supply chain attacks. +Since most Node.js packages do not modify objects in global scope, +many libraries and applications work in Compartments without modification. + +The `importLocation` function runs a compartmentalized application off the file +system. +The `globals` are properties to add to the `globalThis` in the global scope +of the application's main package compartment. +The `modules` are built-in modules to grant the application's main package +compartment. + +```js +import fs from "fs"; +import { importLocation } from "@endo/compartment-mapper"; + +// ... + +const modules = { fs }; +const globals = { console }; + +const read = async location => + fs.promises.readFile(new URL(location).pathname); + +const { namespace } = await importLocation( + read, + moduleLocation, + { + globals, + modules + } +); +``` + +The compartment mapper does nothing to arrange for the realm to be frozen. +The application using the compartment mapper is responsible for applying the +[SES] shim (if necessary) and calling `lockdown` to freeze the realm (if +necessary). +The compartment mapper is also not coupled specifically to Node.js IO and does +not import any powerful modules like `fs`. +The user must provide `read` and `write` functions from whatever IO powers they +have. These powers can be provided as individual functions or as objects +carrying functions. `ReadPowers` has optional functions which can be used to +unlock compatibility features. When `fileURLToPath` is available, `__dirname` +and `__filename` will be provided to CJS modules. If `requireResolve` is +available, it will be called whenever a CJS module calls `require.resolve()`. + +```js +type ReadPowers = { + read: (location: string) => Promise<Uint8Array>, + canonical: (location: string) => Promise<string>, + computeSha512: { (bytes: Uint8Array) => string }?, + fileURLToPath: { (url: string | URL) => string }?, + pathToFileURL: { (path: string) => URL }?, + requireResolve: { (from: string, request: string, options?: {}) => string }? +} +``` + +> TODO +> +> A future version will allow application authors to distribute their choices +> of globals and built-in modules to third-party packages within the +> application, as with [LavaMoat]. + +The `importLocation` function uses `loadLocation`. +Using `loadLocation` directly allows for deferred execution or multiple runs +with different globals or modules in the same process. +Calling `loadLocation` returns an `Application` object with an +`import({ globals?, modules? })` method. + +Use `writeArchive` to capture an application in an archival format. +Archives are `zip` files with a `compartment-map.json` manifest file. + +```js +import fs from "fs"; +import { writeArchive } from "@endo/compartment-mapper"; + +const read = async location => + fs.promises.readFile(new URL(location).pathname); +const write = async (location, content) => + fs.promises.writeFile(new URL(location).pathname, content); + +await writeArchive( + write, + read, + new URL('app.zip', import.meta.url).toString(), // the archive to write + new URL('app.js', import.meta.url).toString() // the application to capture +); +``` + +The `writeArchive` function uses `makeArchive`. +Using `makeArchive` directly gives you the archive bytes. + +Use `importArchive` to run an application from an archive. +Note the similarity to `importLocation`. + +```js +import fs from "fs"; +import { importArchive } from "@endo/compartment-mapper"; + +// ... + +const modules = { fs }; +const globals = { console }; + +const read = async location => + fs.promises.readFile(new URL(location).pathname); + +const { namespace } = await importArchive( + read, + archiveLocation, + { + globals, + modules + } +); +``` + +The `importArchive` function composes `loadArchive` and `parseArchive`. +Use `loadArchive` to defer execution or run multiple times with varying +globals. +Use `parseArchive` to construct a runner from the bytes of an archive. +The `loadArchive` and `parseArchive` functions return an `Application` +object with an `import({ globals?, modules? })` method. + +`loadArchive` and `parseArchive` do not run the archived program, +so they can be used to check the hash of a program without running it. + +# Package Descriptors + +The compartment mapper uses [Compartments], one for each Node.js package your +application needs. +The compartment mapper generates a compartment graph from Node.js packaged +module descriptors: the `package.json` files of the application and all its +dependencies. +Consequently, an application must have a `package.json`. + +Each package has its own descriptor, `package.json`. +Some standard properties of the descriptor are relevant and used by a +compartment map. + +* `name` +* `type` +* `main` +* `exports` +* `browser` +* `dependencies` +* `files` + +The compartment map will contain one compartment for each `package.json` +necessary to build the application. +Like Node.js, the compartment mapper trusts the package manager to arrange the +packages such that a satisfactory version of every package's dependencies rests +in a parent directory, under `node_modules`. + +The `main`, `browser`, and `exports` properties determine the modules each +package exports to other compartments. + +The `exports` property describes [package entry points][] and can be influenced +by build _conditions_. +Currently, the only conditions supported by the compartment mapper are +`import`, `browser`, and `endo`. +The `imports` condition indicates that the module map should use ESM modules +over CommonJS modules or other variants, and `endo`. +The `browser` condition also draws in the `browser` property from +`package.json` instead of `main`. +The `endo` condition only indicates that this tool is in use. + +If no `exports` apply to the root of the compartment namespace (`"."`), +the `main` property serves as a default. + +> TODO +> +> A future version may also respect the `imports` property. + +> TODO +> +> A future version may also respect wildcard patterns in `exports` and +> `imports`. + +The `files` property indicates all of the files in the package that +should be vended out to applications. +The file set implicitly includes all `**.js`, `**.mjs`, and `**.cjs` files. +The file set implicitly excludes anything under `node_modules`. + +With the compartment mapper, just as in Node.js, a module specifier that has no +extension may refer either to the file with the `js` extension, or if that file +does not exist, to the `index.js` file in the directory with the same name. + +> TODO +> +> The compartment mapper does not yet do anything with the `files` globs but a +> future version of the compartment mapper will collect these in archives. +> The compartment mapper should eventually provide the means for any +> compartment to access its own files using an attenuated `fs` module or +> `fetch` global, in conjunction with usable values for `import.meta.url` in +> ECMAScript modules or `__dirname` and `__filename` in CommonJS modules. + +## Language Extensions + +Officially beginning with Node.js 14, Node.js treats `.mjs` files as ECMAScript +modules and `.cjs` files as CommonJS modules. +The `.js` extension indicates a CommonJS module by default, to maintain +backward compatibility. +However, packages that have a `type` property that explicitly says `module` +will treat a `.js` file as an ECMAScript module. + +Many Node.js applications using CommonJS modules expect to be able to `require` +a JSON file like `package.json`. +The compartment mapper supports loading JSON modules from any type of module. +As of Node.js 14, Node does not support importing JSON using ECMAScript +`import` directives, so using this feature may limit compatibility with the +Node.js platform. + +The compartment mapper supports loading CommonJS modules from ECMAScript +modules as well as ECMAScript modules importing CommonJS modules. +This presumes that the CommonJS modules exclusively use `require` calls with a +single string argument, where `require` is not lexically bound, to declare +their shallow dependencies, so that these modules and their transitive +dependencies can be loaded before any module executes. +As of Node.js 14, Node does not support loading ECMAScript modules from +CommonJS modules, so using this feature may limit compatibility with the +Node.js platform. + +The compartment mapper supports language plugins. +The languages supported by default are: + +- `mjs` for ECMAScript modules, +- `cjs` for CommonJS modules, +- `json` for JSON modules, +- `text` for UTF-8 encoded text files, +- `bytes` for any file, exporting a `Uint8Array` as `default`, +- `pre-mjs-json` for pre-compiled ECMAScript modules captured as JSON in + archives, and +- `pre-cjs-json` for pre-compiled CommonJS modules captured as JSON in + archives. + +The compartment mapper accepts extensions to this set of languages with +the `parserForLanguage` option supported by many functions. +See `src/types/external.ts` for the type and expected behavior for +parsers. + +These language identifiers are keys for the `moduleTransforms` and +`syncModuleTransforms` options, which may map each language to a transform +function. +The language identifiers are also the values for a `languageForExtension`, +`moduleLanguageForExtension`, and `commonjsLanguageForExtension` options to +configure additional extension-to-language mappings for a module and its +transitive dependencies. + +For any package that has `type` set to `"module"` in its `package.json`, +`moduleLangaugeForExtension` will precede `languageForExtension`. +Packages with `type` set to `"commonjs"` or simply not set, +`commonjsLanguageForExtension` will precede `languageForExtension`. +This provides an hook for mapping TypeScript's `.ts` to either `.cts` or +`.mts`. + +The analogous `workspaceLanguageForExtension`, +`workspaceCommonjsLanguageForExtension`, and +`workspaceModuleLanguageForExtension` options apply more specifically for +packages that are not under a `node_modules` directory, indicating that they +are in the set of linked workspaces and have not been built or published to +npm. + +In the scope any given package, the `parsers` property in `package.json` may +override the extension-to-language mapping. + +```json +{ + "parsers": { "png": "bytes" } +} +``` + +> TODO +> +> The compartment mapper may elect to respect some properties specified for +> import maps. + +> TODO +> +> A future version of the compartment mapper may add support for +> source-to-source translation in the scope of a package or compartment. +> This would be expressed in `package.json` using a property like +> `translate` that would contain a map from file extension +> to a module that exports a suitable translator. +> +> For browser applications, the compartment mapper would use the translator +> modules in two modes. +> During development, the compartment mapper would be able to load the +> translator in the client, with the `browser` condition. +> The compartment mapper would also be able to run the translator in a separate +> non-browser compartment during bundling, so the translator can be excluded +> from the production application and archived applications. + +> TODO +> +> The compartment mapper may also add support for compartment map plugins that +> would recognize packages in `devDependencies` that need to introduce globals. +> For example, _packages_ that use JSX and a virtual DOM would be able to add a +> module-to-module translator and endow the compartment with the `h` the +> translated modules need. + +# Source Maps + +The `makeArchive`, `makeAndHashArchive`, and `writeArchive` tools can receive a +`sourceMapHook` as one of its options. +The `sourceMapHook` receives a source map `string` for every module it +archives, along with details `compartment`, `module`, `location`, and `sha512`. +The `compartment` is the fully-qualified file URL of the package root. +The `module` is the package-relative module specifier. +The `location` is the fully-qualified file URL of the module file. +The `sha512`, if present, was generated with the `computeSha512` power from the +generated module bytes. + +The functions `importArchive`, `loadArchive`, and `parseArchive` +tools can receive a `computeSourceMapLocation` option that recives the same +details as above and must return a URL. +These will be appended to each module from the archive, for debugging purposes. + +The `@endo/bundle-source` and `@endo/import-bundle` tools integrate source maps +for an end-to-end debugging experience. + +# XS (experimental) + +The Compartment Mapper can use native XS `Compartment` and `ModuleSource` under +certain conditions: + +1. The application must be an XS script that was compiled with the `xs` + package condition. + This causes `ses`, `@endo/module-source`, and `@endo/import-bundle` to + provide slightly different implementations that can fall through to native + behavior. +2. The application must opt-in with the `__native__: true` option on any + of the compartment mapper methods that import modules like `importLocation` + and `importArchive`. + +# Design + +Each of the workflows the compartment mapper executes a portion of one sequence +of underlying internals. + +* search (`search.js`): Scan the parent directories of a given `moduleLocation` + until successfully finding and reading a `package.json` for the containing + application. +* map compartments from Node.js packages (`node-modules.js`): Find and gather + all the `package.json` files for the application's transitive dependencies. + Use these to construct a compartment map describing how to construct a + `Compartment` for each application package and how to link the modules each + exports in the compartments that import them. +* load compartments (`archive.js`): Using `compartment.load`, or + implicitly through `compartment.import`, create a module graph for the + application's entire working set. + When creating an archive, this does not execute any of the modules. + The compartment mapper uses the compartments and a special `importHook` that + records the text of every module the main module needed. +* import modules (`import.js`, `import-archive.js`): Actually execute the + working set. + +Around this sequence, we can enter late or depart early to store or retrieve an +archive. +The compartment mapper provides workflows that use `read` and `write` hooks +when interacting with a filesystem or work with the archive bytes directly. + +This diagram represents the the workflows of each of the public methods like +`importLocation`. +Each column of pipes `|` is a workflow from top to bottom. +Each asterisk `*` denotes a step that is taken by that workflow. +The dotted lines `.'. : '.'` indicate carrying an archive file from the end of +one workflow to the beginning of another, either as bytes or a location. + +In the diagram, "powers" refer to globals and built-in modules that may provide +capabilities to a compartment graph. +For `writeArchive` and `makeArchive`, these may be provided but will be ignored +since the application does not execute. + +``` + loadLocation writeArchive + importLocation | | makeArchive + | | | | + | | | | parseArchive + | | | | | loadArchive + | | | | | | importArchive + | | | | | | |... + search -> * * * * | |'| . ' + map compartments -> * * * * .'.| | |' : : + read archive -> | | | ' | * * : : + unpack archive -> | | | : * * * : : +assemble compartments -> * * * : * : : <- powers + load compartments -> * * * : * : : + import modules -> * | | : * : : + pack archive -> * * ' : : + write archive -> * '.' <- data : : + '..............' : <- files + '...............' +``` + +# Compartment maps + +The compartment mapper works by generating a _compartment map_ from your +application workspace and all of the `node_modules` it needs. +A compartment map is similar to a lock file because it collects information +from all of the installed modules. +A compartment map describes how to construct compartments for each +package in your application and link their module namespaces. + +The compartment map shape: + +```ts +// CompartmentMap describes how to prepare compartments +// to run an application. +type CompartmentMap = { + tags: Conditions, + entry: Entry, + compartments: Record<CompartmentName, Compartment>, + realms: Record<RealmName, Realm>, // TODO +}; + +// Conditions influence which modules are selected +// to represent the implementation of various modules. +// These may include terms like "browser", meaning +// each compartment uses the implementation of each +// module suitable for use in a browser environment. +type Conditions = Array<Condition>; +type Condition = string; + +// Entry is a reference to the module that is the module to initially import. +type Entry = CompartmentModule; + +// CompartmentName is an arbitrary string to name +// a compartment for purposes of inter-compartment linkage. +type CompartmentName = string; + +// Compartment describes where to find the modules +// for a compartment and how to link the compartment +// to modules in other compartments, or to built-in modules. +type Compartment = { + location: Location, + modules: ModuleMap, + parsers: ParserMap, + types: ModuleParserMap, + scopes: ScopeMap, + // The name of the realm to run the compartment within. + // The default is a single frozen realm that has no name. + realm: RealmName? // TODO +}; + +// Location is the URL relative to the compartment-map.json's +// containing location to the compartment's files. +type Location = string; + +// ModuleMap describes modules available in the compartment +// that do not correspond to source files in the same compartment. +type ModuleMap = Record<InternalModuleSpecifier, Module>; + +// Module describes a module in a compartment. +type Module = CompartmentModule | FileModule | ExitModule; + +// CompartmentModule describes a module that isn't in the same +// compartment and how to introduce it to the compartment's +// module namespace. +type CompartmentModule = { + // The name of the foreign compartment: + // TODO an absent compartment name may imply either + // that the module is an internal alias of the + // same compartment, or given by the user. + compartment: CompartmentName?, + // The name of the module in the foreign compartment's + // module namespace: + module: ExternalModuleSpecifier?, +}; + +// FileLocation is a URL for a module's file relative to the location of the +// containing compartment. +type FileLocation = string + +// FileModule is a module from a file. +// When loading modules off a file system (src/import.js), the assembler +// does not need any explicit FileModules, and instead relies on the +// compartment to declare a ParserMap and optionally ModuleParserMap and +// ScopeMap. +// The compartment mapper provides a Compartment importHook and moduleMapHook +// that will search the filesystem for candidate module files and infer the +// type from the extension when necessary. +type FileModule = { + location: FileLocation, + parser: Parser, +}; + +// ExitName is the name of a built-in module, to be threaded in from the +// modules passed to the module executor. +type ExitName = string; + +// ExitModule refers to a module that comes from outside the compartment map. +type ExitModule = { + exit: ExitName +}; + +// InternalModuleSpecifier is the module specifier +// in the namespace of the native compartment. +type InternalModuleSpecifier = string; + +// ExternalModuleSpecifier is the module specifier +// in the namespace of the foreign compartment. +type ExternalModuleSpecifier = string; + +// ParserMap indicates which parser to use to construct module sources +// from sources, for each supported file extension. +// For parity with Node.js, a package with `"type": "module"` in its +// `package.json` would have a parser map of `{"js": "mjs", "cjs": "cjs", +// "mjs": "mjs"}`. +// If `"module"` is not defined in package.json, the legacy parser map // is +// `{"js": "cjs", "cjs": "cjs", "mjs": "mjs"}`. +// The compartment mapper adds `{"json": "json"}` for good measure in both +// cases, although Node.js (as of version 0.14.5) does not support importing +// JSON modules from ESM. +type ParserMap = Record<Extension, Parser>; + +// Extension is a file extension such as "js" for "main.js" or "" for "README". +type Extension = string; + +// Parser is a union of built-in parsers for module sources. +// "mjs" corresponds to ECMAScript modules. +// "cjs" corresponds to CommonJS modules. +// "json" corresponds to JSON. +type Parser = "mjs" | "cjs" | "json"; + +// ModuleParserMap is a table of internal module specifiers +// to the parser that should be used, regardless of that module's +// extension. +// Node.js allows the "module" property in package.json to denote +// a file that is an ECMAScript module, regardless of its extension. +// This is the mechanism that allows the compartment mapper to respect that +// behavior. +type ModuleParserMap = Record<InternalModuleSpecifier, Parser>; + +// ScopeMap is a map from internal module specifier prefixes +// like "dependency" or "@organization/dependency" to another +// compartment. +// The compartment mapper uses this to build a moduleMapHook that can dynamically +// generate entries for a compartment's moduleMap into Node.js packages that do +// not explicitly state their "exports". +// For these modules, any specifier under that prefix corresponds +// to a link into some internal module of the foreign compartment. +>> When the compartment mapper creates an archive, it captures all of the Modules +>> explicitly and erases the scopes entry. +type ScopeMap = Record<InternalModuleSpecifier, Scope>; + +// Scope describes the compartment to use for all ad-hoc +// entries in the compartment's module map. +type Scope = { + compartment: CompartmentName +}; + + +// TODO everything hereafter... + +// Realm describes another realm to contain one or more +// compartments. +// The default realm is frozen by lockdown with no +// powerful references. +type Realm = { + // TODO lockdown options +}; + +// RealmName is an arbitrary identifier for realms +// for reference from any Compartment description. +// No names are reserved; the default realm has no name. +type RealmName = string; + +// ModuleParameter indicates that the module does not come from +// another compartment but must be passed expressly into the +// application by the user. +// For example, the Node.js `fs` built-in module provides +// powers that must be expressly granted to an application +// and may be attenuated or limited by the compartment mapper on behalf of the +// user. +// The string value is the name of the module to be provided +// in the application's given module map. +type ModuleParameter = string; +``` + +# Compartment map policy + +The `policy` option accepted by the compartment-mapper API methods provides means to narrow down the endowments passed to each compartment independently. +The rules defined by policy get preserved in the compartment map and enforced in the application. To explore how policies work, see [Policy Demo]. + +The shape of the `policy` object is based on `policy.json` from LavaMoat. MetaMask's [LavaMoat] generates a `policy.json` file that serves the same purposes, using a tool called TOFU: _trust on first use_. + +> TODO +> +> Endo policy support is intended to reach parity with LavaMoat's policy.json. +> Policy generation may be ported to Endo. + + + [LavaMoat]: https://github.com/LavaMoat/lavamoat + [Compartments]: ../ses/README.md#compartment + [Policy Demo]: ./demo/policy/README.md + [package entry points]: https://nodejs.org/api/esm.html#esm_package_entry_points + +-----END FILE compartment-mapper/README.md----- +-----BEGIN FILE daemon/README.md----- +# Endo Daemon + +This package provides the Endo daemon and controller. +The controller manages the Endo daemon lifecycle. + +The Endo daemon is a persistent host for managing guest programs in hardened +JavaScript worker processes. +The daemon communicates through a Unix domain socket or named pipe associated +with the user, and manages per-user storage and compute access. + +Over that channel, the daemon communicates in CapTP over netstring message +envelopes. +The bootstrap provides the user agent API from which one can derive facets for +other agents. + +-----END FILE daemon/README.md----- +-----BEGIN FILE env-options/README.md----- +# Parameterizing Modules with Environment Options + +JavaScript module semantics resist attempts to parameterize a module's +initialization behavior. A module initializes in order according to +the path by which it is first imported, and then the initialized module +is reused by all the other times it is imported. Compartments give us +the opportunity to bind the same import name to different imported +modules, depending on the package/compartment doing the import. Compartments +also address the difficulty of parameterizing a module's initialization +logic, but not in a pleasant manner. + +A pleasant parameterization would be for a static module to be function-like +with explicit parameters, and for the parameterization to be like +calling the static module with parameters in order to derive from it a +module instance. Compartments instead lets us parameterize the meaning +of a module instance derived from a static module according to the +three namespaces provided by the JavaScript semantics, affecting the +meaning of a module instance. + * The global variable namespaces. + * The global scope, aliased to properties of the global object. + This is necessarily compartment-wide. In our + recommened usage pattern of one compartment per package, + each global would be package-wide. (See LavaMoat) + * The global lexical scope. The SES-shim compartments support + these both compartment-wide as well as per-module. But it is + not yet clear what we will propose in the Compartment proposal. + * The import namespace. + * The host hooks. + +This `@endo/env-options` package follows the Node precedent for +finding Unix environment variable settings: looking for a +global `process` object holding an `env` object, +optionally holding a property with the same name as the option, +whose value is the configuration setting of that option. + +```js +import { getEnvironmentOption } from '@endo/env-options'; +const FooBarOption = getEnvironmentOption('FOO_BAR', 'absent'); +``` + +The first argument to `getEnvironmentOption` is the name of the option. +The value of `FooBarOption` would then be the value of +`globalThis.process.env.FOO_BAR`, if present. +If value is either absent or `undefined`, the second argument, +such as `'absent'`, would be used instead. + +In either case, reflecting Unix environment variable expectations, +the resulting setting must be a string. +This restriction also helps ensure that this channel is used only to pass data, +not authority beyond the ability to read this global state. + +```js +const ENABLED = + getEnvironmentOption('TRACK_TURNS', 'disabled', ['enabled']) === 'enabled'; +``` + +`getEnvironmentOption` also takes an optional third argument, which if present +is an exhaustive list of allowed strings other than the default. If present +and the actual environment option is neither the default nor one of these +allowed strings, then an error is thrown explaining the problem. + +```js +const DEBUG_VALUES = getEnvironmentOptionsList('DEBUG'); +const DEBUG_AGORIC = environmentOptionsListHas('DEBUG', 'agoric'); +``` + +Another common convention is for the value of an option to be a +comma (`','`) separated list of strings. `getEnvironmentOptionsList` will +return this list, or an empty list if the option is absent. +`environmentOptionsListHas` will test if this list contains a specific +value, or return false if the option is absent. + +(Compat note: https://github.com/Agoric/agoric-sdk/issues/8096 explains that +for `DEBUG` specifically, some existing uses split on colon (`':'`) rather +than comma. Once these are fixed, then these uses can be switched to use +`getEnvironmentOptionsList` or `environmentOptionsListHas`.) + +## Tracking used option names + +The `'@endo/env-options'` module also exports a lower-level +`makeEnvironmentCaptor` that you can apply to whatever object you wish to treat +as a global(having a "process" property with its own "env" record), +such as the global of another compartment. It returns an entagled +pair of a `getEnvironmentOption` function as above, and a +`getCapturedEnvironmentOptionNames` function that returns an array of +the option names used by that `getEnvironmentOption` function. This is +useful to give feedback about +which environment variables were actually read, for diagnostic purposes. +For example, the +ses-shim `lockdown` once contained code such as the following, to explain which +environment variables were read to provide `lockdown` settings. + +```js +import { makeEnvironmentCaptor } from '@endo/env-options'; +const { + getEnvironmentOption, + getEnvironmentOptionsList, + environmentOptionsListHas, + getCapturedEnvironmentOptionNames, +} = makeEnvironmentCaptor(globalThis); +... +const capturedEnvironmentOptionNames = getCapturedEnvironmentOptionNames(); +if (capturedEnvironmentOptionNames.length > 0) { + console.warn( + `SES Lockdown using options from environment variables ${enJoin( + arrayMap(capturedEnvironmentOptionNames, q), + 'and', + )}`, + ); +} +``` + +# Note of test migration + +To reduce cyclic dependencies, the tests of this module have been moved to +@endo/ses-ava. Doing `yarn test` here currently does nothing. + +-----END FILE env-options/README.md----- +-----BEGIN FILE errors/README.md----- + + +-----END FILE errors/README.md----- +-----BEGIN FILE eslint-plugin/README.md----- +# @endo/eslint-plugin + +Endo-specific plugin + +## Installation + +You'll first need to install [ESLint](http://eslint.org): + +```sh +npm i eslint --save-dev +``` + +Next, install `@endo/eslint-plugin`: + +```sh +npm install @endo/eslint-plugin --save-dev +``` + +**Note:** If you installed ESLint globally (using the `-g` flag) then you must also install `@endo/eslint-plugin` globally. + +## Usage + +Extend a `plugin:@endo/CONFIG` in your `.eslintrc` configuration file. You can omit the `/eslint-plugin` suffix: + +```json +{ + "extends": [ + "plugin:@endo/recommended" + ] +} +``` + +`CONFIG` can be one of: + +- `recommended` rules for code compatible with Hardened JS +- `imports` opinions on how packages should use imports +- `style` opinions on JS coding style +- `strict` all of the above +- `internal` rules only for packages within the Endo source repository + + +You can configure individual rules you want to use under the rules section. + +```json +{ + "rules": { + "@endo/rule-name": 2 + } +} +``` + +## Supported Rules + +* Fill in provided rules here + +-----END FILE eslint-plugin/README.md----- +-----BEGIN FILE evasive-transform/README.md----- +# @endo/evasive-transform + +> Source transforms for evading censorship in [SES](https://github.com/endojs/endo/tree/master/packages/ses)-enabled applications + +This package provides a function which transforms comments contained in source code which would otherwise be rejected outright by SES. + +## Example + +```js +// ESM example +import { evadeCensor } from '@endo/evasive-transform'; +import fs from 'node:fs/promises'; + +/** + * Imagine this file contains a comment like `@property {import('foo').Bar} bar`. SES will refuse to run this code. + */ +const source = await fs.readFile('./dist/index.js', 'utf8'); +const sourceMap = await fs.readFile('./dist/index.js.map', 'utf8'); +const sourceUrl = 'index.js'; // assuming the source map references index.js +const sourceType = 'script'; + +const { code, map } = await evadeCensor(source, { + sourceMap, + sourceUrl, + sourceType, +}); + +/** + * The resulting file will now contain `@property {ІᛖРΟᏒТ('foo').Bar} bar`, which SES will allow (and TypeScript no longer understands, but that should be fine for the use-case). + * + * Note that this could be avoided entirely by stripping comments during, say, a bundling phase. + */ +await fs.writeFile('./dist/index.ses.js', code); +await fs.writeFile('./dist/index.ses.js.map', JSON.stringify(map)); +``` + +## License + +Apache-2.0 + +-----END FILE evasive-transform/README.md----- +-----BEGIN FILE eventual-send/README.md----- +# HandledPromise + +[![dependency status][deps-svg]][deps-url] +[![dev dependency status][dev-deps-svg]][dev-deps-url] +[![License][license-image]][license-url] + +Create a HandledPromise class to implement the eventual-send API. This API is used by the [ECMAScript eventual-send proposal](https://github.com/tc39/proposal-eventual-send). + +## How to use + +To install the `HandledPromise` global property shim, do: + +```js +import '@agoric/eventual-send/shim'; +``` + +After that, you can use `HandledPromise` in any of your code. If you need access to the `E` proxy maker, do: + +```js +import { E } from '@agoric/eventual-send'; +``` + +[deps-svg]: https://david-dm.org/Agoric/eventual-send.svg +[deps-url]: https://david-dm.org/Agoric/eventual-send +[dev-deps-svg]: https://david-dm.org/Agoric/eventual-send/dev-status.svg +[dev-deps-url]: https://david-dm.org/Agoric/eventual-send?type=dev +[license-image]: https://img.shields.io/badge/License-Apache%202.0-blue.svg +[license-url]: LICENSE + +-----END FILE eventual-send/README.md----- +-----BEGIN FILE exo/README.md----- +# `@endo/exo` + +An Exo object is an exposed Remotable object with methods (aka a Far object) which is normally defined with an InterfaceGuard as a protective outer layer, providing the first layer of defensiveness. + +This `@endo/exo` package defines the APIs for making Exo objects, and for defining ExoClasses and ExoClassKits for making Exo objects. + +See [exo-taxonomy](./docs/exo-taxonomy.md) for the taxonomy and naming conventions for the elements of this API. + +When an exo is defined with an InterfaceGuard, the exo is augmented by default with a meta-method for obtaining the self-describing InterfaceGuard from the exo: + +```js +// `GET_INTERFACE_GUARD` holds the name of the meta-method +import { GET_INTERFACE_GUARD } from '@endo/exo'; +import { getInterfaceMethodKeys } from '@endo/patterns'; + +... + const interfaceGuard = await E(exo)[GET_INTERFACE_GUARD](); + // `methodNames` omits names of automatically added meta-methods like + // the value of `GET_INTERFACE_GUARD`. + // Others may also be omitted if allowed by interfaceGuard options + const methodNames = getInterfaceMethodKeys(interfaceGuard); +... +``` + +-----END FILE exo/README.md----- +-----BEGIN FILE far/README.md----- +# Endo Far Object helpers + +The `@endo/far` package provides a convenient way to use the Endo +[distributed objects system](https://docs.agoric.com/guides/js-programming/) without relying on the underlying messaging +implementation. + +It exists to reduce the boilerplate in Hardened JavaScript vats that are running +in Agoric's SwingSet kernel, +[`@agoric/swingset-vat`](https://github.com/Agoric/agoric-sdk/tree/master/packages/SwingSet), +or arbitrary JS programs using Hardened JavaScript and communicating via +[`@endo/captp`](../captp/README.md). + +You can import any of the following from `@endo/far`: + +```js +import { E, Far, getInterfaceOf, passStyleOf } from '@endo/far'; +``` + +-----END FILE far/README.md----- +-----BEGIN FILE immutable-arraybuffer/README.md----- +# `@endo/immutable-arraybuffer` + +This `@endo/immutable-arraybuffer` package provides both a ponyfill and a shim for a proposed new JavaScript feature: *Immutable ArrayBuffers*. +- A ponyfill just defines and exports new things without modifying old things. The `index.js` file implements the ponyfill, providing the exports of the unqualified `@endo/immutable-arraybuffer` package. +- A shim modifies the existing JavaScript primordials as needed to most closely emulate the feature as proposed. The `shim.js` file uses the exports from `index.js` to modify `ArrayBuffer.prototype` to resemble the API being proposed. Importing `@endo/immutable-arraybuffer/shim.js` will cause these changes. + +Below, we use the term "buffer" to refer informally to an instance of an `ArrayBuffer`, whether immutable or not. + +## Background + +Prior proposals [In-Place Resizable and Growable `ArrayBuffer`s](https://github.com/tc39/proposal-resizablearraybuffer) and [ArrayBuffer.prototype.transfer and friends](https://github.com/tc39/proposal-arraybuffer-transfer) have both reached stage 4, and so are now an official part of JavaScript. Altogether, `ArrayBuffer.prototype` now has the following methods: +- `transfer(newByteLength?: number) :ArrayBuffer` -- move the contents of the original buffer to a new buffer, detach the original buffer, and return the new buffer. The new buffer will be as resizable as the original was. +- `transferToFixedLength(newByteLength?: number) :ArrayBuffer` -- like `transfer` but the new buffer is not resizable. +- `resize(newByteLength: number) :void` -- change the size of this buffer if possible, or throw otherwise. +- `slice(start?: number, end?: number) :ArrayBuffer` -- Return a new buffer whose initial contents are a copy of that region of the original buffer. The original buffer is unmodified. + +and the following read-only accessor properties +- `detached: boolean` -- is this buffer detached, or are its contents still available from this buffer object? +- `resizable: boolean` -- can this buffer be resized, or is it fixed-length? +- `byteLength: number` -- how big are the current contents of this buffer? +- `maxByteLength: number` -- how big could this buffer be resized to be? + +None of the operations above enable the creation of an immutable buffer, i.e., a non-detached buffer whose contents cannot be changed, resized, or detached. + +Both a `DataView` object and a `TypedArray` object are views into a buffer backing store. For a `TypedArray` object, the contents of the backing store appear as indexed data properties of the `TypeArray` object that reflect the current contents of this backing store. Currently, because there is no way to prevent the contents of the backing store from being changed, `TypedArray`s cannot be frozen. + +Some JavaScript implementations, like Moddable XS, bring JavaScript to embedded systems, like device controllers, where ROM is much more plentiful and cheaper than RAM. These systems need to place voluminous fixed data into ROM, and currently do so using semantics outside the official JavaScript standard. + +The [OCapN](https://ocapn.org/) network protocol treats strings and byte-arrays as distinct forms of bulk data to be transmitted by copy. At JavaScript endpoints speaking OCapN such as `@endo/pass-style` + `@endo/marshal`, JavaScript strings represent OCapN strings. The immutability of strings in the JavaScript language reflects their by-copy nature in the protocol. Likewise, to reflect an OCapN byte-array well into the JavaScript language, we need an immutable container of bulk binary data. There currently are none. An Immutable `ArrayBuffer` would provide exactly the low-level machinery we need. + +## Overview of the *Immutable ArrayBuffer* Proposal + +The *Immutable ArrayBuffer* proposal introduces additional methods and read-only accessor properties to `ArrayBuffer.prototype` that fit naturally into those explained above. Just as a buffer can be resizable or not, or detached or not, this proposal enables buffers to be immutable or not. Just as `transferToFixedSize` moves the contents of a original buffer into a newly created non-resizable buffer, this proposal provides a transfer operation that moves the contents of an original original buffer into a newly created immutable buffer. Altogether, this proposal only adds to `ArrayBuffer.prototype` one method +- `transferToImmutable() :ArrayBuffer` -- move the contents of the original buffer into a new immutable buffer, detach the original buffer, and return the new buffer. + +and one read-only accessor +- `immutable: boolean` -- is this buffer immutable, or can its contents be changed? + +An immutable buffer cannot be detached or resized. Its `maxByteLength` is the same as its `byteLength`. A `DataView` or `TypedArray` using an immutable buffer as its backing store can be frozen and immutable. `ArrayBuffer`s, `DataView`s, and `TypedArray`s that are frozen and immutable could be placed in ROM without going beyond JavaScript's official semantics. + +## The Ponyfill + +The proposal would add methods to `ArrayBuffer.prototype`. But a ponyfill, by definition, cannot do so. Instead, it defines and exports two functions corresponding to the two additions above +- `transferBufferToImmutable(buffer: ArrayBuffer) :ArrayBuffer` +- `isBufferImmutable(buffer: ArrayBuffer) :boolean` + +In order for `transferBufferToImmutable` to be able to return something of type `ArrayBuffer` that is actually immutable, that object cannot be an actual `ArrayBuffer` exotic object. Instead, an emulated immutable buffer implements the full proposed `ArrayBuffer` API and ultimately inherits from `ArrayBuffer.prototype`. Thus, `x instanceof ArrayBuffer` will act as proposed. + +The emulated immutable buffers inherit directly from an intermediate prototype we refer to as `immutableArrayBufferPrototype`. This intermediate prototype contains all the methods and read-only accessor properties proposed here, as well as overrides of those inherited from `ArrayBuffer.prototype` as needed to emulate the behavior of an immutable instance. For each emulated immutable buffer, the implementation encapsulates a genuine `ArrayBuffer` that it has exclusive access to, so it can enforce immutability simply by never modifying it. + +## The Shim + +The immutable-arraybuffer shim additionally adds to `ArrayBuffer.prototype` a +- `transferToImmutable` method trivially derived from the ponyfill's `transferBufferToImmutable`. +- `immutable` read-only accessor property trivially derived from the ponyfill's `isBufferImmutable`. + +## Caveats + +The *Immutable ArrayBuffer* shim falls short of the proposal in the following ways +- The ponyfill and shim rely on the underlying platform having either `structuredClone` or `ArrayBuffer.prototype.transfer`. However, Node <= 16 has neither. Node 17 introduces `structuredClone` and Node 21 introduces `ArrayBuffer.prototype.transfer`. Without either, the ponyfill and shim fail to initialize. +- The proposal does not introduce an intermediate prototype, but rather modifies the behavior of the built-in methods on `ArrayBuffer.prototype` itself, to act appropriately on immutable `ArrayBuffer`s. By contrast, the ponyfill's and shim's emulated immutable buffers inherit directly from an intermediate prototype we refer to as `immutableArrayBufferPrototype`. That intermediate prototype directly inherits from `ArrayBuffer.prototype`. All the differential behavior for immutable buffers are provided by overrides found on `immutableArrayBufferPrototype`. +- The `immutableArrayBufferPrototype` intermediate prototype is an artifact of the emulation, but it is not encapsulated. It is trivially discoverable as the object that emulated immutable buffers directly inherit from. +- The shim's emulated immutable buffers are not real `ArrayBuffer` exotic objects. If they were, the shim would not be able to protect them from being written. Even though they implement the full proposed `ArrayBuffer` API, they cannot be plug-compatible -- they cannot be used as the backing stores of `DataView`s or `TypedArray`s. Perhaps follow-on shims might modify `DataView` and `TypedArray` to emulate that as well, but that is hard and beyond the ambition of this ponyfill + shim. +- Unlike genuine `ArrayBuffer` or `SharedArrayBuffer` exotic objects, the shim's emulated immutable buffers cannot be cloned or transfered between JS threads. +- Even after the *Immutable ArrayBuffer* proposal is implemented by the platform, the current code will still replace it with the shim implementation, in accord with shim best practices. See https://github.com/endojs/endo/pull/2311#discussion_r1632607527 . It will require a later manual step to delete the shim, after manual analysis of the compat implications. +- This is a plain *JavaScript* ponyfill/shim, not by itself a *Hardened JavaScript* polyfill/shim. Thus, the objects and function it creates are not hardened by this ponyfill/shim itself. Rather, the ses-shim is expected to import these, and then treat the resulting objects as if they were additional primordials, to be hardened during `lockdown`'s harden phase. + +-----END FILE immutable-arraybuffer/README.md----- +-----BEGIN FILE import-bundle/README.md----- +# import-bundle + +`importBundle` is an async function that evaluates the bundles created by +`bundle-source`, turning them back into callable functions: + +```js +const bundle = await bundleSource('path/to/bundle.js'); +// 'bundle' is JSON-serializable +const options = {}; // filePrefix, endowments, other compartment options +const namespace = await importBundle(bundle); +const { default, namedExport1, namedExport2 } = namespace; +``` + +This must be run in a SES environment: you must install SES before importing +`@endo/import-bundle`. +The conventional way to do this is to import a module (e.g. `@endo/init`) which +does `import 'ses'; lockdown();`. + +The bundle will be loaded into a new Compartment, which does not have access to +platform globals like `document` or `Fetch` or `require`. +The bundle is isolated to only having access to powerless JavaScript facilities +and whatever endowments you provide. + +Each call to `importBundle` creates a new `Compartment`. +The globals of the new Compartment are frozen before any bundle code is +evaluated, to enforce ocap rules. + +## Module Formats + +The source can be bundled in a variety of "formats". + +By default, `bundleSource` uses a format named `endoZipBase64`, in which the +source modules and a "compartment map" are captured in a Zip file and base-64 +encoded. +The compartment map describes how to construct a set of [Hardened +JavaScript](https://hardenedjs.org) compartments and how to load and link the +source modules between them. + +The `endoScript` format captures the sources as a single JavaScript program +that completes with the entry module's namespace object. + +The `getExport` format captures the sources as a single CommonJS-style string, +and wrapped in a callable function that provides the `exports` and +`module.exports` context to which the exports can be attached. + +More sophisticated than `getExport` is named `nestedEvaluate`. +In this mode, the source tree is converted into a table of evaluable strings, +one for each original module. +This table is then encoded and wrapped as before. +The evaluation process uses a separate evaluator call for each module, +providing an opportunity to attach a distinct `sourceMap` to each one. +This preserves relative filenames in subsequent debugging information and stack +traces. + +To set a base prefix for these relative filenames, provide the `filePrefix` +option. + +Note that the `nestedEvaluate` format receives a global endowment named +`require`, although it will only be called if the source tree imported one of +the few modules on the `bundle-source` "external" list. + +## Options + +`importBundle()` takes an options bag and optional additional powers. + +```js +const namespace = await importBundle(bundle, options, powers); +``` + +The most common option is `filePrefix`, which can be provided for +`nestedEvaluate`-format bundles. +This sets the source filename of the top-level module inside the bundle, as +used in debugging messages (like the stack traces displayed in errors). +The other modules will append a suffix to this filename, based upon their +location within the original source tree. + +Another common option is `endowments`, which provides names that will be +available everywhere in the evaluated sources. +By default, the bundle will only get access to the standard JavaScript +primordials (`Array`, `Object`, `Map`, etc). +It will not get `document`, `window`, `Request`, `process`, `require`, or even +`console` unless you provide them as endowments, giving you full control over +what the loaded bundle can do. + +The `bundle-source` tool has a small number of module names marked as +"external". +These modules are not bundled into the source (copied from the filesystem where +`bundleSource` was called). +Instead, the bundler injects a call to `require()` for each external module +that was imported from somewhere in original source graph. +This let the final evaluation environment control what these imports get, +rather than the original source tree. + +To support these "external" imports, you will need to provide a `require` +endowment that can honor any such names. +In addition, the `nestedEvaluate` format always needs a `require` endowment +(although it will only be called if the original sources imported one of the +"external" names). + +For debugging purposes, you should probably provide a `console` endowment. +See `makeConsole.js` in the SwingSet source tree for inspiration. + +The rest of the `options` are passed through to the `Compartment` constructor, +which currently only accepts `transforms`. +For more information, see the `compartment-shim` docs in the SES repository. +Note that `transforms` is defined to be an array of objects which each have a +`rewrite` method. + +Note that `sloppyGlobalsMode` is only accepted by the Compartment's `evaluate` +method, not the Compartment constructor itself, and thus cannot be supplied to +`importBundle`. +To use `sloppyGlobalsMode`, you will probably want to create a Compartment +directly (and not freeze its globals). + +## Source maps + +For an Endo (zip, base64) bundle, `bundleSource` will add source maps to a +per-user cache so they can be debugged if imported on the same host. +To use this facility, pass a `computeSourceMapLocation` capability into +`powers`. + +```js +import 'ses'; +import bundleSource from '@endo/bundle-source'; +import { importBundle } from '@endo/import-bundle'; +import { computeSourceMapLocation } from '@endo/import-bundle/source-map-node.js'; + +lockdown(); +const bundle = await bundleSource('debugme.js'); +await importBundle( + bundle, + { endowments: { console } }, + { computeSourceMapLocation }, +); +``` + +Use `node --inspect-brk` and `debugger` statements. + +-----END FILE import-bundle/README.md----- +-----BEGIN FILE lp32/README.md----- +# LP32 + +Also known as the web extension "Native Host Message" protocol, this package +implements async iterator streams for reading and writing with 32-bit +host-byte-order length-prefix message envelopes for binary data, represented +with Uint8Arrays. + +These streams are "hardened" and depend on Hardened JavaScript. +Most JavaScript environments can be locked down with the +[SES shim](../ses/README.md). + +-----END FILE lp32/README.md----- +-----BEGIN FILE marshal/README.md----- +# @endo/marshal + +"Marshalling" refers to the conversion of structured data (a tree or graph of +objects) into a string, and back again. + +The `marshal` module helps with conversion of "capability-bearing data", in +which some portion of the structured input represents "pass-by-proxy" or +"pass-by-presence" objects that should be serialized into values referencing +special "slot identifiers". The `toCapData()` function returns a "CapData" +structure: an object with a `body` containing a serialization of the input data, +and a `slots` array holding the slot identifiers. `fromCapData()` takes this +CapData structure and returns the object graph. There is no generic way to +convert between pass-by-presence objects and slot identifiers, so the marshaller +is parameterized with a pair of functions to create the slot identifiers and turn +them back into proxies/presences. + +`marshal` uses JSON to serialize the object graph, but knows how to serialize +values that cannot be expressed directly in JSON, such as bigints, `NaN`, and +`undefined`. + +## Usage + +This module exports a `makeMarshal()` function, which can be called with two +optional callbacks (`convertValToSlot` and `convertSlotToVal`), and returns +an object with `toCapData` and `fromCapData` properties. Each callback defaults +to the identity function. + +```js +import '@endo/init'; +import { makeMarshal } from '@endo/marshal'; + +const m = makeMarshal(); +const o = harden({a: 1}); +const s = m.toCapData(o); +console.log(s); +// { body: '{"a":1}', slots: [] } +const o2 = m.fromCapData(s); +console.log(o2); +// { a: 1 } +console.log(o1 === o2); +// false +``` + +Additionally, this module exports a `makePassableKit` function for encoding into +and decoding from a directly-serialized format in which string comparison +corresponds with arbitrary value comparison (cf. +[Patterns: Rank order and key order](https://github.com/endojs/endo/blob/master/packages/patterns/README.md#rank-order-and-key-order). +Rather than accepting `convertValToSlot` and `convertSlotToVal` functions and +keeping a "slots" side table, `makePassableKit` expects +{encode,decode}{Remotable,Promise,Error} functions that directly convert between +instances of the respective pass styles and properly-formatted encodings +(in which Remotable encodings start with "r", Promise encodings start with "?", +Error encodings start with "!", and all other details are left to the provided +functions). +`makePassableKit` supports two variations of this format: "legacyOrdered" and +"compactOrdered". The former is the default for historical reasons (see +https://github.com/endojs/endo/pull/1594 for background) but the latter is +preferred for its better handling of deep structure. The ordering guarantees are +upheld within each format variation, but not across them (i.e., it is not +correct to treat a string comparison of legacyOrdered vs. compactOrdered as a +corresponding value comparison). + +## Frozen Objects Only + +The entire object graph must be "hardened" (recursively frozen), such as done +by the `harden` function installed when importing `@endo/init`. `toCapData` will +refuse to marshal any object graph that contains a non-frozen object. + +## Beyond JSON + +`marshal` uses special values to represent both Presences and data which cannot +be expressed directly in JSON. These special values are usually strings with +reserved prefixes in the preferred "smallcaps" encoding, but in the original +encoding were objects with a property named `@qclass`. For example: + +```js +import '@endo/init'; +import { makeMarshal } from '@endo/marshal'; + +// Smallcaps encoding. +const m1 = makeMarshal(undefined, undefined, { serializeBodyFormat: 'smallcaps' }); +console.log(m1.toCapData(NaN)); +// { body: '#"#NaN"', slots: [] } + +// Original encoding. +const m2 = makeMarshal(); +console.log(m2.toCapData(NaN)); +// { body: '{"@qclass":"NaN"}', slots: [] } +``` + +## Pass-by-Presence vs Pass-by-Copy + +`marshal` relies upon `@endo/pass-style` to distinguish between objects that are +pass-by-presence and those that are pass-by-copy. + +To qualify as pass-by-presence, all properties of an object (and of all objects +in its inheritance hierarchy) must be methods, not data. Pass-by-presence objects +are usually treated as having identity (assuming the `convertValToSlot` and +`convertSlotToVal` callbacks behave well), so passing the same object through +multiple calls will result in multiple references to the same output object. + +To qualify as pass-by-copy, all properties of an object must be string-named and +enumerable and not accessors and not methods: their values can be primitives such +as bigints, booleans, `null`, numbers, and strings, and they can be arrays and +pass-by-copy objects, but they cannot be functions. In addition, the object must +inherit directly from `Object.prototype`. Pass-by-copy objects are not treated as +having identity: `fromCapData` does not produce the same output object for +multiple appearances of the same pass-by-copy serialization. + +Mixed objects having both methods and data properties are rejected. + +Empty objects (which vacuously satisfy requirements for both pass-by-presence and +pass-by-copy) are treated as pass-by-copy, although it is also possible to use +`Far` (from `@endo/far`) for creating empty marker objects which _can_ be +compared for identity and are especially useful as WeakMap keys in the "rights +amplification" pattern. + +## `convertValToSlot` / `convertSlotToVal` + +When `m.toCapData()` encounters a pass-by-presence object, it will call the +`convertValToSlot` callback with the value to be serialized. The return value +will be used as the slot identifier to be placed into the slots array, and the +serialized `body`, in place of the object, will contain a special value +referencing that slot identifier by its index in the slots array. For example: + +```js +import '@endo/init'; +import { makeMarshal } from '@endo/marshal'; + +const slotAssignments = new Map(); +const convertValToSlot = obj => { + let slot = slotAssignments.get(obj); + if (slot === undefined) { + slot = `id1:${(slotAssignments.size + 10).toString(36)}`; + slotAssignments.set(obj, slot); + } + return slot; +}; + +const p = harden(Promise.resolve()); + +// Smallcaps encoding. +const m1 = makeMarshal(convertValToSlot, undefined, { serializeBodyFormat: 'smallcaps' }); +m1.toCapData(p); +// { body: '#"&0"', slots: [ 'id1:a' ] } + +// Original encoding. +const m2 = makeMarshal(convertValToSlot); +m2.toCapData(p); +// { body: '{"@qclass":"slot","index":0}', slots: [ 'id1:a' ] } +``` + +Each time `m.fromCapData()` encounters a slot reference, it calls +`convertSlotToVal` with the value from the slots array. `convertSlotToVal` +should create and return a proxy (or other representative) of the +pass-by-presence object. + +# As a direct alternative to JSON + +This marshal package also exports `stringify` and `parse` functions that are +built on the marshal encoding of passable data. They can serve as direct +substitutes for `JSON.stringify` and `JSON.parse`, respectively, with the +following differences: + +* Compared to JSON, marshal's `stringify` is both more tolerant and less tolerant + of what data it accepts. It is more tolerant in that it will encode `NaN`, + `Infinity`, `-Infinity`, bigints, and `undefined`. It is less tolerant in that + it accepts only pass-by-copy data according to the semantics of our distributed + object model, as enforced by marshal---the `Passable` type exported by the + marshal package. For example, all objects-as-records must be frozen, inherit + from `Object.prototype`, and have only enumerable string-named data properties. + `JSON.stringify` handles unserializable data by skipping it, but marshal's + `stringify` rejects it by throwing an error. +* The JSON functions have parameters for customizing serialization and + deserialization, for example with a *replacer* or *reviver*. The marshal-based + alternatives do not. + +The full marshal package will serialize `Passable` objects containing +presences and promises, because it serializes to a `CapData` structure +containing both a `body` string and a `slots` array. Marshal's `stringify` +function serializes only to a string, and so will not +accept any remotables or promises. If any are found in the input, this +`stringify` will throw an error. + +Any encoding into JSON of data that cannot be represented directly, such as +`NaN`, relies on some kind of escape for the decoding side to detect and use. +For `stringify` and `parse`, this is signaled by an object with a property named +`@qclass` per the original encoding described [above](#beyond-json). + +-----END FILE marshal/README.md----- +-----BEGIN FILE memoize/README.md----- + + +-----END FILE memoize/README.md----- +-----BEGIN FILE module-source/README.md----- +# ModuleSource + +This package provides a ponyfill for the `ModuleSource` constructor, suitable +for use in the SES shim's module descriptors. +The module source accepts a JavaScript module and converts it into +a form that SES can use to emulate and confine JavaScript modules (ESMs, the +`mjs` file format) with compartments. + +```js +import 'ses'; +import { ModuleSource } from '@endo/module-source'; + +const c1 = new Compartment({}, {}, { + name: "first compartment", + resolveHook: (moduleSpecifier, moduleReferrer) => { + return resolve(moduleSpecifier, moduleReferrer); + }, + importHook: async moduleSpecifier => { + const moduleLocation = locate(moduleSpecifier); + const moduleText = await retrieve(moduleLocation); + return new ModuleSource(moduleText, moduleLocation); + }, +}); +``` + +## Source maps + +The `ModuleSource` is a shim for what we hope to eventually call a native +`ModuleSource` constructor. +However, in the absence of a native `ModuleSource`, this produces a +serializable object that emulates the behavior of `ModuleSource` in conjunction +with the `Compartment` constructor from `ses`. +A detail that leaks from the implementation is that the constructor rewrites +the source, from an ESM `[[Module]]` grammar construction to a `[[Program]]` +construction suitable for confining with the compartment's confined evaluator. + +This transform attempts to be unobtrusive, but currently causes some alignment +changes due to (hopefully temporary) limitations to the underlying code +generator. +In the interim, generating a source map can help. + +The `ModuleSource` constructor accepts non-standards-track +`sourceMapHook` and `sourceMapUrl` options. + +Previously, the sole option was a `string` argument for the `sourceUrl`, such +that this would be appended to the generated source. +This change allows for the old or new usage: + +```js +new ModuleSource(source, sourceUrl); +// or +new ModuleSource(source, { sourceUrl, sourceMapUrl, sourceMapHook }); +``` + +The `sourceMapUrl` is necessary for generating a source map. +The URL will appear only in the generated source map, so a fully qualified +source map URL is safe and allows for continuity if the map is generated and +debugged on the same host. +This is important because Endo captures precompiled Static Module Records in +bundles, excluding source maps, such that a relative path is not useful. + +The `sourceMapHook` will receive a string source map and a details bag +including: + +- `source` +- `sourceUrl` +- `sourceMapUrl` + +Such that the receiver can store the source map somewhere as a side-effect. + +Note: the `sourceMapHook` is synchronous and returns `void`. +Exceptions thrown by the hook will propagate up through the constructor. If +the hook returns a promise, it will be dropped and rejections will go uncaught. +If the hook must do async work, these should be queued up as a job that the +caller can later await. + +## XS Specific Variant + +With the `xs` condition, `@endo/module-source` will not entrain Babel and will +just adapt the native `ModuleSource` to the older interface presented by this +package. +That is, the XS native `bindings` will be translated to `imports`, `exports`, +and `reexports` getters. +This form of `ModuleSource` ignores all options. + +## Bug Disclosure + +Please help us practice coordinated security bug disclosure, by using the +instructions in +[SECURITY.md](https://github.com/endojs/endo/blob/master/packages/ses/SECURITY.md) +to report security-sensitive bugs privately. + +For non-security bugs, please use the [regular Issues +page](https://github.com/endojs/endo/issues). + +-----END FILE module-source/README.md----- +-----BEGIN FILE nat/README.md----- +# Nat +[![Build Status][circleci-svg]][circleci-url] +[![dependency status][deps-svg]][deps-url] +[![dev dependency status][dev-deps-svg]][dev-deps-url] +[![License][license-image]][license-url] + +Numbers in a programming language are meaningful because we take them to +represent abstract mathematical numbers. JavaScript has two data types +representing numbers, *JS numbers* (IEEE 64 bit floating point) and *bigints* +(arbitrary precision integers). Not all abstact mathematical +numbers are representable by these data types, and not all values of one of +these data types represent mathematical numbers (The JS number type's `NaN`, +`Infinity`, and `-Infinity`). Because JavaScript uses "number" to refer to +its floating point data type, we'll always say "mathematical number" when +that's what we mean. + +This package is concerned with the mathematical *natural numbers*, the non-negative +integers. All of these can be safely represented as bigints, given enough +memory. Some of these can be represented as JS numbers, and a smaller set can +*safely* be represented as JS numbers, given a specific notion of safety. + +A skippable detail about floating point: + * The JavaScript expression `2**70` evaluates to a JS number that exactly + represents the mathematical number you expect. However, the JavaScript + expression `2**70+1 === 2**70` evaluates to `true` because this JS number is + outside the contiguous range of integers that the JS number type can + represent *unambiguously*. + The contiguous range of exactly representable integers is + `-(2**53)` to `2**53`. However, `2**53+1 === 2**53` is `true`, demonstrating + that other integers will round to `2**53`. The JavaScript standard defines + the [*safe* JS numbers](https://tc39.es/ecma262/#sec-number.issafeinteger) + to be the JS numbers that represent mathematical integers and lie in the + range `-(2**53-1)` to `2**53-1` . The JS safe natural numbers are the + non-negative subset of that, between `0` and `2**53-1`. No other integers + coerce to any of these. If in JavaScript `a + b === c` and all three values + are JS safe integers, then this accurately represents the mathematical sum + of the mathematical numbers they represent. + +The bigint datatype, by contrast, is inherently safe. Every bigint `>= 0n` +safely represents a natural number. + +This package exports two functions, `isNat(allegedNum)` and `Nat(allegedNum)`. + +## isNat(allegedNum: any) => boolean + +```js +isNat(3); // true +isNat(3n); // true +isNat('3'); // false +isNat(2**70); // false +isNat(2n**70n); // true +isNat(-3n); // false +isNat(3.1); // false +``` + +The `isNat` function is a predicate that accepts any input and returns `true` +iff that input safely represents a natural number, i.e., if it is a non-negative +bigint or it is a non-negative JS number safely representing an integer. To the +extent that we consider this abstract notion of mathematical natural number a +type, `isNat` is a *type tester* of possible representations of this type. + +## Nat(allegedNum: bigint | number) => bigint + +```js +Nat(3); // 3n +Nat(3n); // 3n +Nat('3'); // throws TypeError +Nat(2 ** 70); // throws RangeError +Nat(2n ** 70n); // 1180591620717411303424n +Nat(-3); // throws RangeError +Nat(3.1); // throws RangeError +``` + +The `Nat` function accepts exactly those values that pass the `isNat` +predicate. For those it returns a bigint that represents the same natural +number. Otherwise it throws. + +## Validators and Coercers + +Functions like `Nat` and the standard JavaScript `BigInt` can be +classified _validators_ or _coercers_. + +When a validator accepts---returns normally rather than throwing---the caller +knows that their input argument is as expected, and the output is the same as +the input. When a coercer accepts, the caller knows that the output is as +expected, but only knows that the input was one the coercer was willing to +convert from. The `BigInt` function is a coercer. It will even accept +strings as input but its output is always a bigint. `Nat` is an interesting +mixture. It is a coercer at one level of abstraction, and a validator at +another level of abstraction. + +At the level of concrete JavaScript data representations, `Nat` is clearly a +coercer---`Nat` will convert a qualifying JS number into a bigint. At the level +of abstraction of the mathematical number any accepted input represents, `Nat` +is a validator. If `Nat` succeeds the caller knows that their input safely +represents some abstract mathematical natural number, and that the output +safely represents the same abstract mathematical natural number. At this level +of abstraction, on success, the output is the same as the input. + +## History + +`Nat` comes from the Google Caja project, which tested whether a JS number was a +primitive integer within the range of continguously and unambiguously +representable non-negative integers. + +For more, see the [discussion in TC39 notes](https://github.com/rwaldron/tc39-notes/blob/master/es6/2013-07/july-25.md#59-semantics-and-bounds-of-numberisinteger-and-numbermax_integer) + + +[circleci-svg]: https://circleci.com/gh/Agoric/nat.svg?style=svg +[circleci-url]: https://circleci.com/gh/Agoric/nat +[deps-svg]: https://david-dm.org/Agoric/Nat.svg +[deps-url]: https://david-dm.org/Agoric/Nat +[dev-deps-svg]: https://david-dm.org/Agoric/Nat/dev-status.svg +[dev-deps-url]: https://david-dm.org/Agoric/Nat?type=dev +[license-image]: https://img.shields.io/badge/License-Apache%202.0-blue.svg +[license-url]: LICENSE + +-----END FILE nat/README.md----- +-----BEGIN FILE netstring/README.md----- +# netstring + +This is an implementation of asynchronous streams framed as [Netstrings][]. +A netstring is a binary protocol for length-prefixed frames, +using decimal strings as variable width integers. +For example, the frame `5:hello,` corresponds to the message `hello`, +where `5` is the length of `hello` in bytes. + +This implementation relies particularly on the pure JavaScript notion of a +stream, using async iterators of Uint8Arrays. +By convention, these may be ranges of a ring buffer, so a stream owns a byte +range it receives from `next` until the next time it calls `next`. + + +[Netstrings][] <br> +D. J. Bernstein, <djb@pobox.com> <br> +1997-02-01 + +[Netstrings]: https://cr.yp.to/proto/netstrings.txt + +-----END FILE netstring/README.md----- +-----BEGIN FILE pass-style/README.md----- +# `@endo/pass-style` + +Defines the `Passable` objects, and the `passStyleOf` function for classifying them according to their `PassStyle`. + +See [types.js](./src/types.js) for the actual type definitions. See also [CopyRecord guarantees](./doc/copyRecord-guarantees.md) and [CopyArray guarantees](./doc/copyArray-guarantees.md). + +The Passable objects are those that can be passed by the `@endo/marshal` package. Thus `Passable` defines the layer of abstraction on which we need broad agreement for interoperability. One type of `Passable` is the `Tagged` object, which is the extension point for defining higher level data types, which do not need such broad agreement. The main such higher layer of abstraction is provided by the `@endo/patterns` package. + +-----END FILE pass-style/README.md----- +-----BEGIN FILE patterns/README.md----- +# `@endo/patterns` + +The main export from the package is an `M` namespace object, for making a variety of Matchers (hence "M"). For example: + +```js +import '@endo/init/debug.js'; +import { M, mustMatch } from '@endo/patterns'; + +const specimen = harden({ foo: 3, bar: 4 }); + +const pattern = M.splitRecord( + { foo: M.number() }, // required properties + { bar: M.string(), baz: M.number() }, // optional +); + +mustMatch(specimen, pattern); // throws: 'bar?: number 4 - Must be a string' +``` + +See {@link PatternMatchers} for more on `M.splitRecord()`, `M.number()`, and other methods. + +`M` also has {@link GuardMakers} methods to make {@link InterfaceGuard}s that use Patterns to characterize dynamic behavior such as method argument/response signatures and promise awaiting. The {@link @endo/exo!} package uses `InterfaceGuard`s as the first level of defense for Exo objects against malformed input. + +_For best rendering, use the [Endo reference docs](https://endojs.github.io/) site._ + +## Key Equality, Containers + +Builds on {@link @endo/pass-style!} as described in [`kindOf` and `passStyleOf` levels of abstraction](./docs/marshal-vs-patterns-level.md) to define higher level data types as individual refinements of Passable CopyTagged records (PassStyle "tagged"): + - {@link CopySet} -- a collection of unique distinguishable {@link Key}s + - {@link CopyBag} -- a collection of entries associating a unique distinguishable Key with a positive integer count (see [Multiset](https://en.wikipedia.org/wiki/Multiset)). + - {@link CopyMap} -- a collection of entries associating a unique distinguishable Key with a Passable + - {@link Matcher} -- a predicate characterizing a subset of Passables, such as "strings" or "8-bit unsigned integer numbers" or "CopyArrays of Remotables" + +In support of the above, there is also {@link compareKeys} and {@link keyEQ} exposing pass-invariant Key comparison, and two concepts with corresponding TypeScript types: + - {@link Key} -- a Passable arbitrarily deep acyclic data structure in which each non-leaf node is a CopyArray, CopyRecord, CopySet, CopyBag, or CopyMap that is the child of at most one other internal node (forming a possibly-empty tree of containers), and each leaf is either an empty such container or a Passable primitive value or a Remotable (but the same Remotable `r` may be a child of multiple parents, e.g. `{ foo: r, bar: [r] }`). A Key is stable and stably comparable with other Keys via {@link keyEQ}. Key is the most general data type covering valid contents for CopySets and CopyBags and keys for CopyMaps (the last of which explains the "Key" name). + - {@link Pattern} -- a Passable value that can be used to *match* some subset of Passables. Each Pattern is either a Key that matches itself (and any copy of itself --- `keyEQ` considers identity only for Remotables, where it is shared across all local Presences of the same Remotable), or a Key-like structure in which one or more leaves is a Matcher rather than a primitive or Remotable. + +-----END FILE patterns/README.md----- +-----BEGIN FILE promise-kit/README.md----- +# Promise Kit + +The promise-kit package provides a simple abstraction for creating and managing a promise. It exports, `makePromiseKit` which is a utility function used to create a Promise and its associated resolver and rejector functions. This is particularly useful in asynchronous programming, where you might need to create a promise and resolve or reject it at a later point in time. +Note that this serves as a "ponyfill" for `Promise.withResolvers`, making certain accommodations to ensure that the resulting promises can pipeline messages through `@endo/eventual-send`. + +## Usage + +Here’s an example of how `makePromiseKit` might be used in an Agoric smart contract or JavaScript program: + +### Basic Example + +```javascript +import { makePromiseKit } from '@endo/promise-kit'; + +function asyncOperation() { + + const { promise, resolve, reject } = makePromiseKit(); + setTimeout(() => { + const success = true; // Simulating success or failure + if (success) { + resolve("Operation successful!"); + } else { + reject("Operation failed!"); + } + }, 2000); + + return promise; +} + +async function handleAsyncOperation() { + try { + const result = await asyncOperation(); + console.log(result); // "Operation successful!" + } catch (error) { + console.error(error); // "Operation failed!" + } +} + +handleAsyncOperation(); +``` + +### Creating Multiple Promise Kits + +You can create multiple promise kits for managing various asynchronous tasks. + +```javascript +const kit1 = makePromiseKit(); +const kit2 = makePromiseKit(); + +kit1.promise.then(value => console.log('Kit 1 resolved with:', value)); +kit2.promise.then(value => console.log('Kit 2 resolved with:', value)); + +kit1.resolve('First success'); +kit2.resolve('Second success'); + +``` + +## API + +### `makePromiseKit()` +Creates a new promise kit. + +**Returns** +- **`promise`**: The promise object. +- **`resolve`**: The resolve function for the promise. +- **`reject`**: The reject function for the promise. + +## Links +[Repository](https://github.com/endojs/endo/tree/master/packages/promise-kit) + +## License +This package is licensed under the Apache-2.0 License. +-----END FILE promise-kit/README.md----- +-----BEGIN FILE ses-ava/README.md----- +# `@endo/ses-ava` + +*SES-Ava* wraps Ava `test` functions and initializes the SES-shim with options +suitable for debugging tests. This includes logging errors to the console with +- deep stacks of prior turns +- unredacted stack traces +- unredacted error messages + +To use this module, in your Ava test files, replace + +```js +import 'ses'; // or however you initialize the SES-shim +import test from 'ava'; +``` +with +```js +import test from '@endo/ses-ava/prepare-endo.js'; +``` +and add +```json + "devDependencies": { + // ... + "@endo/ses-ava": "...", // for the current version of @endo/ses-ava + // ... + }, +``` +specifically to "devDependencies". @endo/ses-ava itself depends on Ava as +a regular dependency, so it you include @endo/ses-ava as a regular +dependency, bundlers might bundle your code with all of Ava. + +SES-Ava rhymes with Nineveh. + +## Compat note + +If you were already using `@endo/ses-ava` by doing + +```js +import 'ses'; // or however you initialize the SES-shim +import rawTest from 'ava'; +import { wrapTest } from '@endo/ses-ava'; + +const test = wrapTest(rawTest); +``` + +that code will continue to work. But it should be upgraded to the above +pattern if possible. + +-----END FILE ses-ava/README.md----- +-----BEGIN FILE ses/README.md----- +# SES + +SES is a [shim][define shim] for [Hardened JavaScript][] as [proposed][SES +proposal] to ECMA TC39. +SES stands for *fearless cooperation*. +Hardened JavaScript is highly compatible with ordinary JavaScript. +Most existing JavaScript libraries can run on Hardened JavaScript. + +* **Compartments** Compartments are separate execution contexts: each one has + its own global object and global lexical scope. +* **Frozen realm** Compartments share their intrinsics to avoid identity + discontinuity. By freezing the intrinsics, SES protects programs from each + other. By sharing the intrinsics, programs from separate compartments + can recognize each other's arrays, data objects, and so on. +* **Strict mode** SES enforces JavaScript strict mode that enhances security, + for example by changing some silent failures into thrown errors. +* **POLA** (Principle of Least Authority) By default, Compartments receive no + ambient authority. They are created without host-provided APIs, (for example + no `fetch`). Compartments can be selectively endowed with powerful arguments, + globals, or modules. + +SES safely executes third-party JavaScript 'strict' mode programs in +compartments that have no excess authority in their global scope. +SES runs atop an ES6-compliant platform, enabling safe interaction of +mutually-suspicious code, using object-capability -style programming. + +Agoric and MetaMask rely on Hardened JavaScript and this SES shim as part of +systems that sandbox third-party plugins or smart contracts and mitigate supply +chain attacks for production web applications, web extensions, and build +systems. + +[](https://agoric.com/) +[](https://metamask.io/) + +See https://github.com/Agoric/Jessie to see how SES fits into the various +flavors of confined JavaScript execution. And visit +https://ses-demo.agoric.app/demos/ for a demo. + +SES starts where the Caja project left off +https://github.com/google/caja/wiki/SES, and goes on to introduce compartments +and modernize the permitted JavaScript features. + +Please join the conversation on our [Mailing List][SES Strategy Group] and +[Matrix][Endo Matrix]. +We record a [weekly conference call][SES Strategy Recordings] with the Hardened +JavaScript engineering community. + +_Hardened JavaScript_, Kris Kowal: + +[](https://www.youtube.com/watch?v=RZ7bBIU8DRc) + +_Don't add Security, Remove Insecurity_, Mark Miller: + +[](https://www.youtube.com/watch?v=u-XETUbxNUU) + +## Install + +```sh +npm install ses +``` + +## Usage + +The SES shim runs in most engines, either as an ESM module `ses` or as a +`<script>` tag. +For a script tag, the content encoding charset must be UTF-8, either by virtue +of `<head><meta charset="utf-8"></head>` (a general best practice for all HTML +files) or specifically `<script src="node_modules/ses/dist/ses.umd.min.js" +charset="utf-8">`. + +SES can be bundled by Webpack, Browseriy, Rollup, and Parcel, but any of these +tools could be coopted with a supply-chain attack to invalidate the security +properties of SES. We generally recommend installing SES as a separate script +tag. + +### Lockdown + +SES introduces the `lockdown()` function. +Calling `lockdown()` alters the surrounding execution environment, or +**realm**, such that no two programs running in the same realm can observe or +affect each other until they have been introduced, and even then can only +interact through their own exposed interfaces. + +To do this, `lockdown()` tamper-proofs all of the JavaScript intrinsics, to +prevent **prototype pollution**. +After that, no program can subvert the methods of these objects (preventing +some **man in the middle attacks**). +Also, no program can use these mutable objects to pass notes to parties that +haven't been expressly introduced (preventing some **covert communication +channels**). + +Lockdown freezes all objects that are effectively undeniable to programs in the +realm. The set of such objects includes but is not limited to: `globalThis`, +prototype objects of Array, Function, GeneratorFunction, and Object, and objects +accessible from those objects (such as `Object.prototype.toString`). + +The `lockdown()` function also **tames** some objects including regular +expressions, locale methods, and errors. +A tamed `RegExp` does not have the deprecated `compile` method. +A tamed error does not have a V8 `stack`, but the `console` can still see the +stack. +Lockdown replaces locale methods like `String.prototype.localeCompare` with +generic versions that do not reveal the host locale. + +```js +import 'ses'; + +lockdown(); + +console.log(Object.isFrozen([].__proto__)); +// true +``` + +Lockdown does not erase any powerful objects from the initial global scope. +Instead, **Compartments** give complete control over what powerful objects +exist for client code. + +See [`lockdown` options](docs/lockdown.md) for configuration options to +`lockdown`. However, all of these have sensible defaults that should +work for most projects out of the box. + +### Harden + +SES introduces the `harden` function. +*After* calling `lockdown`, the `harden` function ensures that every object in +the transitive closure over property and prototype access starting with that +object has been **frozen** by `Object.freeze`. +This means that the object can be passed among programs and none of those +programs will be able to tamper with the **surface** of that object graph. +They can only read the surface data and call the surface functions. + +```js +import 'ses'; + +lockdown(); + +let counter = 0; +const capability = harden({ + inc() { + counter++; + }, +}); + +console.log(Object.isFrozen(capability)); +// true +console.log(Object.isFrozen(capability.inc)); +// true +``` + +Note that although the **surface** of the capability is frozen, the capability +still closes over the mutable counter. +Hardening an object graph makes the surface immutable, but does not guarantee +that methods are free of side effects. + + +### Compartment + +SES introduces the `Compartment` constructor. +A compartment is an evaluation and execution environment with its own +`globalThis` and wholly independent system of modules, but otherwise shares +the same batch of intrinsics like `Array` with the surrounding compartment. +The concept of a compartment implies an **initial compartment**, the initial +execution environment of a **realm**. + +In the following example, we create a compartment endowed with a `print()` +function on `globalThis`. + +```js +import 'ses'; + +lockdown(); + +const c = new Compartment({ + globals: { + print: harden(console.log), + }, + __options__: true, // temporary migration affordance +}); + +c.evaluate(` + print('Hello! Hello?'); +`); +``` + +The new compartment has a different global object than the start compartment. +The global object is initially mutable. +Locking down the realm hardened the objects in global scope. +After `lockdown`, no compartment can tamper with these **intrinsics** and +**undeniable** objects. +Many of these are identical in the new compartment. + +```js +const c = new Compartment(); +c.globalThis === globalThis; // false +c.globalThis.JSON === JSON; // true +``` + +Other pairs of compartments also share many identical intrinsics and undeniable +objects of the realm. +Each has a unique, initially mutable, global object. + +```js +const c1 = new Compartment(); +const c2 = new Compartment(); +c1.globalThis === c2.globalThis; // false +c1.globalThis.JSON === c2.globalThis.JSON; // true +``` + +The global scope of every compartment includes a shallow, specialized copy of +the JavaScript intrinsics, disabling `Math.random`, `Date.now` and the +behaviors of the `Date` constructor which would reveal the current time. +Compartments leave these out since they can be used as covert communication +channels between programs. +However, a compartment may be expressly given access to these objects +through: + +* the first argument to the compartment constructor or +* by assigning them to the compartment's `globalThis` after construction. + +```js +const powerfulCompartment = new Compartment({ + globals: { Math }, + __options__: true, // temporary migration affordance +}); +powerfulCompartment.globalThis.Date = Date; +``` + +### Compartment + Lockdown + +Together, Compartment and lockdown isolate client code in an environment with +limited powers and communication channels. +A compartment has only the capabilities it is expressly given and cannot modify +any of the shared intrinsics. +Every compartment gets its own globals, including such objects as the +`Function` constructor. +Yet, compartment and lockdown do not break `instanceof` for any of these +intrinsics types! + +All of the evaluators in one compartment are captured by that compartment's +global scope, including `Function`, indirect `eval`, dynamic `import`, and its +own `Compartment` constructor for child compartments. +For example, the `Function` constructor in one compartment creates functions +that evaluate in the global scope of that compartment. + +```js +const c1 = new Compartment(); +const f1 = new c.globalThis.Function('return globalThis'); +f1() === c1.globalThis; // true + +const c2 = new Compartment(); +const f2 = new c.globalThis.Function('return globalThis'); +f2() === c2.globalThis; // true + +f1() === f2(); // false +``` + +Lockdown prepares for compartments with separate globals by freezing +their shared prototypes and replacing their prototype constructors +with powerless dummies. +So, `Function` is different in two compartments, `Function.prototype` is the +same, and `Function` is not the same as `Function.prototype.constructor`. +The `Function.prototype.constructor` can only throw exceptions. +So, a function passed between compartments does not carry access to +its compartment's globals along with it. +Yet, `f instanceof Function` works, even when `f` and `Function` are +from different compartments. + +The `globalThis` in each compartment is mutable. +This can and should be frozen before running any dynamic code in that +compartment, yet is not strictly necessary if the compartment only +runs code from a single party. + +### Modules + +Any code executed within a compartment shares a set of module instances. +For modules to work within a compartment, the creator must provide +module descriptors. +A compartment can be configured with module descriptors, from highest to lowest +precedence: + +- the `modules` map provided to the `Compartment` constructor, +- returned by a `moduleMapHook(specifier)` passed as an option to the + `Compartment` constructor. +- returned by either the `importHook(specifier)` or `importNowHook(specifier)` + option passed to the `Compartment` constructor. Calling + `compartment.import(specifier)` falls through to the `importHook` which may + return a promises, whereas `compartment.importNow(specifier)` falls through + to the synchronous `importNowHook`. + +The `resolveHook` determines how the compartment will infer the full module +specifier for another module from a referrer module and the import specifier. + +```js +import 'ses'; +import { ModuleSource } from '@endo/module-source'; + +const c1 = new Compartment({ + name: "first compartment", + resolveHook: (moduleSpecifier, moduleReferrer) => { + return resolve(moduleSpecifier, moduleReferrer); + }, + importHook: async moduleSpecifier => { + const moduleLocation = locate(moduleSpecifier); + const moduleText = await retrieve(moduleLocation); + return { + source: new ModuleSource(moduleText, moduleLocation) + }; + }, + __options__: true, // temporary migration affordance +}); +``` + +> The Hardened JavaScript language specifies a global `ModuleSource`, but this +> is not provided by the shim because it entrains a full JavaScript parser that +> is an unnecessary performance penalty for the SES runtime. +> Instead, the SES shim accepts a pre-compiled module source duck-type that +> is tightly coupled to the shim implementation. + +A compartment can also link a module in another compartment. + +```js +const c2 = new Compartment({ + name: "second compartment", + modules: { + 'c1': { + source: './main.js', + compartment: c1, + }, + }, + resolveHook, + importHook, + __options__: true, // temporary migration affordance +}); +``` + +#### Module Descriptors + +Comparments can load and initialize module namespaces from module descriptors. +Like property descriptors, module descriptors are ordinary objects with various +forms. + +##### Descriptors with `source` property + +- If fhe value of the `source` property is a string, the parent compartment + loads the module but the compartment itself initializes the module. + +- Otherwise, if the value of the `source` property is the module source, the + module is initialized from the module source. + +- Otherwise, the value of the `source` property must be an object. The module + is loaded and initialized from the object according to the [virtual module + source](#VirtualModuleSource) pattern. + +If the `importMeta` property is present, its value must be an object. The +default `importMeta` object is an empty object. + +Compartments copy the `importMeta` object properties into the module +`import.meta` object like `Object.assign`. + +If the `specifier` property is present, its value is coerced into a string and +becomes the referrer specifier of the module, on which all import specifiers +are resolved using the `resolveHook`. + +##### Descriptors with `namespace` property + +- If fhe value of the `namespace` property is a string, the descriptor shares a + module to be loaded and initialized by the compartment referred by the + `compartment` property. + + - If the `compartment` property is present, its value must be a + compartment. + - If absent, the `compartment` property defaults to the compartment being + constructed in the `modules` option, or being hooked in the `loadHook` + and `loadNowHook` options. + +- Otherwise, if the value of the `namespace` property is a module namepace, the + descriptor shares a module that is already available. + +- Otherwise, the value of `namespace` property must be an object. The module is + loaded and initialized from the object according to the [virtual module + namespace](#VirtualModuleNamespace) pattern. + +#### Redirects + +If a compartment imports a module specified as `"./utility"` but actually +implemented by an alias like `"./utility/index.js"`, the `importHook` may +follow redirects, symbolic links, or search for candidates using its own logic +and return a module that has a different "response specifier" than the original +"request specifier". +The `importHook` may return an "alias" object with `source`, `compartment`, +and `specifier` properties. + +- `source` must be a module source, either a virtual module source + or a compiled module source. +- `compartment` is optional, to be specified if the alias transits to a + the specified different compartment, and +- `specifier` is the full module specifier of the module in its compartment. + This defaults to the request specifier, which is only useful if the + compartment is different. + +In the following example, the importHook searches for a file and returns an +alias. + +```js +const importHook = async specifier => { + const candidates = [specifier, `${specifier}.js`, `${specifier}/index.js`]; + for (const candidate of candidates) { + const source = await maybeImportSource(candidate); + if (source !== undefined) { + return { + source, + specifier: candidate, + compartment, + }; + } + } + throw new Error(`Cannot find module ${specifier}`); +}; + +const compartment = new Compartment({ + resolveHook, + importHook, + __options__: true, // temporary migration affordance +}); +``` + +#### moduleMapHook + +The module map above allows modules to be introduced to a compartment up-front. +Some modules cannot be known that early. +For example, in Node.js, a package might have a dependency that brings in an +entire subtree of modules. +Also, a pair of compartments with cyclic dependencies between modules they each +contain cannot use `compartment.module` to link the second compartment +constructed to the first. +For these cases, the `Compartment` constructor accepts a `moduleMapHook` option +that is like the dynamic version of the static `moduleMap` argument. +This is a function that accepts a module specifier and returns the module +namespace for that module specifier, or `undefined`. +If the `moduleMapHook` returns `undefined`, the compartment proceeds to the +`importHook` to attempt to asynchronously obtain the module's source. + +```js +const moduleMapHook = moduleSpecifier => { + if (moduleSpecifier === 'even') { + return { + source: './index.js', + compartment: even, + }; + } else if (moduleSpecifier === 'odd') { + return { + source: './index.js', + compartment: odd, + }; + } +}; + +const even = new Compartment({ + resolveHook: nodeResolveHook, + importHook: makeImportHook('https://example.com/even'), + moduleMapHook, + __options__: true, // temporary migration affordance +}); + +const odd = new Compartment({ + resolveHook: nodeResolveHook, + importHook: makeImportHook('https://example.com/odd'), + moduleMapHook, + __options__: true, // temporary migration affordance +}); +``` + +#### importNowHook + +Additionally, an `importNowHook` may be provided that the compartment will use +as means to synchronously load modules not seen before in situations where +calling out to asynchronous `importHook` is not possible. +Specifically, when `compartmentInstance.importNow('specifier')` is called, the +compartment will first look up module records it's already aware of and call +`moduleMapHook` and if none of that is successful in finding a module record +matching the specifier, it will call `importNowHook` expecting to synchronously +receive the same record type as from `importHook` or throw if it cannot. + +```js +import 'ses'; +import { ModuleSource } from '@endo/module-source'; + +const compartment = new Compartment({ + name: "first compartment", + modules: { + c: { + source: new ModuleSource(''), + }, + }, + resolveHook: (moduleSpecifier, moduleReferrer) => { + return resolve(moduleSpecifier, moduleReferrer); + }, + importHook: async moduleSpecifier => { + const moduleLocation = locate(moduleSpecifier); + const moduleText = await retrieve(moduleLocation); + return { + source: new ModuleSource(moduleText, moduleLocation), + }; + }, + importNowHook: moduleSpecifier => { + const moduleLocation = locate(moduleSpecifier); + // Platform-specific synchronous read API can be used + const moduleText = fs.readFileSync(moduleLocation); + return { + source: new ModuleSource(moduleText, moduleLocation), + }; + }, + __options__: true, // temporary migration affordance +}); +//... | importHook | importNowHook +await compartment.import('a'); //| called | not called +compartment.importNow('b'); //| not called | called +compartment.importNow('a'); //| not called | not called +compartment.importNow('c'); //| not called | not called +``` + +### <a name="VirtualModuleSource"></a> Virtual Module Source + +To incorporate modules not implemented as JavaScript modules, third-parties may +implement a `VirtualModuleSource` interface. +The object must have an `imports` array and an `execute` method. +The compartment will call `execute` with: + +1. the proxied `exports` namespace object, +2. a `resolvedImports` object that maps import names (from `imports`) to their + corresponding resolved specifiers (through the compartment's `resolveHook`), + and +3. the `compartment`, such that `importNow` can obtain any of the module's + specified `imports`. + +:warning: A future breaking version may allow the `importNow` and the `execute` +method of virtual module sources to return promises, to support +top-level await. + +:warning: The virtual module source interface does not yet agree with the +[XS](https://www.moddable.com/hardening-xs) implementation of [Hardened +JavaScript](https://hardenedjs.org/). + +### Compiled modules + +Instead of the `ModuleSource` constructor specified for the SES language, +the SES shim uses compiled module source records as a stand-in. +These can be created with a `ModuleSource` constructor from a package +like `@endo/module-source`. +We omitted `ModuleSource` from the SES shim because it entrains a heavy +dependency on a JavaScript parser. +The shim depends upon a `ModuleSource` constructor to analyze and +transform the source of a JavaScript module (known as an ESM or a `.mjs` file) +into a JavaScript program suitable for evaluation with `compartment.evaluate` +using a particular calling convention to initialize a module instance. + +A compiled module source record has the following shape: + +- `imports` is a record that maps partial module specifiers to a list of + names imported from the corresponding module. +- `exports` is an array of all the names that the module will export. +- `reexports` is an array of partial module specifier for which this + module exports all imported names. + This field is optional. +- `__syncModuleProgram__` is a string that evaluates to a function that accepts + an initialization record and initializes the module. + This property distinguishes this type of module record. + The name implies a future record type that supports top-level await. + - An initialization record has the properties `imports`, `liveVar`, `importMeta` and + `onceVar`. + - `imports` is a function that accepts a map from partial import + module specifiers to maps from names that the corresponding module + exports to notifier functions. + A notifier function accepts an update function and registers + to receive updates for the value exported by the other module. + - `importMeta` is a null-prototype object with keys transferred from `importMeta` + property in the envelope returned by importHook and/or mutated by + calling `importMetaHook(moduleSpecifier, importMeta)` + - `liveVar` is a record that maps names exported by this module + to a function that may be called to initialize or update + the corresponding value in another module. + - `onceVar` is a record that maps constants exported by this + module to a function that may be called to initialize the + corresponding value in another module. +- `__syncModuleFunctor__` is an optional function that if present is used + instead of the evaluation of the `__syncModuleProgram__` string. It will be + called with the initialization record described above. It is intended to be + used in environments where eval is not available. Sandboxing of the functor is + the responsibility of the author of the ModuleSource. +- `__liveExportsMap__` is a record that maps import names or names in the lexical + scope of the module to export names, for variables that may change after + initialization. Any reexported name is assumed to possibly change. + The exported name is wrapped in a duple array like `["exportedName", true]`. + The second value, a boolean, indicates that the variable has a temporal + dead-zone (a time between creation and initialization) when access to that + name should throw a `ReferenceError`. +- `__fixedExportsMap__` is a record that maps import names to export names + for constants exported by this module. + The fixed exports map is an aesthetic subtype of the live exports map, + so the value is wrapped in a simple array like `["exportedName"]` + +### Transforms + +The `Compartment` constructor accepts a `transforms` option. +This is an array of JavaScript source to source translation functions, +in the order they should be applied. +Passing the source to the first function's input, then from each function's +output to the next's input, the final function's output must be a valid +JavaScript "Program" grammar construction, code that is valid in a `<script>`, +not a module. + +```js +const transforms = [addCodeCoverageInstrumentation]; +const c = new Compartment({ + globals: { console, coverage }, + transforms, + __options__: true, // temporary migration affordance +}); +c.evaluate('console.log("Hello");'); +``` + +The `evaluate` method of a compartment also accepts a `transforms` option. +These apply before and in addition to the compartment-scoped transforms. + +```js +const transform = source => source.replace(/Farewell/g, 'Hello'); +const transforms = [transform]; +c.evaluate('console.log("Farewell, World!")', { transforms }); +// Hello, World! +``` + +These transforms do not apply to modules. +To transform the source of a JavaScript module, the `importHook` must +intercept the source and transform it before passing it to the +`ModuleSource` constructor. +These are distinct because programs and modules have distinct grammar +productions. + +An **internal implementation detail** of the SES-shim is that it +converts modules to programs and evaluates them as programs. +So, only for this implementation of `Compartment`, it is possible for a program +transform to be equally applicable for modules, but that transform will +have a window into the internal translation, will be sensitive to changes to +that translation between any pair of releases, even those that do not disclose +any breaking changes, and will only work on SES-shim, not any other +implementation of `Compartment` like the one provided by XS. + +The SES-shim `Compartment` constructor accepts a `__shimTransforms__` +option for this purpose. +For the `Compartment` to use the same transforms for both evaluated strings +and modules converted to programs, pass them as `__shimTransforms__` +instead of `transforms`. + +```js +const __shimTransforms__ = [addCoverage]; +const c = new Compartment({ + globals: { console, coverage }, + __shimTransforms__, + __options__: true, // temporary migration affordance +}); +c.evaluate('console.log("Hello");'); +``` + +The `__shimTransforms__` feature is designed to uphold the security properties +of compartments, since an attacker may use all available features, whether they +are standard or not. + +### Logging Errors + +`lockdown()` adds new global `assert` and tames the global `console`. The error +taming hides error stacks, accumulating them in side tables. The `assert` +system generates other diagnostic information hidden in side tables. The tamed +console uses these side tables to output more informative diagnostics. +[Logging Errors](./src/error/README.md) explains the design. + +## Security claims and caveats + +The `ses` shim concerns boundaries between programs in the same process and +JavaScript realm. +In terms of the [Taxonomy of Security Issues](https://papers.agoric.com/taxonomy-of-security-issues/), +the `ses` shim creates a boundary that is finer than an operating system +process or thread and facilitates boundaries as fine as individual objects. +While `ses` can interpose at granularities where process isolation is not a +viable boundary, as between an application and its dependencies or between a +platform and a plugin, `ses` combines well with coarser boundaries for defense +in depth. + +For the purposes of these claims and caveats, a "host program" is a program +that arranges `ses`, calls `lockdown`, and orchestrates one or more "guest +programs", providing limited access to its resources. + +### Single-guest Compartment Isolation + +Provided that the `ses` implementation and its +[trusted compute base](#trusted-compute-base) are correct, we claim that a host +program can evaluate a guest program (`program`) in a compartment after +`lockdown` and that the guest program: + +- will initially only have access to one mutable object, the compartment's + `globalThis`, +- specifically cannot modify any shared primordial objects, which are part of + the default execution environment, +- cannot initially perform any I/O (except I/O necessarily performed by the + trusted compute base like paging virtual memory), +- and specifically cannot measure the passage of time at any resolution. + +However, such a program can: + +- execute for an indefinite amount of time, +- allocate arbitrary amounts of memory, +- detect the platform endianness, +- in some JavaScript engines, observe the contents of the stack. + This may include sensitive information about the layout of files on the host + disk. + In cases where the stack is data-dependent, a guest can infer the data. + `ses` occludes the stack on V8 and SpiderMonkey, but cannot on + JavaScriptCore. + +```js +lockdown(); +const compartment = new Compartment(); +compartment.evaluate(program); +``` + +### Multi-guest Compartment Isolation + +If the host program arranges for the compartment's `globalThis` to +be frozen, we additionally claim that the host can evaluate any two guest +programs (`program1` and `program2`) in that compartment such that neither +guest program will: + +- initially share *any* mutable objects. +- be able to observe the relative passage of time of the other program, + as they would had they been given a reference to a working `Date.now()`. +- be able to communicate, as they would if they had shared access to mutable + state like an unfrozen object, a hardened collection like a `Map`, or even + `Math.random()`. + +```js +lockdown(); +const compartment = new Compartment(); +harden(compartment.globaThis); +compartment.evaluate(program1); +compartment.evaluate(program2); +``` + +### Endowment Protection + +The above `program`, `program1`, and `program2` guest programs are only useful +as glorified calculators. +When going beyond that by "endowing" a compartment with extra objects, a host +program is responsible for maintaining any of the invariants above that it +considers necessary. + +For example, a host program may run two guest programs in separate +compartments, giving one the ability to resolve a promise and the other +the ability to observe the settlement (fulfillment or rejection) of +that promise. +The host program is responsible for hardening the objects implementing such +abilities. + +```js +lockdown(); + +const promise = new Promise(resolve => { + const compartmentA = new Compartment({ + globals: harden({ resolve }), + __options__: true, // temporary migration affordance + }); + compartmentA.evaluate(programA); +}); + +const compartmentB = new Compartment({ + globals: harden({ promise }), + __options__: true, // temporary migration affordance +}); +compartmentB.evaluate(programB); +``` + +With `ses`, guest programs are initially powerless. +A host can explicitly share limited powers with guest programs +and provide intentional communication channels between them. + +### Caveats + +Host programs must maintain the `ses` boundary with care in what they present +as endowments. +A host program should take care not to share mutable state with guests, +or distribute mutable state to multiple guests, such as an object that has not +been frozen with `harden` or a collection like a `Map` or `Set` or typed array +(collections retain some mutability even if hardened). + +For the purposes of sharing state, pseudo-random number generators (PRNG) like +`Math.random()` are equivalent to read and write access to shared state, and +any guest can use one to eavesdrop on other guests or the host that share one. + +If a guest program needs a high resolution timer to function, the host should +only invite one guest to a single operating system process and limit the +activity of the host program in the same process. + +Hosts must avoid exposing `SharedArrayBuffer` to guests. +Any two JavaScript programs sharing a `SharedArrayBuffer` can use the shared +buffer to construct a high resolution timer. + +The `ses` shim does not in itself isolate the stack of guest programs, even +when evaluated in separate compartments. +This is relevant when objects are shared between guest programs. + +When a program interacts with an object introduced by another program (as +through the per-compartment `globalThis`, function arguments or returned +values), there are potential risks due to the synchronous nature of object +access. +Even interactions that are not explicit function calls may cause code from +another program, like property accessors or proxy traps, to execute on the same +stack, which may be able to detect the stack height, throw an exception, or call +back into the program in pursuit of a reentrancy attack. + +A host object can defend itself from reentrancy attacks by ensuring that it +interacts with guest objects on a clean stack through the use of promises. + +Within these constraints, a host program can provide objects that grant limited +I/O capabilities to guest programs, and even revoke or suspend those +capabilities at runtime. + +### Trusted Compute Base + +The trusted compute base (TCB) for `ses` includes: + +- the host hardware, +- the host operating system, +- any intermediate virtual operating systems or hypervisors, +- the process memory manager, +- an implementation of JavaScript conforming to ECMAScript 262 as of + 2021, providing no unspecified embedding host behavior like the introduction of syntax + that when evaluated reveals a mutable object. + `ses` accounts for one such host behavior provided by Node.js, namely the `domain` + property on promises, by preventing the use of `ses` in concert with the + `domain` module. +- Also, any attached debugger, and +- any JavaScript that has executed in the same realm before the host program calls + `lockdown`, including JavaScript that executes after `ses` initializes. + +## Audits + +In June 2021, `ses` underwent formal third party vulnerability assessment over a +period of 4 weeks with 3 engineers and a dedicated project manager that +surfaced no unknown security issues or vulnerabilities within the code. As a +result of this assessment, [a single code change was +made](https://github.com/endojs/endo/issues/126) to set a flag to disable the +domain module in Node.js to mitigate a known issue identified in the code. The +code will be the subject of another round of intense application security +review mid-2022 by a reputable application security firm renowned for their +results in security reviews. + +In July 2021, `ses` was the target of an intensive collaborative bug hunt lead by +the MetaMask team. +No critical flaws in the code surfaced during the review. +As a result of the search for flaws, deficiencies, and weaknesses in the code, +a series of small code changes and documentation improvements were made. There +is a report available on the +[Agoric blog](https://agoric.com/blog/technology/purple-teaming-how-metamask-and-agoric-hunted-bugs-to-harden-javascript) +that includes links to recordings of code walk-throughs and technical +discussion, and issues are tagged +[audit-SEStival](https://github.com/endojs/endo/issues?q=label%3Aaudit-sestival). +The [video recordings of the MetaMask and Agoric collaborative +review](https://www.youtube.com/playlist?list=PLzDw4TTug5O2d1XOdB7VNCZbIxRZu3gov). +provide useful background for future audits, reviews, and for learning more +about how the `ses` shim constructs a Hardened JavaScript environment. + +In addition to vulnerability assessments, active efforts to [formally verify +the Agoric kernel](https://agoric.com/blog/technology/the-path-to-verified-blds-how-informal-systems-and-agoric-are-using-formal) +have found the object capability model that `ses` provides to be sound. + +Hardened JavaScript is also within the scope of the [Agoric bug bounty +program](https://hackerone.com/agoric), which rewards researchers for surfacing valid +bugs in our code. We welcome the opportunity to cooperate with researchers, +whose efforts will undoubtedly yield stronger, more resilient code. + +## Bug Disclosure + +Please help us practice coordinated security bug disclosure, by using the +instructions in [SECURITY.md][] to report security-sensitive bugs privately. + +For non-security bugs, please use the [regular Issues page][SES Issues]. + +## Ecosystem Compatibility + +Most ordinary JavaScript can run without issues in a realm locked down by SES. +Exceptions are tracked at [issue #576][incompatibility tracking], and almost +always take the form of assignments that fail because the +"[override mistake][override mistake]" prevents overriding properties inherited +from a frozen intrinsic object in the prototype chain. When that is the case, +the code is often incompatible with *all* environments in which intrinsic +objects are frozen (such as in Node.js with the +[`--frozen-intrinsics`][Node frozen intrinsics] option) and can be fixed by +replacing `<lhs>.<propertyKey> = <rhs>;` or `<lhs>[<propertyKey>] = <rhs>;` with +```js +Object.defineProperties(<lhs>, { + [<propertyKey>]: { + value: <rhs>, + writable: true, + enumerable: true, + configurable: true, + }, +}); +``` + +Upon encountering an incompatibility, we recommend that you add a comment to +[issue #576][incompatibility tracking] and file an issue with the external +project referencing this section. +Projects often have their own unique issue reporting templates, but generally +provide some place to include text like + +> ``` +> This project has some assignments that break in an environment with frozen +> intrinsic objects, such as +> [Hardened JS (a.k.a. SES)](https://github.com/endojs/endo/blob/master/packages/ses#ecosystem-compatibility) +> or Node.js with the +> [`--frozen-intrinsics`](https://nodejs.org/docs/latest/api/cli.html#--frozen-intrinsics) +> option. +> Specifically, [link to source in the project] does not work correctly in such +> an environment. +> +> Please consider increasing support by replacing assignments to object +> properties inherited from intrinsics with use of `Object.defineProperties` +> (thereby working around the JavaScript "override mistake"), and if applicable +> also by avoiding mutation of intrinsic objects. +> If you don't have the capacity but would accept a PR, please comment to that +> effect so that a volunteer knows their efforts would be welcomed. +> ``` + +We find that library authors are generally amenable to making these small changes to increase +compatibility with any environment that protects itself from prototype pollution attacks by freezing +intrinsics, including `ses`. + +[Hardened JavaScript]: https://hardenedjs.org/ +[define shim]: https://en.wikipedia.org/wiki/Shim_(computing) +[Endo Matrix]: https://matrix.to/#/#endojs:matrix.org +[incompatibility tracking]: https://github.com/endojs/endo/issues/576 +[Node frozen intrinsics]: https://nodejs.org/docs/latest/api/cli.html#--frozen-intrinsics +[override mistake]: https://web.archive.org/web/20141230041441/http://wiki.ecmascript.org/doku.php?id=strawman:fixing_override_mistake +[SECURITY.md]: https://github.com/endojs/endo/blob/master/packages/ses/SECURITY.md +[SES Issues]: https://github.com/endojs/endo/issues +[SES proposal]: https://github.com/tc39/proposal-ses +[SES Strategy Group]: https://groups.google.com/g/ses-strategy +[SES Strategy Recordings]: https://www.youtube.com/playlist?list=PLzDw4TTug5O1jzKodRDp3qec8zl88oxGd + +-----END FILE ses/README.md----- +-----BEGIN FILE skel/README.md----- + + +-----END FILE skel/README.md----- +-----BEGIN FILE stream-node/README.md----- +# Endo / Node Stream Adapters + +This package provides `makeNodeReader` and `makeNodeWriter` adapters that adapt Node.js Reader and Writer to Endo's async iterable streams. + +-----END FILE stream-node/README.md----- +-----BEGIN FILE stream/README.md----- +# Endo Streams + +Endo models streams as hardened async iterators. +Async iterators are sufficient to model back-pressure or pacing +since they are channel messages both from producer to consumer +and consumer to producer. +Streams are therefore symmetric. +The same stream type serves for both a reader and a writer. + +These streams depend on full Endo environment initialization, as with `@endo/init` +to ensure that they are run in Hardened JavaScript with remote promise support +(eventual send). + +## Writing + +To write to a stream, give a value to the next method. + +```js +// ... +await writer.next(value); +``` + +Awaiting the returned promise slows the writer to match the pace of the reader. + +## Reading + +To read from a stream, await the value returned by the next method. + +```js +for await (const value of reader) { + // ... +} +``` + +## Map + +To map a reader to a reader through a synchronous value transform, use `mapReader`. + +```js +const doubleReader = mapReader(singleReader, n => n * 2); +``` + +In this example, any value read from doubleReader will be double what was read +from singleReader. + +To map a writer to a writer through a synchronous value transform, use +`mapWriter`. + +```js +const singleWriter = mapWriter(doubleWriter, n => n * 2); +``` + +In this example, any value written to singleWriter will be writ double to +doubleWriter. + +## Pipe + +The `makePipe` function returns an entangled pair of streams. +Use one as a reader and the other as a writer. +Pipes are useful for mocking streams in tests. + +```js +const [writer, reader] = makePipe(); +``` + +Pipes use `makeStream` and `makeQueue`. +`makeQueue` creates an async promise queue: a collection type like a queue +except that `get` returns a promise and `put` accepts a promise, so `get` can +be called before `put`. +An async queue ensures that the promises returned by `get` and accepted by +`put` are matched respectively, but provides no guarantee about the order in +which promises settle. +A stream is consequently a pair of queues that transport iteration results, +one to send messages forward and another to receive acknowledgements. + +## Pump + +The `pump` function pumps iterations from a reader to a writer. +The pump must be primed with the first acknowledgement to send to the reader, +typically `undefined`, as in `reader.next(undefined)`. +This makes the parity of a pump "odd", because the reader needs a free +acknowledgement to start. +This is in contrast to a pipe, which has "even" parity, because the reader and +writer can both proceed initially. + +So, for example, we can implement `cat` in Node.js by pumping stdin to stdout. + +```js +import { makeNodeWriter, makeNodeReader } from '@endo/stream-node'; + +const writer = makeNodeWriter(process.stdout); +const reader = makeNodeReader(process.stdin); +await pump(writer, reader); +``` + +## Prime + +Async generator functions are very useful for making reader adapters. + +```js +async function *double(reader) { + for await (const value of reader) { + yield value * 2; + } + return undefined; +} +``` + +However, async generator functions can also serve as writers, because `yield` +evaluates to the argument passed to `next`. +However, generator writers have odd parity, meaning the first value sent to a +generator function has nowhere to go and gets discarded as the program counter +proceeds from the beginning of the function to the first `yield`, `return`, or +`throw`. + +The `prime` function compensates for this by sending a primer to the generator +once. + +```js +async function *logGenerator() { + for (;;) { + console.log(yield); + } +} + +const writer = prime(logGenerator()); +await writer.next('First message is not discarded'); +``` + +## Hardening + +This library depends on Hardened JavaScript. +The environment must be locked down before initializing this library. +All of the exported functions and the streams they produce are hardened. + +This implementation of streams ensures that iteration results are shallowly +frozen. +The user is responsible for hardening the transported values if that is their +intent. +Some values like array buffers cannot be frozen. + +-----END FILE stream/README.md----- +-----BEGIN FILE syrup/README.md----- +# Syrup + +🚧 Work in progress. + +[Syrup](https://gitlab.com/spritely/syrup) is an binary object marshalling +codec. + +This is a partial implementation of Syrup intended to be future-compatible and +strictly canonical. + +- Single point IEEE floating point, with the "F" prefix, is unsupported because + JavaScript cannot canonically write any kind of floating point number except + a double precision 64 bit float. +- A future version might introduce `null` or `undefined`, possibly both, either + as a record kind supported at a higher layer, or as a single character + optimization. +- A future version might support symbols as JavaScript registered symbols. +- Future versions cannot support Syrup Dictionaries with non-string keys. +- A future version might support Syrup Sets as JavaScript Sets with some + limitation on what keys are expressible. +- A future version might support Syrup Maps as JavaScript Maps, provided Syrup + adds a notation for Maps. + To do this, the Syrup implementation might need to be coupled to + a higher layer protocol like CapTP, where some record traps are built + into the codec, since passing a dictionary (with only string keys) + to a record trap would not be sufficient to express other types of keys. +- A future version might support Syrup records. + +The supported encoding consists of: + +- Syrup booleans: `t` or `f`, + as JavaScript `true` and `false` +- Syrup double flonum: + `D` _big endian IEEE double precision float (64 bits)_, + as JavaScript `number` +- Syrup signed integers: + _whole integer_ `+` or _positive int_ `-`, + like `0+` or `1-`, + as JavaScript `bigint` +- Syrup byte strings: + _whole integer byte length prefix_ `:` _bytes_, + like `3:cat`, + as JavaScript `Uint8Array` +- Syrup strings: + _whole integer byte length prefix_ `"` _UTF-8 encoded bytes_, + like `3"cat`, + as JavaScript `string` +- Syrup dictionary: + `{` _zero or more alternating_ _syrup encoded string key_ _syrup encoded any + value_ `}`, + as frozen JavaScript record objects +- Syrup lists: + `[` _zero or more syrup encoded values_ `]` + as frozen JavaScript arrays + +-----END FILE syrup/README.md----- +-----BEGIN FILE test262-runner/README.md----- +# test262-runner + +Run ECMAScript compliance tests on Node.js, XS, and potentially others, with a +prelude that shims Hardened JavaScript on these platforms. + +## Test262 subset + +The `test262` directory contains + +* a copy of the `tests` and `harness` directories from https://github.com/tc39/test262. +* additional tests from https://github.com/Moddable-OpenSource/moddable +* additional Hardened JavaScript tests + +We currently only run tests expressly marked with the `ses-xs-parity` feature +in their front-matter. + +## Justification + +Maintaining a local copy of tests taken at a given revision provides not only stability, it's also much faster on autobuilds than having to both checkout the test262 git repo and filter for relevant tests, and having to do so at every test run. + +This technique is the same used by all major JavaScript engines: +- https://github.com/WebKit/webkit/tree/master/JSTests/test262 +- https://github.com/v8/v8/tree/master/test/test262 +- https://github.com/mozilla/gecko-dev/tree/master/js/src/tests/test262 +etc. + +-----END FILE test262-runner/README.md----- +-----BEGIN FILE trampoline/README.md----- +# @endo/trampoline + +> Multicolor trampolining using generators + +**@endo/trampoline** is a utility library which helps share code between synchronous and asynchronous variations of the same algorithm. + +## Example Usage + +```js +import { asyncTrampoline, syncTrampoline } from '@endo/trampoline'; + +/** + * This function "reads a file synchronously" and returns "a list of its imports" + * + * @param {string} filepath Source file path + * @returns {string[]} List of imports found in source + */ +const findImportsSync = filepath => { + // read a file, parse it for imports, return a list of import specifiers + // (synchronously) + // ... +}; + +/** + * This function "reads a file asynchronously" and returns "a list of its imports" + * + * @param {string} filepath Source file path + * @returns {Promise<string[]>} List of imports found in source + */ +const findImportsAsync = async filepath => { + // read a file, parse it for imports, return a list of import specifiers + // (asynchronously) + // ... +}; + +/** + * Recursively crawls a dependency tree to find all dependencies + * + * @template {string[] | Promise<string[]>} TResult Type of result (list of imports) + * @param {(filepath: string) => TResult} finder Function which reads a file and returns its imports + * @param {string} filename File to start from; entry point + * @returns {Generator<TResult, string[], string[]>} Generator yielding list of imports + */ +function* findAllImports(finder, filename) { + // it doesn't matter if finder is sync or async! + let specifiers = yield finder(filename); + + // pretend there's some de-duping, caching, + // scrubbing, etc. happening here + + for (const specifier of specifiers) { + // it's okay to be recursive + specifiers = [...specifiers, ...(yield* findAllImports(finder, specifier))]; + } + return specifiers; +} + +// results are an array of all imports found in some.js' dependency tree +const asyncResult = await asyncTrampoline( + findAllImports, + findImports, + './some.js', +); + +// same thing, but synchronously +const syncResult = syncTrampoline( + findAllImports, + findImportsAsync, + './some.js', +); + +asyncResult === syncResult; // true +``` + +In the above example, **@endo/trampoline** allows us to re-use the operations in `loadRecursive()` for _both_ sync and async execution. An implementation _without_ **@endo/trampoline** would need to duplicate the operations into two (2) discrete recursive functions—a synchronous-colored function and an asynchronous-colored function. Over time, this situation commonly leads to diverging implementations. If that _doesn't_ sound like a big deal for _whatever you're trying to do here_, then you probably don't need **@endo/trampoline**. + +## What is this? + +The pattern exposed by this library—known as [trampolining][]—helps manage control flow in a way that avoids deep recursion and potential stack overflows. + +**@endo/trampoline** provides the trampolining pattern, but in such a way that a consumer can execute _either_ synchronous _or_ asynchronous operations _paired with operations common to both_. + +In other words, **@endo/trampoline** can help _reduce code duplication_ when operations must be executed _in both sync and async_ contexts. + +## Install + +The usual sort of thing: + +```sh +npm install @endo/trampoline +``` + +## License + +Apache-2.0 + +## Disclaimer + +By using this library, you agree to indemnify and hold harmless the authors of `@endo/trampoline` from any and all losses, liabilities and risk of bodily injury _including but not limited to_ broken bones, sprains, bruises or other hematomas, fibromas, teratomas, mesotheliomas, cooties, bubonic plague, psychic trauma or warts due to the inherent danger of trampolining. + +[trampolining]: https://raganwald.com/2013/03/28/trampolines-in-javascript.html + +-----END FILE trampoline/README.md----- +-----BEGIN FILE where/README.md----- +# Where is Endo? + +This package provides a utility for finding the user files and Unix domain +socket or Windows named pipe for the Endo daemon. +The Endo user directory stores the per-user runtime data for Endo, +including logs and other application storage. + +Endo attempts to use or infer [Cross-desktop XDG conventions][XDG] paths in +every meaningful way. +Windows named pipes do not appear to fit this model. +Otherwise falls back to the native conventions on Windows and Mac/Darwin. +On Windows, Endo does not use separate state and cache directories and does not +yet sync state between home directories. + +[XDG]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + +-----END FILE where/README.md----- +-----BEGIN FILE zip/README.md----- +# Endo Zip + +This is a modernization and specialization of [JSZip][] (MIT License) that has +no dependencies on any built-in modules and is entirely implemented with +ECMAScript modules. This makes the library suitable for embedding in an XS +binary, bundling for any platform with Rollup, and usable with `node -r esm`. + +Modernization afforded the opportunity to focus on use of TypedArrays and UTF-8 +without reservation, and to use TypeScript JSDoc comments to verify the flow of +information. It also afforded an opportunity to make some security-conscious +decisions, like treating file name spoofing as an integrity error and +requiring a date to be expressly provided instead of reaching for the ambient +original Date constructor, which will pointedly be absent in constructed +compartments in locked-down environments. + +Zip format allows for an arbitrary-length comment and an arbitrary number of +Zip64 headers in the "end of central directory block". +Zip implementations must therefore scan backward from the end for the magic +numbers that introduce the "EOCDB". +However, a specially crafted Zip file may contain those magic numbers +before the end. + +So, for security, this specialized library does not support Zip64 nor +the variable width archive comment. +With some difficulty, Zip64 might be recovered by scanning backward from the +end of the file until we find a coherent EOCDB with no trailing bytes. +Even careful support for the variable width comment at the end of the archive +would always allow for the possibility of a comment that is itself a valid Zip +file with a long prefix, since Zip files allow an arbitrary length prefix. + +For expedience, the specialization dropped support for INFLATE compression. +The dependency would need to be converted to ECMAScript modules, which is not +much effort. Pursuing that intent, one should factor out the shared CRC32 +module. + +JSZip supports an asynchronous mode, that despite the name, is not concurrent. +The mode is intended to keep the main thread lively while emitting progress +reports. For expedience, this mode is omitted, but could be restored using the +same underlying utilities, and I expect async/await and async iterators would +make the feature easier to maintain. + +Provided an async seekable reader, a lazy Zip reader could be built on the same +foundations, deferring decompression and validation until the file is opened. + +For expedience, support for streaming compression and the necessary data +descriptors have been dropped. They are not necessary for synchronous writing. +The data descriptors are also redundant with the central directory for reading, +so they've been omitted, though recovering them for additional integrity +checks would be useful. + +For expedience, explicit directory records are ignored on read and omitted on +write. These would also be straightforward to recover. + +For expedience, there is no API for enumerating the contents of the archive. +This would be straightforward to implement. + + [JSZip]: https://github.com/Stuk/jszip + +-----END FILE zip/README.md----- + +--- + +### Message ID: 3702abde-5512-4742-aa34-1c738711f851 +**Author**: tool + +**Content:** + +Reflecting on the user query. They seek a TS code snippet for NodeJS using Endo. It should download a TypeScript module and run it in a SES compartment, leveraging updated docs from compartment-mapper and Endo. + +**Integrating remote code** + +I’m combining and gathering remote TypeScript code in a NodeJS environment, ensuring it runs correctly using SES. + +**Weighing improvements** + +I'm considering integrating debugging tools, sourceMap handling, and `importMeta` functionalities to enhance the system further, ensuring it aligns with the latest compartment-mapper and Endo documentation. + +**Enhancing debugging** + +I’m working on incorporating `importMetaHook` and `computeSourceMapLocation` to improve remote code debugging and retrieval in a NodeJS environment. Additionally, considering integrating handling for `policy`, `globals`, and `modules`. + +**Integrating source map handling** + +I’m pulling in `computeSourceMapLocation` from `import-bundle` to enhance debugging capabilities by seamlessly incorporating it into `importLocation` as part of the `powers` argument. + +**Enhancing code handling** + +I'm expanding type declaration support and refining code management with `conditions: ['endo']`. This adjustment will guide `importLocation` towards `endo`. + +**Pulling together functionality** + +I'm incorporating `importMetaHook` and `computeSourceMapLocation` into `importLocation`, adding `policy`, setting `conditions: ['endo']`, and refining error handling. This enhances remote code management capabilities. + +--- + +### Message ID: 9ce0ff66-ca31-4cab-97ee-28453c9c3248 +**Author**: assistant + +**Content:** + +Below is an improved version of the previous TypeScript file, incorporating additional features and insights from the Endo documentation. These include setting `importMetaHook` to populate `import.meta.url` for the module, providing a `computeSourceMapLocation` hook for debugging with source maps, and retaining the languageForExtension mapping for `.ts` files. The code also demonstrates how to supply a `console` global and leave room for adding policies or transforms if needed. + +```typescript +// run-remote-module.ts +import 'ses'; +import fetch from 'node-fetch'; +import { importLocation } from '@endo/compartment-mapper'; +import { computeSourceMapLocation } from '@endo/import-bundle/source-map-node.js'; + +// Lock down the environment with verbose stack traces for easier debugging +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }); + +const loadAndRun = async (moduleURL: string) => { + const packageURL = new URL('./package.json', moduleURL).toString(); + + const read = async (location: string): Promise<Uint8Array> => { + if (location === packageURL) { + // Minimal package.json: treat all .ts as ESM + const pkg = { + name: "remote-module", + type: "module", + main: new URL(moduleURL).pathname.replace(/^\/+/, '') + }; + return new TextEncoder().encode(JSON.stringify(pkg)); + } else { + const res = await fetch(location); + if (!res.ok) { + throw new Error(`Failed to fetch ${location}: ${res.statusText}`); + } + const content = await res.arrayBuffer(); + return new Uint8Array(content); + } + }; + + // Provide limited globals and no extra modules by default + const globals = { console }; + const modules = {}; + + // Map .ts files to ESM + const languageForExtension = { ts: 'mjs' }; + + // Hook to set import.meta.url for debugging & referencing + const importMetaHook = (moduleSpecifier: string, importMeta: Record<string, unknown>) => { + importMeta.url = moduleSpecifier; + }; + + // Optionally handle source maps for better debugging experience + // Adjust computeSourceMapLocation as needed for your environment + const { namespace } = await importLocation( + read, + moduleURL, + { + globals, + modules, + languageForExtension, + importMetaHook, + computeSourceMapLocation(details) { + // For debugging: return a URL for the source maps + return `file://${details.location}`; + }, + // In case you want to apply transforms to code, you could add: + // transforms: [yourTransformFunction] + // or __shimTransforms__ if you need to instrument modules + // + // If you want to apply policies, you could do: + // policy: yourPolicyObject + } + ); + + console.log('Loaded module exports:', namespace); +}; + +loadAndRun('https://example.com/yourModule.ts').catch(err => { + console.error(err); + process.exit(1); +}); +``` + +**Key Improvements:** + +1. **importMetaHook:** Populates `import.meta.url` for modules, helpful for modules that rely on `import.meta` for resource location or debugging. + +2. **computeSourceMapLocation:** Integrates source maps for easier local debugging. This is particularly useful if you transform or bundle code before evaluation. + +3. **languageForExtension:** Ensures `.ts` files get treated as ESM modules. This aligns with the `@endo/compartment-mapper` docs for TypeScript integration. + +4. **Globals and Modules:** Demonstrates how to provide a limited global environment and how to supply platform features selectively. Additional `modules` or endowments (like `fs`, `fetch`, or a controlled PRNG) can be included if needed. + +5. **Comments for Future Enhancements:** Leaves hooks for adding transforms, policies, or other features described in the Endo docs as your needs evolve. + +--- + +### Message ID: aaa279ea-48f9-490a-97e6-7c3ca08a6ca7 +**Author**: user + +**Content:** + +now are you sure that ts files are being transpiled or transformed (whatever the term is) so they can run in this environment ? if this was running in deno, would anything need to change or would it be easier ? + +--- + +### Message ID: eafe8dde-96f8-4724-a93f-fc2af36d6a8d +**Author**: tool + +**Content:** + +**Navigating TS transpilation** + +I’m figuring out how to handle TypeScript files in a JavaScript environment, which seems tricky to manage directly. + +**Navigating type erasure** + +It's interesting to see that `importLocation` from `@endo/compartment-mapper` can manage TypeScript type erasure for `.ts` files, as long as they're outside the `node_modules` directory. + +**Leveraging native support** + +OK, let me see. Deno's native TypeScript support simplifies the process, eliminating the need for `@endo/compartment-mapper` and enabling `await import('https://example.com/yourModule.ts')` directly. + +**Transpiling strategies** + +I'm thinking through using `importLocation()` to manage `.ts` files, ensuring `languageForExtension` is set. However, type erasure might need local modifications or pre-compilation. + +**Evaluating Deno's capabilities** + +Deno can handle TypeScript out-of-the-box, simplifying the process. Relying on `@endo/compartment-mapper` is optional for Endo environments. + +--- + +### Message ID: 9d924dbe-2082-4990-b9d9-bf8863f42f8e +**Author**: assistant + +**Content:** + +The snippet as shown does not automatically transpile or type-erase TypeScript files. Simply telling the compartment mapper that `.ts` files use the `mjs` parser with `languageForExtension` does not compile TypeScript source into plain JavaScript. By default, the `@endo/compartment-mapper` does not include a TypeScript-to-JavaScript transform step, and Node.js doesn’t support `.ts` files natively. You would need one of the following: + +- Pre-compile the TypeScript code into JavaScript before using `importLocation`. +- Use a transform hook (like with `@endo/bundle-source` and `ts-blank-space`) that can do type erasure for `.ts` files. +- Provide a custom parser or translator configured via `parserForLanguage` or related compartment mapper options. + +In other words, the code snippet is set up to treat `.ts` files as ESM (`mjs`), but that alone won’t run raw TypeScript. You need a step in the pipeline that converts `.ts` code to JavaScript first. + +**If This Were Running in Deno:** +Deno natively supports TypeScript and automatically transpiles `.ts` files on the fly. If you were using Deno directly (without the compartment mapper’s manual process), then you wouldn’t need these extra steps. Deno would handle the type erasure and compilation under the hood, making it far easier to run `.ts` code without additional transforms or tooling. + +--- + +### Message ID: aaa238c5-dbc1-4b48-80e4-154bed867780 +**Author**: user + +**Content:** + +ok so give me that code, but for running under deno. + +--- + +### Message ID: dbaec262-faee-4aff-8db4-b59518cd279e +**Author**: tool + +**Content:** + +**Adapting to Deno** + +I’m converting `fetch` in a Deno environment, ensuring features are implemented, and removing `node-fetch` references to avoid node-specific dependencies. + +**Crafting Deno code** + +I’m gathering info on using Deno's native `fetch`, adapting to no node-fetch. The code should have arrow functions, and maybe `compartment-mapper` for Deno's environment. + +--- + +### Message ID: 1c7a8b67-f2b9-47de-9a72-ab8e802a620e +**Author**: assistant + +**Content:** + +```typescript +// run-remote-module.ts +// This script runs under Deno with native ESM and TypeScript support. +// It takes a URL to a .ts module, sets up a compartmentalized environment, +// and runs the module in a SES compartment. +// Usage: deno run --allow-net --allow-read run-remote-module.ts https://example.com/yourModule.ts + +import 'ses'; +import { importLocation } from '@endo/compartment-mapper'; + +// Lock down the environment +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }); + +const loadAndRun = async (moduleURL: string): Promise<void> => { + const packageURL = new URL('./package.json', moduleURL).toString(); + + const read = async (location: string): Promise<Uint8Array> => { + if (location === packageURL) { + // Minimal package.json: treat all .ts files as ESM + const pkg = { + name: "remote-module", + type: "module", + main: new URL(moduleURL).pathname.replace(/^\/+/, '') + }; + return new TextEncoder().encode(JSON.stringify(pkg)); + } else { + const res = await fetch(location); + if (!res.ok) { + throw new Error(`Failed to fetch ${location}: ${res.statusText}`); + } + const content = await res.arrayBuffer(); + return new Uint8Array(content); + } + }; + + // Provide limited globals + const globals = { console }; + const modules = {}; + + // Map .ts files to ESM + const languageForExtension = { ts: 'mjs' }; + + // Set import.meta.url for modules + const importMetaHook = (specifier: string, meta: Record<string, unknown>) => { + meta.url = specifier; + }; + + // Import the remote .ts module as if it were a local app entry point + const { namespace } = await importLocation(read, moduleURL, { + globals, + modules, + languageForExtension, + importMetaHook, + }); + + console.log('Loaded module exports:', namespace); +}; + +// Run the script if called directly +if (import.meta.main) { + const moduleURL = Deno.args[0]; + if (!moduleURL) { + console.error('Usage: deno run --allow-net run-remote-module.ts <module_url>'); + Deno.exit(1); + } + await loadAndRun(moduleURL); +} +``` + +--- + +### Message ID: aaa2c028-fd2b-425b-8f07-7acf67a9a1fc +**Author**: user + +**Content:** + +ok, but treat the supplied url as a ts file that might have some imports in it, not as a package.json file, since this might not be the case - I want to treat the file in the url as not part of a package + +--- + +### Message ID: 3fac13d8-4197-4d41-b932-4baac5a33cc9 +**Author**: tool + +**Content:** + +**Mapping module loads** + +I’m piecing together that Deno expects a package.json file for each module load. If dealing with unpackaged TypeScript files, creating a dummy package.json or avoiding package.json is essential. + +**Navigating dependencies** + +I'm considering strategies to bypass `importLocation`'s need for package.json, like using `importArchive`, `parseArchive`, or a minimal ephemeral package.json. It’s intriguing to explore these workarounds. + +**Crafting minimal setups** + +Considering dynamic `package.json` generation or `importArchive` for handling standalone TypeScript files. Exploring ephemeral setups is an intriguing path forward. + +**Crafting an ephemeral environment** + +Creating a minimal ephemeral environment for a single TS file can be achieved with `importArchive`, using a dynamically generated `package.json`. This method sidesteps the need for a pre-existing package.json. + +**Building ephemeral packages** + +Hmm, I’m thinking about creating minimal ephemeral packages, possibly dynamically generated, to handle single TS file imports without requiring a pre-existing package.json file. + +**Piecing together the solution** + +Creating a minimal ephemeral `package.json` inline for `importLocation` seems like a viable fallback, especially when handling single TS files with imports, ensuring it works without a pre-existing package.json. + +**Crafting an ephemeral solution** + +I'm working through providing a minimal ephemeral `package.json` for `importLocation` to address `compartment-mapper` requirements, ensuring seamless handling of single TS files without a pre-existing package.json. + +**Constructing a minimal environment** + +I’m considering using `compartment.import` for single TS file imports, ensuring a basic environment is set up, and dynamically creating a `package.json` from memory. + +**Charting a course** + +I'm delving into `importArchive` with a synthetic in-memory archive to handle single TS file imports without a pre-existing package.json, while ensuring a minimal ephemeral `package.json`. + +**Crafting an ephemeral package** + +OK, let me see. Exploring a minimal ephemeral `package.json` for single TS file imports to meet `compartment-mapper` needs, while treating URL as primary code without a pre-existing `package.json`. + +--- + +### Message ID: 1d89a2a1-7416-464e-90ab-12a790dd6d68 +**Author**: assistant + +**Content:** + +Below is a Deno-based snippet that fetches a remote `.ts` file, transpiles it to JavaScript using Deno’s built-in compiler, and runs it in a SES compartment without treating it as part of a Node-style package. It implements custom `importHook` and `resolveHook` functions to handle module imports. This approach uses the `Compartment` API directly instead of `importLocation`, so no `package.json` is needed. + +**Requirements:** +- Run with `--allow-net` so the code can fetch remote modules +- Run with `--allow-read` if needed (for local caching or other purposes) + +**Usage:** +```bash +deno run --allow-net run-remote-ts-module.ts https://example.com/yourModule.ts +``` + +**Code:** + +```typescript +// run-remote-ts-module.ts +// +// This script runs under Deno, fetches a remote TypeScript file, transpiles it to JavaScript, +// and executes it in a SES compartment. The file may have imports, which are also resolved, +// fetched, and transpiled. +// +// Usage: +// deno run --allow-net run-remote-ts-module.ts https://example.com/yourModule.ts + +import 'ses'; +import { ModuleSource } from '@endo/module-source'; + +// Lock down the environment +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }); + +async function transpileTS(code: string, fileName: string): Promise<string> { + const result = await Deno.transpileOnly({ [fileName]: code }); + const output = result[fileName]; + if (!output || !output.source) { + throw new Error(`Failed to transpile ${fileName}`); + } + return output.source; +} + +const createImportHook = () => { + return async (specifier: string) => { + const res = await fetch(specifier); + if (!res.ok) { + throw new Error(`Failed to fetch ${specifier}: ${res.statusText}`); + } + const sourceCode = await res.text(); + // Transpile TS to JS + const jsCode = await transpileTS(sourceCode, specifier); + return { + // Provide a ModuleSource to SES + source: new ModuleSource(jsCode, specifier), + }; + }; +}; + +const createResolveHook = () => { + return (specifier: string, referrer: string) => { + return new URL(specifier, referrer).toString(); + }; +}; + +// Run the supplied URL in a compartment +async function runModule(entryURL: string) { + const resolveHook = createResolveHook(); + const importHook = createImportHook(); + + // Set up a compartment that can handle imports + const compartment = new Compartment({}, {}, { + name: 'ts-runner', + resolveHook, + importHook, + }); + + // Import the module and log its exports + const ns = await compartment.import(entryURL); + console.log('Loaded module exports:', ns); +} + +if (import.meta.main) { + const moduleURL = Deno.args[0]; + if (!moduleURL) { + console.error('Usage: deno run --allow-net run-remote-ts-module.ts <module_url>'); + Deno.exit(1); + } + + await runModule(moduleURL); +} +``` + +**What This Code Does:** + +1. **Transpile:** Uses `Deno.transpileOnly()` to convert TypeScript to JavaScript on-the-fly. +2. **Fetch & Resolve:** Custom `importHook` and `resolveHook` fetch and resolve imports from URLs. +3. **SES Compartment:** Runs the resulting JavaScript code inside a hardened SES environment. +4. **No package.json Needed:** Unlike `importLocation`, this setup does not rely on any `package.json`. It directly uses `Compartment`, allowing you to treat the URL as just a single module entry point, not part of a package. + +Because Deno supports fetching and transpiling `.ts` modules natively, this approach sidesteps Node-style package assumptions and leverages Deno’s built-in tooling to streamline the process. + +--- + +### Message ID: aaa24c22-0575-43aa-a3b6-1ba3249e7277 +**Author**: user + +**Content:** + +would it work without Deno.transpileOnly() since we are running a deno environment, so wouldn't running the ts file just work ? or if not, can we just strip the ts rather than relying on this function, since we are running in deno deploy and might not have access ? + +--- + +### Message ID: e1a3d481-40ad-4c77-88e1-d1465d0c9363 +**Author**: tool + +**Content:** + +**Transpiling code** + +I'm figuring out if `Deno.transpileOnly()` is needed for TypeScript code in a Deno environment with `importHook`. ESBuild or other transformers could also work. + +**Evaluating methods** + +I'm noting Deno's `Deno.emit()` for TypeScript to JavaScript conversion, which is stable in Deno Deploy, ensuring compatibility with SES compartments. + +**Adopting Deno.emit()** + +I’m inclined towards `Deno.emit()` for converting TypeScript to JavaScript, ensuring compatibility with `ModuleSource` and SES compartments. This method aligns with Deno Deploy's capabilities. + +**Ensuring JS execution** + +To dynamically fetch TypeScript and script evaluations at runtime, it’s crucial to transpile TypeScript to JavaScript, as SES compartments don't natively support TypeScript. + +--- + +### Message ID: 5b50f3bb-7f74-4901-bcbf-0ab8123d095e +**Author**: assistant + +**Content:** + +Deno’s runtime automatically transpiles TypeScript when you use `import` statements directly, but here we're fetching the code as text and passing it to a SES `Compartment`. SES itself cannot parse TypeScript, so we must provide it with JavaScript. Without `Deno.transpileOnly()` you can use `Deno.emit()`—which is available on Deno Deploy—to transpile TypeScript on-the-fly. + +Below is an updated example that uses `Deno.emit()` instead of `Deno.transpileOnly()`. It fetches a `.ts` file, transpiles it to JavaScript using Deno’s built-in compiler infrastructure, and then runs it in a SES `Compartment`. + +**Important:** +- `Deno.emit()` is stable and works on Deno Deploy. +- Make sure to run with `--allow-net` to allow fetching modules. + +**Usage:** +```bash +deno run --allow-net run-remote-ts-module.ts https://example.com/yourModule.ts +``` + +**Code:** +```typescript +// run-remote-ts-module.ts +// +// This script runs under Deno, fetches a remote TypeScript file, uses Deno.emit() to transpile +// it to JavaScript, and executes it in a SES compartment. The file may have imports, which are +// also resolved, fetched, and transpiled. +// +// Usage: +// deno run --allow-net run-remote-ts-module.ts https://example.com/yourModule.ts + +import 'ses'; +import { ModuleSource } from '@endo/module-source'; + +// Lock down the environment +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }); + +async function transpileTS(code: string, fileName: string): Promise<string> { + const { files, diagnostics } = await Deno.emit(fileName, { + sources: { [fileName]: code }, + // Adjust compiler options as needed + compilerOptions: { + // Some sensible defaults for Deno TS + target: "ES2022", + module: "ESNext", + strict: true, + isolatedModules: true, + noEmitOnError: true, + }, + }); + + if (diagnostics?.length) { + // If there are any TypeScript diagnostics, handle them + for (const d of diagnostics) { + console.error(d); + } + throw new Error("Transpilation failed due to TypeScript diagnostics"); + } + + // `Deno.emit` puts emitted files under virtual paths like `deno:///...` + // We need to find the correct emitted file. + // We used `fileName` as key and expect an output keyed by a `.js` extension. + const outFileName = Object.keys(files).find(key => key.endsWith('.js')); + if (!outFileName) { + throw new Error(`No JavaScript output found for ${fileName}`); + } + + return files[outFileName]; +} + +const createImportHook = () => { + return async (specifier: string) => { + const res = await fetch(specifier); + if (!res.ok) { + throw new Error(`Failed to fetch ${specifier}: ${res.statusText}`); + } + const sourceCode = await res.text(); + // Transpile TS to JS + const jsCode = await transpileTS(sourceCode, specifier); + return { + // Provide a ModuleSource to SES + source: new ModuleSource(jsCode, specifier), + }; + }; +}; + +const createResolveHook = () => { + return (specifier: string, referrer: string) => { + return new URL(specifier, referrer).toString(); + }; +}; + +// Run the supplied URL in a compartment +async function runModule(entryURL: string) { + const resolveHook = createResolveHook(); + const importHook = createImportHook(); + + // Provide limited globals + const globals = { console }; + + const compartment = new Compartment(globals, {}, { + name: 'ts-runner', + resolveHook, + importHook, + }); + + // Import the module and log its exports + const ns = await compartment.import(entryURL); + console.log('Loaded module exports:', ns); +} + +if (import.meta.main) { + const moduleURL = Deno.args[0]; + if (!moduleURL) { + console.error('Usage: deno run --allow-net run-remote-ts-module.ts <module_url>'); + Deno.exit(1); + } + + await runModule(moduleURL); +} +``` + +**What This Code Does:** + +- **No `Deno.transpileOnly()`:** Uses `Deno.emit()` which is available in Deno Deploy and can transpile TS to JS in-memory. +- **No package.json Needed:** It treats the fetched `.ts` code as a standalone module, resolving imports recursively. +- **SES Compartment:** Runs the JavaScript code inside a hardened SES environment, ensuring isolation. +- **Deno Native TS:** Still relies on Deno’s built-in TS compiler via `Deno.emit()`, not the runtime auto-transpilation, since we need to manually provide JavaScript to SES. + +--- + +### Message ID: aaa2e176-d8d0-4483-9fcb-cc4b7a969631 +**Author**: user + +**Content:** + +that's good, but instead of Deno.emit(), I want to use the library imported like "import { transpile } from 'jsr:@deno/emit'" + + +The docs for this library are below: + +# emit + +[](https://jsr.io/@deno/emit) + +Transpile and bundle JavaScript and TypeScript in Deno and Deno Deploy. + +> This is an unstable module, where the API is likely to change over time. + +## Transpiling + +Take individual modules that are JavaScript or TypeScript and emit them in a +transpiled fashion. An example of taking some TypeScript and transpiling to +JavaScript: + +```ts +import { transpile } from "jsr:@deno/emit"; + +const url = new URL("./testdata/mod.ts", import.meta.url); +const result = await transpile(url); + +const code = result.get(url.href); +console.log(code?.includes("export default function hello()")); +``` + +## Bundle + +Take a root module and all its dependencies and emit a single JavaScript bundle. +This is similar to the functionality provided by `deno bundle` on the Deno +command line. An example: + +```ts +import { bundle } from "jsr:@deno/emit"; +const result = await bundle( + new URL("https://deno.land/std@0.140.0/examples/chat/server.ts"), +); + +const { code } = result; +console.log(code); +``` + +--- + +### Message ID: 7eeb2af0-d966-4dbc-88fd-74515a76b75f +**Author**: tool + +**Content:** + +**Transpiling Typescript** + +I’m gathering info on the `transpile` method from a hypothetical library, focusing on transforming Typescript to Javascript. It returns a map of module names to their code. + +**Leveraging `transpile(url)`** + +I'm pushing forward with `transpile(url)` to manage fetching and transpiling in one go, making use of a URL. This should streamline the process and reduce complexity. + +**Mapping the approach** + +Starting with `transpile(new URL(entryURL))` to handle the entire dependency graph, and then using an `importHook` to pull code from the result map as needed. + +**Streamlining the process** + +Starting with `transpile` for initial fetching, followed by `importHook` to access the result map. Ensuring `importHook` uses a resolved URL specifier to retrieve code effectively. + +--- + +### Message ID: 2d07ae87-b7f2-4279-b460-f8f8194f273a +**Author**: assistant + +**Content:** + +Below is a revised example that uses `transpile` from `jsr:@deno/emit`. In this scenario, we rely entirely on `transpile()` to fetch and transpile all dependencies starting from the entry URL. Then we use a custom `importHook` that simply picks the already-transpiled JavaScript code from the emitted results map. We do a one-time `transpile()` call before starting the compartment import, and cache the results for all subsequent imports. + +**Requirements:** +- Run with `--allow-net` so the code can fetch modules over the network. + +**Usage:** +```bash +deno run --allow-net run-remote-ts-module.ts https://example.com/yourModule.ts +``` + +**Code:** +```typescript +// run-remote-ts-module.ts +// +// This script runs under Deno, uses "transpile" from "jsr:@deno/emit" to fetch and transpile +// a remote TypeScript file (and its imports) into JavaScript. It then runs the resulting code +// in a SES Compartment. +// +// Usage: +// deno run --allow-net run-remote-ts-module.ts https://example.com/yourModule.ts + +import 'ses'; +import { ModuleSource } from '@endo/module-source'; +import { transpile } from 'jsr:@deno/emit'; // Uses the jsr:@deno/emit library + +// Lock down the environment +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }); + +async function runModule(entryURL: string) { + // Convert the string to a URL object + const rootURL = new URL(entryURL); + + // Transpile the entire dependency graph starting from the entry point + const result = await transpile(rootURL); + + const resolveHook = (specifier: string, referrer: string) => { + // Resolve relative imports + return new URL(specifier, referrer).toString(); + }; + + const importHook = async (specifier: string) => { + // Lookup the transpiled JavaScript code from the map + const jsCode = result.get(specifier); + if (!jsCode) { + throw new Error(`No transpiled code found for ${specifier}`); + } + return { + source: new ModuleSource(jsCode, specifier), + }; + }; + + // Provide limited globals + const globals = { console }; + + const compartment = new Compartment(globals, {}, { + name: 'ts-runner', + resolveHook, + importHook, + }); + + // Import the module and log its exports + const ns = await compartment.import(entryURL); + console.log('Loaded module exports:', ns); +} + +if (import.meta.main) { + const moduleURL = Deno.args[0]; + if (!moduleURL) { + console.error('Usage: deno run --allow-net run-remote-ts-module.ts <module_url>'); + Deno.exit(1); + } + + await runModule(moduleURL); +} +``` + +**What This Code Does:** + +1. **Transpile with `jsr:@deno/emit`:** + We call `transpile(rootURL)` which fetches the entry module and all its dependencies, transpiles them to JavaScript, and returns a map of URLs to code. + +2. **No Manual Fetching or Per-Module Transpiling:** + Since `transpile()` processes the entire dependency graph at once, the `importHook` just picks the correct code from the `result` map. + +3. **SES Compartment Execution:** + After transpilation, the code is run in a SES `Compartment`. No `Deno.emit()` or `Deno.transpileOnly()` is needed. + +4. **No `package.json` or Additional Config:** + We treat the remote `.ts` file purely as a standalone entry point. The `jsr:@deno/emit` library and `transpile()` handle the complexity of fetching and converting TypeScript to JavaScript behind the scenes. + +--- \ No newline at end of file diff --git a/compartment/deno.json b/compartment/deno.json new file mode 100644 index 000000000000..8b3181e2e8c4 --- /dev/null +++ b/compartment/deno.json @@ -0,0 +1,12 @@ +{ + "tasks": { + "dev": "deno run -A --watch main.ts https://jsr.io/@dreamcatcher/concat/0.0.6/main.ts" + }, + "imports": { + "@endo/compartment-mapper": "npm:@endo/compartment-mapper@^1.4.0", + "@endo/import-bundle": "npm:@endo/import-bundle@^1.3.2", + "@endo/module-source": "npm:@endo/module-source@^1.1.2", + "@std/assert": "jsr:@std/assert@1", + "ses": "npm:ses@^1.10.0" + } +} diff --git a/compartment/issues.md b/compartment/issues.md new file mode 100644 index 000000000000..8e23a9999fad --- /dev/null +++ b/compartment/issues.md @@ -0,0 +1,4 @@ +does not work for npm:something or jsr:something dependencies + +does not work when json is imported using +`import denoData from "./deno.json" with { type: "json" };` diff --git a/compartment/main.ts b/compartment/main.ts new file mode 100644 index 000000000000..2424608dc5fa --- /dev/null +++ b/compartment/main.ts @@ -0,0 +1,64 @@ +// run-remote-ts-module.ts +// +// This script runs under Deno, uses "transpile" from "jsr:@deno/emit" to fetch and transpile +// a remote TypeScript file (and its imports) into JavaScript. It then runs the resulting code +// in a SES Compartment. +// +// Usage: +// deno run --allow-net run-remote-ts-module.ts https://example.com/yourModule.ts + +import 'ses' +import { ModuleSource } from '@endo/module-source' +import { transpile } from 'jsr:@deno/emit' // Uses the jsr:@deno/emit library + +// Lock down the environment +lockdown({ errorTaming: 'unsafe', stackFiltering: 'verbose' }) + +async function runModule(entryURL: string) { + // Convert the string to a URL object + const rootURL = new URL(entryURL) + + // Transpile the entire dependency graph starting from the entry point + const result = await transpile(rootURL) + + const resolveHook = (specifier: string, referrer: string) => { + // Resolve relative imports + return new URL(specifier, referrer).toString() + } + + const importHook = async (specifier: string) => { + // Lookup the transpiled JavaScript code from the map + const jsCode = result.get(specifier) + if (!jsCode) { + throw new Error(`No transpiled code found for ${specifier}`) + } + return { + source: new ModuleSource(jsCode, specifier), + } + } + + // Provide limited globals + const globals = { console } + + const compartment = new Compartment(globals, {}, { + name: 'ts-runner', + resolveHook, + importHook, + }) + + // Import the module and log its exports + const ns = await compartment.import(entryURL) + console.log('Loaded module exports:', ns) +} + +if (import.meta.main) { + const moduleURL = Deno.args[0] + if (!moduleURL) { + console.error( + 'Usage: deno run --allow-net run-remote-ts-module.ts <module_url>', + ) + Deno.exit(1) + } + + await runModule(moduleURL) +} diff --git a/deno.json b/deno.json index d57e9d44b0db..f0aa987e7d8b 100644 --- a/deno.json +++ b/deno.json @@ -26,7 +26,8 @@ "cov": "deno coverage --ignore=\"**/*.generated.mjs,**/_test_utils.ts,.github\"", "cov:gen": "deno task cov --lcov --output=cov.lcov", "cov:view": "deno task cov --html", - "ok": "deno task lint && deno fmt --check && deno task test" + "ok": "deno task lint && deno fmt --check && deno task test", + "watch": "clear && DENO_KV_PATH=:memory: deno test --watch --trace-leaks" }, "exclude": [ ".git", @@ -48,14 +49,40 @@ ] } }, + "fmt": { + "semiColons": false, + "singleQuote": true, + "exclude": [ + "_tools", + ".github" + ] + }, "workspace": [ - "./files", - "./openai", + "./actors", + "./api", + "./compartment", "./dumb-bot", - "./reasoner", + "./engine", + "./evals", + "./execution", + "./files", "./fixture", + "./git-kv", "./hamr", "./info", - "./test-file-runner" + "./napp-tools", + "./openai", + "./processes", + "./reasoner", + "./server", + "./shockwave", + "./snapshots", + "./stateboard", + "./stucks", + "./utils/pdfs", + "./utils/youtube", + "./utils/concat", + "./utils/chat", + "./utils/crawl" ] } diff --git a/dumb-bot/README.md b/dumb-bot/README.md index 9fa3181f6baa..d5a97cdc7dcc 100644 --- a/dumb-bot/README.md +++ b/dumb-bot/README.md @@ -1,7 +1,7 @@ # dumb-bot > [!NOTE] -> The system prompt for this agent: [instructions.md](./instructions.md) +> The system prompt for this agent: [instructions.md](dumb-bot/instructions.md) `dumb-bot` is special in that one day, it will replace all other agents. The overall progress of AI models is measured by how many agents are replaced each diff --git a/dumb-bot/deno.json b/dumb-bot/deno.json index 59a73118998a..f3cf9fce7c2d 100644 --- a/dumb-bot/deno.json +++ b/dumb-bot/deno.json @@ -1,7 +1,5 @@ { "name": "@artifact/dumb-bot", - "version": "0.0.8", - "exports": { - ".": "./mod.ts" - } + "version": "0.0.16", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } } diff --git a/dumb-bot/napp.jsonc b/dumb-bot/napp.json similarity index 100% rename from dumb-bot/napp.jsonc rename to dumb-bot/napp.json diff --git a/engine/README.md b/engine/README.md new file mode 100644 index 000000000000..c75b0bffe89c --- /dev/null +++ b/engine/README.md @@ -0,0 +1,3 @@ +# Engine + +What runs execution of napps in a multithreaded environment diff --git a/_import-artifact/isolates/artifact.ts b/engine/artifact.ts similarity index 94% rename from _import-artifact/isolates/artifact.ts rename to engine/artifact.ts index d6312a0ec937..f93156da6cff 100644 --- a/_import-artifact/isolates/artifact.ts +++ b/engine/artifact.ts @@ -1,8 +1,7 @@ import { assert, Debug } from '@utils' -import Executor from '../exe/exe.ts' -import IOChannel from '../io/io-channel.ts' +import Executor from '../../engine/exe/exe.ts' +import IOChannel from '../../engine/io/io-channel.ts' import { - C, ExeResult, freezePid, Functions, @@ -17,13 +16,21 @@ import { QueueMessage, ToApiType, } from '@/constants.ts' -import IA from '../isolate-api.ts' +import IA from '../../execution/napp-api.ts' import { doAtomicBranch, doAtomicCommit, doAtomicPierce } from '@io/io.ts' import DB from '../db.ts' -import FS from '../git/fs.ts' +import FS from '../../engine/git/fs.ts' import { z } from 'zod' const log = Debug('AI:artifact') +/** Artifact Context, including the db and executor */ +export type C = { + db: DB + exe: Executor + aesKey?: string + seed?: Deno.KvEntry<unknown>[] +} + export const parameters = { pierce: z.object({ pierce: pierceSchema }) } export const returns = { pierce: z.void() } diff --git a/_import-artifact/atomic.ts b/engine/atomic.ts similarity index 91% rename from _import-artifact/atomic.ts rename to engine/atomic.ts index 9b5ecc8df3e3..e7e05d04df80 100644 --- a/_import-artifact/atomic.ts +++ b/engine/atomic.ts @@ -1,5 +1,5 @@ import * as keys from './keys.ts' -import { hasPoolables } from '@/db.ts' +import { hasPoolables } from './db.ts' import { isMergeReply, PID, @@ -9,7 +9,9 @@ import { QueueMessageType, sha1, } from '@/constants.ts' -import { assert, Debug, isKvTestMode } from '@utils' +import { assert } from '@std/assert' +import Debug from 'debug' +import { isKvTestMode } from './utils.ts' const log = Debug('AI:db:atomic') @@ -47,7 +49,10 @@ export class Atomic { const poolKey = JSON.stringify(keys.getPoolKeyPrefix(poolable.target)) if (!this.#transmitted.has(poolKey)) { - this.#transmitted.set(poolKey, { pid: poolable.target, count: BigInt(0) }) + this.#transmitted.set(poolKey, { + pid: poolable.target, + count: BigInt(0), + }) } const buffer = this.#transmitted.get(poolKey) assert(buffer) @@ -122,11 +127,14 @@ export class Atomic { } #enqueue(message: QueueMessage) { assert(this.#atomic, 'Atomic not set') - const backoffSchedule = isKvTestMode() ? [] : undefined - this.#atomic = this.#atomic.enqueue(message, { - keysIfUndelivered: [keys.UNDELIVERED], - backoffSchedule, - }) + const options: { + keysIfUndelivered: Deno.KvKey[] + backoffSchedule?: number[] + } = { keysIfUndelivered: [keys.UNDELIVERED] } + if (isKvTestMode()) { + options.backoffSchedule = [] + } + this.#atomic = this.#atomic.enqueue(message, options) return this } #increasePool(pid: PID, amount: bigint) { diff --git a/_import-artifact/benchmarks/bench.ts b/engine/bench.ts similarity index 97% rename from _import-artifact/benchmarks/bench.ts rename to engine/bench.ts index e4aecc9af195..61ba96a138f5 100644 --- a/_import-artifact/benchmarks/bench.ts +++ b/engine/bench.ts @@ -2,7 +2,7 @@ import Benchmark from 'benchmark' import { Debug } from '@utils' import { Engine } from '@/engine.ts' import { Api } from '@/isolates/io-fixture.ts' -import { assert } from '@std/assert' +import { assert } from '@std/assert/assert' import DB from '@/db.ts' import { Crypto } from '../api/crypto.ts' import { Backchat } from '../api/client-backchat.ts' @@ -179,6 +179,8 @@ suite // }, // }) .on('cycle', (event: Benchmark.Event) => { + // console output is how the results to the benchmark are captured + // deno-lint-ignore no-console console.log(String(event.target)) }) .on('complete', async function () { diff --git a/engine/constants.ts b/engine/constants.ts new file mode 100644 index 000000000000..e2b4c18c8888 --- /dev/null +++ b/engine/constants.ts @@ -0,0 +1,95 @@ +import IA from './isolate-api.ts' +export type { IA } +export const IO_PATH = '.io.json' +import { + type Backchat, + Change, + CommitObject, + EngineInterface, + IsolateReturn, + MetaPromise, + Outcome, + Params, + PID, + Pierce, + RemoteRequest, + Request, + SolidRequest, + UnsequencedRequest, +} from './api/types.ts' +import FS from '@/git/fs.ts' +import type DB from '@/db.ts' +import type Executor from '@/exe/exe.ts' +import { assert, equal } from '@utils' +import { JsonSchema7ObjectType, zodToJsonSchema } from 'zod-to-json-schema' +import { ZodObject, ZodSchema, ZodUnknown } from 'zod' + +export const REPO_LOCK_TIMEOUT_MS = 5000 + +export type IsolateLifecycle = { + '@@mount'?: (api: IA) => Promise<IsolateReturn> | IsolateReturn + '@@unmount'?: (api: IA) => Promise<IsolateReturn> | IsolateReturn +} + +export type Solids = { + oid: string + commit: CommitObject + /** Changed files in this commit. Empty change signals deletion. */ + changes: { [key: string]: Change } + exe?: { request: SolidRequest; sequence: number } + branches: number[] + poolables: (MergeReply | RemoteRequest)[] + deletes: { pid: PID; commit: string }[] +} +export type Branched = { + /** The first request in the new branch */ + origin: SolidRequest + /** The branch PID that needs to be created in /.git/refs */ + pid: PID + /** The head of the new branch that needs to be created in /.git/refs */ + head: string +} + +/** + * Messages that go on the queue are one of three types. Each one is an + * operation that will result in a new commit, atomically. Each operation is + * able to detect when it is a duplicate task due to duplicate message delivery. + * Each task will continue to retry until it is successful, as long as its check + * for duplication reassures it to keep trying. + */ +export type QueueMessage = QueuePool | QueueExe | QueueBranch +export enum QueueMessageType { + POOL = 'pool', + EXECUTION = 'exe', + BRANCH = 'branch', +} +export type QueuePool = { + type: QueueMessageType.POOL + pid: PID +} +export type QueueExe = { + type: QueueMessageType.EXECUTION + pid: PID + commit: string + sequence: number +} +export type QueueBranch = { + type: QueueMessageType.BRANCH + parentCommit: string + parentPid: PID + sequence: number +} +export const isQueuePool = (m: QueueMessage): m is QueuePool => { + return m.type === QueueMessageType.POOL +} +export const isQueueExe = (m: QueueMessage): m is QueueExe => { + return m.type === QueueMessageType.EXECUTION +} +export const isQueueBranch = (m: QueueMessage): m is QueueBranch => { + return m.type === QueueMessageType.BRANCH +} +export const isChildOf = (child: PID, parent: PID) => { + const childParent = { ...child, branches: child.branches.slice(0, -1) } + return equal(childParent, parent) +} +export const isBaseRepo = (pid: PID) => pid.branches.length === 1 diff --git a/_import-artifact/cradle-maker.ts b/engine/cradle-maker.ts similarity index 79% rename from _import-artifact/cradle-maker.ts rename to engine/cradle-maker.ts index 85f05ad6f5e6..710148b04206 100644 --- a/_import-artifact/cradle-maker.ts +++ b/engine/cradle-maker.ts @@ -1,7 +1,6 @@ import { Engine } from './engine.ts' import { Crypto } from './api/crypto.ts' import DB from '@/db.ts' -import { type CradleMaker, Provisioner } from '@/constants.ts' import { Backchat } from './api/client-backchat.ts' import { mockCreator } from '@/isolates/utils/mocker.ts' import z from 'zod' @@ -18,6 +17,23 @@ type SeedSet = { } const seeds = new Map<Provisioner | undefined, SeedSet>() +export type Provisioner = (superBackchat: Backchat) => Promise<void> + +export type CradleMaker = ( + t: Deno.TestContext, + /** The file url that the snapshots are associated with */ + snapshotsFor: string, + updateSnapshots?: 'updateSnapshots', + init?: Provisioner, +) => Promise< + { + backchat: Backchat + engine: unknown + privateKey: string + [Symbol.asyncDispose](): Promise<void> + } +> + export const cradleMaker: CradleMaker = async (t, url, update, init) => { const mock = mockCreator(z.unknown()) t.origin = url diff --git a/_import-artifact/db.test.ts b/engine/db.test.ts similarity index 100% rename from _import-artifact/db.test.ts rename to engine/db.test.ts diff --git a/_import-artifact/db.ts b/engine/db.ts similarity index 94% rename from _import-artifact/db.ts rename to engine/db.ts index c7bfdebf3640..09f8ad005b34 100644 --- a/_import-artifact/db.ts +++ b/engine/db.ts @@ -1,21 +1,12 @@ import { pushable } from 'it-pushable' import { BLOB_META_KEY } from '@kitsonk/kv-toolbox/blob' import { CryptoKv, generateKey } from '@kitsonk/kv-toolbox/crypto' -import * as keys from '@/keys.ts' -import { - freezePid, - MergeReply, - PID, - Poolable, - PooledRef, - pooledRef, - print, - RemoteRequest, - REPO_LOCK_TIMEOUT_MS, - sha1, - Splice, -} from '@/constants.ts' -import { assert, Debug, equal, openKv, posix } from '@utils' +import * as keys from './keys.ts' +import { freezePid } from '../processes/processes.ts' +import { assert } from '@std/assert/assert' +import Debug from 'debug' +import equal from 'fast-deep-equal' +import * as posix from '@std/path/posix' import { Atomic } from './atomic.ts' import { QueueMessage } from '@/constants.ts' import { decodeTime, ulid } from 'ulid' @@ -507,3 +498,29 @@ export const hasPoolables = ( } return counter.value > marker.value } + +const isDenoDeploy = Deno.env.get('DENO_DEPLOYMENT_ID') !== undefined +let _isTestMode = false +export const isKvTestMode = () => { + return _isTestMode +} +export const openKv = async () => { + if (isDenoDeploy) { + return Deno.openKv() + } + const KEY = 'DENO_KV_PATH' + let path = ':memory:' + const permission = await Deno.permissions.query({ + name: 'env', + variable: KEY, + }) + if (permission.state === 'granted') { + const env = Deno.env.get(KEY) + if (env) { + path = env + } + } + log('open kv', path) + _isTestMode = path === ':memory:' + return Deno.openKv(path) +} diff --git a/engine/deno.json b/engine/deno.json new file mode 100644 index 000000000000..f4db97f2e2e9 --- /dev/null +++ b/engine/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/engine", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/_import-artifact/engine.test.ts b/engine/engine.test.ts similarity index 100% rename from _import-artifact/engine.test.ts rename to engine/engine.test.ts diff --git a/_import-artifact/engine.ts b/engine/engine.ts similarity index 96% rename from _import-artifact/engine.ts rename to engine/engine.ts index 6b3dc7ecc7ee..72fd4a7f732d 100644 --- a/_import-artifact/engine.ts +++ b/engine/engine.ts @@ -1,5 +1,5 @@ -import { transcribe } from './isolates/ai-completions.ts' -import Compartment from './io/compartment.ts' +import { transcribe } from '../openai/ai-completions.ts' +import Compartment from '../execution/compartment.ts' import '@std/dotenv/load' import { addBranches, @@ -18,7 +18,7 @@ import { Provisioner, } from './constants.ts' import * as actor from './api/isolates/actor.ts' -import IA from './isolate-api.ts' +import NappApi from '../api/napp-api.old.ts' import { assert, Debug, posix } from '@utils' import FS from '@/git/fs.ts' import * as artifact from '@/isolates/artifact.ts' @@ -34,7 +34,7 @@ type Seed = Deno.KvEntry<unknown>[] export class Engine implements EngineInterface { #superuserKey: string #compartment: Compartment - #api: IA<C> + #api: NappApi<C> #pierce: artifact.Api['pierce'] #homeAddress: PID | undefined #githubAddress: PID | undefined @@ -42,7 +42,7 @@ export class Engine implements EngineInterface { private constructor( compartment: Compartment, - api: IA<C>, + api: NappApi<C>, superuserKey: string, ) { this.#compartment = compartment @@ -52,8 +52,8 @@ export class Engine implements EngineInterface { this.#superuserKey = superuserKey } static async boot(superuserKey: string, aesKey: string, seed?: Seed) { - const compartment = await Compartment.create('artifact') - const api = IA.createContext<C>() + const compartment = await Compartment.load('@artifact/engine') + const api = NappApi.createContext<C>() api.context = { aesKey, seed } await compartment.mount(api) const engine = new Engine(compartment, api, superuserKey) @@ -236,10 +236,6 @@ export class Engine implements EngineInterface { // version, deployment location, etc // if you want to ping in a chain, use an isolate } - async apiSchema(isolate: string) { - const compartment = await Compartment.create(isolate) - return compartment.api - } async transcribe(audio: File) { assert(audio instanceof File, 'audio must be a File') const text = await transcribe(audio) diff --git a/_import-artifact/io/io.ts b/engine/io.ts similarity index 100% rename from _import-artifact/io/io.ts rename to engine/io.ts diff --git a/_import-artifact/keys.ts b/engine/keys.ts similarity index 98% rename from _import-artifact/keys.ts rename to engine/keys.ts index f96efe38102d..cb0ffee81642 100644 --- a/_import-artifact/keys.ts +++ b/engine/keys.ts @@ -5,7 +5,7 @@ import { Poolable, SolidRequest, } from '@/constants.ts' -import { assert } from '@utils' +import { assert } from '@std/assert/assert' /** The current count of everything that has been atomically pooled */ export const POOL_COUNTER = 'counter' /** What value the pool has been processed up to */ diff --git a/engine/mod.ts b/engine/mod.ts new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/engine/napp.json b/engine/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/engine/napp.json @@ -0,0 +1 @@ +{} diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 000000000000..5cd479653c33 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,4 @@ +# evals + +> [!NOTE] +> The system prompt for this agent: [instructions.md](evals/instructions.md) diff --git a/evals/deno.json b/evals/deno.json new file mode 100644 index 000000000000..fb682be9aa72 --- /dev/null +++ b/evals/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/evals", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/evals/mod.ts b/evals/mod.ts new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/evals/napp.json b/evals/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/evals/napp.json @@ -0,0 +1 @@ +{} diff --git a/_import-artifact/isolates/test-case-runner.test.ts b/evals/test-case-runner.test.ts similarity index 100% rename from _import-artifact/isolates/test-case-runner.test.ts rename to evals/test-case-runner.test.ts diff --git a/_import-artifact/isolates/test-case-runner.ts b/evals/test-case-runner.ts similarity index 99% rename from _import-artifact/isolates/test-case-runner.ts rename to evals/test-case-runner.ts index 736d1ef29a4d..ea5a67b3dfc7 100644 --- a/_import-artifact/isolates/test-case-runner.ts +++ b/evals/test-case-runner.ts @@ -15,7 +15,7 @@ import * as session from '@/isolates/session.ts' import * as longthread from '@/isolates/longthread.ts' import { loadAgent } from '@/isolates/utils/load-agent.ts' import { getChatParams } from '@/isolates/ai-completions.ts' -import { loadTools } from './utils/ai-load-tools.ts' +import { loadTools } from '../_import-artifact/isolates/utils/ai-load-tools.ts' import { addCaseResult, addIteration, diff --git a/_import-artifact/tests/test-controller.test.ts b/evals/test-controller.test.ts similarity index 100% rename from _import-artifact/tests/test-controller.test.ts rename to evals/test-controller.test.ts diff --git a/_import-artifact/isolates/test-controller.ts b/evals/test-controller.ts similarity index 100% rename from _import-artifact/isolates/test-controller.ts rename to evals/test-controller.ts diff --git a/test-file-runner/test-file-runner.md b/evals/test-file-runner.md similarity index 100% rename from test-file-runner/test-file-runner.md rename to evals/test-file-runner.md diff --git a/test-file-runner/test-file-runner.test.md b/evals/test-file-runner.test.md similarity index 97% rename from test-file-runner/test-file-runner.test.md rename to evals/test-file-runner.test.md index d032f61240d9..a2c8de658673 100644 --- a/test-file-runner/test-file-runner.test.md +++ b/evals/test-file-runner.test.md @@ -1,5 +1,5 @@ --- -target: agents/test-file-runner.md +target: agents/evals.md assessor: agents/test-assessor.md --- diff --git a/_import-artifact/tests/test-file-runner.test.ts b/evals/test-file-runner.test.ts similarity index 100% rename from _import-artifact/tests/test-file-runner.test.ts rename to evals/test-file-runner.test.ts diff --git a/_import-artifact/isolates/test-registry.ts b/evals/test-registry.ts similarity index 100% rename from _import-artifact/isolates/test-registry.ts rename to evals/test-registry.ts diff --git a/_import-artifact/tests/test-requester.test.ts b/evals/test-requester.test.ts similarity index 100% rename from _import-artifact/tests/test-requester.test.ts rename to evals/test-requester.test.ts diff --git a/_import-artifact/isolates/tps-report.ts b/evals/tps-report-creation.ts similarity index 97% rename from _import-artifact/isolates/tps-report.ts rename to evals/tps-report-creation.ts index 0de2cb012521..4919e8f7c401 100644 --- a/_import-artifact/isolates/tps-report.ts +++ b/evals/tps-report-creation.ts @@ -56,6 +56,7 @@ export const functions: Functions<Api> = { await loadAgent(assessor, api) const tpsPath = getTpsPath(path) const hash = await api.readOid(path) + // TODO use the snapshot it / commit id and filepath. let tpsReport = create(path, target, assessor, iterations, hash) for (const { name, promptLists, expectations, dependencies } of cases) { diff --git a/_import-artifact/isolates/tps-report.test.ts b/evals/tps-report.test.ts similarity index 100% rename from _import-artifact/isolates/tps-report.test.ts rename to evals/tps-report.test.ts diff --git a/_import-artifact/api/tps-report.ts b/evals/tps-report.ts similarity index 99% rename from _import-artifact/api/tps-report.ts rename to evals/tps-report.ts index b09d5458aff3..c9c6007f884e 100644 --- a/_import-artifact/api/tps-report.ts +++ b/evals/tps-report.ts @@ -1,4 +1,4 @@ -import { md5 } from './types.ts' +import { md5 } from '../api/types.ts' import { z } from 'zod' export const outcome = z diff --git a/_import-artifact/tests/tps-reports.test.ts b/evals/tps-reports.test.ts similarity index 100% rename from _import-artifact/tests/tps-reports.test.ts rename to evals/tps-reports.test.ts diff --git a/execution/README.md b/execution/README.md new file mode 100644 index 000000000000..a9da3e13f16e --- /dev/null +++ b/execution/README.md @@ -0,0 +1,107 @@ +# Execution + +## Overview + +The execution system provides a repeatable way to handle asynchronous +operations, file system interactions, and remote procedure calls in a +distributed environment. It ensures consistency through a trail-based execution +model with filesystem snapshots. + +## Core Principles + +1. **Action-Based Execution Model** + - Input: JSON action + filesystem snapshot + - Output: JSON reply + JSON actions + filesystem changes + - File IO contents never appear in actions; filesystem is referenced instead + +2. **Trail System** + - Tracks execution paths with sequence numbers + - Manages payloads separately from actions + - Supports pause/resume of executions + - Handles timeouts and activation states + - Manages request sequencing and replies + - Processes pending operations + - Maintains execution order through sequence numbers + +3. **Compartment Isolation** + - Provides isolated execution environments + - Manages mounting/unmounting of napps + - Handles API access and state management + - Supports effect recovery and context management + +## Key Components + +### Executor + +Handles the main execution flow: + +- Function caching +- Trail management +- File system state tracking +- Promise-based JavaScript interface + +### Trail + +Provides execution path tracking: + +- Action sequencing +- Payload management +- State transitions +- Request queuing and reply handling +- Pending operation management +- JSON-based execution state storage +- Historical reply storage for replay capability + +### Compartment + +Provides isolation: + +- Napp loading +- API mounting +- Process management +- Effect handling + +## Error Handling + +1. **Execution Errors** + - Function not found + - Invalid parameters + - Timeout errors + +## Best Practices + +1. **Action Design** + - Keep actions pure and serializable + - Separate file contents from actions + - Use proper sequence numbers + - Handle timeouts appropriately + +2. **State Management** + - Track filesystem changes carefully + - Manage pending operations properly + - Handle effect recovery + - Maintain proper execution order + +3. **Error Handling** + - Implement proper error recovery + - Handle timeout cases + - Track execution failures + +## Implementation Notes + +1. **File Operations** + - File contents are handled through payloads + - Files are referenced by path in actions + - Changes are tracked through snapshots + +2. **Execution Flow** + - Actions are sequenced + - Replies are matched to requests + - Pending operations are tracked + - State is maintained consistently + +3. **Performance Considerations** + - Function caching is available + - Execution can be paused/resumed + - State changes are tracked efficiently + - File operations are optimized diff --git a/execution/api.ts b/execution/api.ts new file mode 100644 index 000000000000..34365f25e114 --- /dev/null +++ b/execution/api.ts @@ -0,0 +1,245 @@ +import { + type NappApi, + stateSchema, + type TreeEntry, +} from '@artifact/api/napp-api' +import Debug from 'debug' +import type { Trail } from './trail.ts' +import { type Action, jsonSchema } from '../api/actions.ts' +import type { z } from 'zod' + +export class Api implements NappApi { + // all this thing does is create actions + // these are then interpreted by whatever is running them + // so it can be used for client side, inside a napp, or for handling different + // processes, branches, and remote repositories + + static create(trail: Trail) { + return new Api(trail) + } + + private constructor(trail: Trail) { + this.#state = createState(trail) + this.#read = createRead(trail) + this.#write = createWrite(trail) + this.#processes = createProcesses(trail) + this.#effects = createEffects(trail) + + Object.freeze(this) + Object.freeze(this.#state) + Object.freeze(this.#read) + Object.freeze(this.#write) + Object.freeze(this.#processes) + Object.freeze(this.#effects) + } + + #state: NappApi['state'] + #read: NappApi['read'] + #write: NappApi['write'] + #processes: NappApi['processes'] + #effects: NappApi['effects'] + + get state() { + return this.#state + } + get read() { + return this.#read + } + get write() { + return this.#write + } + get processes() { + return this.#processes + } + get effects() { + return this.#effects + } +} + +const createState = (trail: Trail) => { + const state: NappApi['state'] = { + get: async (options) => { + const { schema = stateSchema, fallback, ...rest } = options || {} + const action: Action = { + napp: '@artifact/api', + tool: 'state/get', + parameters: { options: rest }, + } + const result = await trail.push(action) + if (result === undefined && fallback !== undefined) { + return fallback + } + return schema.parse(result) as z.infer<typeof schema> + }, + set: async (state, options) => { + const { schema = stateSchema, ...rest } = options || {} + schema.parse(state) + const action: Action = { + napp: '@artifact/api', + tool: 'state/set', + parameters: { options: rest }, + } + await trail.push(action, { json: state }) + }, + } + return state +} + +const createRead = (trail: Trail) => { + const read: NappApi['read'] = { + meta: (path, options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'read/meta', + parameters: { path, options }, + } + return trail.push<TreeEntry>(action) + }, + json: async (path, options) => { + const { schema = jsonSchema, ...rest } = options || {} + const text = await read.text(path, rest) + const object = JSON.parse(text) + return schema.parse(object) as z.infer<typeof schema> + }, + text: async (path, options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'read/text', + parameters: { path, options }, + } + const text = await trail.push<string>(action) + return text + }, + binary: async (path, options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'read/binary', + parameters: { path, options }, + } + const binary = await trail.push<Uint8Array>(action) + return binary + }, + exists: async (path, options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'read/exists', + parameters: { path, options }, + } + const exists = await trail.push<boolean>(action) + return exists + }, + ls: async (path = '.', options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'read/ls', + parameters: { path, options }, + } + const entries = await trail.push<TreeEntry[]>(action) + return entries + }, + } + return read +} + +const createWrite = (trail: Trail) => { + const write: NappApi['write'] = { + json: async (path, content, options) => { + const text = JSON.stringify(content, null, 2) + await write.text(path, text, options) + }, + text: async (path, content, options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'write/text', + parameters: { path, options }, + } + await trail.push(action, { text: content }) + }, + binary: async (path, content, options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'write/binary', + parameters: { path, options }, + } + await trail.push(action, { data: content }) + }, + rm: async (path, options = {}) => { + const action: Action = { + napp: '@artifact/api', + tool: 'write/rm', + parameters: { path, options }, + } + await trail.push(action) + }, + mv: async (from, to) => { + const action: Action = { + napp: '@artifact/api', + tool: 'write/mv', + parameters: { from, to }, + } + await trail.push(action) + }, + cp: async (from, to) => { + const action: Action = { + napp: '@artifact/api', + tool: 'write/cp', + parameters: { from, to }, + } + await trail.push(action) + }, + } + return write +} + +const createProcesses = (trail: Trail) => { + const processes: NappApi['processes'] = { + spawn() { + return Promise.reject('Not implemented') + }, + kill() { + return Promise.reject('Not implemented') + }, + async() { + return Promise.reject('Not implemented') + }, + mv() { + return Promise.reject('Not implemented') + }, + nice() { + return Promise.reject('Not implemented') + }, + dispatch: async (action, options) => { + const _action = { ...action, files: action.files || [] } + const wrapped: Action = { + napp: '@artifact/api', + tool: 'processes/dispatch', + parameters: { action: _action, options }, + } + // ?? how to send up the list of files that were provided ? + // ? what if they were part of tip and had not yet been committed ? + return trail.push(wrapped) + }, + dispatchWithMeta: async (action, options) => { + return Promise.reject('Not implemented') + }, + } + return processes +} + +const createEffects = (trail: Trail) => { + const effects: NappApi['effects'] = { + get signal() { + return trail.signal + }, + get isEffectRecovered() { + return false // TODO + }, + set context(value) { + trail.context = value + }, + get context() { + return trail.context + }, + } + return effects +} diff --git a/execution/compartment.test.ts b/execution/compartment.test.ts new file mode 100644 index 000000000000..66b6c4537b51 --- /dev/null +++ b/execution/compartment.test.ts @@ -0,0 +1,49 @@ +import Compartment from './compartment.ts' +import { expect } from '@std/expect' +import actionCreators from '@artifact/api/actions' +import { Trail, TrailStopReason } from './trail.ts' +import { Tip } from '@artifact/snapshots/tip' +import { MockProvider } from '@artifact/snapshots/mock-provider' +import Debug from 'debug' +import { resolveLocalFiles } from './filesResolver.ts' + +Deno.test('compartment loads a napp', async (t) => { + Debug.enable('@artifact/execution, @artifact/files') + const actions = await actionCreators('@artifact/files') + const action = actions.write({ path: 'test.txt', content: 'hello world' }) + const trail = Trail.create(action) + + await Compartment.load('@artifact/files', trail) + + let stopReason: TrailStopReason + const provider = MockProvider.create() + const tip = Tip.create(provider) + + let withResolves = trail.export() + do { + stopReason = await trail.activate(withResolves) + if (stopReason === TrailStopReason.Triggered) { + console.log('triggered') + console.dir(trail.export()) + + withResolves = await resolveLocalFiles(trail, tip) + } + } while (stopReason !== TrailStopReason.Settled) + + expect(tip.isChanged).toBe(true) + expect(await tip.read.text('test.txt')).toBe('hello world') + + console.dir(trail.export()) +}) + +// test throwing something that is not an error object + +// test returning but having some triggered actions outstanding + +Deno.test('napp api', async (t) => { + // test generating a typed based function call, using a separate library from + // compartment, since compartment is an executor, but actions could be + // generated anywhere. +}) +// schema checking needs to be done by the compartment or the trail ? +// returns checking needs to be done as well diff --git a/execution/compartment.ts b/execution/compartment.ts new file mode 100644 index 000000000000..18b497990ca5 --- /dev/null +++ b/execution/compartment.ts @@ -0,0 +1,88 @@ +import napps from './napps-import.ts' +import type { NappApi } from '@artifact/api/napp-api' +import Debug from 'debug' +import type { Trail } from './trail.ts' +import { assert } from '@std/assert/assert' +import { Api } from './api.ts' + +const log = Debug('@artifact/execution') + +export default class Compartment { + static async load(napp: keyof typeof napps, trail: Trail) { + await Promise.resolve() // simulates loading from the network + const compartment = new Compartment(napp, trail) + compartment.#execute() + return compartment + } + + // TODO figure out how to do types cleanly + // deno-lint-ignore ban-types + #napp: Record<string, Function> + + readonly #trail: Trail + + private constructor(napp: keyof typeof napps, trail: Trail) { + log('load napp:', napp, trail) + if (napp !== trail.origin.napp) { + throw new Error('Napp mismatch: ' + napp + ' !== ' + trail.origin.napp) + } + this.#napp = napps[napp] + this.#trail = trail + trail.signal.onabort = () => this.#tearDown() + } + async #execute() { + await this.#trail.waitForActivation() + try { + const { tool } = this.#trail.origin + log('execute', tool) + if (!(tool in this.#napp)) { + throw new Error('Tool not found: ' + tool) + } + if (typeof this.#napp[tool] !== 'function') { + throw new Error('Tool is not a function: ' + tool) + } + + const api = Api.create(this.#trail) + const result = await this.#napp[tool](this.#trail.origin.parameters, api) + this.#trail.resolve(result) + } catch (error) { + this.#trail.reject(error as Error) + } + } + // what about holding off on the outcome until all the actions were + // transmitted ? + // how to send of actions that have no reply ? + // what if the function completes AFTER the trail trigger ? + + // test firing off some promises then returning, and it should skip those + // promises as well as ignoring their return values + + #tearDown() { + log('tear down') + return Promise.resolve() + } + /** + * Mount the isolate as a side effect, and give it the chance to initialize + * some context that will get passed between different invocations on the + * same mount. + * @param api : IsolateApi + * @returns Promise<void> | void + */ + mount(api: NappApi) { + // TODO use exe to ensure that mount stops working arfter invocation + if ('@@mount' in this.#napp) { + assert(typeof this.#napp['@@mount'] === 'function', 'Invalid mount') + return this.#napp['@@mount'](api) + } + } + /** + * Unmount the isolate as a side effect, and give it the chance to clean up + * @param api : IsolateApi + */ + unmount(api: NappApi) { + if ('@@unmount' in this.#napp) { + assert(typeof this.#napp['@@unmount'] === 'function', 'Invalid unmount') + return this.#napp['@@unmount'](api) + } + } +} diff --git a/execution/deno.json b/execution/deno.json new file mode 100644 index 000000000000..663e5536e80a --- /dev/null +++ b/execution/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/execution", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./trail": "./trail.ts" } +} diff --git a/_import-artifact/exe/exe.ts b/execution/exe.old.ts similarity index 76% rename from _import-artifact/exe/exe.ts rename to execution/exe.old.ts index cc10826c04ee..dbb32142361a 100644 --- a/_import-artifact/exe/exe.ts +++ b/execution/exe.old.ts @@ -1,20 +1,16 @@ -import IOChannel from '../io/io-channel.ts' +import IOChannel from './io-channel.ts' import { getExeId } from '@/keys.ts' -import { - C, - ExeResult, - JsonValue, - Outcome, - SolidReply, - SolidRequest, -} from '@/constants.ts' -import IA from '../isolate-api.ts' -import Compartment from '../io/compartment.ts' +import { C, JsonValue, Outcome, SolidReply, SolidRequest } from '@/constants.ts' +import NappApi from '../../api/napp-api.ts' +import Compartment from '../../execution/compartment.ts' import { assert, Debug, equal, serializeError } from '@utils' -import Accumulator from '@/exe/accumulator.ts' +import type Accumulator from '@/exe/accumulator.ts' import FS from '@/git/fs.ts' const log = Debug('AI:exe') +// Exe interface should simply take an accumulator +// then wrap it in the caching layer to allow executions to be continued + export default class Executor { #functions = new Map<string, Execution>() #functionCacheDisabled = false @@ -53,7 +49,7 @@ export default class Executor { // TODO read side effect config from io.json const { runnable, commit } = io.getExecution() assert(runnable, 'origin not found') - const isolateApi = IA.create(accumulator, runnable, commit, opts) + const isolateApi = NappApi.create(accumulator, runnable, commit, opts) if (req.isolate === 'system') { log('system isolate') isolateApi.context = c @@ -103,7 +99,10 @@ export default class Executor { const trigger = Symbol('🏎️') const accumulatorPromise = execution.accumulator.activate(trigger) - const outcome = await Promise.race([execution.function, accumulatorPromise]) + const outcome = await Promise.race([ + execution.function, + accumulatorPromise, + ]) execution.accumulator.deactivate() let result: ExeResult @@ -133,6 +132,30 @@ export default class Executor { type Execution = { function: Promise<Outcome> accumulator: Accumulator - api: IA + api: NappApi commits: string[] } +export type ExeResult = ExeSettled | ExePending +type ExeResultBase = { + /** + * The last filesystem that was modified during the execution run. The FS + * might have been bumped forwards if accumulations occurred. + */ + fs: FS + /** If this is a side effect request, this is the lock held by for it */ + effectsLock?: Deno.KvEntry<string> +} +type ExeSettled = ExeResultBase & { + reply: SolidReply +} +type ExePending = ExeResultBase & { + pending: Pending +} +export type Pending = { + /** The commit that caused the requests to be generated */ + commit: string + /** The requests that were generated by the latest round of execution */ + requests: UnsequencedRequest[] + /** The sequence number to accumulate the pending requests against */ + sequence: number +} diff --git a/_import-artifact/exe/exe.test.ts b/execution/exe.test.ts similarity index 98% rename from _import-artifact/exe/exe.test.ts rename to execution/exe.test.ts index 6f6530e75194..20ad3c29207a 100644 --- a/_import-artifact/exe/exe.test.ts +++ b/execution/exe.test.ts @@ -1,6 +1,6 @@ -import IOChannel from '../io/io-channel.ts' +import IOChannel from '../engine/exe./../execution/io-channel.ts' import FS from '../git/fs.ts' -import Executor from './exe.ts' +import Executor from '../engine/exe./../execution/exe.ts' import { C, isPierceRequest, diff --git a/execution/files-resolver.ts b/execution/files-resolver.ts new file mode 100644 index 000000000000..cdbccb54c593 --- /dev/null +++ b/execution/files-resolver.ts @@ -0,0 +1 @@ +// resolves requests for files, both local and foreign. diff --git a/execution/filesResolver.ts b/execution/filesResolver.ts new file mode 100644 index 000000000000..9ed708f6ea58 --- /dev/null +++ b/execution/filesResolver.ts @@ -0,0 +1,80 @@ +import { entries, type Trail, type TrailStruct } from './trail.ts' +import type { NappLocal, Tip } from '@artifact/snapshots/tip' +import { assert } from '@std/assert/assert' +import { type Address, addressSchema } from '@artifact/api/napp-api' + +export const resolveLocalFiles = async (trail: Trail, tip: Tip) => { + const withResolves = trail.export() + for (const [index, request] of entries(withResolves)) { + if (isApiRequest(request)) { + // TODO resolve self addresses + console.log('local file request') + + const [category, command] = request.origin.tool.split('/') + switch (category as keyof NappLocal) { + case 'read': { + const { path = '.', options: o = {} } = request.origin.parameters + assert(typeof path === 'string', 'path must be a string') + const options = addressSchema.parse(o) as Address + switch (command as keyof NappLocal['read']) { + case 'meta': { + const result = await tip.read.meta(path, options) + console.log('result', result) + break + } + case 'json': { + const result = await tip.read.json(path, options) + console.log('result', result) + break + } + default: + throw new Error('Command not found: ' + command) + } + break + } + case 'write': { + const { path } = request.origin.parameters + switch (command as keyof NappLocal['write']) { + case 'json': { + assert(typeof path === 'string', 'path must be a string') + const upsert = trail.extractPayload(index) + if ('json' in upsert) { + console.log('upsert', upsert) + await tip.write.json(path, upsert.json) + break + } + throw new Error('json not in upsert') + } + case 'text': { + assert(typeof path === 'string', 'path must be a string') + const upsert = trail.extractPayload(index) + if ('text' in upsert) { + console.log('upsert', upsert) + await tip.write.text(path, upsert.text) + break + } + throw new Error('text not in upsert') + } + } + break + } + default: + throw new Error('Category not found: ' + category) + } + } + + // now the tip is reconciled with the payload buffer, so resolve the + // action + + const snapshot = await tip.snapshots.latest() + request.outcome = { result: { snapshot: snapshot || null } } + } + return withResolves +} + +const isApiRequest = (request: TrailStruct) => { + if (request.origin.napp === '@artifact/api') { + return true + } + return false +} diff --git a/_import-artifact/io/io-channel.test.ts b/execution/io-channel.test.ts similarity index 100% rename from _import-artifact/io/io-channel.test.ts rename to execution/io-channel.test.ts diff --git a/_import-artifact/io/io-channel.ts b/execution/io-channel.ts similarity index 80% rename from _import-artifact/io/io-channel.ts rename to execution/io-channel.ts index 5ee25fc84580..cfa7a8dc4f98 100644 --- a/_import-artifact/io/io-channel.ts +++ b/execution/io-channel.ts @@ -1,11 +1,10 @@ /** * Manages the .io.json file */ - -import { assert, equal } from '@utils' +import { actionSchema } from '@artifact/api/actions' +import equal from 'fast-deep-equal' +import { assert } from '@std/assert/assert' import { - IoStruct, - ioStruct, isMergeReply, isPierceRequest, isRemoteRequest, @@ -21,6 +20,8 @@ import { import Accumulator from '@/exe/accumulator.ts' import FS from '@/git/fs.ts' import { IsolatePromise } from '@/constants.ts' +import { z } from 'zod' +import { outcomeSchema } from '../../api/actions.ts' const createBase = () => ioStruct.parse({ @@ -468,3 +469,89 @@ export const toUnsequenced = ( const { sequence: _, source: __, ...unsequenced } = request return unsequenced } + +const sequenceInteger = z.number().int().gte(0) +const sequenceKey = z.string().refine((data) => { + try { + return sequenceInteger.safeParse(Number.parseInt(data)).success + } catch (error) { + return !error + } +}, 'sequence key must be an integer') + +export const ioStruct = z.object({ + sequence: sequenceInteger, + /** The current sequence of the request being executed serially */ + executing: sequenceInteger.optional(), + /** The sequences of requests that have been executed serially */ + executed: z.record(sequenceKey, z.boolean()), + requests: z.record(sequenceKey, requestSchema), + replies: z.record(sequenceKey, outcomeSchema), + /** If a reply is a merge reply, the commit that carried it is stored here */ + parents: z.record(sequenceKey, md5), + /** + * If a request generates child requests, they are tracked here. The commit + * in each entry is the commit that caused the child requests to be generated. + * This is used to replay by resetting the fs to that commit and doing a + * replay. + */ + pendings: z.record( + sequenceKey, + z.array(z.object({ + commit: md5, + sequences: z.array(sequenceInteger), + })), + ), + /** Active branches are stored here. A branch is a daemon if it is listed + * here but its request has been replied to or it is gone from the requests + * list */ + branches: z.record(sequenceKey, z.string()), + /** + * Isolates can store values here and know they will not leak into other + * branches, and will be quick to access since the io file is always loaded. + */ + state: z.record(jsonSchema), +}) +export type IoStruct = z.infer<typeof ioStruct> + +// the accumulator should be a zod schema data structure, which is persisted. +// when filesystem ops occur, these are recovered at runtime during the replay, +// and not stored with these actions, which are meant for control, not for data +// transmission. +// BUT when replies come in and we want to merge, need to know what commit it +// was that it came in on + +// split io to have an outbox, where all the replies coming in + +// if pendings was a dedicated thing, then when the replies come back in, they +// can easily be assigned to the accumulator directly + +// here's an action, and here's the response, also if you want it, here's the +// changed filesystem snapshot that came back. +// the app should treat the reply as tho it was a remote filesystem call + +// what comes back should be a standard api call, that always returns the result +// as well as access to the filesystem of the returner. + +// the accumulator is just a channel, and each comms with a remote thing should +// be one of these. This might be the thing that gets merged in a comms branch. + +const spawn = z.object({ + proctype: Proctype, + /** + * Allow a custom name for the new branch, if this is a branching request + */ + branch: z.string().optional(), + /** + * If the custom branch name might not be unique, a prefix can be given and + * the sequence number will be appended to the branch name, ensuring + * uniqueness. + */ + branchPrefix: z.string().optional(), + /** + * If the request is a branching request, this will be the name of the new + * branch. If the branch already exists, the request will fail. + */ + branchName: z.string().optional(), +}) +export type Invocation = z.infer<typeof invocation> diff --git a/execution/mod.ts b/execution/mod.ts new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/execution/napps-import.ts b/execution/napps-import.ts new file mode 100644 index 000000000000..eaf303c87dc9 --- /dev/null +++ b/execution/napps-import.ts @@ -0,0 +1,14 @@ +/** + * Ensures that in deno deploy, these packages are available to be read from, as + * deno deploy has no dynamic runtime imports + */ + +import * as files from '@artifact/files' +import * as fixture from '@artifact/fixture' + +const napps = { + '@artifact/files': files, + '@artifact/fixture': fixture, +} + +export default napps diff --git a/execution/trail.test.ts b/execution/trail.test.ts new file mode 100644 index 000000000000..515e53cc5285 --- /dev/null +++ b/execution/trail.test.ts @@ -0,0 +1,160 @@ +import { expect } from '@std/expect/expect' +import { Trail, TrailStopReason } from './trail.ts' +import { assert } from '@std/assert/assert' + +Deno.test('trail', async () => { + const action = { + napp: 'napp', + tool: 'tool', + parameters: {}, + } + const trail = Trail.create(action) + const fake = trail.export() + expect(() => trail.push(action)).toThrow('not active') + + const triggerPromise = trail.activate() + + const pushPromise = trail.push(action) + + expect(pushPromise.then).toBeInstanceOf(Function) + expect(() => trail.export()).toThrow('is active') + await expect(trail.activate(fake)).rejects.toThrow('is active') + expect(await triggerPromise).toEqual(TrailStopReason.Triggered) + + const data = trail.export() + + const resolved = trail.export() + assert(resolved.requests[0]) + resolved.requests[0].outcome = { result: 'test result' } + + const rejected = trail.export() + assert(rejected.requests[0]) + rejected.requests[0].outcome = { error: { message: 'test error' } } + + trail.activate(resolved) + await expect(pushPromise).resolves.toBe('test result') + + await expect(trail.activate(rejected)).rejects.toThrow('Trail is active') + await expect(trail.activate(data)).rejects.toThrow('Trail is active') + + trail.resolve('stop') + + const retrail = Trail.recreate(data) + expect(retrail.export()).toEqual(data) + + const retrailPromise = retrail.activate() + const repush = retrail.push(action) + expect(repush).toBeInstanceOf(Promise) + await expect(retrailPromise).resolves.toEqual(TrailStopReason.Triggered) + + retrail.activate(rejected) + retrail.resolve(undefined) + await expect(repush).rejects.toThrow('test error') +}) + +Deno.test('nested trails - manual data structure', () => { + const childTrailData: TrailStruct = { + sequence: 1, + origin: { + napp: 'child-napp', + tool: 'child-tool', + parameters: {}, + }, + requests: { + 0: { + sequence: 0, + origin: { + napp: 'child-napp', + tool: 'child-action', + parameters: {}, + }, + requests: {}, + activeMs: 50, + options: { timeout: DEFAULT_TIMEOUT }, + outcome: { result: 'child result' }, + }, + }, + activeMs: 100, + options: { timeout: DEFAULT_TIMEOUT }, + outcome: { result: 'child final result' }, + } + + const parentTrailData: TrailStruct = { + sequence: 2, + origin: { + napp: 'parent-napp', + tool: 'parent-tool', + parameters: {}, + }, + requests: { + 0: { + sequence: 1, + origin: { + napp: 'parent-napp', + tool: 'parent-action-1', + parameters: {}, + }, + requests: childTrailData.requests, + activeMs: 150, + options: { timeout: DEFAULT_TIMEOUT }, + outcome: { result: 'parent result 1' }, + }, + 1: { + sequence: 0, + origin: { + napp: 'parent-napp', + tool: 'parent-action-2', + parameters: {}, + }, + requests: {}, + activeMs: 75, + options: { timeout: DEFAULT_TIMEOUT }, + outcome: { result: 'parent result 2' }, + }, + }, + activeMs: 300, + options: { timeout: DEFAULT_TIMEOUT }, + } + + const parentTrail = Trail.recreate(parentTrailData) + + // Test the structure + const exportedData = parentTrail.export() + expect(exportedData).toEqual(parentTrailData) + + // Test nested structure + expect(exportedData.requests[0].requests[0].outcome?.result).toBe( + 'child result', + ) + expect(exportedData.requests[0].outcome?.result).toBe('parent result 1') + expect(exportedData.requests[1].outcome?.result).toBe('parent result 2') + + // Test activation and push + parentTrail.activate().then(async (reason) => { + expect(reason).toBe(TrailStopReason.Triggered) + + const pushResult = await parentTrail.push({ + napp: 'parent-napp', + tool: 'parent-action-1', + parameters: {}, + }) + expect(pushResult).toBe('parent result 1') + + parentTrail.resolve('final result') + }) + + // Test final outcome + expect(parentTrail.waitForOutcome()).resolves.toBeUndefined() + expect(parentTrail.export().outcome?.result).toBe('final result') +}) + +// split out local actions and remote actions + +// get all transmission actions + +// get all replies that need transmission + +// ? should we self reply, so queue up our own actions internally, or use the +// same transmission processing system ? + +// test purging of the structure diff --git a/execution/trail.ts b/execution/trail.ts new file mode 100644 index 000000000000..25fc935ce0ea --- /dev/null +++ b/execution/trail.ts @@ -0,0 +1,349 @@ +import { + type Action, + actionSchema, + type JsonValue, + type Outcome, + outcomeSchema, +} from '@artifact/api/actions' +import type { Upsert } from '@artifact/api/napp-api' +import equal from 'fast-deep-equal' +import { assert } from '@std/assert/assert' +import { z } from 'zod' +import { deserializeError, serializeError } from 'serialize-error' +import Debug from 'debug' +import { expect } from '@std/expect/expect' +const log = Debug('@artifact/execution') + +export const DEFAULT_TIMEOUT = 200 +export enum TrailStopReason { + Triggered, + Settled, +} + +const sequence = z.number().int().gte(0) + +const baseTrailSchema = z.object({ + sequence, + origin: actionSchema, + outcome: outcomeSchema.optional(), + activeMs: z.number().int().gte(0), + options: z.object({ + /** Maximum execution time between async operations that the compartment + * will wait before terminating the execution */ + timeout: z.number().int().gte(0), + }), +}) +export type TrailStruct = z.infer<typeof baseTrailSchema> & { + requests: Record<number, TrailStruct> +} + +const trailSchema: z.ZodType<TrailStruct> = baseTrailSchema.extend({ + requests: z.lazy(() => z.record(sequence, trailSchema)), +}).refine((data) => { + for (const index in data.requests) { + if (Number(index) >= data.sequence) { + return false + } + } + return true +}, { message: 'Request index must be less than sequence' }) + +export class Trail { + static create(origin: Action) { + return new Trail({ + origin, + sequence: 0, + requests: {}, + activeMs: 0, + options: { timeout: DEFAULT_TIMEOUT }, + }) + } + static recreate(data: TrailStruct) { + return new Trail(data) + } + private constructor(data: TrailStruct) { + this.#data = data + } + + #index = 0 + #data: TrailStruct + #trigger: (() => void) | undefined + #hooks: Record<number, ReturnType<typeof hookPromise>> = {} + + #subscribers: Set<() => void> = new Set() + + #abort = new AbortController() + #settled = hookPromise<void>() + + #context: unknown + + #payloads: Record<number, Upsert> = {} + + hasPayload(index: number) { + return index in this.#payloads + } + injectPayload(index: number, upsert: Upsert) { + assert(!this.#payloads[index], 'Payload already exists: ' + index) + Object.freeze(upsert) + this.#payloads[index] = upsert + } + extractPayload(index: number) { + const payload = this.#payloads[index] + if (!payload) { + throw new Error('Payload not found: ' + index) + } + delete this.#payloads[index] + return payload + } + get origin() { + return this.#data.origin + } + get signal() { + return this.#abort.signal + } + get context() { + // TODO check if this is a valid side effect + return this.#context + } + set context(value: unknown) { + this.#context = value + } + + abort() { + assert(!this.#trigger, 'Trail is active') + this.#abort.abort() + } + + waitForOutcome() { + return this.#settled.promise + } + + waitForActivation() { + assert(!this.#trigger, 'Trail is active') + return new Promise<void>((resolve) => { + this.#subscribers.add(resolve) + }) + } + + resolve(result: JsonValue | undefined) { + assert(!this.#data.outcome, 'Trail is already settled') + log('resolving') + this.#settled.resolve() + const outcome: Outcome = {} + if (result !== undefined) { + outcome.result = result + } + this.#data.outcome = outcome + } + reject(error: Error) { + assert(!this.#data.outcome, 'Trail is already settled') + log('rejecting') + this.#settled.resolve() + const outcome: Outcome = { error: serializeError(error) } + this.#data.outcome = outcome + } + + async activate(data?: TrailStruct) { + assert(!this.signal.aborted, 'Trail is aborted') + assert(!this.#trigger, 'Trail is active') + if (this.#data.outcome || data?.outcome) { + throw new Error('Trail is already settled') + } + + if (data) { + this.#absorb(data) + } + + const triggered = Symbol('🔫') + const triggeredPromise = new Promise<symbol>((resolve) => { + this.#trigger = () => resolve(triggered) + }) + const timeout = Symbol('⏰') + let timeoutId: number | undefined + const timeoutPromise = new Promise<symbol>((resolve) => { + const remainingMs = this.#data.options.timeout - this.#data.activeMs + timeoutId = setTimeout(() => resolve(timeout), remainingMs) + }) + const settled = Symbol('🏁') + const settledPromise = this.#settled.promise.then(() => settled) + + for (const resolve of this.#subscribers) { + resolve() + } + this.#subscribers.clear() + + const start = Date.now() + + const result = await Promise.race([ + settledPromise, + triggeredPromise, + timeoutPromise, + ]) + const end = Date.now() + this.#data.activeMs += end - start + this.#deactivate() + log('clearing timeout') + clearTimeout(timeoutId) + + // if timeout, then we should end with an error ? + + if (result === timeout) { + this.reject(new Error(`Timeout of ${this.#data.options.timeout} ms`)) + return TrailStopReason.Settled + } + if (result === triggered) { + return TrailStopReason.Triggered + } + if (result === settled) { + return TrailStopReason.Settled + } + throw new Error('Unknown race result: ' + result.toString()) + } + #deactivate() { + assert(this.#trigger, 'Trail is not active') + this.#trigger = undefined + } + #fireTrigger() { + if (this.#trigger) { + log('firing trigger') + this.#trigger() + return + } + throw new Error('Trail is not active') + } + /** If there is a stored response, will be returned here */ + push<T>(action: Action, upsert?: Upsert) { + assert(!this.signal.aborted, 'Trail is aborted') + assert(this.#trigger, 'Trail is not active') + const index = this.#index++ + const existing = this.#data.requests[index] + if (existing) { + if (!equal(existing.origin, action)) { + expect(action, 'Action mismatch').toEqual(existing.origin) + } + if (existing.outcome) { + if (existing.outcome.error) { + const error = deserializeError(existing.outcome.error) + throw error + } + if (this.hasPayload(index)) { + // TODO check if the upsert is the same using a hash / oid + const { result } = existing.outcome + const isSnapshot = typeof result !== 'object' || result === null || + !('snapshot' in result) + if (isSnapshot) { + throw new Error('Snapshot not found in result') + } + const upsert = this.extractPayload(index) + return Promise.resolve(unwrapUpsert(upsert) as T) + } + return Promise.resolve(existing.outcome.result as T) + } + } else { + this.#data.requests[index] = { + sequence: 0, + origin: action, + requests: {}, + activeMs: 0, + options: { timeout: DEFAULT_TIMEOUT }, + } + if (upsert) { + this.#payloads[index] = upsert + } + } + const hook = this.#hookPromise<T>(index) + return hook + } + #hookPromise<T>(index: number) { + assert(index >= 0, 'Index must be >= 0') + assert(!(index in this.#hooks), 'Promise already exists: ' + index) + this.#fireTrigger() + + const hook = hookPromise<T>() + this.#hooks[index] = hook as ReturnType<typeof hookPromise> + return hook.promise + } + /** + * Used so that an execution can be paused, then receive replies for + * accumulated actions, then continue without restarting the execution. Makes + * it easier to debug these functions, but also can be faster to execute. + * This is a nice to have and the operation is equally capable of starting + * again, if we find ourselves replaying the operation with no existing cache. + */ + #absorb(data: TrailStruct) { + assert(equal(data.origin, this.#data.origin), 'Trail origin mismatch') + assert(data.sequence >= this.#data.sequence, 'Sequence must be >= this') + assert(equal(data.activeMs, this.#data.activeMs), 'Active time mismatch') + + for (const [index, request] of entries(data)) { + const existing = this.#data.requests[index] + + if (existing) { + // TODO test that the incoming change is at least as big + if (!equal(request.origin, existing.origin)) { + expect(request.origin, 'Origin mismatch').toEqual(existing.origin) + } + if (existing.outcome) { + if (!equal(request, existing)) { + expect(request, 'Trail mismatch').toEqual(existing) + } + } else { + if (request.outcome && index < this.#index) { + const hook = this.#hooks[index] + assert(hook, 'Promise not found: ' + index) + if (request.outcome.error) { + const error = deserializeError(request.outcome.error) + hook.reject(error) + } else { + if (this.hasPayload(index)) { + const upsert = this.extractPayload(index) + hook.resolve(unwrapUpsert(upsert)) + } else { + hook.resolve(request.outcome.result) + } + } + delete this.#hooks[index] + } + } + } + this.#data.requests[index] = request + } + this.#data.sequence = data.sequence + } + export() { + assert(!this.#trigger, 'Trail is active') + return structuredClone(this.#data) + } +} + +export const entries = (data: TrailStruct) => { + return Object.entries(data.requests).map(([index, request]) => + [Number(index), request] as const + ) +} + +const hookPromise = <T>() => { + let resolve: (value: T) => void + let reject: (error: Error) => void + const promise = new Promise<T>((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve: resolve!, reject: reject! } +} + +const unwrapUpsert = (upsert: Upsert) => { + if ('meta' in upsert) { + return upsert.meta + } + if ('json' in upsert) { + return upsert.json + } + if ('text' in upsert) { + return upsert.text + } + if ('data' in upsert) { + return upsert.data + } + throw new Error('Invalid upsert') +} diff --git a/files/README.md b/files/README.md new file mode 100644 index 000000000000..4a7ee25b3398 --- /dev/null +++ b/files/README.md @@ -0,0 +1,4 @@ +# files + +> [!NOTE] +> The system prompt for this agent: [instructions.md](instructions.md) diff --git a/files/deno.json b/files/deno.json index ba22f55b4469..64ada2b57e6a 100644 --- a/files/deno.json +++ b/files/deno.json @@ -1,7 +1,5 @@ { "name": "@artifact/files", "version": "0.0.1", - "exports": { - ".": "./mod.ts" - } + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } } diff --git a/files/files.test.md b/files/files.eval.md similarity index 100% rename from files/files.test.md rename to files/files.eval.md diff --git a/files/files.test.ts b/files/files.test.ts new file mode 100644 index 000000000000..0ffe3d68e726 --- /dev/null +++ b/files/files.test.ts @@ -0,0 +1,9 @@ +// import * as files from '@artifact/files' + +// console.log(files) + +// run a test that the napp format is correct, that all the napp functions exist + +Deno.test('files', async (t) => { + console.log('asdf') +}) diff --git a/files/files.ts b/files/files.ts index da4e7003b1cc..294981151b66 100644 --- a/files/files.ts +++ b/files/files.ts @@ -1,17 +1,30 @@ -// import { Debug } from '@utils' -// import { Functions, print } from '@/constants.ts' -// import { type Api, parameters, returns } from '@/api/isolates/files.ts' -// export { type Api, parameters, returns } +import type { NappApi } from '@artifact/api/napp-api' +import type { z, ZodSchema } from 'zod' +import type * as schemas from './zod.ts' +import Debug from 'debug' +const log = Debug('@artifact/files') -// const log = Debug('AI:files') +type ToExternalApi<P extends ZodSchema, R extends ZodSchema> = ( + params: z.infer<P>, +) => z.infer<R> | Promise<z.infer<R>> + +type ToInternalApi<T> = T extends (...args: infer Args) => infer R + ? (...args: [...Args, NappApi]) => R + : never + +type Write = ToExternalApi< + typeof schemas.write.parameters, + typeof schemas.write.returns +> + +type WriteApi = ToInternalApi<Write> + +export const write: WriteApi = async ({ path, content = '' }, api) => { + log('write', path, content) + await api.write.text(path, content) + return { charactersWritten: content.length } +} -// export const functions: Functions<Api> = { -// // TODO this should be a full mirror of the IsolateApi functions -// write: ({ path, content = '' }, api) => { -// log('add', path, content) -// api.write(path, content) -// return { charactersWritten: content.length } -// }, // ls: async ({ path = '.', count, all }, api) => { // log('ls', path) // let result = await api.ls(path) @@ -68,17 +81,12 @@ // return ls.map((path) => ({ path, description: '' })) // }, // } -// export const replace = ( -// contents: string, -// regex: string, -// replacement: string, -// ) => { -// const matches = contents.match(new RegExp(regex, 'g')) || [] -// const result = contents.replace(new RegExp(regex, 'g'), replacement) -// return { matches, result } -// } - -/** - * This is a comment - */ -export const m = 43; +export const replace = ( + contents: string, + regex: string, + replacement: string, +) => { + const matches = contents.match(new RegExp(regex, 'g')) || [] + const result = contents.replace(new RegExp(regex, 'g'), replacement) + return { matches, result } +} diff --git a/files/mod.ts b/files/mod.ts index ca55893e6f97..32f7f3635884 100644 --- a/files/mod.ts +++ b/files/mod.ts @@ -1,2 +1,12 @@ -export * from "./files.ts"; -export * from "./napp.ts"; +export * from './files.ts' + +// this is an attempt to give types to the napp, but this should be auto +// generated from the napp.jsonc file at publish time + +export type NappTypes = { + // import show from the stateboard napp and export this + show: (params: { path: string }) => Promise<void> + write: ( + params: { path: string; content: string }, + ) => { charactersWritten: number } +} diff --git a/files/napp.json b/files/napp.json new file mode 100644 index 000000000000..6023a51087fc --- /dev/null +++ b/files/napp.json @@ -0,0 +1,58 @@ +{ + "name": "@artifact/files", + "version": "0.0.1", + "napp-format": "v1", + "description": "filesystem access using the artifact api", + "runtime": "deno", + "branding": "./avatar.jpg", + "agent": { + "napp": "@artifact/openai", + "parameters": { + "model": "gpt-4o", + "parallel_tool_calls": false + } + }, + "tools": { + "show": "@artifact/stateboard", + "write": { + "parameters": { + "type": "object", + "properties": { + "path": { "type": "string" }, + "content": { "type": "string" } + }, + "required": ["path", "content"] + }, + "returns": { + "type": "object", + "properties": { "charactersWritten": { "type": "number" } } + } + } + }, + "evals": { + "napp": "@artifact/evals", + "parameters": { "files": ["./files.eval.md"] } + }, + "imports": { + "@artifact/evals": { + "version": "0.0.1", + "commit": "some-commit-hash" + }, + "@artifact/stateboard": { + "version": "0.0.1", + "commit": "some-commit-hash" + }, + "@artifact/openai": { + "version": "0.0.1", + "commit": "some-commit-hash" + } + }, + "graphics": { + "googleMapsViewer": { + "type": "react", + "component": "./path/to/component", + "parameters": { + } + } + } +} diff --git a/files/napp.jsonc b/files/napp.jsonc deleted file mode 100644 index ff1306b8d29d..000000000000 --- a/files/napp.jsonc +++ /dev/null @@ -1,127 +0,0 @@ -{ - "name": "@artifact/files", - "version": "0.0.1", - "napp-format": "v1", - "description": "filesystem access using the artifact api", - "runtime": "deno", - "branding": "./avatar.jpg", - "agent": { - // entry point that takes plain text in and returns text plus files - "napp": "napp-runner-name", - "tool": "some-exported-function", // if use a function other than main - "parameters": { - // parameters for the chat runner, based on what the runner is - "model": "gpt-4o", - "parallel_tool_calls": false, - "stop_on_tools": ["tool1", "tool3"] - // content is the reserved name for the input text - } - }, - "main": "some-function", - "tools": { - "some-function": { - "description": "This function does something", - "parameters": { - "param1": { - "description": "This is the first parameter", - "type": "string", - "required": true - }, - "param2": { - "description": "This is the second parameter", - "type": "number", - "required": false - }, - "permissions": [ - // list of permissions required to run this function - ], - "gateways": [ - // list of gateways required to run this function - ] - }, - "returns": { - "description": "This is what the function returns", - "type": "string" - }, - "throws": { - "FileNotFoundError": { - // list of expected error types and what to do about it - } - } - }, - // if the function name is the same, as what is in a napp, - "some-other-function": "dependent-napp-name", - "modified-passthru": { - // passes thru to a tool of the same name - "parameters": {}, - "napp": { - "name": "dependent-napp-name", - "tool": "some-exported-function" - } - } - }, - "help": [ - // list of files that can be read to learn how to interface with this napp - // intended to be read by an AI - ], - "test": { - "napp": "napp-test-runner-name", - "parameters": { - // config for the test runner, based on what the runner is - "files": [ - // list of test files that can be run to verify the napp is working correctly - ] - } - }, - "dependencies": { - // list of napps that this napp depends on, including version numbers and - // commits so that every reference to the napp can be by name. - "dependent-napp-name": { - "version": "0.0.1", - "commit": "some-commit-hash" - } - }, - // list of the api gateway targets that this thing needs to do its job - "sideEffects": { - // default configurations for the production running of side effects - }, - "testSideEffects": { - // special services that can be used for testing - these are still live but - // are somehow test grade, as in might be cheaper, or no credentials - }, - "mockSideEffects": { - // when running in the mock environment, these side effects are supplied - }, - "graphics": { - // a list of exported widgets that can be displayed on the stateboard - "googleMapsViewer": { - "type": "react", - "component": "./path/to/component", - "parameters": { - // mock react props that can be configured by the LLM - } - } - } -} - -// gateways can be just a fetch function with some urls -// can make a moneyworks connector, which could be a napp, running as a gateway -// host -// upstream dependencies ? -// tools being the same exported as what is in the agent -// need an agent config that is a generic way - -// gateways need to have backups and retry policies -// gateways need environments, like mock, test, prod - -// --- -// commands: -// - files:write -// - files:ls -// - files:read -// - files:update -// - files:rm -// - files:mv -// - files:search -// - stateboard:show -// --- diff --git a/files/napp.ts b/files/napp.ts deleted file mode 100644 index c363fa2ed381..000000000000 --- a/files/napp.ts +++ /dev/null @@ -1,105 +0,0 @@ -// import { z } from "zod"; -// const reasoning = z.string().array().optional().describe( -// "reasoning for the operation", -// ); - -// export const parameters = { -// write: z.object({ -// reasoning, -// path: z.string().describe("the relative path to the file"), -// content: z.string().optional().describe( -// "the contents of the file to be written", -// ), -// }).describe( -// "Write to a file with optional contents. Will overwrite existing files. Will create all required parent directories. Returns the number of characters written in the operation", -// ), -// ls: z.object({ -// reasoning, -// path: z.string().optional().describe( -// "the relative path to the directory you want to list", -// ), -// count: z.boolean().optional().describe( -// "count the number of files and return this instead of the names of the files", -// ), -// all: z.boolean().optional().describe( -// "include all files including hidden files in the operation", -// ), -// }).describe( -// 'List files for a given path. Returns file names with directory names ending in "/". The root ("/") is actually just ".". To count the number of files instead of list them, set "count" to true. To include hidden files in the list or count, set "all" to true.', -// ), -// read: z.object({ -// reasoning, -// path: z.string().describe( -// "the relative path to the file you want to read", -// ), -// }).describe("Read a file. The contents will be returned as a string."), -// update: z.object({ -// reasoning, -// expectedMatches: z.number().int().gt(0).describe( -// "the expected number of matches for the regex", -// ), -// path: z.string().describe( -// "the relative path to the file you want to update", -// ), -// regex: z.string().describe("a regular expression string"), -// replacement: z.string().describe("the replacement string"), -// }).refine(({ regex }) => { -// try { -// new RegExp(regex); -// return true; -// } catch (error) { -// return !error; -// } -// }) -// .describe( -// "Update a file using a regex and a replacement string. The number of occurrences replaced will be returned to you as an integer. If you want to append something to a file, you can use a regex to match the end of the file and replace it with the contents you want to append. To delete portions of a file, you can use a regex to match the contents you want to delete and replace it with an empty string. Path must be relative.", -// ), -// rm: z.object({ -// reasoning, -// path: z.string().describe( -// "the relative path to the file you want to remove", -// ), -// }).describe("Remove a file. This is recursive."), -// mv: z.object({ -// reasoning, -// from: z.string().describe( -// "the relative path to the file you want to move", -// ), -// to: z.string().describe( -// "the relative path to the new location of the file", -// ), -// }).describe("Move a file efficiently. This is a rename operation."), -// cp: z.object({ -// reasoning, -// from: z.string().describe( -// "the relative path to the file you want to copy", -// ), -// to: z.string().describe( -// "the relative path to the new location of the file", -// ), -// }).describe("Copy a file efficiently."), -// search: z.object({ -// reasoning, -// query: z.string().describe( -// "the relative path to the file or directory you want to find", -// ), -// }).describe( -// "Search for a file or directory. Returns the relative path to the first match.", -// ), -// }; -// export const returns = { -// /** The number of bytes written */ -// write: z.object({ -// charactersWritten: z.number(), -// }), -// ls: z.union([z.array(z.string()), z.number()]), -// read: z.string(), -// /** The number of occurrences replaced */ -// update: z.object({ matchesUpdated: z.number() }), -// rm: z.void(), -// mv: z.void(), -// cp: z.void(), -// search: z.array( -// z.object({ path: z.string(), description: z.string() }), -// ), -// }; diff --git a/files/zod.ts b/files/zod.ts new file mode 100644 index 000000000000..e097deac61a0 --- /dev/null +++ b/files/zod.ts @@ -0,0 +1,101 @@ +import { z } from 'zod' +const reasoning = z.string().array().optional().describe( + 'reasoning for the operation', +) + +export const write = { + parameters: z.object({ + reasoning, + path: z.string().describe('the relative path to the file'), + content: z.string().optional().describe( + 'the contents of the file to be written', + ), + }).describe( + 'Write to a file with optional contents. Will overwrite existing files. Will create all required parent directories. Returns the number of characters written in the operation', + ), + returns: z.object({ + charactersWritten: z.number(), + }), + throws: z.object({ + name: z.literal('FileNotFound'), + message: z.literal('The file was not found'), + }), +} + +export const ls = { + parameters: z.object({ + reasoning, + path: z.string().optional().describe( + 'the path to the directory you want to list', + ), + count: z.boolean().optional().describe( + 'count the number of files and return this instead of the names of the files', + ), + all: z.boolean().optional().describe( + 'include all files including hidden files in the operation', + ), + }).describe( + 'List files for a given path. Returns file names with directory names ending in "/". The root ("/") is actually just ".". To count the number of files instead of list them, set "count" to true. To include hidden files in the list or count, set "all" to true.', + ), + returns: z.union([z.array(z.string()), z.number()]), +} + +export const read = { + parameters: z.object({ + reasoning, + path: z.string().describe( + 'the path to the file you want to read', + ), + }).describe('Read a file. The contents will be returned as a string.'), + returns: z.string(), +} + +export const rm = { + parameters: z.object({ + reasoning, + path: z.string().describe( + 'the path to the file you want to remove', + ), + }).describe('Remove a file. This is recursive on directories.'), + returns: z.void(), +} + +export const mv = { + parameters: z.object({ + reasoning, + from: z.string().describe( + 'the path to the file you want to move', + ), + to: z.string().describe( + 'the path to the new location of the file', + ), + }).describe('Move a file efficiently. This is a rename operation.'), + returns: z.void(), +} + +export const cp = { + parameters: z.object({ + reasoning, + from: z.string().describe( + 'the path to the file you want to copy', + ), + to: z.string().describe( + 'the path to the new location of the file', + ), + }).describe('Copy a file efficiently.'), + returns: z.void(), +} + +export const search = { + parameters: z.object({ + reasoning, + query: z.string().describe( + 'the path to the file or directory you want to find', + ), + }).describe( + 'Search for a file or directory. Returns the path to the first match.', + ), + returns: z.array( + z.object({ path: z.string(), description: z.string() }), + ), +} diff --git a/fixture/README.md b/fixture/README.md new file mode 100644 index 000000000000..73de11dcee92 --- /dev/null +++ b/fixture/README.md @@ -0,0 +1,4 @@ +# fixture + +> [!NOTE] +> The system prompt for this agent: [instructions.md](fixture/instructions.md) diff --git a/fixture/agent-fixture.md b/fixture/agent-fixture.md deleted file mode 100644 index df5be04771ff..000000000000 --- a/fixture/agent-fixture.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -commands: - - io-fixture:local - - io-fixture:error ---- - -Your name is 'Assistant' - -ALWAYS be as brief as possible diff --git a/fixture/deno.json b/fixture/deno.json index 8e87869af2f7..ad322065948c 100644 --- a/fixture/deno.json +++ b/fixture/deno.json @@ -1,7 +1,5 @@ { "name": "@artifact/fixture", "version": "0.0.1", - "exports": { - ".": "./mod.ts" - } + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } } diff --git a/fixture/instructions.md b/fixture/instructions.md new file mode 100644 index 000000000000..f8b1f21e3076 --- /dev/null +++ b/fixture/instructions.md @@ -0,0 +1,3 @@ +Your name is 'Assistant' + +ALWAYS be as brief as possible diff --git a/_import-artifact/isolates/io-fixture.ts b/fixture/io-fixture.ts similarity index 100% rename from _import-artifact/isolates/io-fixture.ts rename to fixture/io-fixture.ts diff --git a/fixture/mod.ts b/fixture/mod.ts index e69de29bb2d1..efe47ed91549 100644 --- a/fixture/mod.ts +++ b/fixture/mod.ts @@ -0,0 +1,7 @@ +export const ping = () => { + console.log('ping') +} + +export type NappTypes = { + ping: typeof ping +} diff --git a/fixture/napp.json b/fixture/napp.json new file mode 100644 index 000000000000..893f7b5f6110 --- /dev/null +++ b/fixture/napp.json @@ -0,0 +1,11 @@ +{ + "name": "@artifact/fixture", + "version": "0.0.1", + "description": "fixture for testing napp features", + "runtime": "deno", + "tools": { + "ping": { + "description": "ping this fixture" + } + } +} diff --git a/git-kv/README.md b/git-kv/README.md new file mode 100644 index 000000000000..f0fbf7619e26 --- /dev/null +++ b/git-kv/README.md @@ -0,0 +1,73 @@ +# Overview + +This module provides a `.git` directory abstraction on top of a key-value store, +allowing `isomorphic-git` to operate in non-traditional environments without +depending on local disk I/O. Instead of reading and writing `.git` data from the +local filesystem, all interactions are routed through a key-value database, +making it possible to run Git operations in environments like serverless +platforms or multi-tenant, globally distributed systems. + +## Key Characteristics + +- **Key-Value Storage for Git Data:**\ + All repository data that would normally reside in a local `.git` directory is + stored in a key-value database. This includes references, configuration, and + Git object data. Deno’s KV store provides the underlying storage layer. + +- **No Single HEAD Pointer:**\ + There is no global, shared HEAD. Each process or execution context operates on + its own branch reference. HEAD points to `refs/heads/<branch>/...` derived + from the PID (Process ID) context. This design enables concurrent operations + on different branches without conflicts. + +- **Atomic Reference Updates:**\ + Updates to references (e.g., when committing to a branch) are performed + atomically. Either all changes succeed together, or none are applied. This + ensures that distributed or delayed writes never leave the repository in an + inconsistent state. + +- **No Index File:**\ + The traditional Git index file is not used. The system reads objects directly + from the key-value store. This simplifies the implementation and ensures + scalability for large repositories and distributed environments. + +- **No Direct Access to `.git` Directory by Consumers:**\ + The `.git` directory is virtualized and never directly exposed. Consumers only + interact with regular repository paths. When a file is requested, the system + retrieves it from Git objects stored in the database. Paths under `.git` + cannot be read or written by external consumers; they are reserved for + `isomorphic-git` operations through the key-value layer. + +- **Deterministic Path Rules:**\ + Only certain Git-internal paths (`config`, `objects`, `refs`) are stored in + the key-value database. Attempts to read or write other `.git` paths are + rejected. HEAD updates are ignored if attempted directly, and the Git index + file is not supported. + +- **Support for Clone, Fetch, and Merge Operations:**\ + Git operations such as `clone`, `fetch`, and `merge` are implemented via + `isomorphic-git`. These operations operate through this abstraction layer, + with all necessary data (like references and objects) coming from and going + into the key-value store. Merges are performed with a custom merge driver that + uses `diff3` logic to reconcile conflicts. + +- **Caching:**\ + A caching layer prevents unnecessary round trips to the key-value store. + Recently accessed data, such as Git objects, are cached in memory and may + optionally use global caching mechanisms (like `globalThis.caches`) when + available. No additional cache extensions or invalidation strategies are + required beyond what is implemented. + +## Intended Use Case + +This module is designed for environments where a traditional filesystem is not +available or desirable. Serverless functions, containerized builds, and +distributed compute nodes can all benefit from a `.git` abstraction that relies +solely on a key-value store for state. Without a single HEAD pointer and by +removing reliance on local index files, multiple concurrent execution contexts +can work on separate branches, ensuring that operations scale without +introducing file-based race conditions or complex locking schemes. + +Operations like `init`, `clone`, `commit`, `merge`, and `fetch` translate into +atomic database updates, making repository state consistent and reproducible +across arbitrary numbers of execution contexts or physical locations. diff --git a/git-kv/cache.ts b/git-kv/cache.ts new file mode 100644 index 000000000000..8bd1d99ce278 --- /dev/null +++ b/git-kv/cache.ts @@ -0,0 +1,65 @@ +import { assert } from '@std/assert' + +type KvKey = (string | number)[] + +export class Cache { + static create() { + return new Cache() + } + + static #local = new Map<string, Uint8Array>() + #big: globalThis.Cache | undefined + + async #load() { + if ('caches' in globalThis && !this.#big) { + // TODO: If needed, name caches per repo + this.#big = await caches.open('hashbucket') + } + } + + async has(key: KvKey): Promise<boolean> { + const url = toUrl(key) + if (Cache.#local.has(url)) { + return true + } + + await this.#load() + if (this.#big) { + const cached = await this.#big.match(url) + if (cached) { + const cloned = cached.clone() + const bytes = await cloned.arrayBuffer() + const data = new Uint8Array(bytes) + Cache.#local.set(url, data) + return true + } + } + return false + } + + get(key: KvKey): Uint8Array { + const url = toUrl(key) + if (Cache.#local.has(url)) { + const result = Cache.#local.get(url) + assert(result, 'cache inconsistency') + return result + } + throw new Error('not found in cache: ' + key.join('/')) + } + + async set(key: KvKey, value: Uint8Array): Promise<void> { + await this.#load() + const url = toUrl(key) + Cache.#local.set(url, value) + if (this.#big) { + const request = new Request(url) + // We create a new Response with the data + const response = new Response(value) + await this.#big.put(request, response) + } + } +} + +function toUrl(pathKey: KvKey): string { + return 'http://' + pathKey.join('/') +} diff --git a/git-kv/deno.json b/git-kv/deno.json new file mode 100644 index 000000000000..eff2a501f1c9 --- /dev/null +++ b/git-kv/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/git-kv", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/git-kv/errors.ts b/git-kv/errors.ts new file mode 100644 index 000000000000..2ac93402e42e --- /dev/null +++ b/git-kv/errors.ts @@ -0,0 +1,6 @@ +export class FileNotFoundError extends Error { + constructor(message: string) { + super(message); + this.name = 'FileNotFoundError'; + } +} diff --git a/_import-artifact/git/fs.test.ts b/git-kv/git-fs.test.ts similarity index 96% rename from _import-artifact/git/fs.test.ts rename to git-kv/git-fs.test.ts index 3ecb0831d9bf..976a0eada391 100644 --- a/_import-artifact/git/fs.test.ts +++ b/git-kv/git-fs.test.ts @@ -1,8 +1,7 @@ -import * as utils from '@utils' -import DB from '@/db.ts' -import FS from './fs.ts' -import { addBranches, addPeer, IO_PATH, partialFromRepo } from '@/constants.ts' -const { expect } = utils +import { expect } from '@std/expect/expect' +import DB from '../engine/db.ts' +import FS from '../snapshots/tip.ts' + Deno.test('git/init', async (t) => { const db = await DB.create(DB.generateAesKey()) let fs: FS @@ -15,6 +14,8 @@ Deno.test('git/init', async (t) => { ) expect(await db.readHead(fs.pid)).toBe(fs.oid) }) + // BUT should all paths be available, since git is running at a different + // layer ? await t.step('git paths', async () => { const git = 'git paths are forbidden: ' const json = 'path must be *.json' @@ -48,7 +49,14 @@ Deno.test('git/init', async (t) => { } }) await t.step('complicated paths', async () => { - const paths = ['', '.', '/', '/something', '/something/deep', '//something'] + const paths = [ + '', + '.', + '/', + '/something', + '/something/deep', + '//something', + ] for (const path of paths) { if (['', '.', '/'].includes(path)) { expect(() => fs.write(path, ''), path).toThrow() diff --git a/_import-artifact/git/fs.ts b/git-kv/git-fs.ts similarity index 60% rename from _import-artifact/git/fs.ts rename to git-kv/git-fs.ts index f15ace234f17..8966a1a5dd55 100644 --- a/_import-artifact/git/fs.ts +++ b/git-kv/git-fs.ts @@ -1,6 +1,11 @@ +// a git backed implementation of the FS interface +import git, { Errors, type MergeDriverCallback } from 'isomorphic-git' +import http from 'isomorphic-git/http/web/index.js' import diff3Merge from 'diff3' -import http from '$git/http/web/index.js' -import { assert, Debug, equal, posix } from '@utils' +import Debug from 'debug' +import { assert } from '@std/assert/assert' +import equal from 'fast-deep-equal' +import * as posix from '@std/path/posix' import { Change, ENTRY_BRANCH, @@ -13,24 +18,28 @@ import { sha1, type TreeEntry, } from '@/constants.ts' -import git, { Errors, type MergeDriverCallback } from '$git' -import type DB from '@/db.ts' -import { GitKV } from './gitkv.ts' -const log = Debug('git:fs') +import type DB from '../engine/db.ts' +import { GitKV } from './git-kv.ts' +const log = Debug('@artifact/git-kv') const dir = '/' -type Upsert = { oid: string } | { data: string | Uint8Array } +import type { SnapshotsProvider } from '../snapshots/tip.ts' -export default class FS { - readonly #pid: PID +export class GitFS implements SnapshotsProvider { readonly #oid: string readonly #gitkv: GitKV readonly #db: DB - readonly #upserts = new Map<string, Upsert>() - readonly #deletes = new Set<string>() /** If present, the commit oid that overwrote the current FS */ #overwrite: string | undefined + private constructor(pid: PID, oid: string, db: DB) { + assert(sha1.test(oid), 'Commit not SHA-1: ' + oid) + this.#pid = pid + this.#oid = oid + this.#db = db + this.#gitkv = GitKV.recreate(db, pid) + } + static #caches = new Map<string, object>() static #getGitCache(pid: PID) { if (!FS.#caches.has(pid.repoId)) { @@ -47,45 +56,18 @@ export default class FS { const cache = FS.#getGitCache(this.#pid) return { fs: this.fs, dir, cache } } - get pid() { - return this.#pid - } - /** The oid used as the root git directory */ - get #internalOid() { - return this.#overwrite || this.oid - } - /** The commit oid backing this filesystem */ - get oid() { - return this.#oid - } get fs() { return { promises: this.#gitkv } } - get isChanged() { - return this.#overwrite || this.#upserts.size > 0 || this.#deletes.size > 0 - } - get upserts() { - return [...this.#upserts.keys()] - } - get deletes() { - return [...this.#deletes] - } - private constructor(pid: PID, oid: string, db: DB) { - assert(sha1.test(oid), 'Commit not SHA-1: ' + oid) - this.#pid = pid - this.#oid = oid - this.#db = db - this.#gitkv = GitKV.recreate(db, pid) - } static open(pid: PID, commit: string, db: DB) { - return new FS(pid, commit, db) + return new GitFS(pid, commit, db) } static async openHead(pid: PID, db: DB) { const head = await db.readHead(pid) if (!head) { throw new Error('HEAD not found: ' + print(pid)) } - return new FS(pid, head, db) + return new GitFS(pid, head, db) } static async init(partial: PartialPID, db: DB, owner?: PID) { const repoId = generateFakeRepoId() @@ -96,7 +78,7 @@ export default class FS { await git.init({ fs, dir, defaultBranch: pid.branches[0] }) log('init complete') const author = { name: owner ? print(owner) : 'git/init' } - const cache = FS.#getGitCache(pid) + const cache = GitFS.#getGitCache(pid) // TODO insert the .io.json file that includes the consensus config // TODO write owner into config const commit = await git.commit({ @@ -108,7 +90,7 @@ export default class FS { cache, }) await db.atomic().createBranch(pid, commit).commit() - const init = new FS(pid, commit, db) + const init = new GitFS(pid, commit, db) return init } static async clone(repo: string, db: DB) { @@ -123,11 +105,11 @@ export default class FS { const fs = { promises: GitKV.createBlank(db, pid) } fs.promises.oneAtomicWrite = db.atomic() - const cache = FS.#getGitCache(pid) + const cache = GitFS.#getGitCache(pid) await git.clone({ fs, dir, url, http, noCheckout: true, cache }) const commit = await db.readHead(pid) assert(commit, 'HEAD not found: ' + pid.branches.join('/')) - const clone = new FS(pid, commit, db) + const clone = new GitFS(pid, commit, db) return clone } static async fetch(repo: string, pid: PID, db: DB) { @@ -136,8 +118,8 @@ export default class FS { const lockId = await db.lockRepo(pid) try { - const { fs } = await FS.openHead(pid, db) - const cache = FS.#getGitCache(pid) + const { fs } = await GitFS.openHead(pid, db) + const cache = GitFS.#getGitCache(pid) const result = await git.fetch({ fs, http, dir, url, cache }) const { fetchHead } = result assert(fetchHead, 'fetchHead not found') @@ -152,33 +134,14 @@ export default class FS { assert(repo.split('/').length === 2, 'invalid repo: ' + repo) const url = `https://github.com/${repo}.git` assert(isBaseRepo(pid), 'not a base repo: ' + print(pid)) - const { fs } = await FS.openHead(pid, db) - const cache = FS.#getGitCache(pid) + const { fs } = await GitFS.openHead(pid, db) + const cache = GitFS.#getGitCache(pid) const result = await git.push({ fs, http, dir, url, cache }) const { ok } = result if (!ok) { throw new Error('push failed') } } - - tick(commit: string) { - return new FS(this.#pid, commit, this.#db) - } - /** @param the new PID to branch into */ - branch(pid: PID) { - assert(isSameRepo(this.#pid, pid), 'branching into a different repo') - return new FS(pid, this.oid, this.#db) - } - logs(filepath?: string, depth?: number) { - if (filepath) { - filepath = refine(filepath) - } - if (filepath === '.') { - filepath = undefined - } - return git.log({ ...this.#git, filepath, depth, ref: this.oid }) - } - async writeCommitObject(message = '', parents: string[] = []) { assert(this.isChanged, 'empty commit') assert(parents.every((oid) => sha1.test(oid)), 'Parent not SHA-1') @@ -248,33 +211,6 @@ export default class FS { assert(result.oid, 'merge failed') return result.oid } - async exists(path: string) { - path = refine(path) - if (path === '.') { - return true - } - if (this.#deletes.has(path)) { - return false - } - if (this.#upserts.has(path)) { - return true - } - - try { - await this.readOid(path) - return true - } catch (error) { - // TODO move all errors to be FileNotFoundError - if ( - error instanceof Error && 'code' in error && - error.code === 'NotFoundError' - ) { - return false - } - throw error - } - } - async readOid(path: string): Promise<string> { // TODO test how this works for "." path = refine(path) @@ -310,69 +246,6 @@ export default class FS { throw error } } - delete(path: string) { - path = refine(path) - assert(path !== '.', 'cannot delete root') - // TODO delete a whole directory - log('delete', path) - this.#deletes.add(path) - this.#upserts.delete(path) - } - writeJSON(path: string, json: unknown) { - // TODO store json objects specially, only strinify on commit - // then broadcast changes as json object purely - assert(posix.extname(path) === '.json', `path must be *.json: ${path}`) - path = refine(path) - const string = JSON.stringify(json, null, 2) - assert(typeof string === 'string', 'stringify failed') - return this.write(path, string) - } - write(path: string, data: string | Uint8Array) { - path = refine(path) - assert(path !== '.', 'cannot write to root') - // TODO ensure cannot write to a directory - log('write', path, data) - this.#upserts.set(path, { data }) - this.#deletes.delete(path) - } - async readJSON<T>(path: string, commit?: string): Promise<T> { - assert(posix.extname(path) === '.json', `path must be *.json: ${path}`) - const data = await this.read(path, commit) - return JSON.parse(data) - } - async read(path: string, commit?: string) { - const blob = await this.readBinary(path, commit) - return new TextDecoder().decode(blob) - } - async readBinary(path: string, commit?: string): Promise<Uint8Array> { - path = refine(path) - log('readBinary', path, commit) - - if (commit) { - const { blob } = await this.#readBlob(path, commit) - return blob - } - - if (this.#deletes.has(path)) { - throw new Error('Could not find file or directory: ' + path) - } - if (this.#upserts.has(path)) { - const upsert = this.#upserts.get(path) - assert(upsert, 'upsert not found') - if ('data' in upsert) { - if (typeof upsert.data === 'string') { - return new TextEncoder().encode(upsert.data) - } - return upsert.data - } else { - const { oid } = upsert - const { blob } = await this.#safeReadBlob({ oid }) - return blob - } - } - const { blob } = await this.#readBlob(path) - return blob - } async #readBlob(filepath: string, commit?: string) { const { blob, oid } = await this.#safeReadBlob({ oid: commit || this.#internalOid, @@ -395,42 +268,6 @@ export default class FS { throw error } } - - async ls(path: string = '.') { - path = refine(path) - // TODO make a streaming version of this for very large dirs - // TODO handle changes in the directory like deletes and upserts - log('ls', path) - const filepath = path === '.' ? undefined : path - const tree = await this.readTree(filepath) - - tree.sort((a, b) => { - if (a.type === 'tree' && b.type === 'blob') { - return -1 - } - if (a.type === 'blob' && b.type === 'tree') { - return 1 - } - if (a.path.startsWith('.') && !b.path.startsWith('.')) { - return -1 - } - if (!a.path.startsWith('.') && b.path.startsWith('.')) { - return 1 - } - return a.path.localeCompare(b.path) - }) - return tree.map((entry) => { - if (entry.type === 'tree') { - return entry.path + '/' - } - assert(entry.type === 'blob', 'entry type not blob: ' + entry.type) - return entry.path - }) - } - async getCommit() { - const result = await git.readCommit({ ...this.#git, oid: this.oid }) - return result.commit - } copyChanges(from: FS) { assert(!this.isChanged, 'cannot copy changes to a changed FS') assert(equal(this.#pid, from.#pid), 'changes are from different pids') @@ -442,57 +279,7 @@ export default class FS { } this.#overwrite = from.#overwrite } - async isPidExists(pid: PID) { - // TODO maybe if this gets read, it gets stored in the accumulator - // store the root head of our repo in the git file structure, used to - // reference from - return !!await this.#db.readHead(pid) - } - async mv(from: string, to: string) { - await this.cp(from, to) - this.delete(from) - } - async cp(from: string, to: string) { - // TODO check using directories - from = refine(from) - to = refine(to) - assert(from !== to, 'source and destination are the same') - assert(await this.exists(from), 'source does not exist: ' + from) - assert(!await this.exists(to), 'destination already exists: ' + to) - const oid = await this.readOid(from) - this.#upserts.set(to, { oid }) - } - async overwrite(commit: string, ...ignores: string[]) { - // TODO allow changes so long as they are in the ignored set - assert(!this.isChanged, 'Uncommitted changes may be lost') - assert(sha1.test(commit), 'Commit not SHA-1: ' + commit) - assert(this.oid !== commit, 'cannot overwrite with same commit') - assert(this.#overwrite !== commit, 'cannot overwrite the same commit twice') - ignores.push(IO_PATH) - ignores.forEach(refine) - - const result = await git.readCommit({ ...this.#git, oid: commit }) - assert(result, 'commit not found: ' + commit) - - for (const ignore of ignores) { - if (this.#upserts.has(ignore)) { - continue - } - if (this.#deletes.has(ignore)) { - continue - } - if (await this.exists(ignore)) { - const oid = await this.readOid(ignore) - // TODO check if it is unchanged that this is handled - this.#upserts.set(ignore, { oid }) - } else { - // TODO check that deleting something not present is handled - this.#deletes.add(ignore) - } - } - this.#overwrite = commit - } async #flush() { const changes: { [key: string]: Change } = {} const oid = this.#internalOid @@ -553,6 +340,39 @@ export default class FS { return { oid: tree.oid, changes } } } + +const generateFakeRepoId = () => { + // TODO make this genuine based on the genesis commit + return `rep_${randomness()}` +} + +const mergeDriver: MergeDriverCallback = ({ contents, path }) => { + const [baseContent, ourContent, theirContent] = contents + assert(typeof baseContent === 'string', 'base content not found') + assert(typeof ourContent === 'string', 'our content not found') + assert(typeof theirContent === 'string', 'their content not found') + + if (path === IO_PATH) { + return { cleanMerge: true, mergedText: ourContent } + } + + const LINEBREAKS = /^.*(\r?\n|$)/gm + const ours = ourContent.match(LINEBREAKS) + const base = baseContent.match(LINEBREAKS) + const theirs = theirContent.match(LINEBREAKS) + const result = diff3Merge(ours, base, theirs) + let mergedText = '' + for (const item of result) { + if (item.ok) { + mergedText += item.ok.join('') + } + if (item.conflict) { + mergedText += item.conflict.a.join('') + } + } + return { cleanMerge: true, mergedText } +} + type Tree = { oid?: string tree?: TreeObject @@ -649,69 +469,3 @@ const treeToLayers = (tree: Tree, layers: Tree[][] = [], level: number = 0) => { } return layers } -const refine = (path: string) => { - path = path.trim() - while (path.startsWith('/')) { - path = path.slice(1) - } - while (path.endsWith('/')) { - path = path.slice(0, -1) - } - while (path.startsWith('./')) { - path = path.slice(2) - } - if (!path) { - path = '.' - } - path = posix.normalize(path) - assert(path, `path must be relative: ${path}`) - assert(!posix.isAbsolute(path), `path must be relative: ${path}`) - assert(path !== '.git', '.git paths are forbidden: ' + path) - assert(!path.startsWith('.git/'), '.git paths are forbidden: ' + path) - assert(!path.endsWith('/'), 'path must not end with /: ' + path) - assert(!path.startsWith('..'), 'path must not start with ..: ' + path) - return path -} - -const generateFakeRepoId = () => { - // TODO make this genuine based on the genesis commit - return `rep_${randomness()}` -} - -const mergeDriver: MergeDriverCallback = ({ contents, path }) => { - const baseContent = contents[0] - const ourContent = contents[1] - const theirContent = contents[2] - - if (path === IO_PATH) { - return { cleanMerge: true, mergedText: ourContent } - } - - const LINEBREAKS = /^.*(\r?\n|$)/gm - const ours = ourContent.match(LINEBREAKS) - const base = baseContent.match(LINEBREAKS) - const theirs = theirContent.match(LINEBREAKS) - const result = diff3Merge(ours, base, theirs) - let mergedText = '' - for (const item of result) { - if (item.ok) { - mergedText += item.ok.join('') - } - if (item.conflict) { - mergedText += item.conflict.a.join('') - } - } - return { cleanMerge: true, mergedText } -} -const isSameRepo = (a: PID, b: PID) => { - if (a.repoId !== b.repoId) { - return false - } - if (a.account !== b.account) { - return false - } - if (a.repository !== b.repository) { - return false - } - return true -} diff --git a/git-kv/git-kv.test.ts b/git-kv/git-kv.test.ts new file mode 100644 index 000000000000..389e9c740d85 --- /dev/null +++ b/git-kv/git-kv.test.ts @@ -0,0 +1,187 @@ +// test_git_kv.ts +import { GitKV } from './git-kv.ts' +import { FileNotFoundError } from './errors.ts' +import { expect } from '@std/expect/expect' + +// Mock implementations of DB and Atomic for testing +class MockDB { + #store = new Map<string, Uint8Array>() + #heads = new Map<string, string>() + #children = new Map<string, string[]>() + + async blobGet(key: string[]): Promise<{value: Uint8Array, versionstamp?: string}> { + const path = key.join('/') + if (this.#store.has(path)) { + return { value: this.#store.get(path)!, versionstamp: 'test-version' } + } + return { value: new Uint8Array(), versionstamp: undefined } + } + async blobSet(key: string[], value: Uint8Array) { + this.#store.set(key.join('/'), value) + } + async blobExists(key: string[]): Promise<boolean> { + return this.#store.has(key.join('/')) + } + async readHead(pid: {branches: string[]}): Promise<string | null> { + const branchKey = pid.branches.join('/') + return this.#heads.get(branchKey) || null + } + async listImmediateChildren(prefix: string[]): Promise<string[]> { + const joined = prefix.join('/') + return this.#children.get(joined) || [] + } + setHead(pid: {branches: string[]}, refContent: string) { + const branchKey = pid.branches.join('/') + this.#heads.set(branchKey, refContent) + } + addChild(prefix: string[], child: string) { + const joined = prefix.join('/') + const arr = this.#children.get(joined) || [] + arr.push(child) + this.#children.set(joined, arr) + } +} + +class MockAtomic { + #commits: Array<{pid: {branches: string[]}, ref: string}> = [] + createBranch(pid: {branches: string[]}, ref: string) { + this.#commits.push({pid, ref}) + return this + } + async commit(): Promise<void> { + // In reality, would commit atomically, here we just no-op + } + get commits() { + return this.#commits + } +} + +const mockPID = { branches: ['main'] } + +Deno.test("GitKV: readFile HEAD returns correct ref", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + const headRef = await kv.readFile('/.git/HEAD', { encoding: 'utf8' }) + expect(headRef).toBe('ref: refs/heads/main') +}) + +Deno.test("GitKV: readFile non-existent file throws FileNotFoundError", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + await expect(kv.readFile('/.git/config/something', { encoding: 'utf8' })) + .rejects.toThrow(FileNotFoundError) +}) + +Deno.test("GitKV: writeFile and readFile roundtrip on allowed path", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + + const data = "test content" + await kv.writeFile('/.git/config/testfile', data, { encoding: 'utf8' }) + const readBack = await kv.readFile('/.git/config/testfile', { encoding: 'utf8' }) + expect(readBack).toBe(data) +}) + +Deno.test("GitKV: writeFile on refs/heads with atomic commit", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + + const atomic = new MockAtomic() + kv.oneAtomicWrite = atomic + await kv.writeFile('/.git/refs/heads/main', "deadbeef", { encoding: 'utf8' }) + + expect(atomic.commits.length).toBe(1) + expect(atomic.commits[0].ref).toBe('deadbeef') +}) + +Deno.test("GitKV: readFile refs/heads matches PID branch and reads head from DB", async () => { + const db = new MockDB() + db.setHead(mockPID, 'deadbeef') + const kv = GitKV.recreate(db, mockPID) + + const headVal = await kv.readFile('/.git/refs/heads/main', { encoding: 'utf8' }) + expect(headVal).toBe('deadbeef') +}) + +Deno.test("GitKV: readFile from objects throws not found if missing", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + + await expect(() => kv.readFile('/.git/objects/aa/bb')) + .rejects.toThrow(FileNotFoundError) +}) + +Deno.test("GitKV: stat on existing file returns empty object", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + await kv.writeFile('/.git/config/existfile', "exists", { encoding: 'utf8' }) + + const stat = await kv.stat('/.git/config/existfile') + expect(typeof stat).toBe('object') +}) + +Deno.test("GitKV: stat on missing file throws FileNotFoundError", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + + await expect(() => kv.stat('/.git/config/missing')) + .rejects.toThrow(FileNotFoundError) +}) + +Deno.test("GitKV: readdir on .git returns children", async () => { + const db = new MockDB() + const pidPrefix = ['repo', 'main'] + + db.addChild(['repo','main'], 'config') + db.addChild(['repo','main'], 'objects') + + const kv = GitKV.recreate(db, mockPID) + const children = await kv.readdir('/.git') + expect(children.sort()).toEqual(['config','objects'].sort()) +}) + +Deno.test("GitKV: mkdir, rmdir, unlink on shallow are no-ops or not implemented", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + + // mkdir always resolves + await kv.mkdir('/.git/somedirectory') + + // unlink on shallow returns quietly + await kv.unlink('/.git/shallow') + + // unlink elsewhere not implemented + await expect(() => kv.unlink('/.git/config/whatever')) + .rejects.toThrow(Error) + + // rmdir also not implemented + await expect(() => kv.rmdir('/.git/config/dir')) + .rejects.toThrow(Error) +}) + +Deno.test("GitKV: readFile on HEAD tries to emulate ref to PID branches", async () => { + const db = new MockDB() + const pid = { branches: ['feature', 'xyz'] } + const kv = GitKV.recreate(db, pid) + const head = await kv.readFile('/.git/HEAD', { encoding: 'utf8' }) + expect(head).toBe('ref: refs/heads/feature/xyz') +}) + +Deno.test("GitKV: writing to index throws error, HEAD writes ignored", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + + await expect(() => kv.writeFile('/.git/index', "should fail", { encoding: 'utf8' })) + .rejects.toThrow(Error) + + // HEAD writes are ignored + // Let's ensure no error thrown and no DB write attempt + await kv.writeFile('/.git/HEAD', "ignored data", { encoding: 'utf8' }) +}) + +Deno.test("GitKV: unsupported encoding throws error", async () => { + const db = new MockDB() + const kv = GitKV.recreate(db, mockPID) + await expect(() => kv.readFile('/.git/HEAD', { encoding: 'hex' as any })) + .rejects.toThrow(Error) +}) diff --git a/_import-artifact/git/gitkv.ts b/git-kv/git-kv.ts similarity index 68% rename from _import-artifact/git/gitkv.ts rename to git-kv/git-kv.ts index 78b2b683fd55..ba75c795c5c1 100644 --- a/_import-artifact/git/gitkv.ts +++ b/git-kv/git-kv.ts @@ -1,12 +1,20 @@ -import { Debug } from '@utils' -import { getRepoBase, headKeyToPid } from '@/keys.ts' -import type DB from '@/db.ts' -import { assert, AssertionError, equal } from '@utils' -import { PID } from '@/constants.ts' -import { Atomic } from '@/atomic.ts' +import Debug from 'debug' +import { getRepoBase, headKeyToPid } from '../engine/keys.ts' +import type DB from '../engine/db.ts' +import { assert, AssertionError } from '@std/assert' +import equal from 'fast-deep-equal' + +import type { PID } from '../processes/processes.ts' +import type { Atomic } from '../engine/atomic.ts' +import { FileNotFoundError } from './errors.ts' +import { Cache } from './cache.ts' const log = Debug('git:KV') +type EncodingOpts = { + encoding?: 'utf8' +} + export class GitKV { #allowed = ['config', 'objects', 'refs'] #dropWrites = ['HEAD'] @@ -14,6 +22,7 @@ export class GitKV { #pid: PID #exists: Set<string> | undefined #cache = Cache.create() + #oneAtomicWrite: Atomic | undefined private constructor(db: DB, pid: PID, isBlank: boolean = false) { this.#db = db @@ -22,37 +31,57 @@ export class GitKV { this.#exists = new Set() } } + static recreate(db: DB, pid: PID) { return new GitKV(db, pid) } + static createBlank(db: DB, pid: PID) { const isBlankDuringInitAndClone = true return new GitKV(db, pid, isBlankDuringInitAndClone) } - isIgnored(path: string) { + set oneAtomicWrite(atomic: Atomic) { + this.#oneAtomicWrite = atomic + } + + isIgnored(path: string): boolean { + if (!path.startsWith('/.git/')) return false const sliced = path.slice('/.git/'.length) return this.#dropWrites.includes(sliced) } - async readFile(path: string, opts: EncodingOpts) { + + async readFile( + path: string, + opts?: EncodingOpts, + ): Promise<string | Uint8Array> { log('readFile', path, opts) - if (!path && !opts) { - throw new Error('path and opts are required') + if (!path) { + throw new Error('path is required') } + + // Disallow reading the Git index file as per the spec (no Git index) if (path === '/.git/index') { throw new FileNotFoundError('file not found: ' + path) } + + // Emulate a HEAD that points to refs/heads/<branch>... according to PID if (path === '/.git/HEAD') { - let ref = `ref: refs/heads` + // Construct a HEAD ref line + // If multiple branches are present in this PID context, this might need clarification. + let ref = 'ref: refs/heads' for (const branch of this.#pid.branches) { ref += `/${branch}` } log('readFile HEAD ref', ref) return ref } + if (this.#exists && !this.#exists.has(path)) { throw new FileNotFoundError('file not found: ' + path) } + + // Reading refs/heads/... ensures that we are reading the current branch head if (path.startsWith('/.git/refs/heads/')) { // only allow reading heads from the current branch, else what doing ? const rest = path.slice('/.git/refs/heads/'.length) @@ -66,7 +95,7 @@ export class GitKV { return head } - if (opts && opts.encoding && opts.encoding !== 'utf8') { + if (opts?.encoding && opts.encoding !== 'utf8') { throw new Error('only utf8 encoding is supported') } @@ -76,7 +105,6 @@ export class GitKV { result = this.#cache.get(pathKey) } else { const dbResult = await this.#db.blobGet(pathKey) - if (!dbResult.versionstamp) { log('readFile not found', path, opts) throw new FileNotFoundError('file not found: ' + path) @@ -84,7 +112,8 @@ export class GitKV { result = dbResult.value await this.#cache.set(pathKey, result) } - if (opts && opts.encoding === 'utf8') { + + if (opts?.encoding === 'utf8') { const string = new TextDecoder().decode(result) log('readFile', path, opts, string) return string @@ -92,34 +121,42 @@ export class GitKV { log('readFile', path, opts, typeof result) return result } + async writeFile( path: string, data: Uint8Array | string, - opts: EncodingOpts, - ) { + opts?: EncodingOpts, + ): Promise<void> { log('writeFile', path, data, opts) - if (opts && opts.encoding && opts.encoding !== 'utf8') { + if (opts?.encoding && opts.encoding !== 'utf8') { throw new Error('only utf8 encoding is supported') } + + // Ignore writes to HEAD as per the spec (no single HEAD pointer) if (this.isIgnored(path)) { log('writeFile ignored', path) return } + + // No Git index file manipulation if (path === '/.git/index') { throw new Error('will not write to index') } + if (this.#exists) { this.#exists.add(path) } + const pathKey = this.#getAllowedPathKey(path) // TODO skip the remote HEAD writes too ? + // For refs, we assume atomic reference updates if (path.startsWith('/.git/refs/heads/')) { // TODO use the head tool on this.#db to ensure consistency - assert(typeof data === 'string', 'data must be a string') + assert(typeof data === 'string', 'data must be a string for refs') const pid = headKeyToPid(pathKey) // TODO ensure have maintenance while this is being changed - assert(this.#oneAtomicWrite, 'no atomic write provided') + assert(this.#oneAtomicWrite, 'no atomic write provided for ref update') const atomic = this.#oneAtomicWrite this.#oneAtomicWrite = undefined await atomic.createBranch(pid, data.trim()).commit() @@ -133,18 +170,17 @@ export class GitKV { } log('writeFile done:', pathKey) } - #oneAtomicWrite: Atomic | undefined - set oneAtomicWrite(atomic: Atomic) { - this.#oneAtomicWrite = atomic - } - async unlink(path: string) { + + async unlink(path: string): Promise<void> { log('unlink', path) if (path === '/.git/shallow') { + // It's allowed to unlink shallow without error as per git usage return } return await Promise.reject(new Error('not implemented')) } - async readdir(path: string, options?: object) { + + async readdir(path: string, options?: object): Promise<string[]> { log('readdir', path) assert(!options, 'options not supported') let pathKey = getRepoBase(this.#pid) @@ -153,20 +189,22 @@ export class GitKV { } const results = await this.#db.listImmediateChildren(pathKey) log('readdir', path, results) - return results } - mkdir(path: string) { + + mkdir(path: string): Promise<void> { log('mkdir', path) + // Directories are a no-op in a KV store return Promise.resolve() } - async rmdir(path: string) { + + async rmdir(path: string): Promise<void> { log('rmdir', path) return await Promise.reject(new Error('not implemented')) } - async stat(path: string) { + + async stat(path: string): Promise<Record<string, unknown>> { log('stat', path) - // generate the key for the path let pathKey try { pathKey = this.#getAllowedPathKey(path) @@ -176,9 +214,11 @@ export class GitKV { } throw error } + if (this.#exists && !this.#exists.has(path)) { throw new FileNotFoundError('file not found: ' + path) } + log('stat pathKey', pathKey) let exists = false if (path.startsWith('/.git/refs/heads/')) { @@ -190,104 +230,51 @@ export class GitKV { exists = true } else { if (path.startsWith('/.git/objects/')) { - // wastes a round trip to the db otherwise + // Directly throwing not found here to save a DB roundtrip throw new FileNotFoundError('file not found: ' + path) } exists = await this.#db.blobExists(pathKey) } } + if (!exists) { throw new FileNotFoundError('file not found: ' + path) } + + // Return a minimal stat object return {} } - async lstat(path: string) { + + async lstat(path: string): Promise<never> { log('lstat', path) const message = 'not implemented: ' + path return await Promise.reject(new FileNotFoundError(message)) } - async readlink(path: string) { + + async readlink(path: string): Promise<never> { log('readlink', path) return await Promise.reject(new Error('not implemented')) } - async symlink(target: string, path: string, type: string) { + + async symlink(target: string, path: string, type: string): Promise<never> { log('symlink', target, path, type) return await Promise.reject(new Error('not implemented')) } - async chmod(path: string, mode: number) { + + async chmod(path: string, mode: number): Promise<never> { log('chmod', path, mode) return await Promise.reject(new Error('not implemented')) } - #getAllowedPathKey(path: string) { + + #getAllowedPathKey(path: string): string[] { assert(path.startsWith('/.git/'), 'path must start with /.git/') const rest = path.slice('/.git/'.length) assert(rest, 'path must not be bare') const prefix = getRepoBase(this.#pid) const pathKey = rest.split('/') + assert(pathKey[0], 'path must have a first key') assert(this.#allowed.includes(pathKey[0]), 'path not allowed: ' + pathKey) return [...prefix, ...pathKey] } } - -type EncodingOpts = { - encoding?: 'utf8' -} - -export class FileNotFoundError extends Error { - code = 'ENOENT' - constructor(message: string) { - super(message) - this.name = 'FileNotFoundError' - } -} -class Cache { - static create() { - return new Cache() - } - static #local = new Map<string, Uint8Array>() - #big: globalThis.Cache | undefined - async #load() { - if ('caches' in globalThis && !this.#big) { - // TODO name the caches per repo so they can be deleted granularly - this.#big = await caches.open('hashbucket') - } - } - async has(key: Deno.KvKey) { - const url = toUrl(key) - if (Cache.#local.has(url)) { - return true - } - - await this.#load() - if (this.#big) { - const cached = await this.#big.match(url) - if (cached) { - const cloned = cached.clone() - const bytes = await cloned.bytes() - Cache.#local.set(url, bytes) - return true - } - } - return false - } - get(key: Deno.KvKey) { - const url = toUrl(key) - if (Cache.#local.has(url)) { - const result = Cache.#local.get(url) - assert(result, 'cache inconsistency') - return result - } - throw new Error('not found: ' + key.join('/')) - } - async set(key: Deno.KvKey, value: Uint8Array) { - await this.#load() - const url = toUrl(key) - Cache.#local.set(url, value) - if (this.#big) { - const request = new Request(url) - await this.#big.put(request, new Response(value)) - } - } -} -const toUrl = (pathKey: Deno.KvKey) => 'http://' + pathKey.join('/') diff --git a/git-kv/mod.ts b/git-kv/mod.ts new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/git-kv/napp.json b/git-kv/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/git-kv/napp.json @@ -0,0 +1 @@ +{} diff --git a/_import-artifact/guts/__snapshots__/guts-ai-isolates.ts.json b/guts/__snapshots__/guts-ai-isolates.ts.json similarity index 99% rename from _import-artifact/guts/__snapshots__/guts-ai-isolates.ts.json rename to guts/__snapshots__/guts-ai-isolates.ts.json index 62e114617cb9..6a6ba8362f91 100644 --- a/_import-artifact/guts/__snapshots__/guts-ai-isolates.ts.json +++ b/guts/__snapshots__/guts-ai-isolates.ts.json @@ -2009,4 +2009,4 @@ ] } } -} \ No newline at end of file +} diff --git a/_import-artifact/guts/__snapshots__/guts-focus.ts.json b/guts/__snapshots__/guts-focus.ts.json similarity index 99% rename from _import-artifact/guts/__snapshots__/guts-focus.ts.json rename to guts/__snapshots__/guts-focus.ts.json index 3181edcac6d1..b708d6622121 100644 --- a/_import-artifact/guts/__snapshots__/guts-focus.ts.json +++ b/guts/__snapshots__/guts-focus.ts.json @@ -8141,4 +8141,4 @@ ] } } -} \ No newline at end of file +} diff --git a/_import-artifact/guts/__snapshots__/guts-longthread.ts.json b/guts/__snapshots__/guts-longthread.ts.json similarity index 99% rename from _import-artifact/guts/__snapshots__/guts-longthread.ts.json rename to guts/__snapshots__/guts-longthread.ts.json index 2527c126b4f3..1b31fe6dbc68 100644 --- a/_import-artifact/guts/__snapshots__/guts-longthread.ts.json +++ b/guts/__snapshots__/guts-longthread.ts.json @@ -368,4 +368,4 @@ ] } } -} \ No newline at end of file +} diff --git a/_import-artifact/guts/__snapshots__/guts-tps-reports.ts.json b/guts/__snapshots__/guts-tps-reports.ts.json similarity index 99% rename from _import-artifact/guts/__snapshots__/guts-tps-reports.ts.json rename to guts/__snapshots__/guts-tps-reports.ts.json index 4a4bb9dbaf48..717e2e3934c4 100644 --- a/_import-artifact/guts/__snapshots__/guts-tps-reports.ts.json +++ b/guts/__snapshots__/guts-tps-reports.ts.json @@ -1887,4 +1887,4 @@ ] } } -} \ No newline at end of file +} diff --git a/_import-artifact/guts/guts-ai-isolates.ts b/guts/guts-ai-isolates.ts similarity index 100% rename from _import-artifact/guts/guts-ai-isolates.ts rename to guts/guts-ai-isolates.ts diff --git a/_import-artifact/guts/guts-backchats.ts b/guts/guts-backchats.ts similarity index 100% rename from _import-artifact/guts/guts-backchats.ts rename to guts/guts-backchats.ts diff --git a/_import-artifact/guts/guts-benchmarks.ts b/guts/guts-benchmarks.ts similarity index 100% rename from _import-artifact/guts/guts-benchmarks.ts rename to guts/guts-benchmarks.ts diff --git a/_import-artifact/guts/guts-branching.ts b/guts/guts-branching.ts similarity index 100% rename from _import-artifact/guts/guts-branching.ts rename to guts/guts-branching.ts diff --git a/_import-artifact/guts/guts-focus.ts b/guts/guts-focus.ts similarity index 100% rename from _import-artifact/guts/guts-focus.ts rename to guts/guts-focus.ts diff --git a/_import-artifact/guts/guts-git.ts b/guts/guts-git.ts similarity index 100% rename from _import-artifact/guts/guts-git.ts rename to guts/guts-git.ts diff --git a/_import-artifact/guts/guts-longthread.ts b/guts/guts-longthread.ts similarity index 98% rename from _import-artifact/guts/guts-longthread.ts rename to guts/guts-longthread.ts index a63495ad229c..5f54130e9c41 100644 --- a/_import-artifact/guts/guts-longthread.ts +++ b/guts/guts-longthread.ts @@ -1,6 +1,6 @@ import { assert, expect, log } from '@utils' import { CradleMaker, print } from '@/constants.ts' -import { Api } from '../isolates/longthread.ts' +import { Api } from '../openai/longthread.ts' export default (cradleMaker: CradleMaker) => { const prefix = 'longthread: ' diff --git a/_import-artifact/guts/guts-splices.ts b/guts/guts-splices.ts similarity index 100% rename from _import-artifact/guts/guts-splices.ts rename to guts/guts-splices.ts diff --git a/_import-artifact/guts/guts-tps-reports.ts b/guts/guts-tps-reports.ts similarity index 100% rename from _import-artifact/guts/guts-tps-reports.ts rename to guts/guts-tps-reports.ts diff --git a/_import-artifact/guts/guts.ts b/guts/guts.ts similarity index 100% rename from _import-artifact/guts/guts.ts rename to guts/guts.ts diff --git a/hamr/README.md b/hamr/README.md new file mode 100644 index 000000000000..8b8df9a436e2 --- /dev/null +++ b/hamr/README.md @@ -0,0 +1,4 @@ +# hamr + +> [!NOTE] +> The system prompt for this agent: [instructions.md](hamr/instructions.md) diff --git a/hamr/deno.json b/hamr/deno.json index dfc704315fdd..96a4aa336829 100644 --- a/hamr/deno.json +++ b/hamr/deno.json @@ -1,7 +1,5 @@ { "name": "@artifact/hamr", "version": "0.0.1", - "exports": { - ".": "./mod.ts" - } + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } } diff --git a/hamr/napp.json b/hamr/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/hamr/napp.json @@ -0,0 +1 @@ +{} diff --git a/import_map.json b/import_map.json index cce167a56242..dc3909cf1654 100644 --- a/import_map.json +++ b/import_map.json @@ -2,7 +2,6 @@ "imports": { "@deno/graph": "jsr:@deno/graph@^0.82", "@deno/doc": "jsr:@deno/doc@0.148", - "npm:/typescript": "npm:typescript@5.6.2", "graphviz": "npm:node-graphviz@^0.1.1", "@std/semver": "jsr:@std/semver", @@ -12,16 +11,46 @@ "@std/fmt": "jsr:@std/fmt", "@std/collections": "jsr:@std/collections", "@std/expect": "jsr:@std/expect", - + "@std/assert": "jsr:@std/assert", + "@std/dotenv": "jsr:@std/dotenv", + "@std/jsonc": "jsr:@std/jsonc", + "@std/ulid": "jsr:@std/ulid", + "@std/cli": "jsr:@std/cli", + "@std/io": "jsr:@std/io", + "@std/testing": "jsr:@std/testing", + "@artifact/files": "jsr:@artifact/files@^0.0.1", "@artifact/openai": "jsr:@artifact/openai@^0.0.1", - "@artifact/dumb-bot": "jsr:@artifact/dumb-bot@^0.0.8", + "@artifact/dumb-bot": "jsr:@artifact/dumb-bot@^0.0.16", "@artifact/reasoner": "jsr:@artifact/reasoner@^0.0.2", "@artifact/fixture": "jsr:@artifact/fixture@^0.0.1", "@artifact/hamr": "jsr:@artifact/hamr@^0.0.1", "@artifact/info": "jsr:@artifact/info@^0.0.1", - "@artifact/test-file-runner": "jsr:@artifact/test-file-runner@^0.0.1", + "@artifact/evals": "jsr:@artifact/evals@^0.0.1", + "@artifact/napp-tools": "jsr:@artifact/napp-tools@^0.0.1", - "zod": "npm:zod" + "@apideck/better-ajv-errors": "npm:@apideck/better-ajv-errors", + "@kitsonk/kv-toolbox": "jsr:@kitsonk/kv-toolbox", + "@noble/hashes": "npm:@noble/hashes", + "@noble/secp256k1": "npm:@noble/secp256k1", + "@scure/base": "npm:@scure/base", + "ajv": "npm:ajv", + "benchmark": "npm:benchmark", + "chalk": "npm:chalk", + "debug": "npm:debug", + "diff3": "npm:diff3", + "fast-deep-equal": "npm:fast-deep-equal/es6/index.js", + "imagescript": "npm:imagescript", + "isomorphic-git": "npm:isomorphic-git", + "it-pushable": "npm:it-pushable", + "lodash.merge": "npm:lodash.merge", + "multiformats": "npm:multiformats", + "openai": "jsr:@openai/openai", + "serialize-error": "npm:serialize-error", + "youtube-caption-extractor": "npm:youtube-caption-extractor", + "zod": "npm:zod", + "zod-to-json-schema": "npm:zod-to-json-schema", + "gpt-tokenizer": "npm:gpt-tokenizer", + "humanize-number": "npm:humanize-number" } } diff --git a/info/README.md b/info/README.md new file mode 100644 index 000000000000..1fa341a0267c --- /dev/null +++ b/info/README.md @@ -0,0 +1,4 @@ +# info + +> [!NOTE] +> The system prompt for this agent: [instructions.md](info/instructions.md) diff --git a/info/deno.json b/info/deno.json index a2fa1150fe30..129a4df3ef0b 100644 --- a/info/deno.json +++ b/info/deno.json @@ -1,7 +1,5 @@ { "name": "@artifact/info", "version": "0.0.1", - "exports": { - ".": "./mod.ts" - } + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } } diff --git a/info/mod.ts b/info/mod.ts index e69de29bb2d1..a8605f8b03ce 100644 --- a/info/mod.ts +++ b/info/mod.ts @@ -0,0 +1,2 @@ +// TODO make tools that let us resolve external links in instructions files +// using napp paths. diff --git a/info/napp.json b/info/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/info/napp.json @@ -0,0 +1 @@ +{} diff --git a/napp-check.ts b/napp-check.ts deleted file mode 100644 index d1d708648203..000000000000 --- a/napp-check.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Things to check before publishing: - * - name and version match (must be able to pull this over from the deno.json) - * - napp format must pass, and the schema must pass using zod checker - * - branding files must be present and resolvable to a hash - * - tools must all resolve - * - parameters of tools must minimally match the resolution - * - referenced napps must be resolved - * - referenced napps must be in the deno imports field - */ diff --git a/napp-tools/README.md b/napp-tools/README.md new file mode 100644 index 000000000000..dd5e67bc9f3d --- /dev/null +++ b/napp-tools/README.md @@ -0,0 +1,56 @@ +# napp-tools + +The rules of Napps: + +1. the default native code export must contain exactly what is specified in the + napp config file. +2. There must be an export named "napp.json" which contains the napp config + +Requirements of `tools` key: + +1. Be able to merge multiple napps tools together, into this napps tools +2. Cherry pick from multiple imported napps, where we want a combination of some + of the tools of the other napps +3. Include our own tools inside the napp tools as well as imported ones +4. Be simple when we only have our own tools or the tools of another napp +5. Be able to call another napps `agent` function to treat it like a standalone + agent having a plain text conversation with it + +Requirements of the `agent` key: + +1. Be omitted if this is not meant to be chatted with +2. Be called by other napps in pure agent mode, where this is a standalone + agent, talking back to the caller + +Requirements of the `stucks` key: + +1. Define configuration for stuck detection and handling +2. Include title, description, snapshot information, and crypto identifiers +3. Support branch specification and expected conditions + +Requirements of the `evals` key: + +1. Specify test runner configuration +2. Support listing of test files for napp verification + +Requirements of the `dependencies` key: + +1. List all required napps with version information +2. Support optional name overrides for dependencies + +Requirements of the `graphics` key: + +1. Support exported widgets for stateboard display +2. Allow framework specification (currently React) +3. Enable component pass-through and renaming from other napps +4. Support parameter configuration for components + +Requirements of the `effects` key: + +1. Provide mount and unmount hooks for side effects +2. Support tooling for setup and cleanup operations + +Gateway requirements: + +1. Have different configuration for mock, test, and prod environments +2. have failover and retry config information diff --git a/napp-tools/check.ts b/napp-tools/check.ts new file mode 100644 index 000000000000..521686410ff1 --- /dev/null +++ b/napp-tools/check.ts @@ -0,0 +1,26 @@ +/** + * Things to check before publishing: + * - name and version match (must be able to pull this over from the deno.json) + * - napp format must pass, and the schema must pass using zod checker + * - branding files must be present and resolvable to a hash + * - tools must all resolve + * - parameters of tools must minimally match the resolution + * - referenced napps must be resolved + * - referenced napps must be in the deno imports field + * - napp does not reference itself by its own name + * - napp does not reference itself using a dependencies map + */ + +// verify the exported zod schema generates the same json-schema as the +// napp.jsonc, and if so, use the zod schema. +// also allow generation of ajv compiled schemas for speed + +// this would be a convention on the exports of the module, that is specific for +// a particular runtime, like deno can have some defaults like zod schemas or +// ajv compilations, but these can be overridden on the runtime object. + +// if the runtime is a string, use the defaults, else can override and see what +// the defaults are. + +// do we need to make a tooling function that can retrieve an md file from a +// package but importing the package, then reading from the url ? diff --git a/napp-tools/deno.json b/napp-tools/deno.json new file mode 100644 index 000000000000..1dfa5e1cf744 --- /dev/null +++ b/napp-tools/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/napp-tools", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/napp-tools/inject-tools.ts b/napp-tools/inject-tools.ts new file mode 100644 index 000000000000..789065b7c591 --- /dev/null +++ b/napp-tools/inject-tools.ts @@ -0,0 +1,12 @@ +import { JsonSchema7ObjectType, zodToJsonSchema } from 'zod-to-json-schema' +import { ZodSchema } from 'zod' +import type { Napp } from './nappSchema.ts' + +export const inject = (napp: Napp, toolsSchema: ZodSchema) => { + // convert the schema to jsonschema, strip out anything in the object that + // is already in the schema, inject everything back into the object, return + // the new object + + // const schema = zodToJsonSchema(parameters[key]) + // delete schema.$schema +} diff --git a/napp-tools/issues.md b/napp-tools/issues.md new file mode 100644 index 000000000000..7de37688422e --- /dev/null +++ b/napp-tools/issues.md @@ -0,0 +1,27 @@ +how should secrets be handled? + +config overrides model - needs to be reasoned into the module + +```md + Layered Configuration Files and Merging + How It Works: + + The NAPP provides a base configuration (napp.json). + The NAPP may also provide optional environment-specific configuration files (napp.dev.json, napp.prod.json, etc.). + The consumer (or the runtime system) picks which file to load and merges them accordingly. + Consumer Overrides: + + The consumer selects which environment file to include by specifying a parameter (e.g., --env=prod). + Additional final overrides can come from a separate file (like napp.consumer.json) that merges last. +``` + +or you can set these config options in the napp config file, and ignore it in +children ? In the overrides, ANY key is able to be overridden. It can be removed +by setting it to null. Overrides MUST be inside the json file, since they might +be part of the default configuration. In this case, the highest napp wins. + +secrets - make this part of the api, and also override the process.env setting +to enable legacy code to work. recommedned way is using the api. throw if not +present since this is a requirement of the call being made. Might be sent as a +3rd channel in all the actions - so the json action, the filesystem, and the +secrets. diff --git a/napp-tools/mod.ts b/napp-tools/mod.ts new file mode 100644 index 000000000000..3b010c8a7288 --- /dev/null +++ b/napp-tools/mod.ts @@ -0,0 +1,2 @@ +export * from './nappSchema.ts' +export * from './read.ts' diff --git a/napp-tools/napp-example.jsonc b/napp-tools/napp-example.jsonc new file mode 100644 index 000000000000..aac9b36d7235 --- /dev/null +++ b/napp-tools/napp-example.jsonc @@ -0,0 +1,151 @@ +{ + "name": "napp-name", + "version": "0.0.1", + "napp-format": "v1", + "description": "This is a description of the napp", + // what species of runtime needs to be used to execute this napp, such as + // rust, go, python, deno, c++, etc. + "runtime": "deno", + // folder containing the branding assets of the napp + "branding": "path/to/branding/", + + "stucks": { + "napp": "@artifact/stucks", + "tool": "create", + "parameters": { + "title": "This is the title of the stuck", + "description": "This is the description of the stuck", + "snapshotId": "snapshot-id", + "crypto": "crypto-id", // universally and securely id the repo + "branch": "main", + "expections": [ + "Mrs Miggins should have two pies, not one" + ] + } + }, + "agent": { + // entry point that takes plain text in and returns text plus files + "napp": "@artifact/openai", + "tool": "some-exported-function", // if use a function other than main + "parameters": { + // parameters for the chat runner, based on what the runner is. This is + // the exact same format as a tool call. + "model": "gpt-4o", + "parallel_tool_calls": false, + // if any of these tool calls are made, the agent will stop + "stop_on_tools": ["tool1", "tool3"], + // "tools" is a special key that is resolved back to the calling napp if + // not given + "tools": ["some-function"], + // "content" is the reserved name for the input text, so it is an error to + // specify it here + + // the default name for instructions is instructions.md but this can be + // overridden to any other file + "instructions": "instructions.md" + } + }, + "tools": { + "some-function": { + "description": "This function does something", + "parameters": { + "param1": { + "description": "This is the first parameter", + "type": "string", + "required": true + }, + "param2": { + "description": "This is the second parameter", + "type": "number", + "required": false + } + }, + "returns": { + "description": "This is what the function returns", + "type": "string" + }, + // optionally, can include a list of errors + "throws": { + "FileNotFoundError": { + // list of expected error types and what to do about it + } + }, + // if path is not given, then the default export must include a function + // that satisfies this signature and that matches the name of the tool. + // if path is given, the default export must be a tool function + "path": "./path/to/function.ts" + }, + // by referring to a napp directly, it will be called as an agent + "some-other-function": "dependent-napp-name", + // passes thru to a tool of the same name + "modified-passthru": { + // optionally can modify the parameters so long as it is narrowing not + // expanding + "parameters": {}, + "napp": { + "name": "dependent-napp-name", + // if the tool is omitted, then we assume it has the same name + "tool": "some-exported-function" + // this key could be "agent", which would allow rewriting of the agent parameters + } + }, + // if no parameters object is given, an empty object is assumed + // if no returns object is given, the function must return undefined + // if no description is given, an empty string is assumed + "empty-function": {} + }, + "evals": { + "napp": "napp-test-runner-name", + "parameters": { + // config for the test runner, based on what the runner is + "files": [ + // list of eval files that can be run to verify the napp is working correctly + ] + } + }, + "dependencies": { + // list of napps that this napp depends on, including version numbers and + // commits so that every reference to the napp can be by simple name. + "dependent-napp-name": { + // if name is left out, it is assumed to be the same + "version": "0.0.1" + }, + "napp-test-runner-name": { + "name": "@artifact/evals", + "version": "0.0.1" + } + }, + "graphics": { + // a list of exported widgets that can be displayed on the stateboard + "googleMapsViewer": { + // for now, only react is supported, but we can support other frameworks + "type": "react", + // if path is not given, then check the module default export + "path": "./path/to/component.tsx", + "parameters": { + // react props that can be configured by the LLM + // same format as the tool call parameters + } + }, + // same naming conventions as tools apply. This shows passing thru to a + // napp named "@artifact/truck-routes" and using the graphics element named + // "truckRoutes" from that napp + "truckRoutes": "@artifact/truck-routes", + // allows renaming the graphics component during passthru + "weatherDisplay": { + "napp": "@artifact/weather", + "graphics": "weatherDisplayInCelsius" + // could modify the react props here as well + } + }, + "effects": { + "mount": { + // path to a tool that takes no parameters and returns nothing but might + // throw. Used to set up a side effect. + }, + "unmount": { + // path to a tool that takes no parameters and returns nothing but might + // throw. Used to clean up a side effect. + } + } +} diff --git a/napp-tools/napp.json b/napp-tools/napp.json new file mode 100644 index 000000000000..1cc4bf46d83e --- /dev/null +++ b/napp-tools/napp.json @@ -0,0 +1,80 @@ +{ + "name": "@artifact/napp-tools", + "version": "0.0.1", + "napp-format": "v1", + "description": "tooling for working with napps", + "runtime": "deno", + "agent": { + "parameters": { + "model": "gpt-4o" + } + }, + "tools": { + "versions": { + "description": "Lists prior versions of a given napp", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "name of the napp" + } + }, + "required": ["name"] + }, + "returns": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "manifest": { + "description": "Returns the manifest of a given napp, which is an array of all paths of all the files in the napp", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "name of the napp" + }, + "version": { + "type": "string", + "description": "version of the napp. If not given, the latest version will be used by default. This must be a string that came from the 'versions' tool." + } + }, + "required": ["name"] + }, + "returns": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "read": { + "description": "Reads the contents of a file in a given napp", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "name of the napp" + }, + "path": { + "type": "string", + "description": "path of the file to read. This must be a valid path that came from the 'manifest' tool." + }, + "version": { + "type": "string", + "description": "version of the napp. If not given, the latest version will be used by default. This must be a string that came from the 'versions' tool." + } + }, + "required": ["name", "path"] + }, + "returns": { + "type": "string" + } + } + } +} diff --git a/napp-tools/nappSchema.ts b/napp-tools/nappSchema.ts new file mode 100644 index 000000000000..9bc57b74469a --- /dev/null +++ b/napp-tools/nappSchema.ts @@ -0,0 +1,118 @@ +import { z } from 'zod' + +// A napp string: must match format like "@artifact/name" +const nappString = z + .string() + .regex(/^@[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+$/, 'Invalid package name format') + +// Generic parameters schema for tools/functions if needed +const parametersSchema = z.object({ + type: z.literal('object').default('object'), + properties: z.record(z.unknown()).default({}), + required: z.array(z.string()).optional(), + description: z.string().optional(), +}) + +// Schema for a generic JSON function definition +const jsonFunctionSchema = z.object({ + napp: nappString.optional(), + tool: z.string().optional(), + description: z.string().optional(), + parameters: parametersSchema.optional(), + returns: z.record(z.unknown()).optional(), + throws: z.record(z.string()).optional(), +}) + +// Stucks schema +const stucksSchema = z.object({ + napp: nappString, + tool: z.string(), + parameters: z.object({ + title: z.string(), + description: z.string(), + snapshotId: z.string(), + crypto: z.string(), + branch: z.string(), + expections: z.array(z.string()), + }), +}) + +// TODO must have either agent, or main, or tools, else the package is impotent +// TODO if no napp, then must provide parameters. +// TODO if you say tool, then you have to provide napp +// at resolve time, we will resolve to a fully qualified object, where the napp +// can be resolved to self. + +// Agent schema +const agentSchema = z.object({ + napp: nappString, + tool: z.string().optional(), + parameters: z.object({ + model: z.string(), + parallel_tool_calls: z.boolean(), + stop_on_tools: z.array(z.string()), + tools: z.array(z.string()), + instructions: z.string().optional(), + }), +}) + +// Tools schema: each key is either a nappString or a full jsonFunctionSchema +const toolsSchema = z.record(z.union([jsonFunctionSchema, nappString])) + +// Evals schema +const evalsSchema = z.object({ + napp: z.string(), + parameters: z.object({ + files: z.array(z.string()), + }), +}) + +// Dependencies schema +const dependenciesSchema = z.record(z.object({ + name: nappString.optional(), + version: z.string(), +})) + +// Graphics schema: values can be a nappString, a react component definition, +// or a passthrough reference to another napp's graphics +const graphicsItemSchema = z.union([ + nappString, + z.object({ + type: z.literal('react'), + path: z.string().optional(), + parameters: parametersSchema.optional(), + }), + z.object({ + napp: nappString, + graphics: z.string().optional(), + }), +]) +const graphicsSchema = z.record(graphicsItemSchema) + +// Effects schema: mount/unmount each a tool invocation (jsonFunctionSchema) or omitted +const effectsSchema = z.object({ + mount: jsonFunctionSchema.optional(), + unmount: jsonFunctionSchema.optional(), +}) + +// The main Napp schema +export const nappSchema = z.object({ + name: z.string(), + version: z.string(), + 'napp-format': z.literal('v1'), // locked to "v1" as per the example + description: z.string().optional(), + runtime: z.literal('deno'), + + // optional fields + branding: z.string().optional(), + + stucks: stucksSchema.optional(), + agent: agentSchema.optional(), + tools: toolsSchema.optional(), + evals: evalsSchema.optional(), + dependencies: dependenciesSchema.optional(), + graphics: graphicsSchema.optional(), + effects: effectsSchema.optional(), +}) + +export type Napp = z.infer<typeof nappSchema> diff --git a/napp-tools/read.ts b/napp-tools/read.ts new file mode 100644 index 000000000000..46b0172bbefa --- /dev/null +++ b/napp-tools/read.ts @@ -0,0 +1,17 @@ +import { nappSchema } from './nappSchema.ts' + +export const readNappConfig = async (name: string) => { + try { + const imported = await import(name + '/napp.json', { + with: { type: 'json' }, + }) + return nappSchema.parse(imported.default) + } catch (error) { + if (error instanceof Error) { + const msg = 'napp.json not imported from package: ' + name + ':\n' + + error.message + throw new Error(msg) + } + throw error + } +} diff --git a/napp-tools/spec.md b/napp-tools/spec.md new file mode 100644 index 000000000000..80c998e85b57 --- /dev/null +++ b/napp-tools/spec.md @@ -0,0 +1,417 @@ +# NAPP Format Specification + +**Version:** v1 **Status:** Draft + +## Overview + +A **NAPP** (Natural Language Application Package) is a structured, +language-agnostic package definition intended for seamless integration between +deterministic code and AI-driven components. The purpose of a NAPP is to provide +a standardized JSON-based configuration file (referred to as the NAPP manifest) +that describes: + +- The NAPP’s name, version, and runtime environment +- A set of callable tools and agents (functions, possibly AI-backed) +- Dependencies on other NAPPs +- Evaluations, branding assets, user-facing graphical components, and effects +- Stuck detection and handling configuration + +NAPPs are designed to be discovered, understood, and invoked by both humans and +automated systems (including LLMs). + +## File Format and Location + +A NAPP is described by a single JSON file named `napp.json`—that resides at the +root of a NAPP’s source code repository. This file follows the structure defined +in this specification and must validate against the `nappSchema` as defined +below. + +## Top-Level Keys + +The top-level NAPP configuration object must contain the following fields: + +### Required Fields + +- **name** (string):\ + A unique name identifying the NAPP. Typically corresponds to a package name.\ + Example: `"@scope/package-name"` + +- **version** (string):\ + The NAPP’s version in semantic versioning format (e.g., `"0.0.1"`). + +- **napp-format** (string, must be `"v1"`):\ + Indicates the NAPP specification format version. Currently locked to `"v1"`. + +- **runtime** (string):\ + The target runtime environment required to execute the NAPP.\ + Current acceptable values: `"deno"`\ + (Future revisions may allow `"rust"`, `"go"`, `"python"`, etc.) + +### Optional Fields + +- **description** (string):\ + A human-readable description of the NAPP’s purpose. + +- **branding** (string):\ + A path (relative to the NAPP root) to a directory containing branding assets + (e.g., logos, icons). + +- **stucks** (object):\ + Configuration related to stuck detection and handling.\ + See [Stucks Configuration](#stucks-configuration). + +- **agent** (object):\ + Defines an AI agent entry point for chat-based invocation of the NAPP.\ + See [Agent Configuration](#agent-configuration). + +- **tools** (object):\ + A collection of named tools (functions) that can be invoked by the NAPP or its + callers.\ + See [Tools Configuration](#tools-configuration). + +- **evals** (object):\ + Configuration for evaluation/test runners.\ + See [Evals Configuration](#evals-configuration). + +- **dependencies** (object):\ + References to other NAPPs this NAPP depends on.\ + See [Dependencies](#dependencies). + +- **graphics** (object):\ + Configuration describing visual, interactive components (widgets) that can be + displayed.\ + See [Graphics Configuration](#graphics-configuration). + +- **effects** (object):\ + Configuration for side-effect hooks that run on mount/unmount.\ + See [Effects Configuration](#effects-configuration). + +## Detailed Field Specifications + +### Stucks Configuration + +**Key:** `stucks` + +**Structure:** + +- `napp` (string): A reference to another NAPP (in `@artifact/name` format) that + provides stuck-detection functionality. +- `tool` (string): The name of the tool within the referenced NAPP to invoke for + stuck handling. +- `parameters` (object): + - `title` (string): Title describing the stuck scenario. + - `description` (string): Detailed description of the stuck scenario. + - `snapshotId` (string): Identifier for the current snapshot/state. + - `crypto` (string): A cryptographic or unique ID tying this stuck to a + particular revision or repository. + - `branch` (string): The branch name relevant to this stuck scenario. + - `expections` (array of strings): A list of expected conditions or anomalies. + +**Example:** + +```json +"stucks": { + "napp": "@artifact/stucks", + "tool": "create", + "parameters": { + "title": "This is the title of the stuck", + "description": "This is the description of the stuck", + "snapshotId": "snapshot-id", + "crypto": "crypto-id", + "branch": "main", + "expections": ["Mrs Miggins should have two pies, not one"] + } +} +``` + +### Agent Configuration + +**Key:** `agent` + +Defines an agent endpoint that can be queried with natural language to produce +responses (and possibly file outputs). + +**Structure:** + +- `napp` (string, `@artifact/name`): Reference to a NAPP providing the agent + logic. +- `tool` (string, optional): The entry point tool of the agent NAPP if not using + the default. +- `parameters` (object): + - `model` (string): The model or variant of the AI backend to use. + - `parallel_tool_calls` (boolean): Whether the agent can make tool calls in + parallel. + - `stop_on_tools` (array of strings): A list of tool names that, if called, + cause the agent to stop execution. + - `tools` (array of strings): A list of tool names that the agent can call.\ + If omitted, defaults to the current NAPP’s tools. + - `instructions` (string, optional): A path to a file with agent instructions. + +**Example:** + +```json +"agent": { + "napp": "@artifact/openai", + "tool": "some-exported-function", + "parameters": { + "model": "gpt-4o", + "parallel_tool_calls": false, + "stop_on_tools": ["tool1", "tool3"], + "tools": ["some-function"], + "instructions": "instructions.md" + } +} +``` + +### Tools Configuration + +**Key:** `tools` + +A mapping of tool names to their definitions. Each tool: + +- May directly define a function (with parameters, return types, etc.) +- May reference another NAPP’s tool(s) +- May pass-through or wrap another NAPP’s tool with modified parameters. + +**Structure (for each tool key):** + +- If a string is provided and matches `@artifact/name`, the tool is a reference + to another NAPP’s agent or tool. +- Otherwise, an object with keys: + - `napp` (optional, string): Reference to a NAPP providing the tool. + - `tool` (optional, string): Name of the tool in the referenced NAPP. + - `description` (optional, string): Human-readable description of the tool. + - `parameters` (optional, object): JSON schema describing the tool’s input + parameters. + - `type` must be `"object"` if present. + - `properties` is a map of parameter names to schema definitions. + - `required` is an array of parameter names that are required. + - `returns` (optional, object): JSON schema describing the tool’s return + value. + - `throws` (optional, object): Map of error names to error handling metadata. + - `path` (optional, string): A path to a local file implementing the tool + function. If omitted, the default export from the NAPP runtime must provide + the tool logic. + +**Example:** + +```json +"tools": { + "some-function": { + "description": "This function does something", + "parameters": { + "param1": { + "description": "This is the first parameter", + "type": "string", + "required": true + }, + "param2": { + "description": "This is the second parameter", + "type": "number", + "required": false + } + }, + "returns": { + "description": "The return of the function", + "type": "string" + }, + "throws": { + "FileNotFoundError": {} + }, + "path": "./path/to/function.ts" + }, + "some-other-function": "dependent-napp-name" +} +``` + +### Evals Configuration + +**Key:** `evals` + +Specifies a test runner NAPP and configuration for verifying NAPP functionality. + +**Structure:** + +- `napp` (string): A NAPP name or reference that runs tests. +- `parameters` (object): + - `files` (array of strings): Paths to test files. + +**Example:** + +```json +"evals": { + "napp": "napp-test-runner-name", + "parameters": { + "files": ["tests/test1.json", "tests/test2.json"] + } +} +``` + +### Dependencies + +**Key:** `dependencies` + +A map from dependency names to objects specifying version and optional name +overrides. + +**Structure (for each dependency key):** + +- `name` (string, optional): The actual package name if different from the key. +- `version` (string): The version of the dependency NAPP. + +**Example:** + +```json +"dependencies": { + "dependent-napp-name": { + "version": "0.0.1" + }, + "napp-test-runner-name": { + "name": "@artifact/evals", + "version": "0.0.1" + } +} +``` + +### Graphics Configuration + +**Key:** `graphics` + +A collection of graphical components (widgets) that can be displayed on a +stateboard or UI dashboard. + +**Allowed forms:** + +1. A direct reference to another NAPP’s graphics element + (`"@artifact/some-napp"`). +2. A react component definition with `type: "react"`. +3. A passthrough that references another NAPP’s graphics and optionally renames + it. + +**Structure:** + +- React component definition: + - `type` = `"react"` + - `path` (optional, string): Path to the React component source file. + - `parameters` (optional, object): Schema for configurable props. +- NAPP reference: + - `napp` (string): `@artifact/other-napp` + - `graphics` (string, optional): Name of the graphics component in the other + NAPP. + +**Example:** + +```json +"graphics": { + "googleMapsViewer": { + "type": "react", + "path": "./path/to/component.tsx", + "parameters": { + "zoomLevel": { + "type": "number", + "description": "Zoom level of the map", + "required": true + } + } + }, + "truckRoutes": "@artifact/truck-routes", + "weatherDisplay": { + "napp": "@artifact/weather", + "graphics": "weatherDisplayInCelsius" + } +} +``` + +### Effects Configuration + +**Key:** `effects` + +Specifies lifecycle hooks that run when a NAPP is mounted or unmounted. These +are tool invocations that may produce side effects. + +**Structure:** + +- `mount` (optional, object): A tool definition (same structure as in `tools`) + used to set up side effects. +- `unmount` (optional, object): A tool definition used to clean up side effects. + +**Example:** + +```json +"effects": { + "mount": { + "napp": "@artifact/side-effects", + "tool": "setup" + }, + "unmount": { + "napp": "@artifact/side-effects", + "tool": "cleanup" + } +} +``` + +## Validation Rules + +- **Names and References:** + - The `name` field must be a simple string, not referencing itself with + `@artifact/name` style. + - All referenced NAPPs (e.g., in `tools`, `dependencies`, `stucks`, `agent`, + `graphics`) must adhere to the `@namespace/name` format if pointing to + external NAPPs. + - Self-references to the current NAPP by its own name in dependencies or tools + are not allowed. + +- **Schema Compliance:** + - Parameters, returns, and throws objects must conform to JSON schema-like + structures as outlined in the examples. + - The `napp-format` must be `"v1"`. + +- **Runtime Requirements:** + - The `runtime` field currently only supports `"deno"` in this version. + +## Checks Before Publishing + +Implementations should verify: + +1. **Name and Version Match:** Confirm `name` and `version` fields are + consistent and correct. +2. **NAPP Schema Validation:** The entire JSON must validate against the + `nappSchema`. +3. **Branding Files:** If `branding` is specified, ensure referenced files exist + and are hashable. +4. **Tools Resolution:** All `tools` must be resolvable, meaning their `path` or + referenced NAPP/tool must exist. +5. **Parameter Matching:** Tool parameters must match the function signatures at + runtime. +6. **NAPP References in Dependencies:** + - Ensure that dependencies listed in `dependencies` are available. + - If using a runtime requiring import maps, ensure these dependencies are + reflected in import maps. +7. **No Self-Reference:** The NAPP must not reference itself as a dependency or + as a foreign resource. + +## Example + +See `napp-example.jsonc` for a complete annotated example. + +## Extensibility + +Future versions of this specification may introduce: + +- Additional runtimes (Python, Go, Rust). +- Enhanced validation for parameter schemas. +- Support for environment-specific overrides (mock, test, prod). +- Failover and retry configurations for gateway calls. + +## Questions for Clarification + +- Are there any constraints on allowed runtimes beyond `deno`? +- Should we support additional `napp-format` versions in the future (e.g., + `v2`)? +- Is there a standard way to handle environment-based configuration overrides? + +_(Please provide feedback or clarifications as needed.)_ + +--- + +**End of Specification** diff --git a/openai/README.md b/openai/README.md new file mode 100644 index 000000000000..6ca547f4e27e --- /dev/null +++ b/openai/README.md @@ -0,0 +1,4 @@ +# openai + +> [!NOTE] +> The system prompt for this agent: [instructions.md](openai/instructions.md) diff --git a/openai/completions.test.ts b/openai/completions.test.ts index 8bff11d9a73c..d53260eeca47 100644 --- a/openai/completions.test.ts +++ b/openai/completions.test.ts @@ -1,56 +1,50 @@ -// import { loadAgent } from '@/isolates/utils/load-agent.ts' -// import { complete, image } from '@/isolates/ai-completions.ts' -// import { functions, safeAssistantName } from '@/isolates/ai-completions.ts' -// import { expect } from '@utils' -// import { createMockApi } from '@/tests/fixtures/mock-api.ts' +import * as openai from '@artifact/openai' +import { expect } from '@std/expect' +import { loadAgent } from '@/isolates/utils/load-agent.ts' +import { complete, functions, image, safeAssistantName } from './completions.ts' +import { createMockApi } from '@/tests/fixtures/mock-api.ts' -// Deno.test('test the regex for agent name sanitization', () => { -// const result = safeAssistantName({ -// role: 'assistant', -// name: 'agents/o1.md', -// }) -// expect(result).toEqual({ role: 'assistant', name: 'agents_o1_md' }) -// }) -// Deno.test('generate images', async (t) => { -// image.mock.useRecorder(t) -// const { api, stop } = await createMockApi('test/images') -// const path = 'images/test.jpg' -// const result = await functions.image({ -// path, -// prompt: 'a dystopian robot overlord mecha', -// lowQuality: true, -// size: '1024x1024', -// style: 'natural', -// }, api) -// const file = await api.readBinary(path) -// expect(file.byteLength).toBeGreaterThan(0) -// expect(file.byteLength).toEqual(result.size) +Deno.test('test the regex for agent name sanitization', () => { + const result = safeAssistantName({ + role: 'assistant', + name: 'agents/o1.md', + }) + expect(result).toEqual({ role: 'assistant', name: 'agents_o1_md' }) +}) -// image.mock.teardown() -// stop() -// }) +Deno.test('generate images', async (t) => { + image.mock.useRecorder(t) + const { api, stop } = await createMockApi('test/images') + const path = 'images/test.jpg' + const result = await functions.image({ + path, + prompt: 'a dystopian robot overlord mecha', + lowQuality: true, + size: '1024x1024', + style: 'natural', + }, api) + const file = await api.readBinary(path) + expect(file.byteLength).toBeGreaterThan(0) + expect(file.byteLength).toEqual(result.size) -// Deno.test('inject empty', async (t) => { -// complete.mock.useRecorder(t) -// const { api, stop } = await createMockApi('test/inject-single') -// const path = 'fake/agent.md' -// api.write(path, '') -// const agent = await loadAgent(path, api) + image.mock.teardown() + stop() +}) -// const result = await complete(agent, [{ -// role: 'system', -// content: 'say cheese in emoji', -// }], api) -// expect(result).toHaveProperty('assistant') -// expect(result).toHaveProperty('stats') +Deno.test('inject empty', async (t) => { + complete.mock.useRecorder(t) + const { api, stop } = await createMockApi('test/inject-single') + const path = 'fake/agent.md' + api.write(path, '') + const agent = await loadAgent(path, api) -// complete.mock.teardown() -// stop() -// }) + const result = await complete(agent, [{ + role: 'system', + content: 'say cheese in emoji', + }], api) + expect(result).toHaveProperty('assistant') + expect(result).toHaveProperty('stats') -import * as m from "./completions.ts"; -import { expect } from "@std/expect"; - -Deno.test("publish", () => { - expect(m.n).toBe(84); -}); + complete.mock.teardown() + stop() +}) diff --git a/openai/completions.ts b/openai/completions.ts index 27f45ee18504..d0aebf70d40e 100644 --- a/openai/completions.ts +++ b/openai/completions.ts @@ -1,326 +1,334 @@ -// import { mockCreator } from './utils/mocker.ts' -// import { assert, Debug } from '@utils' -// import { base64 } from '@scure/base' -// import '@std/dotenv/load' // load .env variables -// import OpenAI from 'openai' -// import { decode, Image } from 'imagescript' -// import { -// Agent, -// AssistantMessage, -// ChatParams, -// chatParams, -// CompletionMessage, -// Functions, -// getThreadPath, -// type IA, -// messageStatsSchema, -// print, -// printPlain, -// Returns, -// Thread, -// ToApiType, -// } from '@/constants.ts' -// import { loadTools } from './utils/ai-load-tools.ts' -// import { loadAgent } from './utils/load-agent.ts' -// import { z } from 'zod' -// import { assistantMessage } from '@/api/zod.ts' -// import { expect } from '@std/expect/expect' - -// const log = Debug('AI:completions') - -// const apiKey = Deno.env.get('OPENAI_API_KEY') -// if (!apiKey) { -// throw new Error('missing openai api key: OPENAI_API_KEY') -// } -// const ai = new OpenAI({ apiKey, maxRetries: 5 }) - -// export const transcribe = async (file: File) => { -// const transcription = await ai.audio.transcriptions -// .create({ -// file, -// model: 'whisper-1', -// prompt: -// 'Backchat, GPT4, GPT3, Dreamcatcher, CRM, HAL, Deno, Stucks, Redlid, Pijul', -// }) -// return transcription.text -// } - -// export const parameters = { -// /** Complete the thread with the given agent */ -// complete: z.object({ -// path: z.string(), -// }), -// /** Gives slightly quicker feedback to users when waiting for completions */ -// completionEffect: z.object({ -// path: z.string(), -// }), -// image: z.object({ -// path: z.string().regex(/\.jpg$/, { -// message: 'The path must end with .jpg', -// }).describe( -// 'The path to the image to generate, which should be a meaningful name in the images/ directory, and should end in .jpg', -// ), -// prompt: z.string().max(4000), -// lowQuality: z.boolean().optional().describe( -// 'Generate a low quality image as opposed to the default high quality image', -// ), -// size: z.enum(['1024x1024', '1792x1024', '1024x1792']).optional().describe( -// 'The size of the image to generate', -// ), -// style: z.enum(['vivid', 'natural']).optional().describe( -// 'The style of the image to generate, which defaults to vivid', -// ), -// }).describe( -// 'Generate an image using DALL-E-3 from the provided prompt. The image will be saved to the provided path. The revised prompt that the image generator used will be returned, as well as the size of the image in bytes.', -// // TODO add how to show in stateboard -// ), -// } -// export const returns: Returns<typeof parameters> = { -// complete: z.void(), -// completionEffect: z.void(), -// image: z.object({ -// /** The revised prompt that the image generation system used */ -// revisedPrompt: z.string(), -// /** The size in bytes of the written image */ -// size: z.number().int().gte(0), -// }), -// } - -// export type Api = ToApiType<typeof parameters, typeof returns> - -// export const functions: Functions<Api> = { -// async complete({ path }, api) { -// const threadPath = getThreadPath(api.pid) -// log('completing thread %o', threadPath, print(api.pid)) - -// const agent = await loadAgent(path, api) -// const thread = await api.readThread(threadPath) -// thread.messages.push({ role: 'assistant', name: agent.source.path }) -// api.writeJSON(threadPath, thread) - -// const { completionEffect } = await api.actions<Api>('ai-completions') -// await completionEffect({ path }) -// }, -// async completionEffect({ path }, api) { -// const threadPath = getThreadPath(api.pid) -// log('completing thread %o', threadPath, print(api.pid)) - -// const agent = await loadAgent(path, api) - -// const thread = await api.readThread(threadPath) -// const last = thread.messages.pop() -// assert(last, 'no messages in thread') -// assert(last.role === 'assistant', 'last message must be assistant') -// assert(last.name === agent.source.path, 'last message must be from agent') -// assert(!last.content, 'last message must be empty') - -// const { assistant, stats } = await complete(agent, thread.messages, api) - -// const id = thread.messages.length + thread.messageOffset -// thread.messageStats[id] = stats -// thread.messages.push(assistant) -// api.writeJSON(threadPath, thread) -// log('completion complete', assistant.tool_calls?.[0], assistant.content) -// }, -// async image(params, api) { -// const { path } = params -// const { data, response } = await image(params, api) -// log('headers', response.statusText) -// const { b64_json, revised_prompt } = data.data[0] -// if (!b64_json) { -// throw new Error('no image data') -// } -// const imageData = base64.decode(b64_json) -// log('image length', imageData.length) - -// const png = await decode(imageData) -// assert(png instanceof Image, 'image must be an instance of Image') -// const jpg = await png.encodeJPEG() -// log('length', jpg.length) - -// api.write(path, jpg) -// return { revisedPrompt: revised_prompt, size: jpg.length } -// }, -// } - -// export const image = async ( -// params: z.infer<typeof parameters['image']>, -// api: IA, -// ) => { -// const { prompt, lowQuality, size, style } = params -// const id = printPlain(api.pid) -// const recording = imageMock.next(id) -// if (recording) { -// expect(recording.request).toEqual(params) -// return recording.reply as { -// data: OpenAI.Images.ImagesResponse -// response: Response -// } -// } - -// const { data, response } = await ai.images -// .generate({ -// prompt, -// model: 'dall-e-3', -// quality: lowQuality ? 'standard' : 'hd', -// response_format: 'b64_json', -// size, -// style, -// }).withResponse() - -// const squeezed = squeezeMockData(data) -// imageMock.store(id, { request: params, reply: { data: squeezed, response } }) -// return { data, response } -// } - -// const imageReplySchema = z.object({ -// data: z.unknown(), -// response: z.unknown(), -// }) -// const imagePairSchema = z.object({ -// request: parameters['image'], -// reply: imageReplySchema, -// }) - -// const imageMock = mockCreator(imagePairSchema) -// image.mock = imageMock - -// const completionsReplySchema = z.object({ -// assistant: assistantMessage, -// stats: messageStatsSchema, -// }) - -// const completionsPairSchema = z.object({ -// request: chatParams, -// reply: completionsReplySchema, -// }) - -// export const complete = async ( -// agent: Agent, -// messages: Thread['messages'], -// api: IA, -// ) => { -// const tools = await loadTools(agent, api) -// const args = getChatParams(agent, messages, tools) - -// const id = printPlain(api.pid) -// const recording = completionsMock.next(id) -// if (recording) { -// expect(recording.request).toEqual(args) -// return recording.reply -// } - -// log('completion started with model: %o', args.model, print(api.pid)) -// let retries = 0 -// const RETRY_LIMIT = 5 -// let errorMessage = '' -// while (retries++ < RETRY_LIMIT) { -// try { -// const start = Date.now() -// const { data: completion, response: raw } = await ai.chat.completions -// .create(args).withResponse() -// const duration = Date.now() - start -// const openAiProcessingMs = raw.headers.get('openai-processing-ms') -// const { created, model, system_fingerprint, usage } = completion - -// const { message } = completion.choices[0] -// log('completion complete', agent.source.path, message) -// const assistant: AssistantMessage = { -// ...message, -// name: agent.source.path, -// } -// const reply = { -// assistant, -// stats: messageStatsSchema.parse({ -// created, -// model, -// system_fingerprint, -// usage, -// duration, -// openAiProcessingMs: openAiProcessingMs ? +openAiProcessingMs : 0, -// }), -// } -// completionsMock.store(id, { request: args, reply }) - -// return reply -// } catch (error) { -// console.error('ai completion error', error) -// if (error instanceof Error) { -// errorMessage = error.message -// } -// } -// } -// throw new Error(`Failed after ${retries} attempts: ${errorMessage}`) -// } -// const completionsMock = mockCreator(completionsPairSchema) -// complete.mock = completionsMock - -// export const getChatParams = ( -// agent: Agent, -// messages: Thread['messages'], -// tools: OpenAI.ChatCompletionTool[], -// ) => { -// const { config } = agent -// const { -// model, -// temperature, -// tool_choice, -// parallel_tool_calls, -// presence_penalty, -// } = config - -// messages = [...messages] -// const sysprompt: CompletionMessage = { -// role: 'system', -// content: agent.instructions, -// name: agent.source.path, -// } -// if (agent.instructions || agent.commands.length || agent.napps.length) { -// messages.push(sysprompt) -// } -// messages = messages.map(safeAssistantName) - -// let args: ChatParams = { -// model, -// temperature, -// messages, -// seed: 1337, -// tools: tools.length ? tools : undefined, -// tool_choice: tools.length ? tool_choice : undefined, -// presence_penalty, -// parallel_tool_calls: tools.length ? parallel_tool_calls : undefined, -// } -// args = Object.fromEntries( -// Object.entries(args).filter(([_, value]) => value !== undefined), -// ) as ChatParams -// return args -// } -// export const safeAssistantName = (message: CompletionMessage) => { -// if (message.role !== 'assistant' && message.role !== 'system') { -// return message -// } -// if (!message.name) { -// return message -// } -// if (!/^[a-zA-Z0-9_-]+$/.test(message.name)) { -// return { ...message, name: message.name.replaceAll(/[^a-zA-Z0-9_-]/g, '_') } -// } -// return message -// } - -// const squeezeMockData = (result: OpenAI.Images.ImagesResponse) => { -// const fake = -// 'iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAABQUlEQVR4nAE2Acn+AFWFasx5BD3kQfuYK9zNJJXzR81JXGsdYwxOP/ggyAQC+dlbzrmdUZ+Xgo9sBhbNytzXE2IPfxblFuNZ9vYCD9Qw+7UUN/xCErc8I0gk21S+uCU4EX33VN1CFz5pAHaODSC/a8ZsheMYxX37BIOoMcfnpS8k6KwY7zE9VgKU4CsTmvjyE8lur1yLUAo7SKUu07rY8zYwmNASNh0CvvWpIaH1bs1V01RZE2se73fmxRfts48N4bf03otfBDHQ3j52DzyUBguaBfEcXnDOsBuN6HeYeMbrXRzVtADOYHNpl5tMMNBP2PSqx3YI7MFiBfb2wXFXAxH0JSsA1ZsAv2N5fzoy1PrKBJ8201yyqOsPZKnyqJ10ME7oAfSuLGJaACH+38rm67hvIcxsB99FWZb3Ah6CMO1CEnIBlb6tLzIdAAAAAElFTkSuQmCC' - -// assert(result.data.length, 'data length must be 1') -// const { b64_json: _, ...rest } = result.data[0] -// return { -// ...result, -// data: [{ b64_json: fake, ...rest }], -// } -// } - -/** - * The number 84. - */ -export const n = 84; +import { assert } from '@std/assert' +import Debug from 'debug' +import { base64 } from '@scure/base' +import '@std/dotenv/load' // load .env variables +import OpenAI from 'openai' +import { decode, Image } from 'imagescript' + +import { mockCreator } from './utils/mocker.ts' +import { + Agent, + AssistantMessage, + ChatParams, + chatParams, + CompletionMessage, + Functions, + getThreadPath, + type IA, + messageStatsSchema, + print, + printPlain, + Returns, + Thread, + ToApiType, +} from '@/constants.ts' +import { loadTools } from './utils/ai-load-tools.ts' +import { loadAgent } from './utils/load-agent.ts' +import { z } from 'zod' +import { assistantMessage } from '@/api/zod.ts' +import { expect } from '@std/expect/expect' + +const log = Debug('AI:completions') + +const apiKey = Deno.env.get('OPENAI_API_KEY') +if (!apiKey) { + throw new Error('missing openai api key: OPENAI_API_KEY') +} +const ai = new OpenAI({ apiKey, maxRetries: 5 }) + +export const transcribe = async (file: File) => { + const transcription = await ai.audio.transcriptions + .create({ + file, + model: 'whisper-1', + prompt: + 'Backchat, GPT4, GPT3, Dreamcatcher, CRM, HAL, Deno, Stucks, Redlid, Pijul', + }) + return transcription.text +} + +export const parameters = { + /** Complete the thread with the given agent */ + complete: z.object({ + path: z.string(), + }), + /** Gives slightly quicker feedback to users when waiting for completions */ + completionEffect: z.object({ + path: z.string(), + }), + image: z.object({ + path: z.string().regex(/\.jpg$/, { + message: 'The path must end with .jpg', + }).describe( + 'The path to the image to generate, which should be a meaningful name in the images/ directory, and should end in .jpg', + ), + prompt: z.string().max(4000), + lowQuality: z.boolean().optional().describe( + 'Generate a low quality image as opposed to the default high quality image', + ), + size: z.enum(['1024x1024', '1792x1024', '1024x1792']).optional() + .describe( + 'The size of the image to generate', + ), + style: z.enum(['vivid', 'natural']).optional().describe( + 'The style of the image to generate, which defaults to vivid', + ), + }).describe( + 'Generate an image using DALL-E-3 from the provided prompt. The image will be saved to the provided path. The revised prompt that the image generator used will be returned, as well as the size of the image in bytes.', + // TODO add how to show in stateboard + ), +} +export const returns: Returns<typeof parameters> = { + complete: z.void(), + completionEffect: z.void(), + image: z.object({ + /** The revised prompt that the image generation system used */ + revisedPrompt: z.string(), + /** The size in bytes of the written image */ + size: z.number().int().gte(0), + }), +} + +export type Api = ToApiType<typeof parameters, typeof returns> + +export const functions: Functions<Api> = { + async complete({ path }, api) { + const threadPath = getThreadPath(api.pid) + log('completing thread %o', threadPath, print(api.pid)) + + const agent = await loadAgent(path, api) + const thread = await api.readThread(threadPath) + thread.messages.push({ role: 'assistant', name: agent.source.path }) + api.writeJSON(threadPath, thread) + + const { completionEffect } = await api.actions<Api>('ai-completions') + await completionEffect({ path }) + }, + async completionEffect({ path }, api) { + const threadPath = getThreadPath(api.pid) + log('completing thread %o', threadPath, print(api.pid)) + + const agent = await loadAgent(path, api) + + const thread = await api.readThread(threadPath) + const last = thread.messages.pop() + assert(last, 'no messages in thread') + assert(last.role === 'assistant', 'last message must be assistant') + assert( + last.name === agent.source.path, + 'last message must be from agent', + ) + assert(!last.content, 'last message must be empty') + + const { assistant, stats } = await complete(agent, thread.messages, api) + + const id = thread.messages.length + thread.messageOffset + thread.messageStats[id] = stats + thread.messages.push(assistant) + api.writeJSON(threadPath, thread) + log('completion complete', assistant.tool_calls?.[0], assistant.content) + }, + async image(params, api) { + const { path } = params + const { data, response } = await image(params, api) + log('headers', response.statusText) + const { b64_json, revised_prompt } = data.data[0] + if (!b64_json) { + throw new Error('no image data') + } + const imageData = base64.decode(b64_json) + log('image length', imageData.length) + + const png = await decode(imageData) + assert(png instanceof Image, 'image must be an instance of Image') + const jpg = await png.encodeJPEG() + log('length', jpg.length) + + api.write(path, jpg) + return { revisedPrompt: revised_prompt, size: jpg.length } + }, +} + +export const image = async ( + params: z.infer<typeof parameters['image']>, + api: IA, +) => { + const { prompt, lowQuality, size, style } = params + const id = printPlain(api.pid) + const recording = imageMock.next(id) + if (recording) { + expect(recording.request).toEqual(params) + return recording.reply as { + data: OpenAI.Images.ImagesResponse + response: Response + } + } + + const { data, response } = await ai.images + .generate({ + prompt, + model: 'dall-e-3', + quality: lowQuality ? 'standard' : 'hd', + response_format: 'b64_json', + size, + style, + }).withResponse() + + const squeezed = squeezeMockData(data) + imageMock.store(id, { + request: params, + reply: { data: squeezed, response }, + }) + return { data, response } +} + +const imageReplySchema = z.object({ + data: z.unknown(), + response: z.unknown(), +}) +const imagePairSchema = z.object({ + request: parameters['image'], + reply: imageReplySchema, +}) + +const imageMock = mockCreator(imagePairSchema) +image.mock = imageMock + +const completionsReplySchema = z.object({ + assistant: assistantMessage, + stats: messageStatsSchema, +}) + +const completionsPairSchema = z.object({ + request: chatParams, + reply: completionsReplySchema, +}) + +export const complete = async ( + agent: Agent, + messages: Thread['messages'], + api: IA, +) => { + const tools = await loadTools(agent, api) + const args = getChatParams(agent, messages, tools) + + const id = printPlain(api.pid) + const recording = completionsMock.next(id) + if (recording) { + expect(recording.request).toEqual(args) + return recording.reply + } + + log('completion started with model: %o', args.model, print(api.pid)) + let retries = 0 + const RETRY_LIMIT = 5 + let errorMessage = '' + while (retries++ < RETRY_LIMIT) { + try { + const start = Date.now() + const { data: completion, response: raw } = await ai.chat + .completions + .create(args).withResponse() + const duration = Date.now() - start + const openAiProcessingMs = raw.headers.get('openai-processing-ms') + const { created, model, system_fingerprint, usage } = completion + + const { message } = completion.choices[0] + log('completion complete', agent.source.path, message) + const assistant: AssistantMessage = { + ...message, + name: agent.source.path, + } + const reply = { + assistant, + stats: messageStatsSchema.parse({ + created, + model, + system_fingerprint, + usage, + duration, + openAiProcessingMs: openAiProcessingMs ? +openAiProcessingMs : 0, + }), + } + completionsMock.store(id, { request: args, reply }) + + return reply + } catch (error) { + console.error('ai completion error', error) + if (error instanceof Error) { + errorMessage = error.message + } + } + } + throw new Error(`Failed after ${retries} attempts: ${errorMessage}`) +} +const completionsMock = mockCreator(completionsPairSchema) +complete.mock = completionsMock + +export const getChatParams = ( + agent: Agent, + messages: Thread['messages'], + tools: OpenAI.ChatCompletionTool[], +) => { + const { config } = agent + const { + model, + temperature, + tool_choice, + parallel_tool_calls, + presence_penalty, + } = config + + messages = [...messages] + const sysprompt: CompletionMessage = { + role: 'system', + content: agent.instructions, + name: agent.source.path, + } + if (agent.instructions || agent.commands.length || agent.napps.length) { + messages.push(sysprompt) + } + messages = messages.map(safeAssistantName) + + let args: ChatParams = { + model, + temperature, + messages, + seed: 1337, + tools: tools.length ? tools : undefined, + tool_choice: tools.length ? tool_choice : undefined, + presence_penalty, + parallel_tool_calls: tools.length ? parallel_tool_calls : undefined, + } + args = Object.fromEntries( + Object.entries(args).filter(([_, value]) => value !== undefined), + ) as ChatParams + return args +} +export const safeAssistantName = (message: CompletionMessage) => { + if (message.role !== 'assistant' && message.role !== 'system') { + return message + } + if (!message.name) { + return message + } + if (!/^[a-zA-Z0-9_-]+$/.test(message.name)) { + return { + ...message, + name: message.name.replaceAll(/[^a-zA-Z0-9_-]/g, '_'), + } + } + return message +} + +const squeezeMockData = (result: OpenAI.Images.ImagesResponse) => { + const fake = + 'iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAIAAAACUFjqAAABQUlEQVR4nAE2Acn+AFWFasx5BD3kQfuYK9zNJJXzR81JXGsdYwxOP/ggyAQC+dlbzrmdUZ+Xgo9sBhbNytzXE2IPfxblFuNZ9vYCD9Qw+7UUN/xCErc8I0gk21S+uCU4EX33VN1CFz5pAHaODSC/a8ZsheMYxX37BIOoMcfnpS8k6KwY7zE9VgKU4CsTmvjyE8lur1yLUAo7SKUu07rY8zYwmNASNh0CvvWpIaH1bs1V01RZE2se73fmxRfts48N4bf03otfBDHQ3j52DzyUBguaBfEcXnDOsBuN6HeYeMbrXRzVtADOYHNpl5tMMNBP2PSqx3YI7MFiBfb2wXFXAxH0JSsA1ZsAv2N5fzoy1PrKBJ8201yyqOsPZKnyqJ10ME7oAfSuLGJaACH+38rm67hvIcxsB99FWZb3Ah6CMO1CEnIBlb6tLzIdAAAAAElFTkSuQmCC' + + assert(result.data.length, 'data length must be 1') + const { b64_json: _, ...rest } = result.data[0] + return { + ...result, + data: [{ b64_json: fake, ...rest }], + } +} diff --git a/openai/deno.json b/openai/deno.json index b96c114ddd31..6b4eebe582b9 100644 --- a/openai/deno.json +++ b/openai/deno.json @@ -2,6 +2,7 @@ "name": "@artifact/openai", "version": "0.0.1", "exports": { - ".": "./mod.ts" + ".": "./mod.ts", + "./napp.json": "./napp.json" } } diff --git a/openai/openai-instructions.md b/openai/instructions.md similarity index 100% rename from openai/openai-instructions.md rename to openai/instructions.md diff --git a/_import-artifact/isolates/longthread.test.ts b/openai/longthread.test.ts similarity index 100% rename from _import-artifact/isolates/longthread.test.ts rename to openai/longthread.test.ts diff --git a/_import-artifact/isolates/longthread.ts b/openai/longthread.ts similarity index 98% rename from _import-artifact/isolates/longthread.ts rename to openai/longthread.ts index 55c379618b0c..d877602adefa 100644 --- a/_import-artifact/isolates/longthread.ts +++ b/openai/longthread.ts @@ -11,7 +11,7 @@ import { } from '@/constants.ts' import { ToolMessage } from '@/api/zod.ts' import { Functions } from '@/constants.ts' -import { executeTools } from './utils/ai-execute-tools.ts' +import { executeTools } from '../_import-artifact/isolates/utils/ai-execute-tools.ts' import { z } from 'zod' import * as completions from '@/isolates/ai-completions.ts' import * as agents from '@/isolates/agents.ts' diff --git a/openai/mod.ts b/openai/mod.ts index 5a93ec9daf9f..a4ea3c395d44 100644 --- a/openai/mod.ts +++ b/openai/mod.ts @@ -1 +1 @@ -export * from "./completions.ts"; +export * from './completions.ts' diff --git a/openai/napp.jsonc b/openai/napp.json similarity index 50% rename from openai/napp.jsonc rename to openai/napp.json index 39949398b8f7..9259aa475292 100644 --- a/openai/napp.jsonc +++ b/openai/napp.json @@ -6,12 +6,10 @@ "runtime": "deno", "branding": "./avatar.jpg", "agent": { - // an agent that can explain about the openai api and suggest how to use it "parameters": { - "instructions": "./openai-instructions.md", + "instructions": "./instructions.md", "model": "gpt-4o", "tools": [] - // ? how would the tools of the nap package be passed on to this agent ? } }, "main": "completions", @@ -29,12 +27,8 @@ "type": "number", "required": false }, - "permissions": [ - // list of permissions required to run this function - ], - "gateways": [ - // list of gateways required to run this function - ] + "permissions": [], + "gateways": [] }, "returns": { "description": "This is what the function returns", @@ -42,7 +36,6 @@ }, "throws": { "FileNotFoundError": { - // list of expected error types and what to do about it } } }, @@ -61,47 +54,18 @@ } } }, - // list of the api gateway targets that this thing needs to do its job "sideEffects": { - // default configurations for the production running of side effects }, "testSideEffects": { - // special services that can be used for testing - these are still live but - // are somehow test grade, as in might be cheaper, or no credentials }, "mockSideEffects": { - // when running in the mock environment, these side effects are supplied }, "graphics": { - // a list of exported widgets that can be displayed on the stateboard "googleMapsViewer": { "type": "react", "component": "./path/to/component", "parameters": { - // mock react props that can be configured by the LLM } } } } - -// gateways can be just a fetch function with some urls -// can make a moneyworks connector, which could be a napp, running as a gateway -// host -// upstream dependencies ? -// tools being the same exported as what is in the agent -// need an agent config that is a generic way - -// gateways need to have backups and retry policies -// gateways need environments, like mock, test, prod - -// --- -// commands: -// - files:write -// - files:ls -// - files:read -// - files:update -// - files:rm -// - files:mv -// - files:search -// - stateboard:show -// --- diff --git a/openai/zod.ts b/openai/zod.ts new file mode 100644 index 000000000000..dc1b5fb5bb12 --- /dev/null +++ b/openai/zod.ts @@ -0,0 +1,279 @@ +import { z } from 'zod' + +// Define the schema for content parts (text and image) +const ChatCompletionContentPartTextSchema = z.object({ + type: z.literal('text'), + text: z.string(), +}) + +const ChatCompletionContentPartImageSchema = z.object({ + type: z.literal('image_url'), + image_url: z.object({ + url: z.string(), + detail: z.enum(['auto', 'low', 'high']).optional(), + }), +}) + +const ChatCompletionContentPartRefusalSchema = z.object({ + refusal: z.string(), + type: z.literal('refusal'), +}) + +const ChatCompletionContentSchema = z.union([ + z.string(), + z.array( + z.union([ + ChatCompletionContentPartTextSchema, + ChatCompletionContentPartImageSchema, + ]), + ), +]) + +const ChatCompletionContentResponseSchema = z.union([ + z.string(), + z.array(ChatCompletionContentPartTextSchema), +]) + +const systemMessage = z.object({ + content: ChatCompletionContentResponseSchema, + role: z.literal('system'), + name: z.string().optional(), +}) + +// Define the user message schema +const userMessage = z.object({ + content: ChatCompletionContentSchema, + role: z.literal('user'), + name: z.string().optional(), +}) + +// Define the assistant message schema +export type AssistantMessage = z.infer< + typeof assistantMessage +> +export const assistantMessage = z.object({ + role: z.literal('assistant'), + content: z.union([ + z.string(), + z.array(z.union([ + ChatCompletionContentPartTextSchema, + ChatCompletionContentPartRefusalSchema, + ])), + ]).optional().nullable(), + name: z.string().optional(), + refusal: z.string().optional().nullable(), + tool_calls: z + .array( + z.object({ + id: z.string(), + type: z.literal('function'), + function: z.object({ + name: z.string(), + arguments: z.string(), + }), + }), + ) + .optional(), +}) + +export type ToolMessage = z.infer<typeof toolMessage> +const toolMessage = z.object({ + role: z.literal('tool'), + content: ChatCompletionContentResponseSchema, + tool_call_id: z.string(), +}) + +export type CompletionMessage = z.infer<typeof completionMessage> +export const completionMessage = z.union([ + systemMessage, + userMessage, + assistantMessage, + toolMessage, +]) + +export const reasoning = z.array(z.string()).describe( + 'Step by step reasoning why this function was called and what it is trying to achieve. This is working space for clarifying thought and is not passed through to the function', +) + +export const getContent = (message: AssistantsThread['messages'][number]) => { + const { content } = message + if (content[0].type !== 'text') { + throw new Error('content not text') + } + return content[0].text.value +} +export const getThreadPath = (pid: PID) => { + const [, , ...actorChildBranches] = pid.branches + const threadPath = actorChildBranches.join('/') + const path = `threads/${threadPath}.json` + return path +} + +export const agentConfigSchema = z.object({ + model: z.enum([ + 'gpt-3.5-turbo', + 'gpt-4-turbo', + 'gpt-4o', + 'gpt-4o-mini', + 'o1-preview', + 'o1-mini', + ]), + temperature: z.number().gte(0).lte(2).optional(), + presence_penalty: z.number().optional(), + tool_choice: z.enum(['auto', 'none', 'required']).optional().describe( + 'control model behaviour to force it to call a tool or no tool', + ), + parallel_tool_calls: z.boolean().optional().describe( + 'Is the model permitted to call more than one function at a time. Must be false to use strict function calling', + ), +}) + +export const agentSchema = z.object({ + name: z.string().regex(/^[a-zA-Z0-9_-]+$/), + source: triad.describe('Where exactly did this agent come from'), + description: z.string().optional(), + config: agentConfigSchema, + runner: z.enum(['ai-runner']), + commands: z.array(z.string()), + napps: z.array(z.string()), + instructions: z.string().max(256000), +}) +export type Agent = z.infer<typeof agentSchema> + +export const chatParams = agentConfigSchema.extend({ + messages: z.array(completionMessage), + seed: z.literal(1337), + tools: z.array(z.object({ + type: z.literal('function'), + function: z.object({ + name: z.string(), + description: z.string().optional(), + parameters: z.object({}).passthrough().optional(), + strict: z.boolean().optional().nullable(), + }), + })).optional(), +}) +export type ChatParams = z.infer<typeof chatParams> + +const int = z.number().int().gte(0) + +const usageSchema = z.object({ + /** + * Number of tokens in the generated completion. + */ + completion_tokens: int, + /** + * Number of tokens in the prompt. + */ + prompt_tokens: int, + /** + * Total number of tokens used in the request (prompt + completion). + */ + total_tokens: int, + /** + * Breakdown of tokens used in a completion. + */ + completion_tokens_details: z.object({ + /** + * Audio input tokens generated by the model. + */ + audio_tokens: int.optional(), + /** + * Tokens generated by the model for reasoning. + */ + reasoning_tokens: int.optional(), + }).optional(), + /** + * Breakdown of tokens used in the prompt. + */ + prompt_tokens_details: z.object({ + /** + * Audio input tokens present in the prompt. + */ + audio_tokens: int.optional(), + /** + * Cached tokens present in the prompt. + */ + cached_tokens: int.optional(), + }).optional(), +}) + +export const messageStatsSchema = z.object({ + /** The Unix timestamp (in seconds) of when the chat completion was created. */ + created: z.number().int().gte(0), + /** The model used for the completion. */ + model: z.string(), + /** The system fingerprint of the completion. */ + system_fingerprint: z.string(), + /** The duration of the completion in milliseconds from our side. */ + duration: z.number().int().gte(0), + /** The duration of the completion in milliseconds from OpenAI's side. + */ + openAiProcessingMs: z.number().int().gte(0), + /** The usage of the completion. */ + usage: usageSchema, +}) + +export const threadSchema = z.object({ + /** The current agent the conversation is with */ + agent: z.string(), + /** The remote thread the conversation is currently with */ + remote: pidSchema.optional(), + /** If the messages were truncated, this is the offset count */ + messageOffset: int, + messages: z.array(completionMessage), + toolCommits: z.record( + /** The tool call id */ + z.string(), + /** The commit hash the tool ended on */ + md5, + ), + messageStats: z.record( + /** The message index that this stat is for */ + z.string(), + messageStatsSchema, + ), + /** Have any files been changed in this threads branch */ + isDirty: z.boolean().optional(), + summaries: z.array( + z.object({ + title: z.string(), + summary: z.string(), + /** The message index that this summary starts with */ + start: z.number().int().gte(0), + /** The message index that this summary ends with */ + end: z.number().int().gte(0).optional(), + }).refine((data) => data.end === undefined || data.end >= data.start, { + message: "'end' must be greater than or equal to 'start'", + path: ['end'], + }), + ).optional(), + /** History of stateboard changes */ + stateboards: z.array(z.object({ + /** What message number set the stateboard change */ + + setter: int, + commit: z.string(), + })), + /** History of what the focus file path was set to (like the CWD). Allows + * statements like "the previous file", "that other file", and "three files + * ago" */ + focusedFiles: z.array(z.object({ + /** The message number that set the focus */ + setter: int, + focus: z.object({ + // Define the structure of PathTriad here + }), + })), +}) +export type Thread = z.infer<typeof threadSchema> + +export type AssistantsThread = Thread & { + externalId: string + messages: OpenAI.Beta.Threads.Message[] + additionalMessages: OpenAI.Beta.Threads.RunCreateParams.AdditionalMessage[] +} +export type RemoteThread = { + /** The location in the remote repo and the last known commit we have of it */ + triad: Triad +} diff --git a/processes/README.md b/processes/README.md new file mode 100644 index 000000000000..c9a6f1a6b891 --- /dev/null +++ b/processes/README.md @@ -0,0 +1,5 @@ +# Processes + +The programming model is resprented by threads and branches. Branches are an +isolated line of changes in whichever content snapshotting model is employed. +Threads are isolated executions within a given branch. diff --git a/_import-artifact/git/branch.test.ts b/processes/branch.test.ts similarity index 100% rename from _import-artifact/git/branch.test.ts rename to processes/branch.test.ts diff --git a/_import-artifact/git/branch.ts b/processes/branch.ts similarity index 94% rename from _import-artifact/git/branch.ts rename to processes/branch.ts index 4802d28a0826..301a27f14399 100644 --- a/_import-artifact/git/branch.ts +++ b/processes/branch.ts @@ -1,9 +1,9 @@ import { assert, Debug } from '@utils' import { Branched, Proctype } from '@/constants.ts' import { RemoteRequest } from '@/constants.ts' -import IOChannel from '../io/io-channel.ts' +import IOChannel from '../engine/exe/io-channel.ts' import { solidify } from './solidify.ts' -import FS from '@/git/fs.ts' +import FS from '../snapshots/tip.ts' const log = Debug('AI:git:branch') diff --git a/processes/deno.json b/processes/deno.json new file mode 100644 index 000000000000..c7fa3599fe80 --- /dev/null +++ b/processes/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/processes", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/processes/mod.ts b/processes/mod.ts new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/processes/napp.json b/processes/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/processes/napp.json @@ -0,0 +1 @@ +{} diff --git a/processes/processes.ts b/processes/processes.ts new file mode 100644 index 000000000000..7f4753e641ae --- /dev/null +++ b/processes/processes.ts @@ -0,0 +1,162 @@ +import { Chalk } from 'chalk' +import { z } from 'zod' +import { assert } from '@std/assert/assert' + +export const githubRegex = /^[a-zA-Z\d](?:[a-zA-Z\d]|[-.](?=[a-zA-Z\d])){0,38}$/ +export const repoIdRegex = /^rep_[0-9A-HJKMNP-TV-Z]{16}$/ + +export const pidSchema = z.object({ + /** + * The hash of the genesis commit is used to identify this repo in a + * cryptographically secure way. This repoId is used to reference this repo + * unique with strong guarantees that this is the correct repo that + * communication was intended with. + */ + repoId: z.string().regex(repoIdRegex), + account: z.string().regex(githubRegex), + repository: z.string().regex(githubRegex), + branches: z.array(z.string()).min(1), +}) +/** + * The Process Identifier used to address a specific process branch. + */ +export type PID = z.infer<typeof pidSchema> + +export type PartialPID = Omit<PID, 'repoId'> + +export const ENTRY_BRANCH = 'main' + +export const partialFromRepo = (repo: string) => { + const [account, repository] = repo.split('/') + assert(account, 'missing account') + assert(repository, 'missing repository') + const pid: PartialPID = { + account, + repository, + branches: [ENTRY_BRANCH], + } + return pid +} + +export const isPID = (value: unknown): value is PID => { + if (typeof value !== 'object' || value === null) { + return false + } + const pid = value as PID + return ( + typeof pid.account === 'string' && + typeof pid.repository === 'string' && + Array.isArray(pid.branches) && + pid.branches.every((branch) => typeof branch === 'string') + ) +} +const { black, red, green, blue, magenta, cyan, bold } = new Chalk({ level: 1 }) +const colors = [red, green, blue, magenta, cyan, black] +let colorIndex = 0 +const colorMap = new Map<string, number>() +export const colorize = ( + string: string, + noSubstring = false, + noColor = false, +) => { + let sub = string + if (!noSubstring) { + sub = string.substring(0, 7) + } + let index + if (noColor) { + return sub + } + if (colorMap.has(sub)) { + index = colorMap.get(sub)! + } else { + index = colorIndex++ + if (colorIndex === colors.length) { + colorIndex = 0 + } + colorMap.set(sub, index) + } + const color = colors[index] + assert(typeof color === 'function', 'color not a function') + return color(bold(sub)) +} +export const print = (pid?: PID, noColor = false) => { + if (!pid) { + return '(no pid)' + } + const branches = pid.branches.map((segment) => { + const noSubstring = !segment.startsWith('mac_') && + !segment.startsWith('bac_') && + !segment.startsWith('act_') && + !segment.startsWith('rep_') && + !segment.startsWith('the_') + return colorize(segment, noSubstring, noColor) + }) + const noSubstring = false + const repoId = colorize(pid.repoId, noSubstring, noColor) + return `${repoId}/${pid.account}/${pid.repository}:${branches.join('/')}` +} +export const printPlain = (pid?: PID) => { + const noColor = true + return print(pid, noColor) +} +export const freezePid = (pid: PID) => { + if (!pid.repoId) { + throw new Error('repoId is required') + } + if (!repoIdRegex.test(pid.repoId)) { + throw new Error('Invalid repoId: ' + pid.repoId) + } + if (!pid.account) { + throw new Error('account is required') + } + if (!pid.repository) { + throw new Error('repository is required') + } + if (!pid.branches[0]) { + throw new Error('branch is required') + } + if (!githubRegex.test(pid.account) || !githubRegex.test(pid.repository)) { + const repo = `${pid.account}/${pid.repository}` + throw new Error('Invalid GitHub account or repository name: ' + repo) + } + Object.freeze(pid) + Object.freeze(pid.branches) + return pid +} + +export const addBranches = (pid: PID, ...children: string[]) => { + const next = { ...pid, branches: [...pid.branches, ...children] } + return freezePid(next) +} +export const addPeer = (pid: PID, peer: string) => { + const branches = [...pid.branches] + branches.pop() + const next = { ...pid, branches: [...branches, peer] } + return freezePid(next) +} +export const getParent = (pid: PID) => { + const branches = [...pid.branches] + branches.pop() + return freezePid({ ...pid, branches }) +} +export const getRoot = (pid: PID) => { + const root = pid.branches[0] + assert(root, 'root branch is missing') + return freezePid({ ...pid, branches: [root] }) +} +export const getBaseName = (pid: PID) => { + return pid.branches[pid.branches.length - 1] +} +export const sha1 = /^[0-9a-f]{40}$/i + +import { actionSchema } from '../api/actions.ts' + +export const addressedSchema = z.object({ + // what is the addressing ? + // should this be threading, rather than absolute hash space addressing ? + // should all network comes go thru a dedicated branch and get re-addressed + // there ? + action: actionSchema, +}) +export type Addressed = z.infer<typeof addressedSchema> diff --git a/_import-artifact/git/serial.test.ts b/processes/serial.test.ts similarity index 100% rename from _import-artifact/git/serial.test.ts rename to processes/serial.test.ts diff --git a/_import-artifact/git/solidify.ts b/processes/solidify.ts similarity index 99% rename from _import-artifact/git/solidify.ts rename to processes/solidify.ts index 491685d97dec..d824af039a9e 100644 --- a/_import-artifact/git/solidify.ts +++ b/processes/solidify.ts @@ -1,5 +1,5 @@ import { assert, Debug, equal } from '@utils' -import FS from './fs.ts' +import FS from '../snapshots/tip.ts' import { isMergeReply, isPierceRequest, diff --git a/reasoner/PROJECT_MAP_INSTRUCTIONS.md b/reasoner/PROJECT_MAP_INSTRUCTIONS.md new file mode 100644 index 000000000000..d0c0aa28a105 --- /dev/null +++ b/reasoner/PROJECT_MAP_INSTRUCTIONS.md @@ -0,0 +1,85 @@ +# Project Map Instructions + +**Purpose:**\ +`PROJECT_MAP.md` provides a high-level map of the project’s structure, purpose, +and key components. It helps new contributors quickly understand the overall +layout and logic without too much detail. + +**Key Rules:** + +- Start at `📦 PROJECT_ROOT/`. +- Organize top-level folders and files first, with important items at the top. +- Place `dependencies` at the bottom, with `vendor-docs` above it if it exists. +- Notation: + - `📦` for the root project + - `📂` for directories + - `📄` for files + - `🧩` for dependent projects +- Under each file, list: + - Exported functions for code files, including their TS parameter/return + types. + - Top-level sections for knowledge files. + - All tests for test files. +- Add a short rationale note (`ℹ`) beneath each item. +- Skip low-level internal files that aren’t architecturally important. +- Exclude `PROJECT_MAP.md` from its own map. + +**Structure Example:** + +```text +📦 PROJECT_ROOT/ +├─ 📂 src +│ ├─ 📄 main.ts +│ │ ℹ Entry point for CLI +│ │ 1. runCLI(args: string[]): Promise<number> +│ │ ℹ Parses CLI arguments, executes workflow, returns exit code +│ ├─ 📄 utils.ts +│ │ ℹ Shared utilities +│ │ 1. formatOutput(input: string): string +│ │ ℹ Formats CLI output for readability +│ │ 2. handleError(error: Error): void +│ │ ℹ Centralized error handling +│ └─ (Internal helper files omitted) +├─ 📂 tests +│ ├─ 📄 main_test.ts +│ │ ℹ Tests runCLI() +│ │ 1. testRunCLI_withValidArgs() +│ │ ℹ Ensures runCLI() processes valid args correctly +│ │ 2. testRunCLI_withInvalidArgs() +│ │ ℹ Verifies runCLI() handles invalid args gracefully +│ ├─ 📄 utils_test.ts +│ │ ℹ Tests utils.ts functions +│ │ 1. testFormatOutput_valid() +│ │ ℹ Checks correct string formatting +│ │ 2. testHandleError_logsProperly() +│ │ ℹ Confirms handleError() logs error details as expected +├─ 📄 README.md +│ ℹ Quick start and top-level overview +│ 1. Introduction +│ │ ℹ Brief project summary +│ 2. Installation +│ │ ℹ Setup steps +│ 3. Usage +│ ℹ Basic runtime instructions +├─ 📂 vendor-docs +│ ℹ External dependency documentation +│ └─ 📄 commander/ +│ ℹ Docs for the `commander` CLI library +└─ 📂 dependencies + ├─ 📦 @babel/parser + │ ℹ Parses embedded JSON in scripts + ├─ 📦 commander + │ ℹ CLI argument parsing library + └─ 🧩 subproject1 + ℹ Integrated dependent project +``` + +**Why This Format?** + +- Using `ℹ` notes and uniform structure allows automated extraction and + filtering of both structural and rationale information. +- Explicit TS types for functions clarify inputs and outputs. +- Listing all tests under each test file ensures a complete snapshot of the + project’s verification points. +- This approach keeps the overview clean, easy to maintain, and useful at any + level of detail. diff --git a/reasoner/README.md b/reasoner/README.md index 950445f8cf41..ec4147d0ee5b 100644 --- a/reasoner/README.md +++ b/reasoner/README.md @@ -1,4 +1,4 @@ # reasoner > [!NOTE] -> The system prompt for this agent: [instructions.md](./instructions.md) +> The system prompt for this agent: [instructions.md](reasoner/instructions.md) diff --git a/reasoner/SPEC_INSTRUCTIONS.md b/reasoner/SPEC_INSTRUCTIONS.md new file mode 100644 index 000000000000..d6a2ab683872 --- /dev/null +++ b/reasoner/SPEC_INSTRUCTIONS.md @@ -0,0 +1,24 @@ +# Spec Instructions + +How to specify a module in a formal way. + +1. Identify the main input(s) your module accepts, and clearly define them as + variables. +2. Determine the key transformations or operations your module performs and + represent each as a function. Each function should take well-defined inputs + and return well-defined outputs. +3. For each operation, assign a simple name (like $g$, $h$, $i$, etc.) and write + a short English description. +4. Arrange these functions in a logical order, showing how inputs flow from one + function to the next until you get the final output. +5. Represent the entire module’s logic as a mathematical or logical formula by + composing the defined functions. +6. Summarize the workflow in plain English to provide a high-level explanation + without code-level details. +7. For documentation, produce a Markdown file incorporating both the English + summary and the formal formula. Use inline math ($...$) for inline + expressions and block math ($$...$$ + ) for equations, because GitHub natively supports this LaTeX-style math + syntax using KaTeX. +8. Keep the explanations concise, focusing on the essential logic rather than + implementation details. diff --git a/reasoner/deno.json b/reasoner/deno.json index 7708e84a9b48..eaed4ea85419 100644 --- a/reasoner/deno.json +++ b/reasoner/deno.json @@ -1,7 +1,5 @@ { "name": "@artifact/reasoner", "version": "0.0.2", - "exports": { - ".": "./mod.ts" - } + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } } diff --git a/reasoner/napp.json b/reasoner/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/reasoner/napp.json @@ -0,0 +1 @@ +{} diff --git a/_import-artifact/tests/reasoner.test.ts b/reasoner/reasoner.test.ts similarity index 95% rename from _import-artifact/tests/reasoner.test.ts rename to reasoner/reasoner.test.ts index 8ffae877e25b..deb9b5363023 100644 --- a/_import-artifact/tests/reasoner.test.ts +++ b/reasoner/reasoner.test.ts @@ -5,7 +5,7 @@ import { fileRunnerPath, fixture, reasonerTestPath, -} from './fixtures/fixture.ts' +} from '../_import-artifact/tests/fixtures/fixture.ts' import { TestFile } from '@/api/tps-report.ts' import { addBranches } from '@/constants.ts' diff --git a/_import-artifact/server/cloud.ts b/server/cloud.ts similarity index 100% rename from _import-artifact/server/cloud.ts rename to server/cloud.ts diff --git a/_import-artifact/server/db-drop.ts b/server/db-drop.ts similarity index 100% rename from _import-artifact/server/db-drop.ts rename to server/db-drop.ts diff --git a/_import-artifact/server/db-tools.ts b/server/db-tools.ts similarity index 100% rename from _import-artifact/server/db-tools.ts rename to server/db-tools.ts diff --git a/server/deno.json b/server/deno.json new file mode 100644 index 000000000000..3194d6e73056 --- /dev/null +++ b/server/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/server", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/_import-artifact/server/keygen.ts b/server/keygen.ts similarity index 100% rename from _import-artifact/server/keygen.ts rename to server/keygen.ts diff --git a/_import-artifact/main.ts b/server/main.ts similarity index 99% rename from _import-artifact/main.ts rename to server/main.ts index 159a37bf2094..10cffd48b8ad 100644 --- a/_import-artifact/main.ts +++ b/server/main.ts @@ -35,4 +35,3 @@ if (isKvTestMode()) { } Deno.serve(opts, server.fetch) - diff --git a/server/napp.json b/server/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/server/napp.json @@ -0,0 +1 @@ +{} diff --git a/_import-artifact/server/server.test.ts b/server/server.test.ts similarity index 100% rename from _import-artifact/server/server.test.ts rename to server/server.test.ts diff --git a/_import-artifact/server/server.ts b/server/server.ts similarity index 77% rename from _import-artifact/server/server.ts rename to server/server.ts index 0ce90c387f05..040e413bee23 100644 --- a/_import-artifact/server/server.ts +++ b/server/server.ts @@ -7,7 +7,7 @@ import { logger } from 'hono/logger' import { poweredBy } from 'hono/powered-by' import { prettyJSON } from 'hono/pretty-json' import { SSEStreamingApi, streamSSE } from 'hono/streaming' -import { Engine } from '../engine.ts' +import { Engine } from '../engine/engine.ts' import { assert, Debug, delay, serializeError } from '@/utils.ts' import { EventSourceMessage, machineIdRegex, Provisioner } from '@/constants.ts' import '@std/dotenv/load' @@ -154,61 +154,10 @@ export default class Server { }) if (Deno.env.get('GITHUB_CLIENT_ID')) { - const { - signIn, - handleCallback, - signOut, - } = createHelpers(createGitHubOAuthConfig()) - const auth = base.basePath('/auth') - auth.get('/signin', async (c) => { - const { machineId } = c.req.query() - if (!machineIdRegex.test(machineId)) { - // TODO check key is valid using signatures - throw new Error('machineId querystring is required') - } - - const response = await signIn(c.req.raw) - const cookie = response.headers.get('set-cookie') - console.log('cookie', cookie) - // acting as the github actor, pierce the github chain to store this info - - return response - // c.header('set-cookie', response.headers.get('set-cookie')!) - // return c.redirect(response.headers.get('location')!, response.status) - }) - - auth.get('/callback', async (c) => { - const { response, tokens, sessionId } = await handleCallback(c.req.raw) - console.log('tokens', tokens, sessionId) // lol - // acting as the github actor, pierce the github chain to store this info - // as well as storing the token from github - // there should be one PAT per machine id - - // get the userId from github - // move the machine branch to be inside the user branch - // send the new pid down to the browser - - // make a fetch request to get the userId from github - - // pass back an id so the browser knows which pats it has - - return response - // c.header('set-cookie', response.headers.get('set-cookie')!) - // return c.redirect(response.headers.get('location')!, response.status) - }) - - auth.get('/signout', async (c) => { - const response = await signOut(c.req.raw) - return response - // c.header('set-cookie', response.headers.get('set-cookie')!) - // return c.redirect(response.headers.get('location')!, response.status) - }) + // TODO use https://www.npmjs.com/package/@hono/oauth-providers + // used to get repo access, whereas privy is used for sign in } - // TODO set a cookie for the machineId so it doesn't have to prove again - // or get a sig on all pierce actions, and only allow correctly signed ones - // to enter - return server } dump() { @@ -270,3 +219,17 @@ const keepalive = async (stream: SSEStreamingApi, signal: AbortSignal) => { } } } + +// TODO move this to be a universal napp server, where it bridges between +// actions + +// so there would be no special functions or routes, and everything would be a +// json function. + +// if special oauth callback rountes are needed, these should be able to be +// generic. + +// may need middleare type patterns in json functions, so we can pipeline things +// without causing an action to be overly complex. +// so we could check for auth as a seperate function to whatever the main +// function was. diff --git a/_import-artifact/tests/ssl/cert.pem b/server/ssl/cert.pem similarity index 100% rename from _import-artifact/tests/ssl/cert.pem rename to server/ssl/cert.pem diff --git a/_import-artifact/tests/ssl/key.pem b/server/ssl/key.pem similarity index 100% rename from _import-artifact/tests/ssl/key.pem rename to server/ssl/key.pem diff --git a/shockwave/README.md b/shockwave/README.md new file mode 100644 index 000000000000..9a9d746916ca --- /dev/null +++ b/shockwave/README.md @@ -0,0 +1,139 @@ +# Shockwave + +Rapid idempotent fan out of tasks using deno KV Queues. + +## Features + +1. Exactly once task invocation +1. KV Queue overload recovery +1. Fast time to first task +1. Fast time to peak concurrency using amplification actions +1. Nestable - shockwave can be called from within a shockwave task +1. Realtime statistics +1. Optional reduce function +1. Configurable ramp rate to stay friendly to your infrastructure providers +1. Oversize job batches +1. Billing aware +1. Peak queue size limits + +### Exactly once invocation + +By using atomic commits in Deno KV, the results of execution will be returned +exactly once. At worst, the execution may have run twice, but only one result +will be returned. + +### Queue overload recovery + +Deno kv queues have a maximum queue length of 100,000 items. If shockwave tries +to add more items than this and receives an error, it will retry with +exponential backoff. It will watch its own queue length statistics to try to +insert the exact amount of load each time, but will still tolerate competing +queue usage from other applications. + +### Amplification actions + +Technique is that it will create amplification actions for any batches it needs +to send off, then in the last batch it will execute direct task items, then it +will start the final task. + +This means that amplifications take place quickly and rapidly, as well as task +executions, but there is also an initial task that is started, resulting in a +good time to first task metric. + +### Deterministic execution + +Invoking shockwave with the exact same parameters will always give the exact +same results. + +### Peak queue size limits + +By setting amplification actions to redispatch themselves with a delay if the +queue high water mark is reached, the processing rate can be delayed without +compromising reliability of execution. + +### Realtime statistics + +When a jobId is provided, ongoing statistics can be watched to get live +feedback. These stats are returned back with the job in their final form with +some basic statistic analysis on mean times, ramp times, invocation counts, etc. + +Tracked statistics are: + +- **Time to first task** How many ms since the job arrived until the first task + was started + +- **Time to last task** How many ms since the job arrived until the LAST task + was started + +- **Peak task rate** What was the highest observed concurrent task execution + rate + +- **Peak queue length** + + What was the longest the queue was observed to be + +- **Child shockwaves** + + IDs for gathering statistics on shockwaves that were created from within this + shockwave. The shockwave task is not counted as a task itself, but all its + individual components are used in statistics. + +### Result Reduction + +Shockwave supports processing task results through reducer functions that +operate on results as they become available, or processing all results at the +end. The reducer configuration allows for: + +1. **Ordered Processing** + - Optionally process results in the exact order tasks were dispatched + - Uses KV store for buffering out-of-order results + - Configurable error handling: skip errors or stop on first error + +2. **Amplification Boundary Processing** + - Optionally trigger reduction at amplification boundaries + - Natural batching based on the fan-out structure + - Helps maintain memory efficiency during large fan-outs + +Example configuration: + +```typescript +const options = { + // Controls batch size for task processing + batchSize?: number, + + // Use web cache for task data when available + useCache?: boolean, + + // Optional ID for tracking progress and statistics + jobId?: string, + + // Time in ms after which results expire + expireResults?: number, + + // Optional reducer configuration + reducer?: { + // Function that processes an array of task results + fn: async (results: Outcome[]) => { + return results.reduce((acc, curr) => { + if (curr.ok) return acc + curr.value + return acc + }, 0) + }, + // Whether to maintain task dispatch order + ordered?: boolean, + // How to handle failed tasks: 'skip' or 'stop' + handleErrors?: 'skip' | 'stop' + }, + + // Whether to trigger reduction at amplification boundaries + reduceOnAmplify?: boolean +} +``` + +The reducer function is automatically invoked when: + +- An amplification boundary is hit (if reduceOnAmplify is true) +- All tasks have completed + +Results are stored in Deno KV to maintain scalability and handle any potential +memory constraints during processing. diff --git a/shockwave/deno.json b/shockwave/deno.json new file mode 100644 index 000000000000..614d0f0b8fe3 --- /dev/null +++ b/shockwave/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/shockwave", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/shockwave/mod.ts b/shockwave/mod.ts new file mode 100644 index 000000000000..6b265512d816 --- /dev/null +++ b/shockwave/mod.ts @@ -0,0 +1,69 @@ +import type { JsonValue } from '@artifact/api/actions' + +type Outcome = { + ok: boolean + value: JsonValue +} + +type Task = { + type: 'task' + id: number + options: Options +} + +type Amplifer = { + type: 'amplifier' + start: number + count: number + options: Options +} + +type Options = { + /** Controls task batch size and queue load. When reached, amplification + * actions will delay themselves */ + batchSize?: number + + /** Use web cache to retrieve task data when available */ + useCache?: boolean + + /** Optional ID for tracking job progress and statistics. Required for + * watching intermediate results */ + jobId?: string + + /** Time in ms after which results expire from storage */ + expireResults?: number + + /** Optional reducer to process results */ + reducer?: { + /** Process an array of results into a single value */ + fn: (results: Outcome[]) => Promise<JsonValue> + /** Process results in original task order */ + ordered?: boolean + /** How to handle errors: 'skip' = ignore failed tasks, 'stop' = halt on error */ + handleErrors?: 'skip' | 'stop' + } + + /** Trigger reduction at amplification boundaries for memory efficiency */ + reduceOnAmplify?: boolean +} + +export const createQueueListener = <T>(processor: (params: T) => {}) => { + // receive a message from the queue + + // if amplifier, break apart the tasks into further batches +} + +export const enqueue = ( + tasks: Record<string, JsonValue>[], + options: Options, +) => { + // send a message to the queue + + // generate a jobid + + // assert the jobid is not taken already + + // write the tasks to the db + + // send off the first amplifier task, so executes close to the db +} diff --git a/shockwave/napp.json b/shockwave/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/shockwave/napp.json @@ -0,0 +1 @@ +{} diff --git a/snapshots/README.md b/snapshots/README.md new file mode 100644 index 000000000000..9c21713d0cd6 --- /dev/null +++ b/snapshots/README.md @@ -0,0 +1,27 @@ +# Snapshots + +Snapshots are a posix filesystem that works with content addressable files and +is snapshot aware. + +A snapshot represents a tree of posix file paths that represent hashed content. + +Snapshots have linked lineage, and all except the first snapshot in a chain have +at least one parent, and can have several parents if the snapshot represents a +merge of multiple snapshots. + +The intended purpose of this implementation is to be backed by a git repository, +but this abstraction need not use git, and could use any other kind of +blockchain structure underneath, like IPFS or Pijul, for example. + +The purpose of representing a posix filesystem is that all code understands +files, and so this model is time tested, familiar, and broadly compatible with +multiple software languages. + +Snapshots have 3 key concepts. + +1. blobs, which are analogous to files, that are represented by a hash +1. trees, which are analogous to directories, are a data structure containing an + array of blobs and trees +1. snapshots, which are analogous to commits, represent a specific tree with + pointers to the previous snapshots, representing a linear progression of + changes diff --git a/snapshots/deno.json b/snapshots/deno.json new file mode 100644 index 000000000000..e46c22ada193 --- /dev/null +++ b/snapshots/deno.json @@ -0,0 +1,10 @@ +{ + "name": "@artifact/snapshots", + "version": "0.0.1", + "exports": { + ".": "./mod.ts", + "./napp.json": "./napp.json", + "./tip": "./tip.ts", + "./mock-provider": "./mock-provider.ts" + } +} diff --git a/snapshots/mock-provider.ts b/snapshots/mock-provider.ts new file mode 100644 index 000000000000..bc2ec844fdab --- /dev/null +++ b/snapshots/mock-provider.ts @@ -0,0 +1,153 @@ +import { ripemd160 } from '@noble/hashes/ripemd160' +import { base32 } from 'multiformats/bases/base32' +import { jsonSchema } from '../api/actions.ts' +import { Tip } from './tip.ts' +import { + type Address, + FileNotFoundError, + LineageError, + type SnapshotsProvider, +} from '../api/napp-api.ts' +import { ulid } from '@std/ulid/ulid' +import type { z } from 'zod' + +type LocalReadOptions = Pick<Address, 'snapshot'> +type LocalSnapshotsProvider = SnapshotsProvider<LocalReadOptions> + +type Lineage = { tip: Tip; parents: string[] } + +export class MockProvider implements LocalSnapshotsProvider { + static create() { + return new MockProvider() + } + private constructor() {} + + #snapshots = new Map<string, Lineage>() + + #latest: string | undefined + + commit: LocalSnapshotsProvider['commit'] = async (upserts, deletes) => { + await Promise.resolve() + if (!upserts.size && !deletes.size) { + throw new Error('No changes to commit') + } + const parents = [] + if (this.#latest) { + parents.push(this.#latest) + } + const id = ulid() + this.#latest = id + const tip = Tip.createWith(this, upserts, deletes) + this.#snapshots.set(id, { tip, parents }) + } + read: LocalSnapshotsProvider['read'] = { + meta: async (path, options = {}) => { + await Promise.resolve() + if (!this.#latest) { + throw new FileNotFoundError(path) + } + const { snapshot = this.#latest } = options + if (!snapshot) { + throw new FileNotFoundError(path) + } + const lineage = this.#snapshots.get(snapshot) + if (!lineage) { + throw new LineageError(path) + } + // TODO handle directories + const binary = await lineage.tip.read.binary(path) + const hash = ripemd160(binary) + return { + snapshot, + mode: '100644', + path, + oid: base32.encode(hash), + type: 'blob', + } + }, + json: async (path, options) => { + const text = await this.read.text(path, options) + const result = JSON.parse(text) + const schema = options?.schema || jsonSchema + return schema.parse(result) as z.infer<typeof schema> + }, + text: async (path, options = {}) => { + const blob = await this.read.binary(path, options) + return new TextDecoder().decode(blob) + }, + binary: async (path, options = {}) => { + // mock latest needs to not move forwards for the ancestors + const { snapshot = this.#latest } = options + if (!snapshot) { + throw new FileNotFoundError(path) + } + const lineage = this.#snapshots.get(snapshot) + if (!lineage) { + throw new LineageError(path) + } + return lineage.tip.read.binary(path) + }, + exists: async (path, options = {}) => { + const { snapshot = this.#latest } = options + if (!snapshot) { + return false + } + const lineage = this.#snapshots.get(snapshot) + if (!lineage) { + // TODO make a lineage fault error + throw new LineageError(snapshot) + } + return lineage.tip.read.exists(path) + }, + ls: async (path, options = {}) => { + await Promise.resolve() + return [] + }, + } + snapshots: LocalSnapshotsProvider['snapshots'] = { + latest: () => { + return Promise.resolve(this.#latest) + }, + parents: async (options = {}) => { + const { snapshot = this.#latest } = options + if (!snapshot) { + return [] + } + const lineage = this.#snapshots.get(snapshot) + if (!lineage) { + throw new LineageError(snapshot) + } + + return lineage.parents + }, + history: async (options = {}) => { + await Promise.resolve() + const { count = 1, snapshot = this.#latest } = options + if (count <= 0) { + throw new Error('count must be greater than 0') + } + if (!snapshot) { + return [] + } + const lineage = this.#snapshots.get(snapshot) + if (!lineage) { + throw new LineageError(snapshot) + } + let current = snapshot + const history = [] + for (let i = 0; i < count; i++) { + history.push(current) + const lineage = this.#snapshots.get(current) + if (!lineage) { + break + } + const parent = lineage.parents[0] + if (!parent) { + break + } + current = parent + } + return history + }, + } +} diff --git a/snapshots/mod.ts b/snapshots/mod.ts new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/snapshots/napp.json b/snapshots/napp.json new file mode 100644 index 000000000000..ba468978e97a --- /dev/null +++ b/snapshots/napp.json @@ -0,0 +1,70 @@ +{ + "name": "@artifact/snapshots", + "version": "1.0.0", + "description": "A snapshot is a point-in-time representation of a set of files. It can also be statefully changed before being committed.", + "tools": { + "snapshotId": { + "description": "The snapshot ID that allows this snapshot to be retrieved. In git, this is the commit hash.", + "returns": { "type": "string" } + }, + "parents": { + "description": "The parent snapshot IDs that this snapshot is based on, ordered by most important first. In git, these is the parent commit hashes, with incoming merges being less important than the merge target.", + "returns": { "type": "array", "items": { "type": "string" } } + }, + "log": { + "description": "The log of changes that have been made to this snapshot, ordered by time", + "parameters": { + "since": { + "type": "string", + "description": "The snapshot ID to start the log from. If not provided, the log will start from the beginning." + }, + "count": { + "type": "integer", + "description": "The number of log entries to return. If not provided, all log entries will be returned." + } + }, + "returns": { + "type": "array", + "items": { + "type": "object", + "properties": { + "snapshotId": { "type": "string" }, + "parents": { + "type": "array", + "items": { "type": "string" } + } + } + } + } + }, + "exists": { + "description": "Given a file path, determines if the file exists in this snapshot.", + "parameters": { + "path": { + "type": "string", + "description": "The path to the file to check." + } + } + }, + "ls": {}, + "upsert": {}, + "rm": {}, + "mv": {}, + "cp": {}, + "upserts": { + }, + "deletes": { + }, + "isChanged": { + "description": "Determines if the snapshot has been changed since the last commit.", + "returns": { "type": "boolean" } + }, + "commit": { + "description": "Commits the snapshot to the underlying snapshot store.", + "returns": { + "type": "string", + "description": "The snapshot ID of the committed snapshot." + } + } + } +} diff --git a/snapshots/tip.test.ts b/snapshots/tip.test.ts new file mode 100644 index 000000000000..1df8ebf7d65c --- /dev/null +++ b/snapshots/tip.test.ts @@ -0,0 +1,57 @@ +import { Tip } from './tip.ts' +import { MockProvider } from './mock-provider.ts' +import { expect } from '@std/expect/expect' +import { assert } from '@std/assert/assert' + +Deno.test('mock provider', async (t) => { + const provider = MockProvider.create() + expect(await provider.snapshots.latest()).toBeUndefined() + expect(await provider.snapshots.parents()).toEqual([]) + expect(await provider.snapshots.history()).toEqual([]) + const commit = provider.commit(new Map(), new Set()) + await expect(commit).rejects.toThrow('No changes to commit') + + await expect(provider.read.binary('a')).rejects.toThrow('Could not find') + + await provider.commit(new Map([['a', { text: 'hello' }]]), new Set()) + const id = await provider.snapshots.latest() + assert(id, 'id should be defined') + + expect(await provider.read.text('a')).toEqual('hello') + + await provider.commit(new Map(), new Set(['a'])) + expect(await provider.read.exists('a')).toEqual(false) + + const recovered = await provider.read.text('a', { snapshot: id }) + expect(recovered).toEqual('hello') + + const latest = await provider.snapshots.latest() + expect(latest).not.toEqual(id) + expect(await provider.snapshots.parents()).toEqual([id]) + expect(await provider.snapshots.parents({ snapshot: id })).toEqual([]) +}) + +Deno.test('snapshot', async (t) => { + const provider = MockProvider.create() + + const tip = Tip.create(provider) + expect(tip.isChanged).toEqual(false) + await expect(tip.commit()).rejects.toThrow('No changes to commit') + + await tip.write.text('a', 'hello') + expect(await tip.read.text('a')).toEqual('hello') + expect(tip.isChanged).toEqual(true) + + tip.write.rm('a') + expect(await tip.read.exists('a')).toEqual(false) + await expect(tip.read.text('a')).rejects.toThrow('Could not find') + expect(tip.isChanged).toEqual(false) + + await expect(tip.read.meta('a')).rejects.toThrow('Could not find') + + await tip.write.text('a', 'hello again') + await tip.commit() + expect(tip.isChanged).toEqual(false) + await expect(tip.read.exists('a')).resolves.toEqual(true) + expect(await tip.read.text('a')).toEqual('hello again') +}) diff --git a/snapshots/tip.ts b/snapshots/tip.ts new file mode 100644 index 000000000000..d570aa5cecb3 --- /dev/null +++ b/snapshots/tip.ts @@ -0,0 +1,341 @@ +import * as posix from '@std/path/posix' +import { assert } from '@std/assert/assert' +import type { + Address, + NappRead, + NappSnapshots, + NappWrite, + SnapshotsProvider, + TreeEntry, + Upsert, +} from '@artifact/api/napp-api' +import { FileNotFoundError } from '@artifact/api/napp-api' +import Debug from 'debug' +import { jsonSchema } from '@artifact/api/actions' + +type NoAddressingOptions = Record<string, never> + +const log = Debug('@artifact/snapshots') + +type LocalReadOptions = Pick<Address, 'snapshot'> + +export interface NappLocal { + readonly read: NappRead<LocalReadOptions> + readonly write: NappWrite<NoAddressingOptions> + readonly snapshots: NappSnapshots<LocalReadOptions> +} + +export class Tip implements NappLocal { + static create(snapshotsProvider: SnapshotsProvider) { + return new Tip(snapshotsProvider) + } + static createWith( + snaps: SnapshotsProvider, + upserts: Map<string, Upsert>, + deletes: Set<string>, + ) { + const tip = new Tip(snaps) + for (const [path, upsert] of upserts) { + tip.#upserts.set(path, upsert) + } + for (const path of deletes) { + if (upserts.has(path)) { + throw new Error('cannot delete and upsert at the same time: ' + path) + } + tip.#deletes.add(path) + } + return tip + } + + readonly #snapshotsProvider: SnapshotsProvider + readonly #upserts = new Map<string, Upsert>() + readonly #deletes = new Set<string>() + + readonly #read: NappLocal['read'] = { + meta: async (path, options) => { + path = refine(path) + return await this.#snapshotsProvider.read.meta(path, options) + }, + json: async (path, options) => { + path = refine(path) + assert(posix.extname(path) === '.json', `path must be *.json: ${path}`) + + const upsert = this.#upserts.get(path) + if (!options?.snapshot && upsert !== undefined) { + if ('json' in upsert) { + return structuredClone(upsert.json) + } + } + + const text = await this.read.text(path, options) + const result = JSON.parse(text) + const { schema = jsonSchema } = options || {} + return schema.parse(result) + }, + text: async (path, options) => { + path = refine(path) + const upsert = this.#upserts.get(path) + if (!options?.snapshot && upsert !== undefined) { + if ('text' in upsert) { + return upsert.text + } + } + + const blob = await this.read.binary(path, options) + return new TextDecoder().decode(blob) + }, + binary: async (path, options) => { + path = refine(path) + if (options?.snapshot) { + return this.#snapshotsProvider.read.binary(path, options) + } + if (this.#deletes.has(path)) { + throw new FileNotFoundError(path) + } + + const upsert = this.#upserts.get(path) + if (upsert !== undefined) { + let text: string | undefined + if ('json' in upsert) { + text = JSON.stringify(upsert.json) + } + if ('text' in upsert) { + text = upsert.text + } + if (text !== undefined) { + return new TextEncoder().encode(text) + } + if ('data' in upsert) { + return upsert.data + } + if ('meta' in upsert) { + const { snapshot, path } = upsert.meta + return this.#snapshotsProvider.read.binary(path, { snapshot }) + } + throw new Error('invalid upsert: ' + JSON.stringify(upsert)) + } + return await this.#snapshotsProvider.read.binary(path) + }, + exists: async (path, options) => { + path = refine(path) + if (options?.snapshot) { + return this.#snapshotsProvider.read.exists(path, options) + } + if (this.#deletes.has(path)) { + return false + } + if (this.#upserts.has(path)) { + return true + } + if (this.#snapshotsProvider) { + return await this.#snapshotsProvider.read.exists(path) + } + return false + }, + ls: async (path = '.', options) => { + // TODO make a streaming version of this for very large dirs + path = refine(path) + log('ls', path) + if (options?.snapshot) { + return this.#snapshotsProvider.read.ls(path, options) + } + + let entries: TreeEntry[] = [] + if (this.#deletes.has(path)) { + if (path === '.') { + return entries + } + throw new FileNotFoundError(path) + } + if (this.#snapshotsProvider) { + entries = await this.#snapshotsProvider.read.ls(path) + } + + entries = entries.filter(({ path }) => { + return !this.#deletes.has(path) + }) + + const dir = posix.dirname(path) + const has = new Set(entries.map(({ path }) => path)) + + for (const path of this.#upserts.keys()) { + if (posix.dirname(path) === dir) { + if (!has.has(path)) { + // TODO handle directories + entries.push({ + path, + type: 'blob', + oid: '', + mode: '00644', + snapshot: '', + }) + } + } + } + return entries + }, + } as const + #write: NappLocal['write'] = { + json: (path, json) => { + // TODO store json objects specially, only stringify on commit + path = refine(path) + assert(posix.extname(path) === '.json', `path must be *.json: ${path}`) + + const string = JSON.stringify(json, null, 2) + return this.write.text(path, string) + }, + text: (path: string, content: string) => { + path = refine(path) + assert(path !== '.', 'cannot write to root') + // TODO ensure cannot write to a directory + log('writeText', path, content) + this.#upserts.set(path, { text: content }) + this.#deletes.delete(path) + return Promise.resolve() + }, + binary: (path: string, content: Uint8Array) => { + path = refine(path) + assert(path !== '.', 'cannot write to root') + // TODO ensure cannot write to a directory + log('writeBinary', path, content) + this.#upserts.set(path, { data: content }) + this.#deletes.delete(path) + return Promise.resolve() + }, + rm: (path) => { + path = refine(path) + assert(path !== '.', 'cannot rm root') + // TODO delete a whole directory + log('rm', path) + if (this.#upserts.has(path)) { + this.#upserts.delete(path) + } else { + this.#deletes.add(path) + } + return Promise.resolve() + }, + mv: async (from, to) => { + const fromPath = refine(from.path) + const toPath = refine(to.path) + if (fromPath === toPath) { + return + } + await this.write.cp(from, to) + await this.write.rm(from.path) + }, + cp: async (from, to) => { + const fromPath = refine(from.path) + const toPath = refine(to.path) + if (fromPath === toPath) { + return + } + + const upsert = this.#upserts.get(fromPath) + if (upsert !== undefined) { + this.#upserts.delete(fromPath) + this.#upserts.set(toPath, upsert) + return + } + if (this.#deletes.has(fromPath)) { + throw new FileNotFoundError(fromPath) + } + const meta = await this.read.meta(fromPath) + if (meta.type === 'tree') { + // TODO cp a directory + throw new Error('cannot cp a directory') + } + this.#deletes.delete(toPath) + // BUT need to get the snapshot id back from it + this.#upserts.set(toPath, { + meta: { snapshot: meta.oid, path: fromPath }, + }) + }, + } as const + #snapshots: NappLocal['snapshots'] = { + latest: async (options) => { + return await this.#snapshotsProvider.snapshots.latest(options) + }, + parents: async (options) => { + return await this.#snapshotsProvider.snapshots.parents(options) + }, + history: async (options) => { + return await this.#snapshotsProvider.snapshots.history(options) + }, + } as const + + private constructor(snapshotsProvider: SnapshotsProvider) { + this.#snapshotsProvider = snapshotsProvider + Object.freeze(this.#read) + Object.freeze(this.#write) + Object.freeze(this.#snapshots) + } + get read() { + return this.#read + } + get write() { + return this.#write + } + get snapshots() { + return this.#snapshots + } + get isChanged() { + return this.#upserts.size > 0 || this.#deletes.size > 0 + } + async commit() { + if (!this.isChanged) { + throw new Error('No changes to commit') + } + await this.#snapshotsProvider.commit(this.#upserts, this.#deletes) + this.#upserts.clear() + this.#deletes.clear() + } +} + +const refine = (path: string) => { + path = path.trim() + while (path.startsWith('/')) { + path = path.slice(1) + } + while (path.endsWith('/')) { + path = path.slice(0, -1) + } + while (path.startsWith('./')) { + path = path.slice(2) + } + if (!path) { + path = '.' + } + path = posix.normalize(path) + assert(path, `path must be relative: ${path}`) + assert(!posix.isAbsolute(path), `path must be relative: ${path}`) + assert(path !== '.git', '.git paths are forbidden: ' + path) + assert(!path.startsWith('.git/'), '.git paths are forbidden: ' + path) + assert(!path.endsWith('/'), 'path must not end with /: ' + path) + assert(!path.startsWith('..'), 'path must not start with ..: ' + path) + return path +} + +export const sortTreeEntries = (tree: TreeEntry[]) => { + tree.sort((a, b) => { + if (a.type === 'tree' && b.type === 'blob') { + return -1 + } + if (a.type === 'blob' && b.type === 'tree') { + return 1 + } + if (a.path.startsWith('.') && !b.path.startsWith('.')) { + return -1 + } + if (!a.path.startsWith('.') && b.path.startsWith('.')) { + return 1 + } + return a.path.localeCompare(b.path) + }) + return tree.map((entry) => { + if (entry.type === 'tree') { + return entry.path + '/' + } + assert(entry.type === 'blob', 'entry type not blob: ' + entry.type) + return entry.path + }) +} diff --git a/stateboard/deno.json b/stateboard/deno.json new file mode 100644 index 000000000000..53f2473f37ac --- /dev/null +++ b/stateboard/deno.json @@ -0,0 +1,5 @@ +{ + "name": "@artifact/stateboard", + "version": "0.0.1", + "exports": { ".": "./mod.ts", "./napp.json": "./napp.json" } +} diff --git a/stateboard/mod.ts b/stateboard/mod.ts new file mode 100644 index 000000000000..099d891dc95d --- /dev/null +++ b/stateboard/mod.ts @@ -0,0 +1,13 @@ +import { z } from 'zod' + +export type STATEBOARD_WIDGETS = z.infer<typeof STATEBOARD_WIDGETS> +export const STATEBOARD_WIDGETS = z.enum([ + 'TPS_REPORT', + 'FILE_EXPLORER', + 'MARKDOWN_EDITOR', + 'BRANCH_EXPLORER', + 'COMMIT_GRAPH', + 'COMMIT_INFO', + 'THREADS', + 'REPOS', +]) diff --git a/stateboard/napp.json b/stateboard/napp.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/stateboard/napp.json @@ -0,0 +1 @@ +{} diff --git a/_import-artifact/isolates/stateboard.ts b/stateboard/stateboard.ts similarity index 97% rename from _import-artifact/isolates/stateboard.ts rename to stateboard/stateboard.ts index 49a98eba9a8b..e56c8d29cc27 100644 --- a/_import-artifact/isolates/stateboard.ts +++ b/stateboard/stateboard.ts @@ -53,6 +53,8 @@ export const functions: Functions<Api> = { }, } +// ERROR too dependent on openai message structure + function getLastAssistantMessageId(thread: Thread) { const messages = [...thread.messages] while (messages.length > 0) { diff --git a/stucks/seed.md b/stucks/seed.md new file mode 100644 index 000000000000..80b70aa9bb2e --- /dev/null +++ b/stucks/seed.md @@ -0,0 +1,20 @@ +the napp used to modify stucks, and to search for them. + +holds within it the json schema format for each stuck. + +knows how to store them on the filesystem, in a way that is scalable. + +manages a large folder of them, in a way that is efficient. + +stucks are stored in a json object. + +Enables wrappers around the outside, such as expressing the relationships +between them, which need not change the stuck itself, and can be subject to many +different forked views. + +Funding is another wrapper, which depends on who is QA. + +solutions are a similar structure to a stuck, but they point to some stucks as +their targets. + +Solutions can reach different lifecycles / milestones, like passed by QA or not. diff --git a/test-file-runner/deno.json b/test-file-runner/deno.json deleted file mode 100644 index 2a270a1553ac..000000000000 --- a/test-file-runner/deno.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "@artifact/test-file-runner", - "version": "0.0.1", - "exports": { - ".": "./mod.ts" - } -} diff --git a/utils/chat/README.md b/utils/chat/README.md new file mode 100644 index 000000000000..fe6ec1f62c77 --- /dev/null +++ b/utils/chat/README.md @@ -0,0 +1,95 @@ +# @dreamcatcher/chat + +## Overview + +**@dreamcatcher/chat** extracts and reconstructs shared conversations from AI +chat interfaces. It focuses on retrieving single conversations via public shared +URLs. **Note:** This tool only works if the chat has been publicly shared on the +ChatGPT web app. + +## Installation + +To use this tool as a CLI command, first ensure you have Deno installed. Then +run: + +```sh +deno install --global --allow-read --allow-write --allow-env --allow-net=jsr.io,chatgpt.com jsr:@dreamcatcher/chat +``` + +After this, `chat` will be available as a system-wide command. + +> Note: The `--allow-net` permission is required to check for package updates +> from the jsr.io registry and to fetch conversations from chatgpt.com. + +## Usage + +Once installed as a CLI command, simply execute: + +```bash +chat https://chatgpt.com/share/675e1d90-0408-800b-b7e8-db1ded459182 [--output filename] +``` + +Where: + +- **URL** is the link to a publicly shared conversation HTML page (of the form + `https://chatgpt.com/share/<GUID>`). +- **--output filename** (optional) lets you specify a custom output filename. By + default, the conversation will be saved as `chat_<timestamp>_<guid>.md`, where + `timestamp` is the conversation's ISO 8601 creation date (YYYY-MM-DD) and + `guid` is a shortened version of the conversation ID extracted from the URL. + +## What It Does + +1. **Fetch HTML**: The tool requests the provided URL. +2. **Extract JSON**: It searches the first `<script>` tag containing the state + JSON object. +3. **Identify Server Response**: Using the `sharedConversationId` derived from + the URL's GUID, the tool locates the associated `serverResponse` object. +4. **Reconstruct Conversation**: The `extract` function returns + `[timestamp, conversationMarkdown]`. The timestamp is derived from the + conversation's creation time, and `conversationMarkdown` is a clean, readable + Markdown representation of the conversation. +5. **Output as Markdown**: If no custom filename is provided, the output file + follows the pattern: + + ``` + chat_<timestamp>_<guid>.md + ``` + + The `<timestamp>` and `<guid>` form a unique, descriptive filename for the + conversation. + +## Example + +```bash +chat https://chatgpt.com/share/675e1d90-0408-800b-b7e8-db1ded459182 +``` + +This command will produce a file like: + +``` +chat_2024-03-20_abcd.md +``` + +You can also specify a custom filename: + +```bash +chat https://chatgpt.com/share/675e1d90-0408-800b-b7e8-db1ded459182 --output my_conversation.md +``` + +This will produce `my_conversation.md` containing the fully reconstructed chat. + +## Requirements + +- Deno (latest stable) +- Permissions: + - `--allow-read` to read files + - `--allow-write` if using `--output` option + - `--allow-net` to fetch the conversation data from chatgpt.com and check for + updates from jsr.io + +## Upgrading the CLI command + +```sh +deno install --global --reload --force --allow-read --allow-write --allow-env --allow-net=jsr.io,chatgpt.com jsr:@dreamcatcher/chat +``` diff --git a/utils/chat/cli.ts b/utils/chat/cli.ts new file mode 100644 index 000000000000..6d44865895e4 --- /dev/null +++ b/utils/chat/cli.ts @@ -0,0 +1,21 @@ +import { program } from 'commander' +import { main } from './mod.ts' + +program + .name('@dreamcatcher/chat') + .description( + 'Extract and reconstruct a shared AI conversation from a given URL.', + ) + .argument('<url>', 'The shared conversation URL') + .option('-o, --output <filename>', 'Specify an output filename') + .action(async (url: string, options: { output?: string }) => { + try { + await main(url, options.output) + } catch (e) { + // deno-lint-ignore no-console + console.error(e) + Deno.exit(1) + } + }) + +program.parse(Deno.args, { from: 'user' }) diff --git a/utils/chat/deno.json b/utils/chat/deno.json new file mode 100644 index 000000000000..f1663963edd5 --- /dev/null +++ b/utils/chat/deno.json @@ -0,0 +1,19 @@ +{ + "name": "@dreamcatcher/chat", + "version": "0.0.6", + "tasks": { + "test": "deno test --allow-read --allow-write" + }, + "exports": "./cli.ts", + "imports": { + "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.48", + "@babel/parser": "npm:@babel/parser@^7.26.3", + "@babel/types": "npm:@babel/types@^7.26.3", + "@c4spar/mock-fetch": "jsr:@c4spar/mock-fetch@^1.0.0", + "@std/assert": "jsr:@std/assert@^1.0.9", + "@std/expect": "jsr:@std/expect@^1.0.9", + "commander": "npm:commander@^12.1.0", + "jsdom": "npm:jsdom@^25.0.1", + "json5": "npm:json5@^2.2.3" + } +} diff --git a/utils/chat/example.html b/utils/chat/example.html new file mode 100644 index 000000000000..6b8cb1f8db5d --- /dev/null +++ b/utils/chat/example.html @@ -0,0 +1,13735 @@ +<!DOCTYPE html> +<html + data-build="prod-11173c28d7974347784902fe1d57aae3a20e661b" + dir="ltr" + class="" +> + <head> + <meta charSet="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <link rel="preconnect" href="https://cdn.oaistatic.com" /> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> + <meta name="apple-itunes-app" content="app-id=6448311069" /> + <link + rel="apple-touch-icon" + sizes="180x180" + href="https://cdn.oaistatic.com/assets/apple-touch-icon-mz9nytnj.webp" + /> + <link + rel="icon" + type="image/png" + sizes="32x32" + href="https://cdn.oaistatic.com/assets/favicon-32x32-p4ktpm1x.webp" + media="(prefers-color-scheme: light)" + /> + <link + rel="icon" + type="image/png" + sizes="32x32" + href="https://cdn.oaistatic.com/assets/favicon-dark-32x32-gt5kfzyp.webp" + media="(prefers-color-scheme: dark)" + /> + <link + rel="icon" + type="image/svg+xml" + sizes="32x32" + href="https://cdn.oaistatic.com/assets/favicon-o20kmmos.svg" + /> + <title>ChatGPT - Git as Supply Chain</title> + <meta property="og:site_name" content="ChatGPT" /> + <meta name="robots" content="noindex,nofollow" /> + <meta property="og:title" content="ChatGPT - Git as Supply Chain" /> + <meta property="og:description" content="Shared via ChatGPT" /> + <meta + property="og:image" + content="https://cdn.oaistatic.com/assets/chatgpt-share-og-u7j5uyao.webp" + /> + <meta + property="og:url" + content="https://chatgpt.com/share/6758f810-286c-800b-8e2c-490e634ca002" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/manifest-7d43a138.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/mxkyxjre6muko6z4.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/nqo5y2f0dorhrqsr.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/fpwmsu1awpj0g2ko.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/dh0yl0m9q337gmci.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/ihbvqeoqhbkuefft.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/ty18ymk2wf2zvd35.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/ktiwgucld5a8s55m.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/k6el7ghyym9hl80v.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/dgcxf4c1lo6y3h3a.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/nb34aa8izknzna97.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/l697z2ouob9b6hw7.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/odhzv2q19wc7ynyt.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/k56enwh74zn4hbwt.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/mfdhaorj0sghs4gl.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/fy8orb1z72ox92eg.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/cnv95wiq9am77uyo.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/gtbc1g1q4ztw05rv.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/dvl2tfqalthh42cv.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/cb0x1wlgm93n2hpu.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/buun9i8g5c97ea0e.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/ezyyguzloavludkz.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/gxhifror9unq5n3g.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/lrt1lzalv2692ks0.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/lln33rcqgeiu1vmy.js" + /> + <link + rel="modulepreload" + href="https://cdn.oaistatic.com/assets/e8aqfi7hhkvyz94l.js" + /> + <link + rel="stylesheet" + href="https://cdn.oaistatic.com/assets/root-e6p3mfos.css" + /> + <link + rel="stylesheet" + href="https://cdn.oaistatic.com/assets/conversation-small-kq10986g.css" + /> + </head> + <body class=""> + <script> + !function () { + try { + var d = document.documentElement, c = d.classList; + c.remove("light", "dark"); + var e = localStorage.getItem("theme"); + if ("system" === e || (!e && true)) { + var t = "(prefers-color-scheme: dark)", + m = window.matchMedia(t); + if (m.media !== t || m.matches) { + d.style.colorScheme = "dark"; + c.add("dark"); + } else { + d.style.colorScheme = "light"; + c.add("light"); + } + } else if (e) c.add(e || ""); + if (e === "light" || e === "dark") d.style.colorScheme = e; + } catch (e) {} + }(); + </script> + <div + class="relative flex h-full w-full overflow-hidden transition-colors z-0" + > + <div + class="relative flex h-full max-w-full flex-1 flex-col overflow-hidden" + > + <main + class="relative h-full w-full flex-1 overflow-auto transition-width" + > + <div + role="presentation" + class="composer-parent flex h-full flex-col focus-visible:outline-0" + > + <div class="flex-1 overflow-hidden @container/thread"> + <div class="relative h-full"> + <div class="absolute left-0 right-0"> + <div + class="draggable no-draggable-children sticky top-0 p-3 mb-1.5 flex items-center justify-between z-10 h-header-height font-semibold bg-token-main-surface-primary max-md:hidden" + > + <div + class="absolute start-1/2 ltr:-translate-x-1/2 rtl:translate-x-1/2" + > + </div> + <div class="flex items-center gap-0 overflow-hidden"> + <div class="flex items-center"> + <span class="hidden"></span> + <span class="flex" data-state="closed"><button + aria-label="New chat" + class="h-10 rounded-lg px-2 text-token-text-secondary focus-visible:outline-0 disabled:text-token-text-quaternary focus-visible:bg-token-sidebar-surface-secondary enabled:hover:bg-token-sidebar-surface-secondary" + > + <svg + width="24" + height="24" + viewBox="0 0 24 24" + fill="currentColor" + xmlns="http://www.w3.org/2000/svg" + class="icon-xl-heavy" + > + <path + d="M15.6729 3.91287C16.8918 2.69392 18.8682 2.69392 20.0871 3.91287C21.3061 5.13182 21.3061 7.10813 20.0871 8.32708L14.1499 14.2643C13.3849 15.0293 12.3925 15.5255 11.3215 15.6785L9.14142 15.9899C8.82983 16.0344 8.51546 15.9297 8.29289 15.7071C8.07033 15.4845 7.96554 15.1701 8.01005 14.8586L8.32149 12.6785C8.47449 11.6075 8.97072 10.615 9.7357 9.85006L15.6729 3.91287ZM18.6729 5.32708C18.235 4.88918 17.525 4.88918 17.0871 5.32708L11.1499 11.2643C10.6909 11.7233 10.3932 12.3187 10.3014 12.9613L10.1785 13.8215L11.0386 13.6986C11.6812 13.6068 12.2767 13.3091 12.7357 12.8501L18.6729 6.91287C19.1108 6.47497 19.1108 5.76499 18.6729 5.32708ZM11 3.99929C11.0004 4.55157 10.5531 4.99963 10.0008 5.00007C9.00227 5.00084 8.29769 5.00827 7.74651 5.06064C7.20685 5.11191 6.88488 5.20117 6.63803 5.32695C6.07354 5.61457 5.6146 6.07351 5.32698 6.63799C5.19279 6.90135 5.10062 7.24904 5.05118 7.8542C5.00078 8.47105 5 9.26336 5 10.4V13.6C5 14.7366 5.00078 15.5289 5.05118 16.1457C5.10062 16.7509 5.19279 17.0986 5.32698 17.3619C5.6146 17.9264 6.07354 18.3854 6.63803 18.673C6.90138 18.8072 7.24907 18.8993 7.85424 18.9488C8.47108 18.9992 9.26339 19 10.4 19H13.6C14.7366 19 15.5289 18.9992 16.1458 18.9488C16.7509 18.8993 17.0986 18.8072 17.362 18.673C17.9265 18.3854 18.3854 17.9264 18.673 17.3619C18.7988 17.1151 18.8881 16.7931 18.9393 16.2535C18.9917 15.7023 18.9991 14.9977 18.9999 13.9992C19.0003 13.4469 19.4484 12.9995 20.0007 13C20.553 13.0004 21.0003 13.4485 20.9999 14.0007C20.9991 14.9789 20.9932 15.7808 20.9304 16.4426C20.8664 17.116 20.7385 17.7136 20.455 18.2699C19.9757 19.2107 19.2108 19.9756 18.27 20.455C17.6777 20.7568 17.0375 20.8826 16.3086 20.9421C15.6008 21 14.7266 21 13.6428 21H10.3572C9.27339 21 8.39925 21 7.69138 20.9421C6.96253 20.8826 6.32234 20.7568 5.73005 20.455C4.78924 19.9756 4.02433 19.2107 3.54497 18.2699C3.24318 17.6776 3.11737 17.0374 3.05782 16.3086C2.99998 15.6007 2.99999 14.7266 3 13.6428V10.3572C2.99999 9.27337 2.99998 8.39922 3.05782 7.69134C3.11737 6.96249 3.24318 6.3223 3.54497 5.73001C4.02433 4.7892 4.78924 4.0243 5.73005 3.54493C6.28633 3.26149 6.88399 3.13358 7.55735 3.06961C8.21919 3.00673 9.02103 3.00083 9.99922 3.00007C10.5515 2.99964 10.9996 3.447 11 3.99929Z" + fill="currentColor" + > + </path> + </svg> + </button></span> + </div> + </div> + <div class="gap-2 flex items-center pr-1 leading-[0]"> + <div class="flex items-center justify-center gap-2"> + <button + class="btn relative btn-primary btn-small" + data-testid="login-button" + > + <div class="flex items-center justify-center"> + Log in + </div> + </button> + <button + class="btn relative btn-secondary btn-small" + data-testid="signup-button" + > + <div class="flex items-center justify-center"> + Sign up + </div> + </button> + </div> + </div> + </div> + </div> + <div + class="flex h-full flex-col items-center justify-center text-token-text-primary" + > + <div class="h-full w-full lg:py-[18px]"> + <div + class="m-auto text-base px-3 md:px-4 w-full md:px-5 lg:px-4 xl:px-5 h-full" + > + <div + class="mx-auto flex h-full w-full flex-col text-base lg:justify-center md:max-w-3xl" + > + <div class="mb-7 hidden text-center lg:block"> + <div + class="relative inline-flex justify-center text-center text-2xl font-semibold leading-9" + > + <h1>What can I help with?</h1> + </div> + </div> + <div + class="justify-center mt-[var(--screen-optical-compact-offset-amount)] flex h-full flex-shrink flex-col items-center overflow-hidden text-token-text-primary lg:hidden" + style="opacity: 0; will-change: opacity" + > + <div + class="relative inline-flex justify-center text-center text-2xl font-semibold leading-9" + > + <h1>What can I help with?</h1> + </div> + <div + class="h-[116px]" + style="opacity: 0; will-change: opacity" + > + </div> + </div> + <div + class="lg:absolute lg:bottom-8 lg:left-0 lg:w-full" + > + <div + class="mx-auto flex h-full w-full flex-col text-base lg:justify-center md:max-w-3xl" + > + <div class="block z-20"></div> + </div> + </div> + <div class="w-full"> + <div class="flex justify-center"></div> + <form + class="w-full" + type="button" + aria-haspopup="dialog" + aria-expanded="false" + aria-controls="radix-:R7nkkuiiklj5:" + data-state="closed" + > + <div + class="relative flex h-full max-w-full flex-1 flex-col" + > + <div + class="group relative flex w-full items-center" + > + <div + id="composer-background" + class="flex w-full cursor-text flex-col rounded-3xl px-2.5 py-1 transition-colors contain-inline-size bg-[#f4f4f4] dark:bg-token-main-surface-secondary" + > + <div + class="flex min-h-[44px] items-start pl-2" + > + <div class="min-w-0 max-w-full flex-1"> + <div + class="_prosemirror-parent_15ceg_1 text-token-text-primary max-h-[25dvh] max-h-52 overflow-auto default-browser" + > + <textarea + class="block h-10 w-full resize-none border-0 bg-transparent px-0 py-2 text-token-text-primary placeholder:text-token-text-secondary" + autofocus="" + placeholder="Message ChatGPT" + ></textarea> + <script + nonce="52bc4634-5139-448c-b73b-89e4eef4691c" + > + window + .__oai_logHTML + ? window + .__oai_logHTML() + : window + .__oai_SSR_HTML = + window + .__oai_SSR_HTML || + Date + .now(); + requestAnimationFrame( + function () { + window + .__oai_logTTI + ? window + .__oai_logTTI() + : window + .__oai_SSR_TTI = + window + .__oai_SSR_TTI || + Date + .now(); + }, + ); + </script> + </div> + </div> + <div class="w-[32px] pt-1"></div> + </div> + <div + class="flex h-[44px] items-center justify-between" + > + <div class="flex gap-x-1"> + <div class="relative"> + <div class="relative"> + <span class="hidden"></span> + <span + class="flex" + data-state="closed" + ><div class="flex"> + <button + disabled="" + aria-label="Attach files is unavailable" + class="flex items-center justify-center h-8 w-8 rounded-lg rounded-bl-xl text-token-text-primary dark:text-white focus-visible:outline-black dark:focus-visible:outline-white opacity-30" + > + <svg + width="24" + height="24" + viewBox="0 0 24 24" + fill="none" + xmlns="http://www.w3.org/2000/svg" + > + <path + fill-rule="evenodd" + clip-rule="evenodd" + d="M9 7C9 4.23858 11.2386 2 14 2C16.7614 2 19 4.23858 19 7V15C19 18.866 15.866 22 12 22C8.13401 22 5 18.866 5 15V9C5 8.44772 5.44772 8 6 8C6.55228 8 7 8.44772 7 9V15C7 17.7614 9.23858 20 12 20C14.7614 20 17 17.7614 17 15V7C17 5.34315 15.6569 4 14 4C12.3431 4 11 5.34315 11 7V15C11 15.5523 11.4477 16 12 16C12.5523 16 13 15.5523 13 15V9C13 8.44772 13.4477 8 14 8C14.5523 8 15 8.44772 15 9V15C15 16.6569 13.6569 18 12 18C10.3431 18 9 16.6569 9 15V7Z" + fill="currentColor" + > + </path> + </svg> + </button> + <input + disabled="" + multiple="" + type="file" + style="display: none" + tabindex="-1" + class="hidden" + /> + </div></span> + </div> + </div> + </div> + <span class="hidden"></span> + <span class="" data-state="closed"><button + disabled="" + aria-label="Send prompt" + data-testid="send-button" + class="flex h-8 w-8 items-center justify-center rounded-full transition-colors hover:opacity-70 focus-visible:outline-none focus-visible:outline-black disabled:text-[#f4f4f4] disabled:hover:opacity-100 dark:focus-visible:outline-white disabled:dark:bg-token-text-quaternary dark:disabled:text-token-main-surface-secondary bg-black text-white dark:bg-white dark:text-black disabled:bg-[#D7D7D7]" + > + <svg + width="32" + height="32" + viewBox="0 0 32 32" + fill="none" + xmlns="http://www.w3.org/2000/svg" + class="icon-2xl" + > + <path + fill-rule="evenodd" + clip-rule="evenodd" + d="M15.1918 8.90615C15.6381 8.45983 16.3618 8.45983 16.8081 8.90615L21.9509 14.049C22.3972 14.4953 22.3972 15.2189 21.9509 15.6652C21.5046 16.1116 20.781 16.1116 20.3347 15.6652L17.1428 12.4734V22.2857C17.1428 22.9169 16.6311 23.4286 15.9999 23.4286C15.3688 23.4286 14.8571 22.9169 14.8571 22.2857V12.4734L11.6652 15.6652C11.2189 16.1116 10.4953 16.1116 10.049 15.6652C9.60265 15.2189 9.60265 14.4953 10.049 14.049L15.1918 8.90615Z" + fill="currentColor" + > + </path> + </svg> + </button></span> + </div> + </div> + </div> + </div> + </form> + </div> + <div + class="hidden lg:block h-[70px]" + style="opacity: 0; will-change: opacity" + > + </div> + <div + class="w-full px-2 py-2 text-center text-xs text-token-text-secondary empty:hidden md:px-[60px] lg:absolute lg:bottom-0 lg:left-0" + > + <div class="min-h-4"> + <div> + ChatGPT can make mistakes. Check important info. + </div> + </div> + </div> + </div> + </div> + </div> + </div> + </div> + </div> + <div + class="md:pt-0 dark:border-white/20 md:border-transparent md:dark:border-transparent w-full" + > + <div> + <div + class="relative w-full px-2 py-2 text-center text-xs text-token-text-secondary empty:hidden md:px-[60px]" + > + </div> + </div> + </div> + </div> + <div + class="group absolute bottom-2 end-2 z-20 flex flex-col gap-1 md:flex lg:bottom-3 lg:end-3" + > + <button + class="flex h-6 w-6 items-center justify-center rounded-full border border-token-border-light text-xs text-token-text-secondary" + type="button" + id="radix-:R9eiiklj5:" + aria-haspopup="menu" + aria-expanded="false" + data-state="closed" + data-testid="undefined-button" + > + ? + </button> + </div> + </main> + </div> + </div> + <div aria-live="assertive" aria-atomic="true" class="sr-only"></div> + <div aria-live="polite" aria-atomic="true" class="sr-only"></div> + <audio + class="fixed bottom-0 left-0 hidden h-0 w-0" + autoPlay="" + crossorigin="anonymous" + > + </audio> + <script nonce="52bc4634-5139-448c-b73b-89e4eef4691c"> + window.__remixContext = { + "basename": "/", + "future": { + "v3_fetcherPersist": false, + "v3_relativeSplatPath": false, + "v3_throwAbortReason": false, + "unstable_singleFetch": false, + "unstable_lazyRouteDiscovery": false, + "unstable_optimizeDeps": true, + }, + "isSpaMode": false, + "state": { + "loaderData": { + "root": { + 'rq:["session"]': { + "data": null, + "dataUpdateCount": 1, + "dataUpdatedAt": 1733905443121, + "error": null, + "errorUpdateCount": 0, + "errorUpdatedAt": 0, + "fetchFailureCount": 0, + "fetchFailureReason": null, + "fetchMeta": null, + "isInvalidated": false, + "status": "success", + "fetchStatus": "idle", + }, + 'rq:["account-status"]': {}, + "clientBootstrap": { + "session": {}, + "user": { + "email": "", + "groups": [], + "id": "ua-d76fbeb9-aa53-4dcb-b666-8358452854be", + "name": "", + "intercom_hash": + "ua-d76fbeb9-aa53-4dcb-b666-8358452854be", + "image": "", + "picture": "", + "mfa": false, + }, + "cluster": "unified-21", + "userCountry": "CA", + "userRegion": null, + "userRegionCode": null, + "cfConnectingIp": "2001:df0:a280:1001::6:1", + "cfIpLatitude": "43.63190", + "cfIpLongitude": "-79.37160", + "cfTimezone": "America/Toronto", + "cfIpCity": null, + "userLocale": null, + "authStatus": "logged_out", + "statsig": { + "payload": { + "feature_gates": { + "14938527": { + "name": "14938527", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "16325302": { + "name": "16325302", + "value": false, + "rule_id": "dOGUtq0Id8KhSPYCfoxRE", + "secondary_exposures": [], + }, + "29898433": { + "name": "29898433", + "value": true, + "rule_id": "5hlR2AjP7h2nJXOxZHKzKH:100.00:2", + "secondary_exposures": [], + }, + "44045625": { + "name": "44045625", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "61299031": { + "name": "61299031", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "73891701": { + "name": "73891701", + "value": true, + "rule_id": "4M51IcwRuyJAV7JLf2RvzR:100.00:1", + "secondary_exposures": [], + }, + "80186230": { + "name": "80186230", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "87687712": { + "name": "87687712", + "value": true, + "rule_id": "2qhKk5ydXFvhD5pios2h1S", + "secondary_exposures": [], + }, + "106692620": { + "name": "106692620", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "138956718": { + "name": "138956718", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1132191510", + "gateValue": "false", + "ruleID": "dHE9Ru37UQ9JP4Jaewul2", + }], + }, + "156153730": { + "name": "156153730", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "166966726": { + "name": "166966726", + "value": true, + "rule_id": "5VFEe5M4mDSyQU0M9U5hTm", + "secondary_exposures": [], + }, + "174366048": { + "name": "174366048", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1923022511", + "gateValue": "false", + "ruleID": "default", + }], + }, + "181161203": { + "name": "181161203", + "value": true, + "rule_id": "6NmIF8U6DauDDclcytkCkc", + "secondary_exposures": [], + }, + "187267097": { + "name": "187267097", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "193142637": { + "name": "193142637", + "value": false, + "rule_id": "7eSgKGxxnDNbWyzYM0IMHe:0.00:3", + "secondary_exposures": [], + }, + "211801712": { + "name": "211801712", + "value": true, + "rule_id": "2wyraagVOBvRaQh0LWLKgh", + "secondary_exposures": [], + }, + "218782548": { + "name": "218782548", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "226799327": { + "name": "226799327", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "232791851": { + "name": "232791851", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "3922476776", + "gateValue": "false", + "ruleID": "4XHkj4xXCwlkAzMMY0VSn4:0.00:2", + }, { + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + }, + "263833443": { + "name": "263833443", + "value": true, + "rule_id": "1mdaBiHva650rqO0InYHjm:100.00:2", + "secondary_exposures": [], + }, + "286716193": { + "name": "286716193", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "314203997": { + "name": "314203997", + "value": false, + "rule_id": "1xxCxFqZdreszzk4pe7fVb", + "secondary_exposures": [], + }, + "359124149": { + "name": "359124149", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "374328013": { + "name": "374328013", + "value": false, + "rule_id": "5xNC0SZHSEfRGwLRCyEofs:0.00:1", + "secondary_exposures": [{ + "gate": "2046333385", + "gateValue": "false", + "ruleID": "4nM2ehmgoDQIv69B0zohb6", + }], + }, + "389809369": { + "name": "389809369", + "value": false, + "rule_id": "5WK3s5Ax4bJcCZUXG1aCaM:0.00:2", + "secondary_exposures": [], + }, + "437245079": { + "name": "437245079", + "value": true, + "rule_id": "1KKPtXmOjxZamlgM2DzXlg", + "secondary_exposures": [], + }, + "458009956": { + "name": "458009956", + "value": true, + "rule_id": "6LgEwykI8hHnFAQ5EkKTX2:100.00:1", + "secondary_exposures": [], + }, + "462735957": { + "name": "462735957", + "value": true, + "rule_id": "1SCzY8HZjgbrVUviddAgUM:100.00:1", + "secondary_exposures": [], + }, + "467934241": { + "name": "467934241", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "491279851": { + "name": "491279851", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "543687013": { + "name": "543687013", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "544659310": { + "name": "544659310", + "value": true, + "rule_id": "1Opz1br1XsEM6Uajp6F1XU:100.00:3", + "secondary_exposures": [], + }, + "562926978": { + "name": "562926978", + "value": true, + "rule_id": "5NtGLuwEGvKkFrz0jD9Jgb:100.00:1", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "566128514": { + "name": "566128514", + "value": true, + "rule_id": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + "secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }], + }, + "568802918": { + "name": "568802918", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "582612297": { + "name": "582612297", + "value": true, + "rule_id": "5censDsCfS2zQeYtTIui2s:100.00:2", + "secondary_exposures": [], + }, + "584022615": { + "name": "584022615", + "value": true, + "rule_id": "7xLTX1iWf4Q14u604BUFhJ:100.00:2", + "secondary_exposures": [], + }, + "589604007": { + "name": "589604007", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "633009675", + "gateValue": "false", + "ruleID": "15DR4R8pnag2oqKIvQOVni", + }], + }, + "603035019": { + "name": "603035019", + "value": false, + "rule_id": "3LIHLisyFlEpdQBDU6f5A0", + "secondary_exposures": [{ + "gate": "2547456621", + "gateValue": "false", + "ruleID": "jDZnsHB4L995XxJ7J5vtK", + }], + }, + "605040022": { + "name": "605040022", + "value": true, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "614413305": { + "name": "614413305", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "633009675": { + "name": "633009675", + "value": false, + "rule_id": "15DR4R8pnag2oqKIvQOVni", + "secondary_exposures": [], + }, + "645560164": { + "name": "645560164", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "711369489": { + "name": "711369489", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "716475478": { + "name": "716475478", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "716722001": { + "name": "716722001", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "719574156": { + "name": "719574156", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "730579107": { + "name": "730579107", + "value": true, + "rule_id": "2D13jk8X6i5dXQlM6FWS9N", + "secondary_exposures": [], + }, + "730601627": { + "name": "730601627", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "740954505": { + "name": "740954505", + "value": false, + "rule_id": "6m5kbcBHvQbkbeBzwquYFp", + "secondary_exposures": [], + }, + "756982148": { + "name": "756982148", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "771993664": { + "name": "771993664", + "value": true, + "rule_id": "1dDdxrXy1KadHtYusuGRgR:100.00:6", + "secondary_exposures": [], + }, + "774838244": { + "name": "774838244", + "value": false, + "rule_id": "6iWAMGfVhFkUDAeK2ldKZa", + "secondary_exposures": [], + }, + "808722915": { + "name": "808722915", + "value": false, + "rule_id": "1HBUgTG7kZsvlP01c71Pbv", + "secondary_exposures": [], + }, + "809675868": { + "name": "809675868", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "857491970": { + "name": "857491970", + "value": true, + "rule_id": "2LpyNTIfbYC4R6XS6dfdIL", + "secondary_exposures": [], + }, + "859327441": { + "name": "859327441", + "value": false, + "rule_id": "2kLMbEEhNzrRXoN7Lw2PmZ", + "secondary_exposures": [], + }, + "874794369": { + "name": "874794369", + "value": true, + "rule_id": "2VpsO0nGBUK2dZXmYhPA0B", + "secondary_exposures": [], + }, + "889823676": { + "name": "889823676", + "value": true, + "rule_id": "IbtsgDIEUaa3M0uovI8VE", + "secondary_exposures": [], + }, + "910189142": { + "name": "910189142", + "value": true, + "rule_id": "3epNymlQdEWHnDUZEmtWUC", + "secondary_exposures": [], + }, + "967447444": { + "name": "967447444", + "value": true, + "rule_id": "2mOBTIuCpZKSBYUIdPaATY", + "secondary_exposures": [], + }, + "989108178": { + "name": "989108178", + "value": false, + "rule_id": "4sTodKrNyByM4guZ68MORR", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "989226566": { + "name": "989226566", + "value": true, + "rule_id": "6yqqYAWKtmfU8A7QGdiky4", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "1016364891": { + "name": "1016364891", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1026575464": { + "name": "1026575464", + "value": true, + "rule_id": "1f4cSeenpcmWQ9eMKKQhF5", + "secondary_exposures": [], + }, + "1028288328": { + "name": "1028288328", + "value": true, + "rule_id": "N8aqJyROPpTrVV5VygNPc:100.00:3", + "secondary_exposures": [], + }, + "1028682714": { + "name": "1028682714", + "value": true, + "rule_id": "735n03snBvba4AEhd2Qwqu:100.00:3", + "secondary_exposures": [], + }, + "1070272635": { + "name": "1070272635", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1074323483": { + "name": "1074323483", + "value": false, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "1103845153": { + "name": "1103845153", + "value": true, + "rule_id": "2ACgZQuQHKLrOqIdaMq7y", + "secondary_exposures": [], + }, + "1118624160": { + "name": "1118624160", + "value": true, + "rule_id": "401MRIAwDUgxfVJedMgoze", + "secondary_exposures": [], + }, + "1123641887": { + "name": "1123641887", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2173548801", + "gateValue": "true", + "ruleID": "22nVhoL17eyMvGWgFrDfZe", + }], + }, + "1123642877": { + "name": "1123642877", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2173548801", + "gateValue": "true", + "ruleID": "22nVhoL17eyMvGWgFrDfZe", + }], + }, + "1132191510": { + "name": "1132191510", + "value": false, + "rule_id": "dHE9Ru37UQ9JP4Jaewul2", + "secondary_exposures": [], + }, + "1159399595": { + "name": "1159399595", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1198030896": { + "name": "1198030896", + "value": true, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "1236320153": { + "name": "1236320153", + "value": true, + "rule_id": "7Bq8t59h8MTmfDfmTxiCBa:100.00:6", + "secondary_exposures": [], + }, + "1239490451": { + "name": "1239490451", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1239490452": { + "name": "1239490452", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1291584794": { + "name": "1291584794", + "value": false, + "rule_id": "6ZsiDEUpOSTbG0NkCUShbJ", + "secondary_exposures": [], + }, + "1306782371": { + "name": "1306782371", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1318146997": { + "name": "1318146997", + "value": true, + "rule_id": "2AclmEgqaQBVFbxz37XKzy:100.00:5", + "secondary_exposures": [], + }, + "1334159177": { + "name": "1334159177", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1382475798": { + "name": "1382475798", + "value": true, + "rule_id": "3P8OsGy1e5tQlR5dsTIWbL", + "secondary_exposures": [], + }, + "1413999995": { + "name": "1413999995", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "697158486", + "gateValue": "false", + "ruleID": "5KvTiw548r4tjqy7oHEcAL", + }, { + "gate": "3199899666", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2048457345", + "gateValue": "false", + "ruleID": "default", + }], + }, + "1422501431": { + "name": "1422501431", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1426009137": { + "name": "1426009137", + "value": true, + "rule_id": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }], + }, + "1428536080": { + "name": "1428536080", + "value": true, + "rule_id": "2lFzVvuye5InI9X01xArgp", + "secondary_exposures": [], + }, + "1444106524": { + "name": "1444106524", + "value": true, + "rule_id": "6Kd1cWwMI8EXiyQ7bqo8f3:50.00:1", + "secondary_exposures": [], + }, + "1468311859": { + "name": "1468311859", + "value": false, + "rule_id": "7tfl8ZUhwr5pzErE3ikBej", + "secondary_exposures": [], + }, + "1487207580": { + "name": "1487207580", + "value": false, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "1489221567": { + "name": "1489221567", + "value": false, + "rule_id": "z06JxGlMXQVrwG418FqnU", + "secondary_exposures": [], + }, + "1524207076": { + "name": "1524207076", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1526010272": { + "name": "1526010272", + "value": true, + "rule_id": "680tNuHpCDKDg9DEYqeqv3:100.00:1", + "secondary_exposures": [], + }, + "1542198993": { + "name": "1542198993", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1567586774": { + "name": "1567586774", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "1568195116": { + "name": "1568195116", + "value": false, + "rule_id": "KKhehNjRxNHoIWKNkqpU2", + "secondary_exposures": [], + }, + "1578703058": { + "name": "1578703058", + "value": false, + "rule_id": "2l4nEVMUnPuXkgprUm5zzs:1.00:2", + "secondary_exposures": [], + }, + "1611573287": { + "name": "1611573287", + "value": true, + "rule_id": "159rwM3sBnviE9XWH24azn:100.00:2", + "secondary_exposures": [], + }, + "1656345175": { + "name": "1656345175", + "value": true, + "rule_id": "2CwIChuIr7SLQ2CyqRegF2", + "secondary_exposures": [], + }, + "1684776951": { + "name": "1684776951", + "value": true, + "rule_id": "nnOtXYe2wzehNEU8MfPam", + "secondary_exposures": [{ + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }], + }, + "1788948501": { + "name": "1788948501", + "value": false, + "rule_id": "40Z1SWSk3MWJlOarI38p6Z", + "secondary_exposures": [], + }, + "1789593108": { + "name": "1789593108", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1794124947": { + "name": "1794124947", + "value": true, + "rule_id": "6Xp2LEMkm5UEgaW5ZGIN3M", + "secondary_exposures": [], + }, + "1820151945": { + "name": "1820151945", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1825130190": { + "name": "1825130190", + "value": false, + "rule_id": "YLtE1lbpb7dOUfvjJEC3x:0.00:4", + "secondary_exposures": [], + }, + "1828361965": { + "name": "1828361965", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1847911009": { + "name": "1847911009", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2304807207", + "gateValue": "false", + "ruleID": "default", + }], + }, + "1860647109": { + "name": "1860647109", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1877321278": { + "name": "1877321278", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1883608273": { + "name": "1883608273", + "value": true, + "rule_id": "2fjZsiHb0X34cxyryHit5D", + "secondary_exposures": [], + }, + "1923022511": { + "name": "1923022511", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1938289220": { + "name": "1938289220", + "value": false, + "rule_id": "79O8DQPDmTKxnLdAH9loVk", + "secondary_exposures": [], + }, + "2000076788": { + "name": "2000076788", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2033872549": { + "name": "2033872549", + "value": false, + "rule_id": "7dScmNU0bu2UQuzCNtva50", + "secondary_exposures": [], + }, + "2048332657": { + "name": "2048332657", + "value": false, + "rule_id": "7zy1mC6ANDLQQEC55KcGXC", + "secondary_exposures": [], + }, + "2055244389": { + "name": "2055244389", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2067628123": { + "name": "2067628123", + "value": false, + "rule_id": "3CuBjEMi97tY3EGnq0NA9s", + "secondary_exposures": [], + }, + "2091463435": { + "name": "2091463435", + "value": true, + "rule_id": "5t78GUS68KOn3bHZd8z7ii:100.00:1", + "secondary_exposures": [], + }, + "2104868687": { + "name": "2104868687", + "value": true, + "rule_id": "2MrZhodWRXji2DjHVx1UwZ", + "secondary_exposures": [], + }, + "2128055372": { + "name": "2128055372", + "value": false, + "rule_id": "6m0Rlf0uNfNc4HscDdpcUA:0.00:3", + "secondary_exposures": [], + }, + "2173548801": { + "name": "2173548801", + "value": true, + "rule_id": "22nVhoL17eyMvGWgFrDfZe", + "secondary_exposures": [], + }, + "2192543539": { + "name": "2192543539", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "4206244917", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2216175490": { + "name": "2216175490", + "value": true, + "rule_id": "1n25S96Hm7ICEdb3VyBfqT", + "secondary_exposures": [], + }, + "2220211676": { + "name": "2220211676", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "2304807207": { + "name": "2304807207", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2307260085": { + "name": "2307260085", + "value": true, + "rule_id": "4v47Zgj07BoXsAC6nwCwgU", + "secondary_exposures": [], + }, + "2381300469": { + "name": "2381300469", + "value": true, + "rule_id": "1XW58zJg16b7h0clyVhEW3:100.00:10", + "secondary_exposures": [], + }, + "2420509970": { + "name": "2420509970", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2436427643": { + "name": "2436427643", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "697158486", + "gateValue": "false", + "ruleID": "5KvTiw548r4tjqy7oHEcAL", + }, { + "gate": "3199899666", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2048457345", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2437036977": { + "name": "2437036977", + "value": true, + "rule_id": "1tfSWj1wjQlwP4QQ4eD5Qu:100.00:4", + "secondary_exposures": [], + }, + "2445152477": { + "name": "2445152477", + "value": true, + "rule_id": "5qtlunRMswJX2JGoF8GikC", + "secondary_exposures": [], + }, + "2449630478": { + "name": "2449630478", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2492340648": { + "name": "2492340648", + "value": false, + "rule_id": "7vWEOrlZ07LCdewUnumW3C:0.00:9", + "secondary_exposures": [], + }, + "2494375100": { + "name": "2494375100", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2522578969": { + "name": "2522578969", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2542814321": { + "name": "2542814321", + "value": true, + "rule_id": "1m1xZtjqqNjZ3o8AOuRHeU", + "secondary_exposures": [], + }, + "2544616245": { + "name": "2544616245", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2546838099": { + "name": "2546838099", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2547456621": { + "name": "2547456621", + "value": false, + "rule_id": "jDZnsHB4L995XxJ7J5vtK", + "secondary_exposures": [], + }, + "2558436080": { + "name": "2558436080", + "value": false, + "rule_id": "2iIp4XMISbfnSy5NsEpKaN", + "secondary_exposures": [], + }, + "2562876640": { + "name": "2562876640", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2634628831": { + "name": "2634628831", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "2689530298": { + "name": "2689530298", + "value": true, + "rule_id": "37djhtjwy17C2tz3RWPj8i", + "secondary_exposures": [], + }, + "2742826311": { + "name": "2742826311", + "value": false, + "rule_id": "3IJuxWy2vbHjpJGwVraJQM", + "secondary_exposures": [], + }, + "2747189243": { + "name": "2747189243", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2756095923": { + "name": "2756095923", + "value": true, + "rule_id": "6jPp6nW1wQVJbfY0uwQgmv:100.00:1", + "secondary_exposures": [], + }, + "2775516364": { + "name": "2775516364", + "value": true, + "rule_id": "62xFU2LRHp4dPsOwj2jVUT", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "2813404493": { + "name": "2813404493", + "value": false, + "rule_id": "4jCRuoOxK8tLEGn9ngylXq", + "secondary_exposures": [{ + "gate": "1074323483", + "gateValue": "false", + "ruleID": "disabled", + }], + }, + "2824504080": { + "name": "2824504080", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2830958549": { + "name": "2830958549", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2833534668": { + "name": "2833534668", + "value": true, + "rule_id": "7uYkibMYlCPSnoWmmYNanm", + "secondary_exposures": [], + }, + "2848981774": { + "name": "2848981774", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1074323483", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "2849024256": { + "name": "2849024256", + "value": true, + "rule_id": "5m34P11wI6H8E5sAMtsEXj", + "secondary_exposures": [], + }, + "2851018874": { + "name": "2851018874", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2865702999": { + "name": "2865702999", + "value": true, + "rule_id": "31nbFDkAl6BHUyVaOtsonb", + "secondary_exposures": [], + }, + "2865703032": { + "name": "2865703032", + "value": true, + "rule_id": "2JsPzQuM9307I2E2pNdWHL", + "secondary_exposures": [], + }, + "2881332567": { + "name": "2881332567", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1410082514", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2893406386": { + "name": "2893406386", + "value": false, + "rule_id": "32QpYlIXk5Wr45idZ2iRkG", + "secondary_exposures": [], + }, + "2919213474": { + "name": "2919213474", + "value": false, + "rule_id": "6HLlb6nSjJk5ADynHucWgP", + "secondary_exposures": [], + }, + "2923133196": { + "name": "2923133196", + "value": false, + "rule_id": "1HnipyUiP0nxZWtbaIXTlK", + "secondary_exposures": [{ + "gate": "2048332657", + "gateValue": "false", + "ruleID": "7zy1mC6ANDLQQEC55KcGXC", + }], + }, + "2935021756": { + "name": "2935021756", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2940166566": { + "name": "2940166566", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3006519778": { + "name": "3006519778", + "value": true, + "rule_id": "15jPMyA4orCotRzcwO6hXn", + "secondary_exposures": [], + }, + "3011415004": { + "name": "3011415004", + "value": false, + "rule_id": "7pUMK6uci7sslAj8bP7VEA", + "secondary_exposures": [{ + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3033842386": { + "name": "3033842386", + "value": true, + "rule_id": "P6Yw3pPRSdCV9khAOA5zA", + "secondary_exposures": [], + }, + "3051378794": { + "name": "3051378794", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "3134329717": { + "name": "3134329717", + "value": false, + "rule_id": "2UWRV6bObClvvj8jLkP7kJ", + "secondary_exposures": [], + }, + "3168919911": { + "name": "3168919911", + "value": true, + "rule_id": "4ZR6dSHrJ9rEydRKbpzGfr:100.00:2", + "secondary_exposures": [], + }, + "3180974288": { + "name": "3180974288", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3213153699": { + "name": "3213153699", + "value": true, + "rule_id": "46yaCNlYIQAFy8P7hV5tXO:100.00:1", + "secondary_exposures": [], + }, + "3320547351": { + "name": "3320547351", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3323077373": { + "name": "3323077373", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3325813340": { + "name": "3325813340", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2304807207", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3376455464": { + "name": "3376455464", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3419264204": { + "name": "3419264204", + "value": false, + "rule_id": "1SLt7ZIEzNa581wiqmmbrJ:0.00:1", + "secondary_exposures": [{ + "gate": "1132191510", + "gateValue": "false", + "ruleID": "dHE9Ru37UQ9JP4Jaewul2", + }, { + "gate": "138956718", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3435450078": { + "name": "3435450078", + "value": true, + "rule_id": "2qCdHpFuWOOkibzLRL0zgn", + "secondary_exposures": [], + }, + "3483957186": { + "name": "3483957186", + "value": true, + "rule_id": "6g82ij32XX2iKDaEpwNXZt:100.00:1", + "secondary_exposures": [], + }, + "3501550327": { + "name": "3501550327", + "value": false, + "rule_id": "HEU9zVkuwUfmXHhGcckPQ", + "secondary_exposures": [], + }, + "3504894861": { + "name": "3504894861", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3511639124": { + "name": "3511639124", + "value": true, + "rule_id": "7o5NDbkivFFnos7rz9JSKL", + "secondary_exposures": [], + }, + "3636405501": { + "name": "3636405501", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3643162575": { + "name": "3643162575", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3650277116": { + "name": "3650277116", + "value": false, + "rule_id": "3Txp6tA3tgTjsKGkRbaKVM:0.00:26", + "secondary_exposures": [], + }, + "3655058338": { + "name": "3655058338", + "value": false, + "rule_id": "2eakzH7Djowztq0c8xUBjk:0.00:13", + "secondary_exposures": [], + }, + "3664702598": { + "name": "3664702598", + "value": false, + "rule_id": "7x9wS41bRDCji9ns8x5Oej", + "secondary_exposures": [], + }, + "3678527908": { + "name": "3678527908", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3680626616": { + "name": "3680626616", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "633009675", + "gateValue": "false", + "ruleID": "15DR4R8pnag2oqKIvQOVni", + }], + }, + "3680956616": { + "name": "3680956616", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3700195277": { + "name": "3700195277", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3700204850": { + "name": "3700204850", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3700615661": { + "name": "3700615661", + "value": false, + "rule_id": "66covmutTYx82FWVUlZAqF", + "secondary_exposures": [], + }, + "3768008236": { + "name": "3768008236", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3825730354": { + "name": "3825730354", + "value": false, + "rule_id": "3TOe1n7juAS2TMXzLqNTfj", + "secondary_exposures": [], + }, + "3894844102": { + "name": "3894844102", + "value": false, + "rule_id": "3QXis04GA1OiER0i1tq5MT:50.00:4", + "secondary_exposures": [], + }, + "3912844760": { + "name": "3912844760", + "value": true, + "rule_id": "2PBMw1F9LSIuyNtzThg3PP", + "secondary_exposures": [], + }, + "3922145230": { + "name": "3922145230", + "value": false, + "rule_id": "14DZA2LumaPqAdCo52CrUB", + "secondary_exposures": [], + }, + "3922476776": { + "name": "3922476776", + "value": false, + "rule_id": "4XHkj4xXCwlkAzMMY0VSn4:0.00:2", + "secondary_exposures": [], + }, + "3941367332": { + "name": "3941367332", + "value": true, + "rule_id": "4NngvQ5h3bIzSutqTQ4CeM", + "secondary_exposures": [{ + "gate": "716475478", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3954884439": { + "name": "3954884439", + "value": true, + "rule_id": "5rqjCf7T9KpJtLnaE73Kum:100.00:4", + "secondary_exposures": [], + }, + "3978920525": { + "name": "3978920525", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3999712942": { + "name": "3999712942", + "value": true, + "rule_id": "3tcEyACMDCatpAcpodrJfN", + "secondary_exposures": [], + }, + "4017530922": { + "name": "4017530922", + "value": true, + "rule_id": "26y7j9aeW6IfykfWOedJm7:100.00:3", + "secondary_exposures": [], + }, + "4038001028": { + "name": "4038001028", + "value": false, + "rule_id": "7sPHYLu6jyA6IUXgqQQsJW", + "secondary_exposures": [], + }, + "4039254565": { + "name": "4039254565", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4137442097": { + "name": "4137442097", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4182851027": { + "name": "4182851027", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4206244917": { + "name": "4206244917", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4215082624": { + "name": "4215082624", + "value": true, + "rule_id": "6oaDIks7YZl9qNION8SRYD", + "secondary_exposures": [], + }, + "4226692983": { + "name": "4226692983", + "value": true, + "rule_id": "6sEu91zwlBGSKOqFiNpGlA:100.00:2", + "secondary_exposures": [], + }, + "4242486290": { + "name": "4242486290", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4245811603": { + "name": "4245811603", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + }, + "dynamic_configs": { + "51287004": { + "name": "51287004", + "value": { "enable": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["enable"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "94117254": { + "name": "94117254", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_onboarding_flow", + "new_onboarding_flow_qualified_start_date", + "use_plus_rl_during_onboarding", + "personalized_onboarding", + "write_custom_instructions_in_onboarding", + "plus_rl_during_onboarding_minutes_after_creation", + "onboarding_show_custom_instructions_page", + "onboarding_flow_tool_steps", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "101132775": { + "name": "101132775", + "value": { "show_new_ui": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "171317775": { + "name": "171317775", + "value": { + "should_see_splash_screen_title": true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "186644516": { + "name": "186644516", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_team_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "387752763": { + "name": "387752763", + "value": { + "enable_slash_commands": true, + "enable_rich_text_composer": true, + }, + "group": "5UE8g4T56yxUBUYancL7KB:override", + "rule_id": "5UE8g4T56yxUBUYancL7KB:override", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + "explicit_parameters": [ + "enable_rich_text_composer", + "enable_slash_commands", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "410076910": { + "name": "410076910", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": ["enable_new_mobile"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "457207600": { + "name": "457207600", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_primary_btn_blue"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "459502094": { + "name": "459502094", + "value": { "is_sticky_toggle_off": false }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["is_sticky_toggle_off"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "475896730": { + "name": "475896730", + "value": { "should-enable-hojicha": false }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "should-enable-hojicha", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "488365967": { + "name": "488365967", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["forwardToAuthApi"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "505180994": { + "name": "505180994", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4038001028", + "gateValue": "false", + "ruleID": "7sPHYLu6jyA6IUXgqQQsJW", + }], + "explicit_parameters": [ + "show_share_button_inline", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "506086779": { + "name": "506086779", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "1860647109", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "is_direct_continue_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "548098440": { + "name": "548098440", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + "show_composer_header", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "550560761": { + "name": "550560761", + "value": { + "history_results_limit": 6, + "local_results_limit": 2, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "653115894": { + "name": "653115894", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_indexing", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + "wave", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "668618468": { + "name": "668618468", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_hardcoded_onboarding_prompt", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "743318407": { + "name": "743318407", + "value": {}, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1885925757", + "gateValue": "false", + "ruleID": "default", + }], + "is_user_in_experiment": false, + "is_experiment_active": true, + }, + "754469756": { + "name": "754469756", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["signup_cta_copy"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "790041403": { + "name": "790041403", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "855925714": { + "name": "855925714", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_simplified_sharing_modal_enabled", + "is_social_share_options_enabled", + "is_update_shared_links_enabled", + "is_discoverability_toggle_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "917528628": { + "name": "917528628", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "login_allow_phone", + "signup_allow_phone", + "use_phone_number", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "923531957": { + "name": "923531957", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "login_allow_phone", + "signup_allow_phone", + "use_phone_number", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "938456440": { + "name": "938456440", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1221279314", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "wave", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "944459048": { + "name": "944459048", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_produce_color_scheme", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "987538661": { + "name": "987538661", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["is_produce_design"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1009155978": { + "name": "1009155978", + "value": { + "enable_mobile_app_upsell_banner": true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_mobile_app_upsell_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "1013114931": { + "name": "1013114931", + "value": { + "show_follow_ups": false, + "follow_ups_prompt_key": "default", + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1028722647": { + "name": "1028722647", + "value": { + "enable_indexing": false, + "backfill_completed": false, + "enable_local_indexing": false, + "enable_ux": false, + "enable_us_rollout": false, + "enable_ux_rollout": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "2372319800", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "enable_local_indexing", + "backfill_completed", + "enable_ux", + "enable_ux_rollout", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "1065013594": { + "name": "1065013594", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_new_mobile"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1134365732": { + "name": "1134365732", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "should_show_disclaimer_only_once_per_device", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1193714591": { + "name": "1193714591", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1193714592": { + "name": "1193714592", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "config", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1193714593": { + "name": "1193714593", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "config", + "show_message_regenerate_model_selector_on_every_message", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1237212031": { + "name": "1237212031", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_country_selector_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1277879515": { + "name": "1277879515", + "value": { "show_ui": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1070272635", + "gateValue": "false", + "ruleID": "default", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1320801051": { + "name": "1320801051", + "value": { + "hide_new_at_workspace_section": false, + "hide_section_new_at_workspace": false, + "gpt_discovery_experiment_enabled": true, + "popular_at_my_workspace_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "gpt_discovery_experiment_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1321202851": { + "name": "1321202851", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "show_share_button_text", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1388643772": { + "name": "1388643772", + "value": { + "is_mobile_enterprise_enabled": false, + "is_enterprise_enabled": true, + }, + "group": "3INu3qkV6QoN42TYoP3gja:override", + "rule_id": "3INu3qkV6QoN42TYoP3gja:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1426679726": { + "name": "1426679726", + "value": { + "is-enabled": true, + "use-rtl-layout": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["use-rtl-layout"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1547924522": { + "name": "1547924522", + "value": {}, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "2183784321", + "gateValue": "false", + "ruleID": "5WAOcp7L45Zgr7TxHv69Bv", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1597388338": { + "name": "1597388338", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1722608525": { + "name": "1722608525", + "value": {}, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1826752011": { + "name": "1826752011", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": "v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_hardcoded_vision_prompts", + "enable_hardcoded_file_document_prompts", + "enable_hardcoded_data_vis_prompts", + "enable_hardcoded_browse_prompts", + "starter_prompt_ranking_algorithm", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1856338298": { + "name": "1856338298", + "value": { "forward_to_authapi": false }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "14938527", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": ["forward_to_authapi"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1873047021": { + "name": "1873047021", + "value": { + "search_scoring_dyconfig_name": + "gizmo_search_score_config", + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "search_scoring_dyconfig_name", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1904139466": { + "name": "1904139466", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_secondary_auth_banner_button_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1908072088": { + "name": "1908072088", + "value": { + "blocking_modal": true, + "single_button_variant": true, + "dropdown_tooltip": false, + "conversation_model_switcher_badge": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "1930243550": { + "name": "1930243550", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": ["enable_new_mobile"], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "1934819659": { + "name": "1934819659", + "value": { "is_redesign_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1934819688": { + "name": "1934819688", + "value": { + "is_redesign_enabled": true, + "secondary_button_type": "login", + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1977905538": { + "name": "1977905538", + "value": { + "forwardToAuthApi": false, + "directToGoogle": false, + "directToApple": false, + "directToMsft": false, + }, + "group": "disabled", + "rule_id": "disabled", + "is_device_based": true, + "secondary_exposures": [], + }, + "2068644600": { + "name": "2068644600", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "login_allow_phone", + "signup_allow_phone", + "use_phone_number", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2121003545": { + "name": "2121003545", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "2211924604": { + "name": "2211924604", + "value": { "is_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "3483957186", + "gateValue": "true", + "ruleID": "6g82ij32XX2iKDaEpwNXZt:100.00:1", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "2212993488": { + "name": "2212993488", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "model_talks_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2214103875": { + "name": "2214103875", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["signup_cta_copy"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2246721288": { + "name": "2246721288", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_indexing", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2272050751": { + "name": "2272050751", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2294210794": { + "name": "2294210794", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "enable_new_mobile", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2331692225": { + "name": "2331692225", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "226799327", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2335908103": { + "name": "2335908103", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + "enable_new_autocomplete_homepage", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2335908104": { + "name": "2335908104", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2335908105": { + "name": "2335908105", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 36, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2346290881": { + "name": "2346290881", + "value": { + "has_sidekick_access": false, + "show_nux_banner": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1938289220", + "gateValue": "false", + "ruleID": "79O8DQPDmTKxnLdAH9loVk", + }, { + "gate": "2033872549", + "gateValue": "false", + "ruleID": "7dScmNU0bu2UQuzCNtva50", + }], + "explicit_parameters": [ + "has_sidekick_access", + "show_nux_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2380789656": { + "name": "2380789656", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "2558701922", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1347376961", + "gateValue": "false", + "ruleID": "", + }], + "explicit_parameters": [ + "include_legacy_sidebar_contents", + "enable_arch_updates", + "include_floating_state", + "include_share_on_mobile", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2395035283": { + "name": "2395035283", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "show_share_button_text", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2423967026": { + "name": "2423967026", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2424145028": { + "name": "2424145028", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "starter_prompt_ranking_algorithm", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2424412160": { + "name": "2424412160", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "starter_prompt_ranking_algorithm", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2497137017": { + "name": "2497137017", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "signup_allow_phone", + "use_new_phone_ui", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2499513047": { + "name": "2499513047", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "2530795349": { + "name": "2530795349", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["enable_v2_cleanup"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2533035210": { + "name": "2533035210", + "value": { + "snowflake_composer_entry_point": false, + "use_broad_rate_limit_language": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "snowflake_composer_entry_point", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2555105235": { + "name": "2555105235", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "1414026179", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": ["login_allow_phone"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2555870456": { + "name": "2555870456", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["signup_cta_copy"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2567214617": { + "name": "2567214617", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "starter_prompt_ranking_algorithm", + "autocomplete_mode", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2575540972": { + "name": "2575540972", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2635734272": { + "name": "2635734272", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_simplified_sharing_modal_enabled", + "is_social_share_options_enabled", + "is_update_shared_links_enabled", + "is_discoverability_toggle_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2663462255": { + "name": "2663462255", + "value": {}, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "164447571", + "gateValue": "false", + "ruleID": "5p0MgR5OkQRqM3bl2fUW74", + }], + "is_user_in_experiment": false, + "is_experiment_active": true, + }, + "2664741443": { + "name": "2664741443", + "value": { "enable": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "2683915289": { + "name": "2683915289", + "value": { + "use_greeting": false, + "use_juice_ui": true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "3483957186", + "gateValue": "true", + "ruleID": "6g82ij32XX2iKDaEpwNXZt:100.00:1", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "2715504799": { + "name": "2715504799", + "value": { "source": "rockset" }, + "group": "3hVfTKFf19aVXd409tUXcS:100.00:2", + "rule_id": "3hVfTKFf19aVXd409tUXcS:100.00:2", + "is_device_based": false, + "secondary_exposures": [], + }, + "2724953007": { + "name": "2724953007", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_mobile_app_upsell_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745767556": { + "name": "2745767556", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745768482": { + "name": "2745768482", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745768485": { + "name": "2745768485", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745768542": { + "name": "2745768542", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745770409": { + "name": "2745770409", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "7xj42ZY6EtniGNkumF3vQ3:override", + "rule_id": "7xj42ZY6EtniGNkumF3vQ3:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2773808134": { + "name": "2773808134", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + "enable_new_mobile", + "search_autocomplete_mode", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2778287751": { + "name": "2778287751", + "value": { + "is_enterprise_enabled": false, + "is_enterprise_desktop_enabled": false, + "is_desktop_enterprise_enabled": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2795141704": { + "name": "2795141704", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2818189709": { + "name": "2818189709", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "2l2l6SrM9hXnjCjkr95Ibl:override", + "rule_id": "2l2l6SrM9hXnjCjkr95Ibl:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": ["is_produce_design"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2892794518": { + "name": "2892794518", + "value": {}, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "2909749120": { + "name": "2909749120", + "value": { "is_gating_fix_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_gating_fix_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2919366999": { + "name": "2919366999", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "716475478", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "show_copylink_state_if_no_updates", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2980740905": { + "name": "2980740905", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_yearly_plus_subscription_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983618516": { + "name": "2983618516", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983618523": { + "name": "2983618523", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983619414": { + "name": "2983619414", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2983619420": { + "name": "2983619420", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983619445": { + "name": "2983619445", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983619448": { + "name": "2983619448", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2983619449": { + "name": "2983619449", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3021134725": { + "name": "3021134725", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_hardcoded_vision_prompts", + "enable_hardcoded_file_document_prompts", + "enable_hardcoded_data_vis_prompts", + "enable_hardcoded_browse_prompts", + "starter_prompt_ranking_algorithm", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3022018663": { + "name": "3022018663", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_team_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3074105577": { + "name": "3074105577", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + "remove_edit_in_signup": false, + "remove_edit_in_login": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "remove_edit_in_signup", + "remove_edit_in_login", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3123427997": { + "name": "3123427997", + "value": { + "enable_indexing": false, + "backfill_completed": false, + "enable_local_indexing": false, + "enable_ux": false, + "enable_us_rollout": false, + "enable_ux_rollout": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_indexing", + "backfill_completed", + "enable_local_indexing", + "enable_ux", + "enable_ux_rollout", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3130493514": { + "name": "3130493514", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["is_continue_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3148769572": { + "name": "3148769572", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "HARDCODED", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 0, + "autocomplete_max_char": 0, + "headline_option": "WHERE_TO_START", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2024-09-16T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + "enable_new_autocomplete_homepage", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3168246095": { + "name": "3168246095", + "value": { "gizmo_ids": [] }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "3218121151": { + "name": "3218121151", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3230069703": { + "name": "3230069703", + "value": { "expirySeconds": 15 }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "3286693234": { + "name": "3286693234", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + "enable_new_autocomplete_homepage", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3295035639": { + "name": "3295035639", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "include_account_settings_move", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3297250902": { + "name": "3297250902", + "value": { "use_new_query": true }, + "group": "53DF4oF9AvHTpJTxQ5ZLLI:100.00:2", + "rule_id": "53DF4oF9AvHTpJTxQ5ZLLI:100.00:2", + "is_device_based": false, + "secondary_exposures": [], + }, + "3299284219": { + "name": "3299284219", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "show_message_regenerate_model_selector_on_every_message", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3311396813": { + "name": "3311396813", + "value": { + "is_enterprise_enabled": true, + "is_enterprise_desktop_enabled": false, + "is_desktop_enterprise_enabled": false, + }, + "group": "3HX7vpdJsUkuyCUEL4V9cE:override", + "rule_id": "3HX7vpdJsUkuyCUEL4V9cE:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3336092451": { + "name": "3336092451", + "value": { "is_upgrade_button_blue": false }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_upgrade_button_blue", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3337889731": { + "name": "3337889731", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["use_email_otp"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3414159263": { + "name": "3414159263", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_no_auth_welcome_back_modal_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3444737088": { + "name": "3444737088", + "value": { + "is_mobile_web_toggle_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_mobile_web_toggle_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3460472607": { + "name": "3460472607", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_onboarding_flow", + "new_onboarding_flow_qualified_start_date", + "use_plus_rl_during_onboarding", + "personalized_onboarding", + "keep_onboarding_after_dismiss", + "write_custom_instructions_in_onboarding", + "onboarding_show_custom_instructions_page", + "plus_rl_during_onboarding_minutes_after_creation", + "onboarding_show_other_option", + "onboarding_flow_tool_steps", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3514586323": { + "name": "3514586323", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3591528489": { + "name": "3591528489", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_login_primary_button", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3600059992": { + "name": "3600059992", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["use_email_otp"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3623422562": { + "name": "3623422562", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + "headline_option", + "disclaimer_color_adjust", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3656846979": { + "name": "3656846979", + "value": {}, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "3701102259": { + "name": "3701102259", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": ["is_two_line"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3758098899": { + "name": "3758098899", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_team_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3762501946": { + "name": "3762501946", + "value": { "disable-ssr": false }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["disable-ssr"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3814631162": { + "name": "3814631162", + "value": { "is_shimmer_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "3851747685": { + "name": "3851747685", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "one_announcement_tooltip_per_session", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3917128856": { + "name": "3917128856", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "config", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3947925598": { + "name": "3947925598", + "value": { "is_upgrade_in_settings": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_upgrade_in_settings", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3947925629": { + "name": "3947925629", + "value": { "is_upgrade_in_settings": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_upgrade_in_settings", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3972089454": { + "name": "3972089454", + "value": { + "search_scoring_dyconfig_name": + "gizmo_search_score_config", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "search_scoring_dyconfig_name", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3983984123": { + "name": "3983984123", + "value": { "is_memory_undo_enabled": false }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_memory_undo_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4108016587": { + "name": "4108016587", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "is_continue_enabled", + "is_direct_continue_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "4198218833": { + "name": "4198218833", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "remove_early_access_upsell", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4211831761": { + "name": "4211831761", + "value": { "enabled": false }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4252260557": { + "name": "4252260557", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_v2_toggle_labels_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4252260612": { + "name": "4252260612", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_v2_toggle_labels_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4252261542": { + "name": "4252261542", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_v2_toggle_labels_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4268016031": { + "name": "4268016031", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "optimize_initial_modals", + "defer_memory_modal", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4278549316": { + "name": "4278549316", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + }, + "layer_configs": { + "51287004": { + "name": "51287004", + "value": { "enable": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "229662723": { + "name": "229662723", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "387752763": { + "name": "387752763", + "value": { + "enable_slash_commands": true, + "enable_rich_text_composer": true, + }, + "group": "5UE8g4T56yxUBUYancL7KB:override", + "rule_id": "5UE8g4T56yxUBUYancL7KB:override", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + "explicit_parameters": [ + "enable_rich_text_composer", + "enable_slash_commands", + ], + "allocated_experiment_name": "387752763", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + }, + "468168202": { + "name": "468168202", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "7xj42ZY6EtniGNkumF3vQ3:override", + "rule_id": "7xj42ZY6EtniGNkumF3vQ3:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "allocated_experiment_name": "2745770409", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + "660512088": { + "name": "660512088", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "2558701922", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1347376961", + "gateValue": "false", + "ruleID": "", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "2558701922", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1347376961", + "gateValue": "false", + "ruleID": "", + }], + }, + "685344542": { + "name": "685344542", + "value": { + "is_mobile_enterprise_enabled": false, + "is_enterprise_enabled": true, + }, + "group": "3INu3qkV6QoN42TYoP3gja:override", + "rule_id": "3INu3qkV6QoN42TYoP3gja:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "allocated_experiment_name": "1388643772", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + "717266490": { + "name": "717266490", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "871635014": { + "name": "871635014", + "value": { + "snowflake_composer_entry_point": false, + "use_broad_rate_limit_language": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "snowflake_composer_entry_point", + ], + "allocated_experiment_name": "2533035210", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [], + }, + "1238742812": { + "name": "1238742812", + "value": { "should_enable_zh_tw": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1320801051": { + "name": "1320801051", + "value": { + "hide_new_at_workspace_section": false, + "hide_section_new_at_workspace": false, + "gpt_discovery_experiment_enabled": true, + "popular_at_my_workspace_enabled": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1346366956": { + "name": "1346366956", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1358849452": { + "name": "1358849452", + "value": { "disable-ssr": false }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1547743984": { + "name": "1547743984", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4038001028", + "gateValue": "false", + "ruleID": "7sPHYLu6jyA6IUXgqQQsJW", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "4038001028", + "gateValue": "false", + "ruleID": "7sPHYLu6jyA6IUXgqQQsJW", + }], + }, + "1578749296": { + "name": "1578749296", + "value": { "is_sticky_toggle_off": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1630876919": { + "name": "1630876919", + "value": { + "enable_indexing": false, + "backfill_completed": false, + "enable_local_indexing": false, + "enable_ux": false, + "enable_us_rollout": false, + "enable_ux_rollout": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "2372319800", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "enable_local_indexing", + "backfill_completed", + "enable_ux", + "enable_ux_rollout", + ], + "allocated_experiment_name": "1028722647", + "is_experiment_active": true, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [], + }, + "1696863369": { + "name": "1696863369", + "value": { + "has_sidekick_access": false, + "show_nux_banner": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1938289220", + "gateValue": "false", + "ruleID": "79O8DQPDmTKxnLdAH9loVk", + }, { + "gate": "2033872549", + "gateValue": "false", + "ruleID": "7dScmNU0bu2UQuzCNtva50", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "1938289220", + "gateValue": "false", + "ruleID": "79O8DQPDmTKxnLdAH9loVk", + }, { + "gate": "2033872549", + "gateValue": "false", + "ruleID": "7dScmNU0bu2UQuzCNtva50", + }], + }, + "1846737571": { + "name": "1846737571", + "value": { "is_upgrade_button_blue": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1914829685": { + "name": "1914829685", + "value": { "forward_to_authapi": false }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "14938527", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "14938527", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2670443078": { + "name": "2670443078", + "value": { "is_gating_fix_enabled": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "2723963139": { + "name": "2723963139", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "2840731323": { + "name": "2840731323", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "1860647109", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "1860647109", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2888142241": { + "name": "2888142241", + "value": { "is_upgrade_in_settings": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "2932223118": { + "name": "2932223118", + "value": { + "is_mobile_web_toggle_enabled": true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3048336830": { + "name": "3048336830", + "value": { + "is-enabled": true, + "use-rtl-layout": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3922145230", + "gateValue": "false", + "ruleID": "14DZA2LumaPqAdCo52CrUB", + }, { + "gate": "3700615661", + "gateValue": "false", + "ruleID": "66covmutTYx82FWVUlZAqF", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "3922145230", + "gateValue": "false", + "ruleID": "14DZA2LumaPqAdCo52CrUB", + }, { + "gate": "3700615661", + "gateValue": "false", + "ruleID": "66covmutTYx82FWVUlZAqF", + }], + }, + "3119715334": { + "name": "3119715334", + "value": { "should-enable-hojicha": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3436367576": { + "name": "3436367576", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1221279314", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "wave", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + ], + "allocated_experiment_name": "938456440", + "is_experiment_active": true, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [], + }, + "3517133692": { + "name": "3517133692", + "value": { "is_memory_undo_enabled": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3533083032": { + "name": "3533083032", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3637408529": { + "name": "3637408529", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + }, + "3768341700": { + "name": "3768341700", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "2l2l6SrM9hXnjCjkr95Ibl:override", + "rule_id": "2l2l6SrM9hXnjCjkr95Ibl:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": ["is_produce_design"], + "allocated_experiment_name": "2818189709", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + "3927927759": { + "name": "3927927759", + "value": { + "enable_mobile_app_upsell_banner": true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3972089454": { + "name": "3972089454", + "value": { + "search_scoring_dyconfig_name": + "gizmo_search_score_config", + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "4031588851": { + "name": "4031588851", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + }, + "4211831761": { + "name": "4211831761", + "value": { "enabled": false }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "4250072504": { + "name": "4250072504", + "value": { + "is_enterprise_enabled": true, + "is_enterprise_desktop_enabled": false, + "is_desktop_enterprise_enabled": false, + }, + "group": "3HX7vpdJsUkuyCUEL4V9cE:override", + "rule_id": "3HX7vpdJsUkuyCUEL4V9cE:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "allocated_experiment_name": "3311396813", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + }, + "sdkParams": {}, + "has_updates": true, + "generator": "statsig-node-sdk", + "sdkInfo": { + "sdkType": "statsig-node", + "sdkVersion": "5.26.0", + }, + "time": 1733904874794, + "evaluated_keys": { + "customIDs": { + "WebAnonymousCookieID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "DeviceId": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "stableID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + }, + }, + "hash_used": "djb2", + "user": { + "country": "CA", + "custom": { + "has_logged_in_before": false, + "user_agent": "Wget/1.25.0", + "is_punch_out_user": false, + "auth_status": "logged_out", + }, + "customIDs": { + "WebAnonymousCookieID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "DeviceId": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "stableID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + }, + "ip": "2001:df0:a280:1001::6:1", + "userAgent": "Wget/1.25.0", + "statsigEnvironment": { "tier": "production" }, + }, + }, + "user": { + "country": "CA", + "custom": { + "has_logged_in_before": false, + "user_agent": "Wget/1.25.0", + "is_punch_out_user": false, + "auth_status": "logged_out", + }, + "customIDs": { + "WebAnonymousCookieID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "DeviceId": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "stableID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + }, + "ip": "2001:df0:a280:1001::6:1", + "privateAttributes": {}, + "userAgent": "Wget/1.25.0", + }, + }, + "accountsStatus": null, + }, + "isIos": false, + "isAndroidChrome": false, + "isElectron": false, + "cspScriptNonce": "52bc4634-5139-448c-b73b-89e4eef4691c", + }, + "routes/share.$shareId.($action)": { + "sharedConversationId": + "6758f810-286c-800b-8e2c-490e634ca002", + "serverResponse": { + "type": "data", + "data": { + "title": "Git as Supply Chain", + "create_time": 1733883920.159243, + "update_time": 1733905184.927294, + "mapping": { + "aaa1615c-1f40-439e-8b5d-a779ee9a1192": { + "id": "aaa1615c-1f40-439e-8b5d-a779ee9a1192", + "children": [ + "2ae52d22-dba5-457d-8147-faade76900dd", + ], + }, + "2ae52d22-dba5-457d-8147-faade76900dd": { + "id": "2ae52d22-dba5-457d-8147-faade76900dd", + "message": { + "id": "2ae52d22-dba5-457d-8147-faade76900dd", + "author": { "role": "system", "metadata": {} }, + "content": { + "content_type": "text", + "parts": [""], + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 0, + "metadata": { + "is_visually_hidden_from_conversation": true, + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaa1615c-1f40-439e-8b5d-a779ee9a1192", + "children": [ + "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + ], + }, + "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074": { + "id": "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "message": { + "id": "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "author": { "role": "user", "metadata": {} }, + "create_time": 1733880310.052759, + "content": { + "content_type": "text", + "parts": [ + "here's a chat with my friend. what concepts might we be trying to hint at ? we need clarity of thought since we are poking around at things we don't fully understand. Give us a few options. Clarify our thinking. Be creative. Be bold. Be profound.\n\n11/12/24, 11:51 - Ben Hoffman: Hey feel like the mood was a bit off when we talked. Wanted to say one more thing I think is really neat about git being the fabric in which the system natively thinks. There’s an unexplored interface between the system and the software supply chain. The system can “think” files into places that make its execution space consumable as an npm module hosted on GitHub for example \u003cThis message was edited\u003e\n11/12/24, 11:52 - Ben Hoffman: I’m sure that’s something you considered already, but I didn’t quite visualize it properly in my head until after that call\n11/12/24, 11:55 - Ben Hoffman: It’s possibly a weird way to couple into current systems, but it feels more direct than wrapping them all in APIs and then consuming those APIs from a chatbot adapter\n11/12/24, 12:45 - Tom Thompson 🧴: so you're talking about the benefit of publishing packages as units of work, rather than these live and mutable systems ?\n11/12/24, 12:45 - Tom Thompson 🧴: as well as having the bots be provenance aware, where they know how the software got there, and how to iterate on the history to produce the next version ?\n11/12/24, 12:46 - Tom Thompson 🧴: like a kind of next level introspection\n11/12/24, 12:47 - Tom Thompson 🧴: but I think the real thing you're getting at is if it stays in git, then it doesn't break that whole impure compilation problem\n11/12/24, 12:47 - Ben Hoffman: I guess what I mean is, if one of the bots decides to set up a remote on GitHub of its own “mind”, it can “think” code directly into software supply chains by just cherry-picking commits onto that remote that adhere to package format for whatever ecosystem it’s trying to supply into\n11/12/24, 12:49 - Tom Thompson 🧴: ok that's cool, but what is the benefit you're seeing there ?\n11/12/24, 12:49 - Ben Hoffman: Yeah I mean I think that’s more or less it, you don’t have to have a separate build step to make the outcome compatible with a foreign software system\n11/12/24, 12:49 - Tom Thompson 🧴: you effectively killed the build step completely\n11/12/24, 12:49 - Tom Thompson 🧴: there is no build, there is only commit\n11/12/24, 12:50 - Ben Hoffman: I’m not sure I’m thinking in terms of benefits yet, just an interface I never really considered before\n11/12/24, 12:50 - Tom Thompson 🧴: publishing to npm and jsr are just transmissions of something that could have stayed pure, if it wasn't for the need for backwards compatibility\n11/12/24, 12:50 - Ben Hoffman: Like it might be a lot easier to make this system plug in fairly directly into the software of the world without adapters and wrappers\n11/12/24, 12:52 - Tom Thompson 🧴: if the context of the LLM was git, then the context can be shared more readily, is what you're saying ?\n11/12/24, 12:52 - Ben Hoffman: Yeah\n11/12/24, 12:52 - Tom Thompson 🧴: as an anti example, if the context was all stored on Ethereum, then sharing anywhere is a copy paste rather than sharing a git commit, which is pretty universal, you mean ?\n11/12/24, 12:53 - Ben Hoffman: Yeah I think so\n11/12/24, 12:53 - Ben Hoffman: Let me sit with it a bit more and come up with a clearer expression of what I’m trying to communicate here\n11/12/24, 12:54 - Ben Hoffman: If the agent presented itself in such a way on GitHub, I guess ultimately what would need to be figured out is what does it think of commits coming in from other humans and agents onto those files\n11/12/24, 12:55 - Ben Hoffman: I’m using the word “think” a lot in an ambiguous way here\n11/12/24, 12:55 - Ben Hoffman: That’s what I need to fix\n11/12/24, 12:56 - Tom Thompson 🧴: so to conclude, there is value in having git be the blockchain, since it completely removes a transformation step, which adds impedance to flows, and hinders integration with the rest of the knowledge based economy\n11/12/24, 12:56 - Ben Hoffman: Yeah that’s part of it for sure\n11/12/24, 12:56 - Ben Hoffman: Most of it probably\n11/12/24, 12:56 - Tom Thompson 🧴: that means that, it is frictionless, or native, to transmit thoughts out from the model, and to receive thoughts in, using the most common format for knowledge transmission, which is git, with no translation whatsoever\n11/12/24, 12:57 - Tom Thompson 🧴: so its sort of like a git repo got up and started talking\n11/12/24, 12:57 - Tom Thompson 🧴: in a fanciful sort of metaphor\n11/12/24, 12:58 - Tom Thompson 🧴: so what I'm hearing then, is there is still value to pursue our own blockchain, purely for the sake of the compatibility a pure git model gives everyone\n11/12/24, 12:59 - Ben Hoffman: Yes this\n11/12/24, 12:59 - Ben Hoffman: Well look the world is not going to want to model time and synchrony less in the future than it does now\n11/12/24, 13:00 - Ben Hoffman: Since that’s probably the biggest source of WTF in big systems\n11/12/24, 13:00 - Ben Hoffman: There’s a reason the whole software industry jumped on git within a few years, make and sftp are kinda shit by comparison\n11/12/24, 13:05 - Ben Hoffman: Also I maintain that the most interesting links between git/blockchain/ai will not emerge until training is fully on chain and there’s a taxonomy of models cooked right into the source tree\n11/12/24, 13:06 - Ben Hoffman: That and AIs just simply won’t obviate the need for “basic plumbing” style software, so better to make them more aware of how the plumbing is maintained than less\n11/12/24, 13:07 - Ben Hoffman: I’m still not convinced you need to carry a blockchain around with you to build v1 of what you’re thinking, but I am still completely convinced of the long term utility of having one in mind that fits the outcome you’re aiming at\n11/12/24, 13:09 - Ben Hoffman: They just feel qualitatively super complementary to me. AIs obfuscate context, blockchains freeze context and keep it retrievable forever. You can trust an AI a lot more that you can see what it gobbled into its latent space \u003cThis message was edited\u003e\n11/12/24, 13:32 - Tom Thompson 🧴: so training being git native as a series of commits, and then calls being commits too, against a long term stable commit which is the training epoc ?\n\nSo that would mean that a model being trained like this could be consumed early, while it was still being trained, to encourage more resources to be put in it, whether you sponsor it with money, or with compute resources.\n11/12/24, 13:33 - Tom Thompson 🧴: so getting training on chain is something we should totally be going for, you're saying ?\n11/12/24, 13:33 - Tom Thompson 🧴: and then using ambient attribution to pay back to people who did training work\n11/12/24, 13:35 - Tom Thompson 🧴: yes, and further they should be the plumbing themselves, with actual software only used in cases where scale, speed, cost are an issue\n11/12/24, 13:38 - Tom Thompson 🧴: I think this is the key concept\n11/12/24, 13:39 - Tom Thompson 🧴: I think the only reason not to carry one with us is the build cost. After the last few days, I feel like the cost just fell thru the floor with o1-pro\n11/12/24, 13:39 - Ben Hoffman: Yes, training data being committed into a payload layer or git lfs for hash provenance followed by commit triggered compute that results in a new model being checked in. Training is deterministic I think, so it’s a natural fit for on-chain, and then you get a free taxonomy of models baked into the source tree structure by just cherry picking training data cross branch for new models\n11/12/24, 13:40 - Ben Hoffman: Very easy to create “cousin” models and have good training governance in such a system\n11/12/24, 13:40 - Tom Thompson 🧴: training and execution can be deterministic if configured to be so. I was hoping to use this as a proof of work system to fortify the network consensus\n11/12/24, 13:40 - Ben Hoffman: Hell yes\n11/12/24, 13:41 - Tom Thompson 🧴: rather than having consensus dorkily separate, and worse having its own economics like a token issuance that is all tangled up in there\n11/12/24, 13:41 - Ben Hoffman: It will happen eventually when people realize hugging face and GitHub could be the same thing\n11/12/24, 13:42 - Ben Hoffman: This is a god tier idea. Proof of actually useful work\n11/12/24, 13:42 - Tom Thompson 🧴: or rather, should be the same thing\n11/12/24, 13:42 - Ben Hoffman: Yes\n11/12/24, 13:42 - Tom Thompson 🧴: yeah well that's what I've been trying to fucking do this whole time 😅\n11/12/24, 13:42 - Tom Thompson 🧴: but baking ambient attribution in there too\n11/12/24, 13:42 - Ben Hoffman: Yes I think you’re seeing it\n11/12/24, 13:43 - Ben Hoffman: What I was seeing I mean\n11/12/24, 13:43 - Tom Thompson 🧴: so its part of the actual next tick of the global system, is attribution\n11/12/24, 13:43 - Ben Hoffman: No better way to make them “be the plumbing” than making them be the source repos\n11/12/24, 13:44 - Tom Thompson 🧴: so enabling the llms to be the plumbing workforce, by connecting up to git, npm, etc ?\n11/12/24, 13:44 - Ben Hoffman: Yeah build cost for sure\n11/12/24, 13:44 - Tom Thompson 🧴: so if a Napp is also the repo it lives in, then it is sort of a repo that is alive, and can commit to itself, as well as manage other repos\n11/12/24, 13:44 - Ben Hoffman: Yeah\n11/12/24, 13:46 - Ben Hoffman: And, crucially, other peoples’ commits can become part of its domain of analysis, which should eventually teach AIs to “understand” human commit streams, which are a type of political information\n11/12/24, 13:46 - Tom Thompson 🧴: ok\n11/12/24, 13:46 - Tom Thompson 🧴: I was thinking after our call too, and whilst we are seeing point solutions pop up for pieces of the dreamcatcher, there is still no unified solution, which I think is essential. Furthermore, the unified solution should have zero friction back into the established knowledge net, so I'm thinking there is a benefit to keep going with our own pure blockchain piece.\n\nYesterday I used o1 pro to produce a software module entirely from prompting. https://jsr.io/@dreamcatcher/concat\n\nThis gives me hope that actually, if we do things right, building artifact might not be nearly as hard as previously thought, since I think I can prompt large chunks of it. It's almost as tho not being able to prompt it is a fatal sign ? like code should be able to be generated from prompting a smart model, or else the design is bad.\n11/12/24, 13:47 - Ben Hoffman: I think learning how to understand commits and eventually incorporate them into the training data will give AIs a better shot at understanding consequence, time, politics, group dynamics\n11/12/24, 13:48 - Ben Hoffman: And if the AI processes via commit, then understanding commits gives it a type of introspection\n11/12/24, 13:50 - Tom Thompson 🧴: they could even begin to understand themselves, as they see their effect on that plane\n11/12/24, 13:51 - Ben Hoffman: That’s pretty exciting\n11/12/24, 13:51 - Ben Hoffman: Yes!\n11/12/24, 13:51 - Ben Hoffman: And yes, agree to “not being able to prompt is a fatal sign”\n11/12/24, 13:51 - Ben Hoffman: I really like that\n11/12/24, 13:52 - Ben Hoffman: Smart right from the outset, no bits that can’t be queried in natural language\n11/12/24, 13:52 - Ben Hoffman: It’s a type of transparency guarantee\n11/12/24, 13:52 - Ben Hoffman: Hopefully the ai doesn’t lie about stuff tho lol\n11/12/24, 13:53 - Tom Thompson 🧴: lol at least if it does, it will do so repeatably. \n\nI think this is the stongest case for on chain training - its not open ai unless the training process is repeatable too\n11/12/24, 13:54 - Tom Thompson 🧴: ok well I think you've given me the confidence to continue building artifact\n11/12/24, 13:54 - Tom Thompson 🧴: its VERY SUBTLE, what you say\n11/12/24, 13:54 - Tom Thompson 🧴: but it is a genuine\n11/12/24, 13:54 - Tom Thompson 🧴: advantage\n11/12/24, 13:55 - Tom Thompson 🧴: so the punchline is, an execution framework that is git native\n11/12/24, 13:55 - Tom Thompson 🧴: where the LLMs don't just do the commits, they ARE the commits\n11/12/24, 13:55 - Ben Hoffman: Yeah I think so. Just knowing how important git is, and how it’s really the structure that we all politically align on\n11/12/24, 13:56 - Ben Hoffman: Yes!\n11/12/24, 13:56 - Tom Thompson 🧴: yeah its so common its forgotten about hey\n11/12/24, 13:56 - Tom Thompson 🧴: its like air\n11/12/24, 13:56 - Ben Hoffman: Yeah\n11/12/24, 13:56 - Ben Hoffman: That’s a good way of thinking about it\n11/12/24, 13:56 - Tom Thompson 🧴: ok thank you 🙏\n11/12/24, 13:57 - Tom Thompson 🧴: such rough times these days - feels like sound barrier type of problems all the time\n11/12/24, 13:57 - Ben Hoffman: Yeah I think it’s the singularity\n11/12/24, 13:57 - Ben Hoffman: I guess I never thought of the singularity like a sound barrier\n11/12/24, 13:57 - Ben Hoffman: Have the right design or get crushed by the energy\n11/12/24, 13:58 - Ben Hoffman: But it seems obvious once you frame it that way\n11/12/24, 13:59 - Tom Thompson 🧴: lolololo\n11/12/24, 13:59 - Tom Thompson 🧴: that is actually 100% how I feel every day\n11/12/24, 13:59 - Tom Thompson 🧴: oscillating rapidly between crushing energy waves and serenity\n11/12/24, 13:59 - Ben Hoffman: Also I still get too attached to things so I hope I’m not pushing you in the wrong direction\n11/12/24, 13:59 - Tom Thompson 🧴: lol\n11/12/24, 13:59 - Tom Thompson 🧴: no I don't think you are\n11/12/24, 13:59 - Tom Thompson 🧴: the difference is clear\n11/12/24, 13:59 - Ben Hoffman: Ain’t it the truth\n11/12/24, 14:00 - Tom Thompson 🧴: and then when you remind me about the utility of genuine proof of computational work\n11/12/24, 14:00 - Tom Thompson 🧴: all the other blockchains have some dorky consensus with a token and they're always wedlocked to it\n11/12/24, 14:00 - Tom Thompson 🧴: like it is their whole identity,\n11/12/24, 14:00 - Tom Thompson 🧴: and they can never be upgraded to change that core thing\n11/12/24, 14:00 - Ben Hoffman: And they all create extremely toxic cultures because of it\n11/12/24, 14:01 - Tom Thompson 🧴: yeah its a totally unnecessary thing, too - like it is entirely fictitious - physics does not demand it in the least\n11/12/24, 14:01 - Ben Hoffman: Lots of people stressed by the big quantum breakthrough Google just posted about because the crypto in btc and eth are both not quantum proof lol\n11/12/24, 14:01 - Tom Thompson 🧴: I don't think they have currencies in heaven, for example\n11/12/24, 14:01 - Ben Hoffman: I will literally kill myself if I get there and that’s what it is\n11/12/24, 14:02 - Tom Thompson 🧴: well with artifact, giving it quantum computation services via an api is easy, might be a fun thing to tack on lol\n11/12/24, 14:03 - Ben Hoffman: Yeah it should be way less punishing to change the crypto when you don’t have an army of shitcoin ruggers trying to stop you\n11/12/24, 14:03 - Ben Hoffman: Totally", + ], + }, + "status": "finished_successfully", + "weight": 1, + "metadata": { + "request_id": "8f01adde8942d9be-AKL", + "message_source": null, + "timestamp_": "absolute", + "message_type": null, + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "2ae52d22-dba5-457d-8147-faade76900dd", + "children": [ + "e9410778-39b9-464a-a0f5-cc461cb28e85", + ], + }, + "e9410778-39b9-464a-a0f5-cc461cb28e85": { + "id": "e9410778-39b9-464a-a0f5-cc461cb28e85", + "message": { + "id": "e9410778-39b9-464a-a0f5-cc461cb28e85", + "author": { + "role": "tool", + "name": "a8km123", + "metadata": {}, + }, + "create_time": 1733880397.691115, + "content": { + "content_type": "text", + "parts": [ + "**Exploring AI integrations**\n\nI’m examining how Git, npm modules, software supply chain, AI, and blockchains intersect, focusing on knowledge, reproducible model training, and traceability in AI development.", + ], + }, + "status": "finished_successfully", + "weight": 0, + "metadata": { + "finished_text": "Thought for 1m 26s", + "initial_text": "Thinking", + "summarization_headline": + "Git, Blockchain, and AI Integration", + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "request_id": "8f01adde8942d9be-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "children": [ + "46cc8de2-2860-4c13-9083-32117c314a3a", + ], + }, + "46cc8de2-2860-4c13-9083-32117c314a3a": { + "id": "46cc8de2-2860-4c13-9083-32117c314a3a", + "message": { + "id": "46cc8de2-2860-4c13-9083-32117c314a3a", + "author": { + "role": "assistant", + "metadata": {}, + }, + "create_time": 1733880397.691219, + "content": { + "content_type": "text", + "parts": [ + 'Below are a few conceptual frameworks and themes that might help clarify the directions you’re exploring. These are just options—bold thought experiments and profound re-interpretations of what you’ve discussed. Each tries to distill the underlying "essence" of what you\'re circling around:\n\n1. **Git as a Cognitive Substrate** \n *Concept*: Treat Git not just as a version-control system, but as a universal medium for knowledge representation—a canonical substrate into which intelligence can “think” its outputs directly. \n *Idea*: Instead of thinking of Git as infrastructure hiding behind the scenes, imagine it as the actual “mindspace” of an AI. Commits become “thoughts,” branches become “possible worlds,” and merges become negotiated alignments of reality. This would mean that interacting with systems wouldn’t require separately building APIs or wrapping logic in adapters; the “thought process” would already be in the universal language of code and data tracked by Git.\n\n2. **Provenance-Aware Execution and Training** \n *Concept*: Embrace full provenance tracking of models—every piece of training data, every parameter update, every intermediate artifact—being anchored in a Git-like or blockchain-like structure. \n *Idea*: On-chain (or on-git) training ensures that models, their lineage, and their transformations are fully transparent and repeatable. This is “Proof of Useful Work” done right: no contrived token games, just a history of refinement. Imagine a Hugging Face that merges seamlessly with GitHub: training datasets as commits, parameter updates as commits, final models as tagged releases. The model’s “mind” isn’t some opaque black box, but an ever-growing, fully versioned garden of knowledge and code.\n\n3. **Frictionless Integration With the Wider Software Supply Chain** \n *Concept*: The AI doesn’t produce code that must then be adapted or compiled into external ecosystems; it directly “commits” code into existing supply chains (e.g., npm modules) as a natural extension of its reasoning. \n *Idea*: By removing the “build step” or “adapter step” entirely, the AI’s output is not something that must be transformed—its output is directly usable. Code changes flow seamlessly into the world’s largest ecosystems of shared code, enabling the AI not only to contribute but also to continuously learn from incoming commits by humans and other agents.\n\n4. **Time, Context, and Political Awareness Through Commit Histories** \n *Concept*: Commit histories are fundamentally stories about human and machine collaboration, negotiation, conflict, and resolution. They are a record of political dynamics, group decisions, priorities shifting over time. \n *Idea*: By letting the AI “live” in this world of commits, it can learn to interpret the subtle signals encoded in version histories—understanding governance, consequence, and social context. Training on commit histories gives AI a vantage point to reason about time, intention, and human organizational structures. In other words, by modeling itself as a participant in Git’s political tapestry, the AI begins to understand what it means to cooperate, to iterate, to govern.\n\n5. **Merging Knowledge Networks: From Fragmented Tools to a Single Universal Medium** \n *Concept*: Git, package managers, blockchains, AI models—these have historically been disparate systems, each with its own protocols and friction points. \n *Idea*: Your conversation hints at merging these concepts into a unified environment: a single substrate (like Git) where data, code, models, and their entire genealogy live side by side. Instead of separate infrastructures for AI training, code versioning, supply chain management, and trust enforcement, you get one substrate that can host them all. This reduces conceptual overhead and unleashes a new kind of “global system,” where knowledge flows freely and every increment of intelligence is recorded and trusted.\n\n---\n\nAll of these concepts circle around the core theme: using version-controlled code repositories, possibly combined with blockchain-style invariants, to eliminate friction, ensure trust, enable provenance, and integrate AI deeply into the world’s software and knowledge ecosystems—no artificial boundaries, no hidden build steps, no murky provenances. Instead, a continuous, transparent, and universal environment of incremental, queryable, and trustworthy “thought commits.”', + ], + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 1, + "metadata": { + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "e9410778-39b9-464a-a0f5-cc461cb28e85", + "request_id": "8f01adde8942d9be-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "e9410778-39b9-464a-a0f5-cc461cb28e85", + "children": [ + "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + ], + }, + "aaa2979c-fd1f-4f49-bfd7-051c15a258b3": { + "id": "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "message": { + "id": "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "author": { "role": "user", "metadata": {} }, + "create_time": 1733881250.145666, + "content": { + "content_type": "text", + "parts": [ + "if that conversation was being turned into a set of comparisons to make against competing blockchains, name what concepts are missing from the below comparison, and say how you might improve this comparator:\n\n1. Token dependency:\n \n - Does the chain have a built-in token?\n - How tightly coupled is that token to the consensus?\n - How easy is it to remove, replace, or adjust that token?\n - Is the consensus mechanism itself token-agnostic or inherently token-bound?\n - is the token stable or fluctuatable\n\n1. Integration format:\n \n - Can the chain’s data and consensus be naturally represented in Git commits or NPM packages?\n - Are translations or bridging layers required to interface with standard tools, and if so, do these weaken the consensus?\n3. Portability:\n \n - Can the system run in isolated parts, completely on your own infrastructure?\n - Can these isolated instances later merge consensus without loss of trust?\n\nFor the demonstration of a sharded or “splintered” proof-of-work system, consider calling it “Shattered Proof-of-Work” (S-PoW). This name reflects how the workload can be broken into isolated segments and later merged based on verifiable computational work.", + ], + }, + "status": "finished_successfully", + "weight": 1, + "metadata": { + "serialization_metadata": { + "custom_symbol_offsets": [], + }, + "request_id": "8f01c4d23b851c5e-AKL", + "message_source": null, + "timestamp_": "absolute", + "message_type": null, + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "46cc8de2-2860-4c13-9083-32117c314a3a", + "children": [ + "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + ], + }, + "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6": { + "id": "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "message": { + "id": "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "author": { + "role": "tool", + "name": "a8km123", + "metadata": {}, + }, + "create_time": 1733881301.561146, + "content": { + "content_type": "text", + "parts": [ + '**Gathering blockchain insights**\n\nI\'m compiling info on blockchain developments, pinpointing gaps like "Community involvement" and "Transparency and trust." Also, looking at concepts such as "Provenance and lineage" and "Natural language promptability," considering how they can enrich the comparison.', + ], + }, + "status": "finished_successfully", + "weight": 0, + "metadata": { + "finished_text": "Thought for 50 seconds", + "initial_text": "Thinking", + "summarization_headline": + "Blockchain Comparison Concepts", + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "request_id": "8f01c4d23b851c5e-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "children": [ + "c132e044-dade-4946-95a7-b1c942539af4", + ], + }, + "c132e044-dade-4946-95a7-b1c942539af4": { + "id": "c132e044-dade-4946-95a7-b1c942539af4", + "message": { + "id": "c132e044-dade-4946-95a7-b1c942539af4", + "author": { + "role": "assistant", + "metadata": {}, + }, + "create_time": 1733881301.56126, + "content": { + "content_type": "text", + "parts": [ + "**Missing Concepts and How to Improve the Comparison:**\n\n1. **Provenance and Lineage of Data/Models:** \n **What’s missing:** \n - How does the chain handle the version history of data, code, or model parameters? \n - Can it track the full lineage of a model’s training process or a software artifact’s evolution? \n **Improvement:** \n Add criteria examining whether each system can embed transparent, verifiable version histories—like Git commit graphs—directly in its consensus. For example: \n - *“Can the chain immutably record the origin and version history of data and code as a first-class citizen?”* \n - *“Does it allow for ‘on-chain training’ records or a taxonomy of artifacts that mirror Git’s branching, merging, and tagging semantics?”*\n\n2. **Deterministic Computation and Reproducibility:** \n **What’s missing:** \n - Are computations (e.g., model training, code transformations) guaranteed to be deterministic and reproducible from on-chain data? \n - Can the entire state (including the model’s parameters) be reconstructed from on-chain commits? \n **Improvement:** \n Add criteria around determinism and replayability: \n - *“Is the chain’s consensus logic and stored data sufficient to deterministically reproduce any result, such as model parameters after training?”* \n - *“Does the system enable ‘proof-of-useful-work’ where the work is fully re-executable and verifiable from the recorded state?”*\n\n3. **Interoperability With Existing Ecosystems (Beyond Git \u0026 NPM):** \n **What’s missing:** \n - Beyond just Git commits and npm packages, how well does the system integrate with other major ecosystems (e.g., Docker images, PyPI packages, Hugging Face model hubs)? \n **Improvement:** \n Add criteria that measure the breadth of ecosystem compatibility: \n - *“Can the chain’s artifacts be directly consumed by standard developer tools, CI/CD pipelines, model registries, or container registries without translation?”* \n - *“Is the system extensible to support new tooling integrations as standards evolve?”*\n\n4. **Governance and Social Dynamics:** \n **What’s missing:** \n - How does the chain model governance over code and data? \n - Can it represent the political dimension of commits—who proposed changes, who merged them, and the rationale behind decisions? \n **Improvement:** \n Add criteria related to governance and social aspects: \n - *“Does the chain encode and preserve metadata of contributors, allowing AIs and humans to understand political, social, and organizational contexts of commits?”* \n - *“Can we trace not just the code but also the discussions, proposals, and decision-making processes that led to a given state?”*\n\n5. **Economic Neutrality and Flexible Incentive Structures:** \n **What’s missing:** \n - If the chain employs incentives, can those be non-financial or proof-of-actual-work rather than bound to a volatile token economy? \n **Improvement:** \n Add criteria around incentive flexibility and neutrality: \n - *“Are consensus and network security achievable without relying on a native token’s economic incentives?”* \n - *“Is it possible to introduce alternate reward mechanisms, like reputation, stable credits, or proof-of-completion of useful tasks?”*\n\n6. **Upgradability and Evolution Over Time:** \n **What’s missing:** \n - How easy is it to evolve the protocol over time, add new features, or adapt to new technological standards without causing ecosystem turbulence? \n **Improvement:** \n Include questions about extensibility and protocol migration: \n - *“Can the system’s core protocols be upgraded easily and cleanly, preserving full historical integrity?”* \n - *“Does it support branching/forking processes similar to Git, allowing for experimental consensus changes that can later be merged back?”*\n\n7. **Data Availability and Scalability Constraints:** \n **What’s missing:** \n - How does the system ensure that the entire state is always available and scalable as the code/data base grows large? \n **Improvement:** \n Add criteria regarding scaling and data availability: \n - *“Can the chain scale with increasing amounts of data and computations without degrading verification or trust?”* \n - *“Does it support ‘shattering’ (sharding) into separate chains or segments that can be recombined, ensuring data availability and trust remain intact?”*\n\n---\n\n**Improved Comparator Example:**\n\n- **Token Dependency \u0026 Incentive Neutrality:** \n - Does consensus rely on an intrinsic token or can it operate without one? \n - Can incentives be reconfigured (e.g., non-monetary proofs of useful work)?\n\n- **Integration Format \u0026 Ecosystem Compatibility:** \n - Beyond Git and NPM, can the chain represent and serve artifacts to other ecosystems (Hugging Face models, Docker, PyPI)? \n - Are commits, model parameters, and artifact states directly usable by standard devops and AI tooling?\n\n- **Provenance, Governance \u0026 Social Dynamics:** \n - Does it record full artifact lineage, enabling complete reproducibility? \n - Can the platform encode and understand governance (commit authors, merging policies) and social context for changes?\n\n- **Deterministic \u0026 Reproducible Computation:** \n - Are computations fully deterministic, so any artifact (including AI models) can be re-derived from on-chain data? \n - Are training processes and model outputs repeatably verifiable as a form of proof-of-useful-work?\n\n- **Upgradability \u0026 Evolution:** \n - Can the system’s core mechanics be upgraded without loss of trust or data fidelity? \n - Does it support branching and merging of different consensus versions, akin to code versioning in Git?\n\n- **Portability, Data Availability \u0026 Scalability:** \n - Can it run in isolated modes and then later reconcile states without losing trust? \n - Does “Shattered Proof-of-Work” support scaling by splitting workloads and merging them without compromising determinism or provenance?\n\nBy including these additional angles—provenance, governance, determinism, ecosystem compatibility, incentive neutrality, and upgradability—you create a more comprehensive set of criteria for evaluating next-generation chains against conventional blockchains.", + ], + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 1, + "metadata": { + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "request_id": "8f01c4d23b851c5e-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "children": [], + }, + }, + "moderation_results": [], + "current_node": + "c132e044-dade-4946-95a7-b1c942539af4", + "conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + "is_archived": false, + "safe_urls": ["https://jsr.io/@dreamcatcher/concat"], + "default_model_slug": "o1-pro", + "disabled_tool_ids": [], + "is_public": true, + "linear_conversation": [{ + "id": "aaa1615c-1f40-439e-8b5d-a779ee9a1192", + "children": [ + "2ae52d22-dba5-457d-8147-faade76900dd", + ], + }, { + "id": "2ae52d22-dba5-457d-8147-faade76900dd", + "message": { + "id": "2ae52d22-dba5-457d-8147-faade76900dd", + "author": { "role": "system", "metadata": {} }, + "content": { + "content_type": "text", + "parts": [""], + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 0, + "metadata": { + "is_visually_hidden_from_conversation": true, + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaa1615c-1f40-439e-8b5d-a779ee9a1192", + "children": [ + "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + ], + }, { + "id": "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "message": { + "id": "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "author": { "role": "user", "metadata": {} }, + "create_time": 1733880310.052759, + "content": { + "content_type": "text", + "parts": [ + "here's a chat with my friend. what concepts might we be trying to hint at ? we need clarity of thought since we are poking around at things we don't fully understand. Give us a few options. Clarify our thinking. Be creative. Be bold. Be profound.\n\n11/12/24, 11:51 - Ben Hoffman: Hey feel like the mood was a bit off when we talked. Wanted to say one more thing I think is really neat about git being the fabric in which the system natively thinks. There’s an unexplored interface between the system and the software supply chain. The system can “think” files into places that make its execution space consumable as an npm module hosted on GitHub for example \u003cThis message was edited\u003e\n11/12/24, 11:52 - Ben Hoffman: I’m sure that’s something you considered already, but I didn’t quite visualize it properly in my head until after that call\n11/12/24, 11:55 - Ben Hoffman: It’s possibly a weird way to couple into current systems, but it feels more direct than wrapping them all in APIs and then consuming those APIs from a chatbot adapter\n11/12/24, 12:45 - Tom Thompson 🧴: so you're talking about the benefit of publishing packages as units of work, rather than these live and mutable systems ?\n11/12/24, 12:45 - Tom Thompson 🧴: as well as having the bots be provenance aware, where they know how the software got there, and how to iterate on the history to produce the next version ?\n11/12/24, 12:46 - Tom Thompson 🧴: like a kind of next level introspection\n11/12/24, 12:47 - Tom Thompson 🧴: but I think the real thing you're getting at is if it stays in git, then it doesn't break that whole impure compilation problem\n11/12/24, 12:47 - Ben Hoffman: I guess what I mean is, if one of the bots decides to set up a remote on GitHub of its own “mind”, it can “think” code directly into software supply chains by just cherry-picking commits onto that remote that adhere to package format for whatever ecosystem it’s trying to supply into\n11/12/24, 12:49 - Tom Thompson 🧴: ok that's cool, but what is the benefit you're seeing there ?\n11/12/24, 12:49 - Ben Hoffman: Yeah I mean I think that’s more or less it, you don’t have to have a separate build step to make the outcome compatible with a foreign software system\n11/12/24, 12:49 - Tom Thompson 🧴: you effectively killed the build step completely\n11/12/24, 12:49 - Tom Thompson 🧴: there is no build, there is only commit\n11/12/24, 12:50 - Ben Hoffman: I’m not sure I’m thinking in terms of benefits yet, just an interface I never really considered before\n11/12/24, 12:50 - Tom Thompson 🧴: publishing to npm and jsr are just transmissions of something that could have stayed pure, if it wasn't for the need for backwards compatibility\n11/12/24, 12:50 - Ben Hoffman: Like it might be a lot easier to make this system plug in fairly directly into the software of the world without adapters and wrappers\n11/12/24, 12:52 - Tom Thompson 🧴: if the context of the LLM was git, then the context can be shared more readily, is what you're saying ?\n11/12/24, 12:52 - Ben Hoffman: Yeah\n11/12/24, 12:52 - Tom Thompson 🧴: as an anti example, if the context was all stored on Ethereum, then sharing anywhere is a copy paste rather than sharing a git commit, which is pretty universal, you mean ?\n11/12/24, 12:53 - Ben Hoffman: Yeah I think so\n11/12/24, 12:53 - Ben Hoffman: Let me sit with it a bit more and come up with a clearer expression of what I’m trying to communicate here\n11/12/24, 12:54 - Ben Hoffman: If the agent presented itself in such a way on GitHub, I guess ultimately what would need to be figured out is what does it think of commits coming in from other humans and agents onto those files\n11/12/24, 12:55 - Ben Hoffman: I’m using the word “think” a lot in an ambiguous way here\n11/12/24, 12:55 - Ben Hoffman: That’s what I need to fix\n11/12/24, 12:56 - Tom Thompson 🧴: so to conclude, there is value in having git be the blockchain, since it completely removes a transformation step, which adds impedance to flows, and hinders integration with the rest of the knowledge based economy\n11/12/24, 12:56 - Ben Hoffman: Yeah that’s part of it for sure\n11/12/24, 12:56 - Ben Hoffman: Most of it probably\n11/12/24, 12:56 - Tom Thompson 🧴: that means that, it is frictionless, or native, to transmit thoughts out from the model, and to receive thoughts in, using the most common format for knowledge transmission, which is git, with no translation whatsoever\n11/12/24, 12:57 - Tom Thompson 🧴: so its sort of like a git repo got up and started talking\n11/12/24, 12:57 - Tom Thompson 🧴: in a fanciful sort of metaphor\n11/12/24, 12:58 - Tom Thompson 🧴: so what I'm hearing then, is there is still value to pursue our own blockchain, purely for the sake of the compatibility a pure git model gives everyone\n11/12/24, 12:59 - Ben Hoffman: Yes this\n11/12/24, 12:59 - Ben Hoffman: Well look the world is not going to want to model time and synchrony less in the future than it does now\n11/12/24, 13:00 - Ben Hoffman: Since that’s probably the biggest source of WTF in big systems\n11/12/24, 13:00 - Ben Hoffman: There’s a reason the whole software industry jumped on git within a few years, make and sftp are kinda shit by comparison\n11/12/24, 13:05 - Ben Hoffman: Also I maintain that the most interesting links between git/blockchain/ai will not emerge until training is fully on chain and there’s a taxonomy of models cooked right into the source tree\n11/12/24, 13:06 - Ben Hoffman: That and AIs just simply won’t obviate the need for “basic plumbing” style software, so better to make them more aware of how the plumbing is maintained than less\n11/12/24, 13:07 - Ben Hoffman: I’m still not convinced you need to carry a blockchain around with you to build v1 of what you’re thinking, but I am still completely convinced of the long term utility of having one in mind that fits the outcome you’re aiming at\n11/12/24, 13:09 - Ben Hoffman: They just feel qualitatively super complementary to me. AIs obfuscate context, blockchains freeze context and keep it retrievable forever. You can trust an AI a lot more that you can see what it gobbled into its latent space \u003cThis message was edited\u003e\n11/12/24, 13:32 - Tom Thompson 🧴: so training being git native as a series of commits, and then calls being commits too, against a long term stable commit which is the training epoc ?\n\nSo that would mean that a model being trained like this could be consumed early, while it was still being trained, to encourage more resources to be put in it, whether you sponsor it with money, or with compute resources.\n11/12/24, 13:33 - Tom Thompson 🧴: so getting training on chain is something we should totally be going for, you're saying ?\n11/12/24, 13:33 - Tom Thompson 🧴: and then using ambient attribution to pay back to people who did training work\n11/12/24, 13:35 - Tom Thompson 🧴: yes, and further they should be the plumbing themselves, with actual software only used in cases where scale, speed, cost are an issue\n11/12/24, 13:38 - Tom Thompson 🧴: I think this is the key concept\n11/12/24, 13:39 - Tom Thompson 🧴: I think the only reason not to carry one with us is the build cost. After the last few days, I feel like the cost just fell thru the floor with o1-pro\n11/12/24, 13:39 - Ben Hoffman: Yes, training data being committed into a payload layer or git lfs for hash provenance followed by commit triggered compute that results in a new model being checked in. Training is deterministic I think, so it’s a natural fit for on-chain, and then you get a free taxonomy of models baked into the source tree structure by just cherry picking training data cross branch for new models\n11/12/24, 13:40 - Ben Hoffman: Very easy to create “cousin” models and have good training governance in such a system\n11/12/24, 13:40 - Tom Thompson 🧴: training and execution can be deterministic if configured to be so. I was hoping to use this as a proof of work system to fortify the network consensus\n11/12/24, 13:40 - Ben Hoffman: Hell yes\n11/12/24, 13:41 - Tom Thompson 🧴: rather than having consensus dorkily separate, and worse having its own economics like a token issuance that is all tangled up in there\n11/12/24, 13:41 - Ben Hoffman: It will happen eventually when people realize hugging face and GitHub could be the same thing\n11/12/24, 13:42 - Ben Hoffman: This is a god tier idea. Proof of actually useful work\n11/12/24, 13:42 - Tom Thompson 🧴: or rather, should be the same thing\n11/12/24, 13:42 - Ben Hoffman: Yes\n11/12/24, 13:42 - Tom Thompson 🧴: yeah well that's what I've been trying to fucking do this whole time 😅\n11/12/24, 13:42 - Tom Thompson 🧴: but baking ambient attribution in there too\n11/12/24, 13:42 - Ben Hoffman: Yes I think you’re seeing it\n11/12/24, 13:43 - Ben Hoffman: What I was seeing I mean\n11/12/24, 13:43 - Tom Thompson 🧴: so its part of the actual next tick of the global system, is attribution\n11/12/24, 13:43 - Ben Hoffman: No better way to make them “be the plumbing” than making them be the source repos\n11/12/24, 13:44 - Tom Thompson 🧴: so enabling the llms to be the plumbing workforce, by connecting up to git, npm, etc ?\n11/12/24, 13:44 - Ben Hoffman: Yeah build cost for sure\n11/12/24, 13:44 - Tom Thompson 🧴: so if a Napp is also the repo it lives in, then it is sort of a repo that is alive, and can commit to itself, as well as manage other repos\n11/12/24, 13:44 - Ben Hoffman: Yeah\n11/12/24, 13:46 - Ben Hoffman: And, crucially, other peoples’ commits can become part of its domain of analysis, which should eventually teach AIs to “understand” human commit streams, which are a type of political information\n11/12/24, 13:46 - Tom Thompson 🧴: ok\n11/12/24, 13:46 - Tom Thompson 🧴: I was thinking after our call too, and whilst we are seeing point solutions pop up for pieces of the dreamcatcher, there is still no unified solution, which I think is essential. Furthermore, the unified solution should have zero friction back into the established knowledge net, so I'm thinking there is a benefit to keep going with our own pure blockchain piece.\n\nYesterday I used o1 pro to produce a software module entirely from prompting. https://jsr.io/@dreamcatcher/concat\n\nThis gives me hope that actually, if we do things right, building artifact might not be nearly as hard as previously thought, since I think I can prompt large chunks of it. It's almost as tho not being able to prompt it is a fatal sign ? like code should be able to be generated from prompting a smart model, or else the design is bad.\n11/12/24, 13:47 - Ben Hoffman: I think learning how to understand commits and eventually incorporate them into the training data will give AIs a better shot at understanding consequence, time, politics, group dynamics\n11/12/24, 13:48 - Ben Hoffman: And if the AI processes via commit, then understanding commits gives it a type of introspection\n11/12/24, 13:50 - Tom Thompson 🧴: they could even begin to understand themselves, as they see their effect on that plane\n11/12/24, 13:51 - Ben Hoffman: That’s pretty exciting\n11/12/24, 13:51 - Ben Hoffman: Yes!\n11/12/24, 13:51 - Ben Hoffman: And yes, agree to “not being able to prompt is a fatal sign”\n11/12/24, 13:51 - Ben Hoffman: I really like that\n11/12/24, 13:52 - Ben Hoffman: Smart right from the outset, no bits that can’t be queried in natural language\n11/12/24, 13:52 - Ben Hoffman: It’s a type of transparency guarantee\n11/12/24, 13:52 - Ben Hoffman: Hopefully the ai doesn’t lie about stuff tho lol\n11/12/24, 13:53 - Tom Thompson 🧴: lol at least if it does, it will do so repeatably. \n\nI think this is the stongest case for on chain training - its not open ai unless the training process is repeatable too\n11/12/24, 13:54 - Tom Thompson 🧴: ok well I think you've given me the confidence to continue building artifact\n11/12/24, 13:54 - Tom Thompson 🧴: its VERY SUBTLE, what you say\n11/12/24, 13:54 - Tom Thompson 🧴: but it is a genuine\n11/12/24, 13:54 - Tom Thompson 🧴: advantage\n11/12/24, 13:55 - Tom Thompson 🧴: so the punchline is, an execution framework that is git native\n11/12/24, 13:55 - Tom Thompson 🧴: where the LLMs don't just do the commits, they ARE the commits\n11/12/24, 13:55 - Ben Hoffman: Yeah I think so. Just knowing how important git is, and how it’s really the structure that we all politically align on\n11/12/24, 13:56 - Ben Hoffman: Yes!\n11/12/24, 13:56 - Tom Thompson 🧴: yeah its so common its forgotten about hey\n11/12/24, 13:56 - Tom Thompson 🧴: its like air\n11/12/24, 13:56 - Ben Hoffman: Yeah\n11/12/24, 13:56 - Ben Hoffman: That’s a good way of thinking about it\n11/12/24, 13:56 - Tom Thompson 🧴: ok thank you 🙏\n11/12/24, 13:57 - Tom Thompson 🧴: such rough times these days - feels like sound barrier type of problems all the time\n11/12/24, 13:57 - Ben Hoffman: Yeah I think it’s the singularity\n11/12/24, 13:57 - Ben Hoffman: I guess I never thought of the singularity like a sound barrier\n11/12/24, 13:57 - Ben Hoffman: Have the right design or get crushed by the energy\n11/12/24, 13:58 - Ben Hoffman: But it seems obvious once you frame it that way\n11/12/24, 13:59 - Tom Thompson 🧴: lolololo\n11/12/24, 13:59 - Tom Thompson 🧴: that is actually 100% how I feel every day\n11/12/24, 13:59 - Tom Thompson 🧴: oscillating rapidly between crushing energy waves and serenity\n11/12/24, 13:59 - Ben Hoffman: Also I still get too attached to things so I hope I’m not pushing you in the wrong direction\n11/12/24, 13:59 - Tom Thompson 🧴: lol\n11/12/24, 13:59 - Tom Thompson 🧴: no I don't think you are\n11/12/24, 13:59 - Tom Thompson 🧴: the difference is clear\n11/12/24, 13:59 - Ben Hoffman: Ain’t it the truth\n11/12/24, 14:00 - Tom Thompson 🧴: and then when you remind me about the utility of genuine proof of computational work\n11/12/24, 14:00 - Tom Thompson 🧴: all the other blockchains have some dorky consensus with a token and they're always wedlocked to it\n11/12/24, 14:00 - Tom Thompson 🧴: like it is their whole identity,\n11/12/24, 14:00 - Tom Thompson 🧴: and they can never be upgraded to change that core thing\n11/12/24, 14:00 - Ben Hoffman: And they all create extremely toxic cultures because of it\n11/12/24, 14:01 - Tom Thompson 🧴: yeah its a totally unnecessary thing, too - like it is entirely fictitious - physics does not demand it in the least\n11/12/24, 14:01 - Ben Hoffman: Lots of people stressed by the big quantum breakthrough Google just posted about because the crypto in btc and eth are both not quantum proof lol\n11/12/24, 14:01 - Tom Thompson 🧴: I don't think they have currencies in heaven, for example\n11/12/24, 14:01 - Ben Hoffman: I will literally kill myself if I get there and that’s what it is\n11/12/24, 14:02 - Tom Thompson 🧴: well with artifact, giving it quantum computation services via an api is easy, might be a fun thing to tack on lol\n11/12/24, 14:03 - Ben Hoffman: Yeah it should be way less punishing to change the crypto when you don’t have an army of shitcoin ruggers trying to stop you\n11/12/24, 14:03 - Ben Hoffman: Totally", + ], + }, + "status": "finished_successfully", + "weight": 1, + "metadata": { + "request_id": "8f01adde8942d9be-AKL", + "message_source": null, + "timestamp_": "absolute", + "message_type": null, + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "2ae52d22-dba5-457d-8147-faade76900dd", + "children": [ + "e9410778-39b9-464a-a0f5-cc461cb28e85", + ], + }, { + "id": "e9410778-39b9-464a-a0f5-cc461cb28e85", + "message": { + "id": "e9410778-39b9-464a-a0f5-cc461cb28e85", + "author": { + "role": "tool", + "name": "a8km123", + "metadata": {}, + }, + "create_time": 1733880397.691115, + "content": { + "content_type": "text", + "parts": [ + "**Exploring AI integrations**\n\nI’m examining how Git, npm modules, software supply chain, AI, and blockchains intersect, focusing on knowledge, reproducible model training, and traceability in AI development.", + ], + }, + "status": "finished_successfully", + "weight": 0, + "metadata": { + "finished_text": "Thought for 1m 26s", + "initial_text": "Thinking", + "summarization_headline": + "Git, Blockchain, and AI Integration", + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "request_id": "8f01adde8942d9be-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074", + "children": [ + "46cc8de2-2860-4c13-9083-32117c314a3a", + ], + }, { + "id": "46cc8de2-2860-4c13-9083-32117c314a3a", + "message": { + "id": "46cc8de2-2860-4c13-9083-32117c314a3a", + "author": { "role": "assistant", "metadata": {} }, + "create_time": 1733880397.691219, + "content": { + "content_type": "text", + "parts": [ + 'Below are a few conceptual frameworks and themes that might help clarify the directions you’re exploring. These are just options—bold thought experiments and profound re-interpretations of what you’ve discussed. Each tries to distill the underlying "essence" of what you\'re circling around:\n\n1. **Git as a Cognitive Substrate** \n *Concept*: Treat Git not just as a version-control system, but as a universal medium for knowledge representation—a canonical substrate into which intelligence can “think” its outputs directly. \n *Idea*: Instead of thinking of Git as infrastructure hiding behind the scenes, imagine it as the actual “mindspace” of an AI. Commits become “thoughts,” branches become “possible worlds,” and merges become negotiated alignments of reality. This would mean that interacting with systems wouldn’t require separately building APIs or wrapping logic in adapters; the “thought process” would already be in the universal language of code and data tracked by Git.\n\n2. **Provenance-Aware Execution and Training** \n *Concept*: Embrace full provenance tracking of models—every piece of training data, every parameter update, every intermediate artifact—being anchored in a Git-like or blockchain-like structure. \n *Idea*: On-chain (or on-git) training ensures that models, their lineage, and their transformations are fully transparent and repeatable. This is “Proof of Useful Work” done right: no contrived token games, just a history of refinement. Imagine a Hugging Face that merges seamlessly with GitHub: training datasets as commits, parameter updates as commits, final models as tagged releases. The model’s “mind” isn’t some opaque black box, but an ever-growing, fully versioned garden of knowledge and code.\n\n3. **Frictionless Integration With the Wider Software Supply Chain** \n *Concept*: The AI doesn’t produce code that must then be adapted or compiled into external ecosystems; it directly “commits” code into existing supply chains (e.g., npm modules) as a natural extension of its reasoning. \n *Idea*: By removing the “build step” or “adapter step” entirely, the AI’s output is not something that must be transformed—its output is directly usable. Code changes flow seamlessly into the world’s largest ecosystems of shared code, enabling the AI not only to contribute but also to continuously learn from incoming commits by humans and other agents.\n\n4. **Time, Context, and Political Awareness Through Commit Histories** \n *Concept*: Commit histories are fundamentally stories about human and machine collaboration, negotiation, conflict, and resolution. They are a record of political dynamics, group decisions, priorities shifting over time. \n *Idea*: By letting the AI “live” in this world of commits, it can learn to interpret the subtle signals encoded in version histories—understanding governance, consequence, and social context. Training on commit histories gives AI a vantage point to reason about time, intention, and human organizational structures. In other words, by modeling itself as a participant in Git’s political tapestry, the AI begins to understand what it means to cooperate, to iterate, to govern.\n\n5. **Merging Knowledge Networks: From Fragmented Tools to a Single Universal Medium** \n *Concept*: Git, package managers, blockchains, AI models—these have historically been disparate systems, each with its own protocols and friction points. \n *Idea*: Your conversation hints at merging these concepts into a unified environment: a single substrate (like Git) where data, code, models, and their entire genealogy live side by side. Instead of separate infrastructures for AI training, code versioning, supply chain management, and trust enforcement, you get one substrate that can host them all. This reduces conceptual overhead and unleashes a new kind of “global system,” where knowledge flows freely and every increment of intelligence is recorded and trusted.\n\n---\n\nAll of these concepts circle around the core theme: using version-controlled code repositories, possibly combined with blockchain-style invariants, to eliminate friction, ensure trust, enable provenance, and integrate AI deeply into the world’s software and knowledge ecosystems—no artificial boundaries, no hidden build steps, no murky provenances. Instead, a continuous, transparent, and universal environment of incremental, queryable, and trustworthy “thought commits.”', + ], + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 1, + "metadata": { + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "e9410778-39b9-464a-a0f5-cc461cb28e85", + "request_id": "8f01adde8942d9be-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "e9410778-39b9-464a-a0f5-cc461cb28e85", + "children": [ + "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + ], + }, { + "id": "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "message": { + "id": "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "author": { "role": "user", "metadata": {} }, + "create_time": 1733881250.145666, + "content": { + "content_type": "text", + "parts": [ + "if that conversation was being turned into a set of comparisons to make against competing blockchains, name what concepts are missing from the below comparison, and say how you might improve this comparator:\n\n1. Token dependency:\n \n - Does the chain have a built-in token?\n - How tightly coupled is that token to the consensus?\n - How easy is it to remove, replace, or adjust that token?\n - Is the consensus mechanism itself token-agnostic or inherently token-bound?\n - is the token stable or fluctuatable\n\n1. Integration format:\n \n - Can the chain’s data and consensus be naturally represented in Git commits or NPM packages?\n - Are translations or bridging layers required to interface with standard tools, and if so, do these weaken the consensus?\n3. Portability:\n \n - Can the system run in isolated parts, completely on your own infrastructure?\n - Can these isolated instances later merge consensus without loss of trust?\n\nFor the demonstration of a sharded or “splintered” proof-of-work system, consider calling it “Shattered Proof-of-Work” (S-PoW). This name reflects how the workload can be broken into isolated segments and later merged based on verifiable computational work.", + ], + }, + "status": "finished_successfully", + "weight": 1, + "metadata": { + "serialization_metadata": { + "custom_symbol_offsets": [], + }, + "request_id": "8f01c4d23b851c5e-AKL", + "message_source": null, + "timestamp_": "absolute", + "message_type": null, + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "46cc8de2-2860-4c13-9083-32117c314a3a", + "children": [ + "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + ], + }, { + "id": "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "message": { + "id": "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "author": { + "role": "tool", + "name": "a8km123", + "metadata": {}, + }, + "create_time": 1733881301.561146, + "content": { + "content_type": "text", + "parts": [ + '**Gathering blockchain insights**\n\nI\'m compiling info on blockchain developments, pinpointing gaps like "Community involvement" and "Transparency and trust." Also, looking at concepts such as "Provenance and lineage" and "Natural language promptability," considering how they can enrich the comparison.', + ], + }, + "status": "finished_successfully", + "weight": 0, + "metadata": { + "finished_text": "Thought for 50 seconds", + "initial_text": "Thinking", + "summarization_headline": + "Blockchain Comparison Concepts", + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "request_id": "8f01c4d23b851c5e-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaa2979c-fd1f-4f49-bfd7-051c15a258b3", + "children": [ + "c132e044-dade-4946-95a7-b1c942539af4", + ], + }, { + "id": "c132e044-dade-4946-95a7-b1c942539af4", + "message": { + "id": "c132e044-dade-4946-95a7-b1c942539af4", + "author": { "role": "assistant", "metadata": {} }, + "create_time": 1733881301.56126, + "content": { + "content_type": "text", + "parts": [ + "**Missing Concepts and How to Improve the Comparison:**\n\n1. **Provenance and Lineage of Data/Models:** \n **What’s missing:** \n - How does the chain handle the version history of data, code, or model parameters? \n - Can it track the full lineage of a model’s training process or a software artifact’s evolution? \n **Improvement:** \n Add criteria examining whether each system can embed transparent, verifiable version histories—like Git commit graphs—directly in its consensus. For example: \n - *“Can the chain immutably record the origin and version history of data and code as a first-class citizen?”* \n - *“Does it allow for ‘on-chain training’ records or a taxonomy of artifacts that mirror Git’s branching, merging, and tagging semantics?”*\n\n2. **Deterministic Computation and Reproducibility:** \n **What’s missing:** \n - Are computations (e.g., model training, code transformations) guaranteed to be deterministic and reproducible from on-chain data? \n - Can the entire state (including the model’s parameters) be reconstructed from on-chain commits? \n **Improvement:** \n Add criteria around determinism and replayability: \n - *“Is the chain’s consensus logic and stored data sufficient to deterministically reproduce any result, such as model parameters after training?”* \n - *“Does the system enable ‘proof-of-useful-work’ where the work is fully re-executable and verifiable from the recorded state?”*\n\n3. **Interoperability With Existing Ecosystems (Beyond Git \u0026 NPM):** \n **What’s missing:** \n - Beyond just Git commits and npm packages, how well does the system integrate with other major ecosystems (e.g., Docker images, PyPI packages, Hugging Face model hubs)? \n **Improvement:** \n Add criteria that measure the breadth of ecosystem compatibility: \n - *“Can the chain’s artifacts be directly consumed by standard developer tools, CI/CD pipelines, model registries, or container registries without translation?”* \n - *“Is the system extensible to support new tooling integrations as standards evolve?”*\n\n4. **Governance and Social Dynamics:** \n **What’s missing:** \n - How does the chain model governance over code and data? \n - Can it represent the political dimension of commits—who proposed changes, who merged them, and the rationale behind decisions? \n **Improvement:** \n Add criteria related to governance and social aspects: \n - *“Does the chain encode and preserve metadata of contributors, allowing AIs and humans to understand political, social, and organizational contexts of commits?”* \n - *“Can we trace not just the code but also the discussions, proposals, and decision-making processes that led to a given state?”*\n\n5. **Economic Neutrality and Flexible Incentive Structures:** \n **What’s missing:** \n - If the chain employs incentives, can those be non-financial or proof-of-actual-work rather than bound to a volatile token economy? \n **Improvement:** \n Add criteria around incentive flexibility and neutrality: \n - *“Are consensus and network security achievable without relying on a native token’s economic incentives?”* \n - *“Is it possible to introduce alternate reward mechanisms, like reputation, stable credits, or proof-of-completion of useful tasks?”*\n\n6. **Upgradability and Evolution Over Time:** \n **What’s missing:** \n - How easy is it to evolve the protocol over time, add new features, or adapt to new technological standards without causing ecosystem turbulence? \n **Improvement:** \n Include questions about extensibility and protocol migration: \n - *“Can the system’s core protocols be upgraded easily and cleanly, preserving full historical integrity?”* \n - *“Does it support branching/forking processes similar to Git, allowing for experimental consensus changes that can later be merged back?”*\n\n7. **Data Availability and Scalability Constraints:** \n **What’s missing:** \n - How does the system ensure that the entire state is always available and scalable as the code/data base grows large? \n **Improvement:** \n Add criteria regarding scaling and data availability: \n - *“Can the chain scale with increasing amounts of data and computations without degrading verification or trust?”* \n - *“Does it support ‘shattering’ (sharding) into separate chains or segments that can be recombined, ensuring data availability and trust remain intact?”*\n\n---\n\n**Improved Comparator Example:**\n\n- **Token Dependency \u0026 Incentive Neutrality:** \n - Does consensus rely on an intrinsic token or can it operate without one? \n - Can incentives be reconfigured (e.g., non-monetary proofs of useful work)?\n\n- **Integration Format \u0026 Ecosystem Compatibility:** \n - Beyond Git and NPM, can the chain represent and serve artifacts to other ecosystems (Hugging Face models, Docker, PyPI)? \n - Are commits, model parameters, and artifact states directly usable by standard devops and AI tooling?\n\n- **Provenance, Governance \u0026 Social Dynamics:** \n - Does it record full artifact lineage, enabling complete reproducibility? \n - Can the platform encode and understand governance (commit authors, merging policies) and social context for changes?\n\n- **Deterministic \u0026 Reproducible Computation:** \n - Are computations fully deterministic, so any artifact (including AI models) can be re-derived from on-chain data? \n - Are training processes and model outputs repeatably verifiable as a form of proof-of-useful-work?\n\n- **Upgradability \u0026 Evolution:** \n - Can the system’s core mechanics be upgraded without loss of trust or data fidelity? \n - Does it support branching and merging of different consensus versions, akin to code versioning in Git?\n\n- **Portability, Data Availability \u0026 Scalability:** \n - Can it run in isolated modes and then later reconcile states without losing trust? \n - Does “Shattered Proof-of-Work” support scaling by splitting workloads and merging them without compromising determinism or provenance?\n\nBy including these additional angles—provenance, governance, determinism, ecosystem compatibility, incentive neutrality, and upgradability—you create a more comprehensive set of criteria for evaluating next-generation chains against conventional blockchains.", + ], + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 1, + "metadata": { + "message_type": null, + "model_slug": "o1-pro", + "default_model_slug": "o1-pro", + "parent_id": + "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "request_id": "8f01c4d23b851c5e-AKL", + "timestamp_": "absolute", + "shared_conversation_id": + "6758f810-286c-800b-8e2c-490e634ca002", + }, + "recipient": "all", + }, + "parent": "aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6", + "children": [], + }], + "has_user_editable_context": false, + "continue_conversation_url": + "https://chatgpt.com/share/6758f810-286c-800b-8e2c-490e634ca002/continue", + "moderation_state": { + "has_been_moderated": false, + "has_been_blocked": false, + "has_been_accepted": false, + "has_been_auto_blocked": false, + "has_been_auto_moderated": false, + }, + "is_indexable": false, + "is_better_metatags_enabled": true, + }, + }, + "continueMode": false, + "moderationMode": false, + "chatPageProps": { + "session": {}, + "user": { + "email": "", + "groups": [], + "id": "ua-d76fbeb9-aa53-4dcb-b666-8358452854be", + "name": "", + "intercom_hash": + "ua-d76fbeb9-aa53-4dcb-b666-8358452854be", + "image": "", + "picture": "", + "mfa": false, + }, + "cluster": "unified-21", + "userCountry": "CA", + "userRegion": null, + "userRegionCode": null, + "cfConnectingIp": "2001:df0:a280:1001::6:1", + "cfIpLatitude": "43.63190", + "cfIpLongitude": "-79.37160", + "cfTimezone": "America/Toronto", + "cfIpCity": null, + "userLocale": null, + "authStatus": "logged_out", + "statsig": { + "payload": { + "feature_gates": { + "14938527": { + "name": "14938527", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "16325302": { + "name": "16325302", + "value": false, + "rule_id": "dOGUtq0Id8KhSPYCfoxRE", + "secondary_exposures": [], + }, + "29898433": { + "name": "29898433", + "value": true, + "rule_id": "5hlR2AjP7h2nJXOxZHKzKH:100.00:2", + "secondary_exposures": [], + }, + "44045625": { + "name": "44045625", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "61299031": { + "name": "61299031", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "73891701": { + "name": "73891701", + "value": true, + "rule_id": "4M51IcwRuyJAV7JLf2RvzR:100.00:1", + "secondary_exposures": [], + }, + "80186230": { + "name": "80186230", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "87687712": { + "name": "87687712", + "value": true, + "rule_id": "2qhKk5ydXFvhD5pios2h1S", + "secondary_exposures": [], + }, + "106692620": { + "name": "106692620", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "138956718": { + "name": "138956718", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1132191510", + "gateValue": "false", + "ruleID": "dHE9Ru37UQ9JP4Jaewul2", + }], + }, + "156153730": { + "name": "156153730", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "166966726": { + "name": "166966726", + "value": true, + "rule_id": "5VFEe5M4mDSyQU0M9U5hTm", + "secondary_exposures": [], + }, + "174366048": { + "name": "174366048", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1923022511", + "gateValue": "false", + "ruleID": "default", + }], + }, + "181161203": { + "name": "181161203", + "value": true, + "rule_id": "6NmIF8U6DauDDclcytkCkc", + "secondary_exposures": [], + }, + "187267097": { + "name": "187267097", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "193142637": { + "name": "193142637", + "value": false, + "rule_id": "7eSgKGxxnDNbWyzYM0IMHe:0.00:3", + "secondary_exposures": [], + }, + "211801712": { + "name": "211801712", + "value": true, + "rule_id": "2wyraagVOBvRaQh0LWLKgh", + "secondary_exposures": [], + }, + "218782548": { + "name": "218782548", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "226799327": { + "name": "226799327", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "232791851": { + "name": "232791851", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "3922476776", + "gateValue": "false", + "ruleID": "4XHkj4xXCwlkAzMMY0VSn4:0.00:2", + }, { + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + }, + "263833443": { + "name": "263833443", + "value": true, + "rule_id": "1mdaBiHva650rqO0InYHjm:100.00:2", + "secondary_exposures": [], + }, + "286716193": { + "name": "286716193", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "314203997": { + "name": "314203997", + "value": false, + "rule_id": "1xxCxFqZdreszzk4pe7fVb", + "secondary_exposures": [], + }, + "359124149": { + "name": "359124149", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "374328013": { + "name": "374328013", + "value": false, + "rule_id": "5xNC0SZHSEfRGwLRCyEofs:0.00:1", + "secondary_exposures": [{ + "gate": "2046333385", + "gateValue": "false", + "ruleID": "4nM2ehmgoDQIv69B0zohb6", + }], + }, + "389809369": { + "name": "389809369", + "value": false, + "rule_id": "5WK3s5Ax4bJcCZUXG1aCaM:0.00:2", + "secondary_exposures": [], + }, + "437245079": { + "name": "437245079", + "value": true, + "rule_id": "1KKPtXmOjxZamlgM2DzXlg", + "secondary_exposures": [], + }, + "458009956": { + "name": "458009956", + "value": true, + "rule_id": "6LgEwykI8hHnFAQ5EkKTX2:100.00:1", + "secondary_exposures": [], + }, + "462735957": { + "name": "462735957", + "value": true, + "rule_id": "1SCzY8HZjgbrVUviddAgUM:100.00:1", + "secondary_exposures": [], + }, + "467934241": { + "name": "467934241", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "491279851": { + "name": "491279851", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "543687013": { + "name": "543687013", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "544659310": { + "name": "544659310", + "value": true, + "rule_id": "1Opz1br1XsEM6Uajp6F1XU:100.00:3", + "secondary_exposures": [], + }, + "562926978": { + "name": "562926978", + "value": true, + "rule_id": "5NtGLuwEGvKkFrz0jD9Jgb:100.00:1", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "566128514": { + "name": "566128514", + "value": true, + "rule_id": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + "secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }], + }, + "568802918": { + "name": "568802918", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "582612297": { + "name": "582612297", + "value": true, + "rule_id": "5censDsCfS2zQeYtTIui2s:100.00:2", + "secondary_exposures": [], + }, + "584022615": { + "name": "584022615", + "value": true, + "rule_id": "7xLTX1iWf4Q14u604BUFhJ:100.00:2", + "secondary_exposures": [], + }, + "589604007": { + "name": "589604007", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "633009675", + "gateValue": "false", + "ruleID": "15DR4R8pnag2oqKIvQOVni", + }], + }, + "603035019": { + "name": "603035019", + "value": false, + "rule_id": "3LIHLisyFlEpdQBDU6f5A0", + "secondary_exposures": [{ + "gate": "2547456621", + "gateValue": "false", + "ruleID": "jDZnsHB4L995XxJ7J5vtK", + }], + }, + "605040022": { + "name": "605040022", + "value": true, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "614413305": { + "name": "614413305", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "633009675": { + "name": "633009675", + "value": false, + "rule_id": "15DR4R8pnag2oqKIvQOVni", + "secondary_exposures": [], + }, + "645560164": { + "name": "645560164", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "711369489": { + "name": "711369489", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "716475478": { + "name": "716475478", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "716722001": { + "name": "716722001", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "719574156": { + "name": "719574156", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "730579107": { + "name": "730579107", + "value": true, + "rule_id": "2D13jk8X6i5dXQlM6FWS9N", + "secondary_exposures": [], + }, + "730601627": { + "name": "730601627", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "740954505": { + "name": "740954505", + "value": false, + "rule_id": "6m5kbcBHvQbkbeBzwquYFp", + "secondary_exposures": [], + }, + "756982148": { + "name": "756982148", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "771993664": { + "name": "771993664", + "value": true, + "rule_id": "1dDdxrXy1KadHtYusuGRgR:100.00:6", + "secondary_exposures": [], + }, + "774838244": { + "name": "774838244", + "value": false, + "rule_id": "6iWAMGfVhFkUDAeK2ldKZa", + "secondary_exposures": [], + }, + "808722915": { + "name": "808722915", + "value": false, + "rule_id": "1HBUgTG7kZsvlP01c71Pbv", + "secondary_exposures": [], + }, + "809675868": { + "name": "809675868", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "857491970": { + "name": "857491970", + "value": true, + "rule_id": "2LpyNTIfbYC4R6XS6dfdIL", + "secondary_exposures": [], + }, + "859327441": { + "name": "859327441", + "value": false, + "rule_id": "2kLMbEEhNzrRXoN7Lw2PmZ", + "secondary_exposures": [], + }, + "874794369": { + "name": "874794369", + "value": true, + "rule_id": "2VpsO0nGBUK2dZXmYhPA0B", + "secondary_exposures": [], + }, + "889823676": { + "name": "889823676", + "value": true, + "rule_id": "IbtsgDIEUaa3M0uovI8VE", + "secondary_exposures": [], + }, + "910189142": { + "name": "910189142", + "value": true, + "rule_id": "3epNymlQdEWHnDUZEmtWUC", + "secondary_exposures": [], + }, + "967447444": { + "name": "967447444", + "value": true, + "rule_id": "2mOBTIuCpZKSBYUIdPaATY", + "secondary_exposures": [], + }, + "989108178": { + "name": "989108178", + "value": false, + "rule_id": "4sTodKrNyByM4guZ68MORR", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "989226566": { + "name": "989226566", + "value": true, + "rule_id": "6yqqYAWKtmfU8A7QGdiky4", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "1016364891": { + "name": "1016364891", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1026575464": { + "name": "1026575464", + "value": true, + "rule_id": "1f4cSeenpcmWQ9eMKKQhF5", + "secondary_exposures": [], + }, + "1028288328": { + "name": "1028288328", + "value": true, + "rule_id": "N8aqJyROPpTrVV5VygNPc:100.00:3", + "secondary_exposures": [], + }, + "1028682714": { + "name": "1028682714", + "value": true, + "rule_id": "735n03snBvba4AEhd2Qwqu:100.00:3", + "secondary_exposures": [], + }, + "1070272635": { + "name": "1070272635", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1074323483": { + "name": "1074323483", + "value": false, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "1103845153": { + "name": "1103845153", + "value": true, + "rule_id": "2ACgZQuQHKLrOqIdaMq7y", + "secondary_exposures": [], + }, + "1118624160": { + "name": "1118624160", + "value": true, + "rule_id": "401MRIAwDUgxfVJedMgoze", + "secondary_exposures": [], + }, + "1123641887": { + "name": "1123641887", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2173548801", + "gateValue": "true", + "ruleID": "22nVhoL17eyMvGWgFrDfZe", + }], + }, + "1123642877": { + "name": "1123642877", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2173548801", + "gateValue": "true", + "ruleID": "22nVhoL17eyMvGWgFrDfZe", + }], + }, + "1132191510": { + "name": "1132191510", + "value": false, + "rule_id": "dHE9Ru37UQ9JP4Jaewul2", + "secondary_exposures": [], + }, + "1159399595": { + "name": "1159399595", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1198030896": { + "name": "1198030896", + "value": true, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "1236320153": { + "name": "1236320153", + "value": true, + "rule_id": "7Bq8t59h8MTmfDfmTxiCBa:100.00:6", + "secondary_exposures": [], + }, + "1239490451": { + "name": "1239490451", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1239490452": { + "name": "1239490452", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1291584794": { + "name": "1291584794", + "value": false, + "rule_id": "6ZsiDEUpOSTbG0NkCUShbJ", + "secondary_exposures": [], + }, + "1306782371": { + "name": "1306782371", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1318146997": { + "name": "1318146997", + "value": true, + "rule_id": "2AclmEgqaQBVFbxz37XKzy:100.00:5", + "secondary_exposures": [], + }, + "1334159177": { + "name": "1334159177", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1382475798": { + "name": "1382475798", + "value": true, + "rule_id": "3P8OsGy1e5tQlR5dsTIWbL", + "secondary_exposures": [], + }, + "1413999995": { + "name": "1413999995", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "697158486", + "gateValue": "false", + "ruleID": "5KvTiw548r4tjqy7oHEcAL", + }, { + "gate": "3199899666", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2048457345", + "gateValue": "false", + "ruleID": "default", + }], + }, + "1422501431": { + "name": "1422501431", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1426009137": { + "name": "1426009137", + "value": true, + "rule_id": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }], + }, + "1428536080": { + "name": "1428536080", + "value": true, + "rule_id": "2lFzVvuye5InI9X01xArgp", + "secondary_exposures": [], + }, + "1444106524": { + "name": "1444106524", + "value": true, + "rule_id": "6Kd1cWwMI8EXiyQ7bqo8f3:50.00:1", + "secondary_exposures": [], + }, + "1468311859": { + "name": "1468311859", + "value": false, + "rule_id": "7tfl8ZUhwr5pzErE3ikBej", + "secondary_exposures": [], + }, + "1487207580": { + "name": "1487207580", + "value": false, + "rule_id": "disabled", + "secondary_exposures": [], + }, + "1489221567": { + "name": "1489221567", + "value": false, + "rule_id": "z06JxGlMXQVrwG418FqnU", + "secondary_exposures": [], + }, + "1524207076": { + "name": "1524207076", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1526010272": { + "name": "1526010272", + "value": true, + "rule_id": "680tNuHpCDKDg9DEYqeqv3:100.00:1", + "secondary_exposures": [], + }, + "1542198993": { + "name": "1542198993", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1567586774": { + "name": "1567586774", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "1568195116": { + "name": "1568195116", + "value": false, + "rule_id": "KKhehNjRxNHoIWKNkqpU2", + "secondary_exposures": [], + }, + "1578703058": { + "name": "1578703058", + "value": false, + "rule_id": "2l4nEVMUnPuXkgprUm5zzs:1.00:2", + "secondary_exposures": [], + }, + "1611573287": { + "name": "1611573287", + "value": true, + "rule_id": "159rwM3sBnviE9XWH24azn:100.00:2", + "secondary_exposures": [], + }, + "1656345175": { + "name": "1656345175", + "value": true, + "rule_id": "2CwIChuIr7SLQ2CyqRegF2", + "secondary_exposures": [], + }, + "1684776951": { + "name": "1684776951", + "value": true, + "rule_id": "nnOtXYe2wzehNEU8MfPam", + "secondary_exposures": [{ + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }], + }, + "1788948501": { + "name": "1788948501", + "value": false, + "rule_id": "40Z1SWSk3MWJlOarI38p6Z", + "secondary_exposures": [], + }, + "1789593108": { + "name": "1789593108", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1794124947": { + "name": "1794124947", + "value": true, + "rule_id": "6Xp2LEMkm5UEgaW5ZGIN3M", + "secondary_exposures": [], + }, + "1820151945": { + "name": "1820151945", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1825130190": { + "name": "1825130190", + "value": false, + "rule_id": "YLtE1lbpb7dOUfvjJEC3x:0.00:4", + "secondary_exposures": [], + }, + "1828361965": { + "name": "1828361965", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1847911009": { + "name": "1847911009", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2304807207", + "gateValue": "false", + "ruleID": "default", + }], + }, + "1860647109": { + "name": "1860647109", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1877321278": { + "name": "1877321278", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1883608273": { + "name": "1883608273", + "value": true, + "rule_id": "2fjZsiHb0X34cxyryHit5D", + "secondary_exposures": [], + }, + "1923022511": { + "name": "1923022511", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "1938289220": { + "name": "1938289220", + "value": false, + "rule_id": "79O8DQPDmTKxnLdAH9loVk", + "secondary_exposures": [], + }, + "2000076788": { + "name": "2000076788", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2033872549": { + "name": "2033872549", + "value": false, + "rule_id": "7dScmNU0bu2UQuzCNtva50", + "secondary_exposures": [], + }, + "2048332657": { + "name": "2048332657", + "value": false, + "rule_id": "7zy1mC6ANDLQQEC55KcGXC", + "secondary_exposures": [], + }, + "2055244389": { + "name": "2055244389", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2067628123": { + "name": "2067628123", + "value": false, + "rule_id": "3CuBjEMi97tY3EGnq0NA9s", + "secondary_exposures": [], + }, + "2091463435": { + "name": "2091463435", + "value": true, + "rule_id": "5t78GUS68KOn3bHZd8z7ii:100.00:1", + "secondary_exposures": [], + }, + "2104868687": { + "name": "2104868687", + "value": true, + "rule_id": "2MrZhodWRXji2DjHVx1UwZ", + "secondary_exposures": [], + }, + "2128055372": { + "name": "2128055372", + "value": false, + "rule_id": "6m0Rlf0uNfNc4HscDdpcUA:0.00:3", + "secondary_exposures": [], + }, + "2173548801": { + "name": "2173548801", + "value": true, + "rule_id": "22nVhoL17eyMvGWgFrDfZe", + "secondary_exposures": [], + }, + "2192543539": { + "name": "2192543539", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "4206244917", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2216175490": { + "name": "2216175490", + "value": true, + "rule_id": "1n25S96Hm7ICEdb3VyBfqT", + "secondary_exposures": [], + }, + "2220211676": { + "name": "2220211676", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1259585210", + "gateValue": "false", + "ruleID": "3cQqu8Odku3b2Mx7Ba4Lq0", + }], + }, + "2304807207": { + "name": "2304807207", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2307260085": { + "name": "2307260085", + "value": true, + "rule_id": "4v47Zgj07BoXsAC6nwCwgU", + "secondary_exposures": [], + }, + "2381300469": { + "name": "2381300469", + "value": true, + "rule_id": "1XW58zJg16b7h0clyVhEW3:100.00:10", + "secondary_exposures": [], + }, + "2420509970": { + "name": "2420509970", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2436427643": { + "name": "2436427643", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "697158486", + "gateValue": "false", + "ruleID": "5KvTiw548r4tjqy7oHEcAL", + }, { + "gate": "3199899666", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2048457345", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2437036977": { + "name": "2437036977", + "value": true, + "rule_id": "1tfSWj1wjQlwP4QQ4eD5Qu:100.00:4", + "secondary_exposures": [], + }, + "2445152477": { + "name": "2445152477", + "value": true, + "rule_id": "5qtlunRMswJX2JGoF8GikC", + "secondary_exposures": [], + }, + "2449630478": { + "name": "2449630478", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2492340648": { + "name": "2492340648", + "value": false, + "rule_id": "7vWEOrlZ07LCdewUnumW3C:0.00:9", + "secondary_exposures": [], + }, + "2494375100": { + "name": "2494375100", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2522578969": { + "name": "2522578969", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2542814321": { + "name": "2542814321", + "value": true, + "rule_id": "1m1xZtjqqNjZ3o8AOuRHeU", + "secondary_exposures": [], + }, + "2544616245": { + "name": "2544616245", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2546838099": { + "name": "2546838099", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2547456621": { + "name": "2547456621", + "value": false, + "rule_id": "jDZnsHB4L995XxJ7J5vtK", + "secondary_exposures": [], + }, + "2558436080": { + "name": "2558436080", + "value": false, + "rule_id": "2iIp4XMISbfnSy5NsEpKaN", + "secondary_exposures": [], + }, + "2562876640": { + "name": "2562876640", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2634628831": { + "name": "2634628831", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "2689530298": { + "name": "2689530298", + "value": true, + "rule_id": "37djhtjwy17C2tz3RWPj8i", + "secondary_exposures": [], + }, + "2742826311": { + "name": "2742826311", + "value": false, + "rule_id": "3IJuxWy2vbHjpJGwVraJQM", + "secondary_exposures": [], + }, + "2747189243": { + "name": "2747189243", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2756095923": { + "name": "2756095923", + "value": true, + "rule_id": "6jPp6nW1wQVJbfY0uwQgmv:100.00:1", + "secondary_exposures": [], + }, + "2775516364": { + "name": "2775516364", + "value": true, + "rule_id": "62xFU2LRHp4dPsOwj2jVUT", + "secondary_exposures": [{ + "gate": "1457171347", + "gateValue": "false", + "ruleID": "2EjTipm6C4kk4fuvcHMzZe", + }, { + "gate": "1426009137", + "gateValue": "true", + "ruleID": "7D8EAif25E3Y8A3zkg6ljp:100.00:2", + }], + }, + "2813404493": { + "name": "2813404493", + "value": false, + "rule_id": "4jCRuoOxK8tLEGn9ngylXq", + "secondary_exposures": [{ + "gate": "1074323483", + "gateValue": "false", + "ruleID": "disabled", + }], + }, + "2824504080": { + "name": "2824504080", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2830958549": { + "name": "2830958549", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2833534668": { + "name": "2833534668", + "value": true, + "rule_id": "7uYkibMYlCPSnoWmmYNanm", + "secondary_exposures": [], + }, + "2848981774": { + "name": "2848981774", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1074323483", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "2849024256": { + "name": "2849024256", + "value": true, + "rule_id": "5m34P11wI6H8E5sAMtsEXj", + "secondary_exposures": [], + }, + "2851018874": { + "name": "2851018874", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2865702999": { + "name": "2865702999", + "value": true, + "rule_id": "31nbFDkAl6BHUyVaOtsonb", + "secondary_exposures": [], + }, + "2865703032": { + "name": "2865703032", + "value": true, + "rule_id": "2JsPzQuM9307I2E2pNdWHL", + "secondary_exposures": [], + }, + "2881332567": { + "name": "2881332567", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1410082514", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2893406386": { + "name": "2893406386", + "value": false, + "rule_id": "32QpYlIXk5Wr45idZ2iRkG", + "secondary_exposures": [], + }, + "2919213474": { + "name": "2919213474", + "value": false, + "rule_id": "6HLlb6nSjJk5ADynHucWgP", + "secondary_exposures": [], + }, + "2923133196": { + "name": "2923133196", + "value": false, + "rule_id": "1HnipyUiP0nxZWtbaIXTlK", + "secondary_exposures": [{ + "gate": "2048332657", + "gateValue": "false", + "ruleID": "7zy1mC6ANDLQQEC55KcGXC", + }], + }, + "2935021756": { + "name": "2935021756", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "2940166566": { + "name": "2940166566", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3006519778": { + "name": "3006519778", + "value": true, + "rule_id": "15jPMyA4orCotRzcwO6hXn", + "secondary_exposures": [], + }, + "3011415004": { + "name": "3011415004", + "value": false, + "rule_id": "7pUMK6uci7sslAj8bP7VEA", + "secondary_exposures": [{ + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3033842386": { + "name": "3033842386", + "value": true, + "rule_id": "P6Yw3pPRSdCV9khAOA5zA", + "secondary_exposures": [], + }, + "3051378794": { + "name": "3051378794", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "1456438623", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1745914872", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "2546838099", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3805873235", + "gateValue": "false", + "ruleID": "5KvGWtWJiHRmVbxFXsA9KS:0.00:1", + }], + }, + "3134329717": { + "name": "3134329717", + "value": false, + "rule_id": "2UWRV6bObClvvj8jLkP7kJ", + "secondary_exposures": [], + }, + "3168919911": { + "name": "3168919911", + "value": true, + "rule_id": "4ZR6dSHrJ9rEydRKbpzGfr:100.00:2", + "secondary_exposures": [], + }, + "3180974288": { + "name": "3180974288", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3213153699": { + "name": "3213153699", + "value": true, + "rule_id": "46yaCNlYIQAFy8P7hV5tXO:100.00:1", + "secondary_exposures": [], + }, + "3320547351": { + "name": "3320547351", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3323077373": { + "name": "3323077373", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3325813340": { + "name": "3325813340", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "2304807207", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3376455464": { + "name": "3376455464", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3419264204": { + "name": "3419264204", + "value": false, + "rule_id": "1SLt7ZIEzNa581wiqmmbrJ:0.00:1", + "secondary_exposures": [{ + "gate": "1132191510", + "gateValue": "false", + "ruleID": "dHE9Ru37UQ9JP4Jaewul2", + }, { + "gate": "138956718", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3435450078": { + "name": "3435450078", + "value": true, + "rule_id": "2qCdHpFuWOOkibzLRL0zgn", + "secondary_exposures": [], + }, + "3483957186": { + "name": "3483957186", + "value": true, + "rule_id": "6g82ij32XX2iKDaEpwNXZt:100.00:1", + "secondary_exposures": [], + }, + "3501550327": { + "name": "3501550327", + "value": false, + "rule_id": "HEU9zVkuwUfmXHhGcckPQ", + "secondary_exposures": [], + }, + "3504894861": { + "name": "3504894861", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3511639124": { + "name": "3511639124", + "value": true, + "rule_id": "7o5NDbkivFFnos7rz9JSKL", + "secondary_exposures": [], + }, + "3636405501": { + "name": "3636405501", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3643162575": { + "name": "3643162575", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3650277116": { + "name": "3650277116", + "value": false, + "rule_id": "3Txp6tA3tgTjsKGkRbaKVM:0.00:26", + "secondary_exposures": [], + }, + "3655058338": { + "name": "3655058338", + "value": false, + "rule_id": "2eakzH7Djowztq0c8xUBjk:0.00:13", + "secondary_exposures": [], + }, + "3664702598": { + "name": "3664702598", + "value": false, + "rule_id": "7x9wS41bRDCji9ns8x5Oej", + "secondary_exposures": [], + }, + "3678527908": { + "name": "3678527908", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3680626616": { + "name": "3680626616", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "633009675", + "gateValue": "false", + "ruleID": "15DR4R8pnag2oqKIvQOVni", + }], + }, + "3680956616": { + "name": "3680956616", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3700195277": { + "name": "3700195277", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3700204850": { + "name": "3700204850", + "value": false, + "rule_id": "default", + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3700615661": { + "name": "3700615661", + "value": false, + "rule_id": "66covmutTYx82FWVUlZAqF", + "secondary_exposures": [], + }, + "3768008236": { + "name": "3768008236", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3825730354": { + "name": "3825730354", + "value": false, + "rule_id": "3TOe1n7juAS2TMXzLqNTfj", + "secondary_exposures": [], + }, + "3894844102": { + "name": "3894844102", + "value": false, + "rule_id": "3QXis04GA1OiER0i1tq5MT:50.00:4", + "secondary_exposures": [], + }, + "3912844760": { + "name": "3912844760", + "value": true, + "rule_id": "2PBMw1F9LSIuyNtzThg3PP", + "secondary_exposures": [], + }, + "3922145230": { + "name": "3922145230", + "value": false, + "rule_id": "14DZA2LumaPqAdCo52CrUB", + "secondary_exposures": [], + }, + "3922476776": { + "name": "3922476776", + "value": false, + "rule_id": "4XHkj4xXCwlkAzMMY0VSn4:0.00:2", + "secondary_exposures": [], + }, + "3941367332": { + "name": "3941367332", + "value": true, + "rule_id": "4NngvQ5h3bIzSutqTQ4CeM", + "secondary_exposures": [{ + "gate": "716475478", + "gateValue": "false", + "ruleID": "default", + }], + }, + "3954884439": { + "name": "3954884439", + "value": true, + "rule_id": "5rqjCf7T9KpJtLnaE73Kum:100.00:4", + "secondary_exposures": [], + }, + "3978920525": { + "name": "3978920525", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "3999712942": { + "name": "3999712942", + "value": true, + "rule_id": "3tcEyACMDCatpAcpodrJfN", + "secondary_exposures": [], + }, + "4017530922": { + "name": "4017530922", + "value": true, + "rule_id": "26y7j9aeW6IfykfWOedJm7:100.00:3", + "secondary_exposures": [], + }, + "4038001028": { + "name": "4038001028", + "value": false, + "rule_id": "7sPHYLu6jyA6IUXgqQQsJW", + "secondary_exposures": [], + }, + "4039254565": { + "name": "4039254565", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4137442097": { + "name": "4137442097", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4182851027": { + "name": "4182851027", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4206244917": { + "name": "4206244917", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4215082624": { + "name": "4215082624", + "value": true, + "rule_id": "6oaDIks7YZl9qNION8SRYD", + "secondary_exposures": [], + }, + "4226692983": { + "name": "4226692983", + "value": true, + "rule_id": "6sEu91zwlBGSKOqFiNpGlA:100.00:2", + "secondary_exposures": [], + }, + "4242486290": { + "name": "4242486290", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + "4245811603": { + "name": "4245811603", + "value": false, + "rule_id": "default", + "secondary_exposures": [], + }, + }, + "dynamic_configs": { + "51287004": { + "name": "51287004", + "value": { "enable": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["enable"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "94117254": { + "name": "94117254", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_onboarding_flow", + "new_onboarding_flow_qualified_start_date", + "use_plus_rl_during_onboarding", + "personalized_onboarding", + "write_custom_instructions_in_onboarding", + "plus_rl_during_onboarding_minutes_after_creation", + "onboarding_show_custom_instructions_page", + "onboarding_flow_tool_steps", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "101132775": { + "name": "101132775", + "value": { "show_new_ui": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "171317775": { + "name": "171317775", + "value": { + "should_see_splash_screen_title": true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "186644516": { + "name": "186644516", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_team_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "387752763": { + "name": "387752763", + "value": { + "enable_slash_commands": true, + "enable_rich_text_composer": true, + }, + "group": "5UE8g4T56yxUBUYancL7KB:override", + "rule_id": "5UE8g4T56yxUBUYancL7KB:override", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + "explicit_parameters": [ + "enable_rich_text_composer", + "enable_slash_commands", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "410076910": { + "name": "410076910", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": ["enable_new_mobile"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "457207600": { + "name": "457207600", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_primary_btn_blue"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "459502094": { + "name": "459502094", + "value": { "is_sticky_toggle_off": false }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["is_sticky_toggle_off"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "475896730": { + "name": "475896730", + "value": { "should-enable-hojicha": false }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "should-enable-hojicha", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "488365967": { + "name": "488365967", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["forwardToAuthApi"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "505180994": { + "name": "505180994", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4038001028", + "gateValue": "false", + "ruleID": "7sPHYLu6jyA6IUXgqQQsJW", + }], + "explicit_parameters": [ + "show_share_button_inline", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "506086779": { + "name": "506086779", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "1860647109", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "is_direct_continue_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "548098440": { + "name": "548098440", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + "show_composer_header", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "550560761": { + "name": "550560761", + "value": { + "history_results_limit": 6, + "local_results_limit": 2, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "653115894": { + "name": "653115894", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_indexing", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + "wave", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "668618468": { + "name": "668618468", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_hardcoded_onboarding_prompt", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "743318407": { + "name": "743318407", + "value": {}, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "44045625", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "1885925757", + "gateValue": "false", + "ruleID": "default", + }], + "is_user_in_experiment": false, + "is_experiment_active": true, + }, + "754469756": { + "name": "754469756", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["signup_cta_copy"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "790041403": { + "name": "790041403", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "855925714": { + "name": "855925714", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_simplified_sharing_modal_enabled", + "is_social_share_options_enabled", + "is_update_shared_links_enabled", + "is_discoverability_toggle_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "917528628": { + "name": "917528628", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "login_allow_phone", + "signup_allow_phone", + "use_phone_number", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "923531957": { + "name": "923531957", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "login_allow_phone", + "signup_allow_phone", + "use_phone_number", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "938456440": { + "name": "938456440", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1221279314", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "wave", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "944459048": { + "name": "944459048", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_produce_color_scheme", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "987538661": { + "name": "987538661", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["is_produce_design"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1009155978": { + "name": "1009155978", + "value": { + "enable_mobile_app_upsell_banner": true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_mobile_app_upsell_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "1013114931": { + "name": "1013114931", + "value": { + "show_follow_ups": false, + "follow_ups_prompt_key": "default", + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1028722647": { + "name": "1028722647", + "value": { + "enable_indexing": false, + "backfill_completed": false, + "enable_local_indexing": false, + "enable_ux": false, + "enable_us_rollout": false, + "enable_ux_rollout": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "2372319800", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "enable_local_indexing", + "backfill_completed", + "enable_ux", + "enable_ux_rollout", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "1065013594": { + "name": "1065013594", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_new_mobile"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1134365732": { + "name": "1134365732", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "should_show_disclaimer_only_once_per_device", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1193714591": { + "name": "1193714591", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1193714592": { + "name": "1193714592", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "config", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1193714593": { + "name": "1193714593", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "config", + "show_message_regenerate_model_selector_on_every_message", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1237212031": { + "name": "1237212031", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_country_selector_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1277879515": { + "name": "1277879515", + "value": { "show_ui": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1070272635", + "gateValue": "false", + "ruleID": "default", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1320801051": { + "name": "1320801051", + "value": { + "hide_new_at_workspace_section": false, + "hide_section_new_at_workspace": false, + "gpt_discovery_experiment_enabled": true, + "popular_at_my_workspace_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "gpt_discovery_experiment_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1321202851": { + "name": "1321202851", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "show_share_button_text", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1388643772": { + "name": "1388643772", + "value": { + "is_mobile_enterprise_enabled": false, + "is_enterprise_enabled": true, + }, + "group": "3INu3qkV6QoN42TYoP3gja:override", + "rule_id": "3INu3qkV6QoN42TYoP3gja:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1426679726": { + "name": "1426679726", + "value": { + "is-enabled": true, + "use-rtl-layout": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["use-rtl-layout"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1547924522": { + "name": "1547924522", + "value": {}, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "2183784321", + "gateValue": "false", + "ruleID": "5WAOcp7L45Zgr7TxHv69Bv", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1597388338": { + "name": "1597388338", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1722608525": { + "name": "1722608525", + "value": {}, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1826752011": { + "name": "1826752011", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": "v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_hardcoded_vision_prompts", + "enable_hardcoded_file_document_prompts", + "enable_hardcoded_data_vis_prompts", + "enable_hardcoded_browse_prompts", + "starter_prompt_ranking_algorithm", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1856338298": { + "name": "1856338298", + "value": { "forward_to_authapi": false }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "14938527", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": ["forward_to_authapi"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1873047021": { + "name": "1873047021", + "value": { + "search_scoring_dyconfig_name": + "gizmo_search_score_config", + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "search_scoring_dyconfig_name", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1904139466": { + "name": "1904139466", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_secondary_auth_banner_button_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "1908072088": { + "name": "1908072088", + "value": { + "blocking_modal": true, + "single_button_variant": true, + "dropdown_tooltip": false, + "conversation_model_switcher_badge": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "1930243550": { + "name": "1930243550", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": ["enable_new_mobile"], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "1934819659": { + "name": "1934819659", + "value": { "is_redesign_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1934819688": { + "name": "1934819688", + "value": { + "is_redesign_enabled": true, + "secondary_button_type": "login", + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "1977905538": { + "name": "1977905538", + "value": { + "forwardToAuthApi": false, + "directToGoogle": false, + "directToApple": false, + "directToMsft": false, + }, + "group": "disabled", + "rule_id": "disabled", + "is_device_based": true, + "secondary_exposures": [], + }, + "2068644600": { + "name": "2068644600", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "login_allow_phone", + "signup_allow_phone", + "use_phone_number", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2121003545": { + "name": "2121003545", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "2211924604": { + "name": "2211924604", + "value": { "is_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "3483957186", + "gateValue": "true", + "ruleID": "6g82ij32XX2iKDaEpwNXZt:100.00:1", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "2212993488": { + "name": "2212993488", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "model_talks_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2214103875": { + "name": "2214103875", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["signup_cta_copy"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2246721288": { + "name": "2246721288", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_indexing", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2272050751": { + "name": "2272050751", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2294210794": { + "name": "2294210794", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "enable_new_mobile", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2331692225": { + "name": "2331692225", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "226799327", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2335908103": { + "name": "2335908103", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + "enable_new_autocomplete_homepage", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2335908104": { + "name": "2335908104", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2335908105": { + "name": "2335908105", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 36, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2346290881": { + "name": "2346290881", + "value": { + "has_sidekick_access": false, + "show_nux_banner": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1938289220", + "gateValue": "false", + "ruleID": "79O8DQPDmTKxnLdAH9loVk", + }, { + "gate": "2033872549", + "gateValue": "false", + "ruleID": "7dScmNU0bu2UQuzCNtva50", + }], + "explicit_parameters": [ + "has_sidekick_access", + "show_nux_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2380789656": { + "name": "2380789656", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "2558701922", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1347376961", + "gateValue": "false", + "ruleID": "", + }], + "explicit_parameters": [ + "include_legacy_sidebar_contents", + "enable_arch_updates", + "include_floating_state", + "include_share_on_mobile", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2395035283": { + "name": "2395035283", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "show_share_button_text", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2423967026": { + "name": "2423967026", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2424145028": { + "name": "2424145028", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "starter_prompt_ranking_algorithm", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2424412160": { + "name": "2424412160", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "starter_prompt_ranking_algorithm", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2497137017": { + "name": "2497137017", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "signup_allow_phone", + "use_new_phone_ui", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2499513047": { + "name": "2499513047", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "2530795349": { + "name": "2530795349", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": ["enable_v2_cleanup"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2533035210": { + "name": "2533035210", + "value": { + "snowflake_composer_entry_point": false, + "use_broad_rate_limit_language": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "snowflake_composer_entry_point", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2555105235": { + "name": "2555105235", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "1414026179", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": ["login_allow_phone"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2555870456": { + "name": "2555870456", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["signup_cta_copy"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2567214617": { + "name": "2567214617", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_new_homepage", + "starter_prompt_ranking_algorithm", + "autocomplete_mode", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2575540972": { + "name": "2575540972", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2635734272": { + "name": "2635734272", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_simplified_sharing_modal_enabled", + "is_social_share_options_enabled", + "is_update_shared_links_enabled", + "is_discoverability_toggle_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2663462255": { + "name": "2663462255", + "value": {}, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "164447571", + "gateValue": "false", + "ruleID": "5p0MgR5OkQRqM3bl2fUW74", + }], + "is_user_in_experiment": false, + "is_experiment_active": true, + }, + "2664741443": { + "name": "2664741443", + "value": { "enable": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "2683915289": { + "name": "2683915289", + "value": { + "use_greeting": false, + "use_juice_ui": true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "3483957186", + "gateValue": "true", + "ruleID": "6g82ij32XX2iKDaEpwNXZt:100.00:1", + }], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "2715504799": { + "name": "2715504799", + "value": { "source": "rockset" }, + "group": "3hVfTKFf19aVXd409tUXcS:100.00:2", + "rule_id": "3hVfTKFf19aVXd409tUXcS:100.00:2", + "is_device_based": false, + "secondary_exposures": [], + }, + "2724953007": { + "name": "2724953007", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_mobile_app_upsell_banner", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745767556": { + "name": "2745767556", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745768482": { + "name": "2745768482", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745768485": { + "name": "2745768485", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745768542": { + "name": "2745768542", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2745770409": { + "name": "2745770409", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "7xj42ZY6EtniGNkumF3vQ3:override", + "rule_id": "7xj42ZY6EtniGNkumF3vQ3:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2773808134": { + "name": "2773808134", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + "enable_new_mobile", + "search_autocomplete_mode", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2778287751": { + "name": "2778287751", + "value": { + "is_enterprise_enabled": false, + "is_enterprise_desktop_enabled": false, + "is_desktop_enterprise_enabled": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2795141704": { + "name": "2795141704", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enable_arch_updates"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2818189709": { + "name": "2818189709", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "2l2l6SrM9hXnjCjkr95Ibl:override", + "rule_id": "2l2l6SrM9hXnjCjkr95Ibl:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": ["is_produce_design"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2892794518": { + "name": "2892794518", + "value": {}, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "2909749120": { + "name": "2909749120", + "value": { "is_gating_fix_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_gating_fix_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2919366999": { + "name": "2919366999", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "716475478", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "show_copylink_state_if_no_updates", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2980740905": { + "name": "2980740905", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_yearly_plus_subscription_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983618516": { + "name": "2983618516", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983618523": { + "name": "2983618523", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983619414": { + "name": "2983619414", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2983619420": { + "name": "2983619420", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983619445": { + "name": "2983619445", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "2983619448": { + "name": "2983619448", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "2983619449": { + "name": "2983619449", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_anon_chat_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3021134725": { + "name": "3021134725", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "enable_hardcoded_vision_prompts", + "enable_hardcoded_file_document_prompts", + "enable_hardcoded_data_vis_prompts", + "enable_hardcoded_browse_prompts", + "starter_prompt_ranking_algorithm", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3022018663": { + "name": "3022018663", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_team_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3074105577": { + "name": "3074105577", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + "remove_edit_in_signup": false, + "remove_edit_in_login": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "remove_edit_in_signup", + "remove_edit_in_login", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3123427997": { + "name": "3123427997", + "value": { + "enable_indexing": false, + "backfill_completed": false, + "enable_local_indexing": false, + "enable_ux": false, + "enable_us_rollout": false, + "enable_ux_rollout": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_indexing", + "backfill_completed", + "enable_local_indexing", + "enable_ux", + "enable_ux_rollout", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3130493514": { + "name": "3130493514", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["is_continue_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3148769572": { + "name": "3148769572", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "HARDCODED", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 0, + "autocomplete_max_char": 0, + "headline_option": "WHERE_TO_START", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2024-09-16T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + "enable_new_autocomplete_homepage", + "headline_option", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3168246095": { + "name": "3168246095", + "value": { "gizmo_ids": [] }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "3218121151": { + "name": "3218121151", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3230069703": { + "name": "3230069703", + "value": { "expirySeconds": 15 }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + }, + "3286693234": { + "name": "3286693234", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + "enable_new_autocomplete_homepage", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3295035639": { + "name": "3295035639", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "include_account_settings_move", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3297250902": { + "name": "3297250902", + "value": { "use_new_query": true }, + "group": "53DF4oF9AvHTpJTxQ5ZLLI:100.00:2", + "rule_id": "53DF4oF9AvHTpJTxQ5ZLLI:100.00:2", + "is_device_based": false, + "secondary_exposures": [], + }, + "3299284219": { + "name": "3299284219", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "show_message_regenerate_model_selector_on_every_message", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3311396813": { + "name": "3311396813", + "value": { + "is_enterprise_enabled": true, + "is_enterprise_desktop_enabled": false, + "is_desktop_enterprise_enabled": false, + }, + "group": "3HX7vpdJsUkuyCUEL4V9cE:override", + "rule_id": "3HX7vpdJsUkuyCUEL4V9cE:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3336092451": { + "name": "3336092451", + "value": { "is_upgrade_button_blue": false }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_upgrade_button_blue", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3337889731": { + "name": "3337889731", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["use_email_otp"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3414159263": { + "name": "3414159263", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_no_auth_welcome_back_modal_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3444737088": { + "name": "3444737088", + "value": { + "is_mobile_web_toggle_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_mobile_web_toggle_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3460472607": { + "name": "3460472607", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_onboarding_flow", + "new_onboarding_flow_qualified_start_date", + "use_plus_rl_during_onboarding", + "personalized_onboarding", + "keep_onboarding_after_dismiss", + "write_custom_instructions_in_onboarding", + "onboarding_show_custom_instructions_page", + "plus_rl_during_onboarding_minutes_after_creation", + "onboarding_show_other_option", + "onboarding_flow_tool_steps", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3514586323": { + "name": "3514586323", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "starter_prompt_ranking_algorithm", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "3591528489": { + "name": "3591528489", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_login_primary_button", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3600059992": { + "name": "3600059992", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["use_email_otp"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3623422562": { + "name": "3623422562", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "enable_new_homepage_anon", + "filter_prompt_by_model", + "headline_option", + "disclaimer_color_adjust", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3656846979": { + "name": "3656846979", + "value": {}, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "3701102259": { + "name": "3701102259", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": ["is_two_line"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3758098899": { + "name": "3758098899", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": ["is_team_enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3762501946": { + "name": "3762501946", + "value": { "disable-ssr": false }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["disable-ssr"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3814631162": { + "name": "3814631162", + "value": { "is_shimmer_enabled": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "is_user_in_experiment": false, + "is_experiment_active": false, + }, + "3851747685": { + "name": "3851747685", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "one_announcement_tooltip_per_session", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3917128856": { + "name": "3917128856", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_dynamic_model_enabled", + "show_message_regenerate_model_selector", + "is_conversation_model_switching_allowed", + "show_rate_limit_downgrade_banner", + "config", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3947925598": { + "name": "3947925598", + "value": { "is_upgrade_in_settings": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_upgrade_in_settings", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3947925629": { + "name": "3947925629", + "value": { "is_upgrade_in_settings": true }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_upgrade_in_settings", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3972089454": { + "name": "3972089454", + "value": { + "search_scoring_dyconfig_name": + "gizmo_search_score_config", + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "search_scoring_dyconfig_name", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "3983984123": { + "name": "3983984123", + "value": { "is_memory_undo_enabled": false }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "is_memory_undo_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4108016587": { + "name": "4108016587", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [ + "is_continue_enabled", + "is_direct_continue_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + "4198218833": { + "name": "4198218833", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "remove_early_access_upsell", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4211831761": { + "name": "4211831761", + "value": { "enabled": false }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": ["enabled"], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4252260557": { + "name": "4252260557", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_v2_toggle_labels_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4252260612": { + "name": "4252260612", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "abandoned", + "rule_id": "abandoned", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_v2_toggle_labels_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4252261542": { + "name": "4252261542", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1395530911", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [ + "is_v2_toggle_labels_enabled", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4268016031": { + "name": "4268016031", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "launchedGroup", + "rule_id": "launchedGroup", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "optimize_initial_modals", + "defer_memory_modal", + ], + "is_user_in_experiment": false, + "is_experiment_active": false, + "is_in_layer": true, + }, + "4278549316": { + "name": "4278549316", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "layerAssignment", + "rule_id": "layerAssignment", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [ + "autocomplete_mode", + "autocomplete_min_char", + "autocomplete_max_char", + "autocomplete_qualified_start_date", + ], + "is_user_in_experiment": false, + "is_experiment_active": true, + "is_in_layer": true, + }, + }, + "layer_configs": { + "51287004": { + "name": "51287004", + "value": { "enable": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "229662723": { + "name": "229662723", + "value": {}, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "387752763": { + "name": "387752763", + "value": { + "enable_slash_commands": true, + "enable_rich_text_composer": true, + }, + "group": "5UE8g4T56yxUBUYancL7KB:override", + "rule_id": "5UE8g4T56yxUBUYancL7KB:override", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + "explicit_parameters": [ + "enable_rich_text_composer", + "enable_slash_commands", + ], + "allocated_experiment_name": "387752763", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "749124420", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "566128514", + "gateValue": "true", + "ruleID": "5hCRKi4Gs5QJkOanmdVvHU:100.00:4", + }], + }, + "468168202": { + "name": "468168202", + "value": { + "is_team_enabled": true, + "is_yearly_plus_subscription_enabled": false, + "is_split_between_personal_and_business_enabled": + true, + "is_modal_fullscreen": true, + "is_v2_toggle_labels_enabled": false, + "is_bw": false, + "is_produce_colors": false, + "is_produce_color_scheme": false, + "is_mobile_web_toggle_enabled": false, + "is_enterprise_enabled": false, + "is_produce_text": false, + }, + "group": "7xj42ZY6EtniGNkumF3vQ3:override", + "rule_id": "7xj42ZY6EtniGNkumF3vQ3:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_split_between_personal_and_business_enabled", + "is_modal_fullscreen", + ], + "allocated_experiment_name": "2745770409", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + "660512088": { + "name": "660512088", + "value": { + "enable_arch_updates": false, + "include_legacy_sidebar_contents": false, + "include_floating_state": false, + "include_share_on_mobile": true, + "include_account_settings_move": true, + "include_scrolling_behavior_update": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "2558701922", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1347376961", + "gateValue": "false", + "ruleID": "", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "2558701922", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3765213438", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "491279851", + "gateValue": "false", + "ruleID": "default", + }, { + "gate": "3011415004", + "gateValue": "false", + "ruleID": "7pUMK6uci7sslAj8bP7VEA", + }, { + "gate": "1347376961", + "gateValue": "false", + "ruleID": "", + }], + }, + "685344542": { + "name": "685344542", + "value": { + "is_mobile_enterprise_enabled": false, + "is_enterprise_enabled": true, + }, + "group": "3INu3qkV6QoN42TYoP3gja:override", + "rule_id": "3INu3qkV6QoN42TYoP3gja:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "allocated_experiment_name": "1388643772", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + "717266490": { + "name": "717266490", + "value": { + "optimize_initial_modals": true, + "defer_memory_modal": true, + "enable_v2_cleanup": true, + "new_onboarding_flow_qualified_start_date": + "2099-11-04T00:00:00Z", + "enable_new_onboarding_flow": false, + "use_plus_rl_during_onboarding": false, + "personalized_onboarding": false, + "keep_onboarding_after_dismiss": false, + "write_custom_instructions_in_onboarding": + false, + "plus_rl_during_onboarding_minutes_after_creation": + 0, + "enable_mobile_app_upsell_banner": false, + "onboarding_show_custom_instructions_page": + false, + "one_tooltip_per_session": false, + "one_announcement_tooltip_per_session": true, + "onboarding_show_other_option": false, + "onboarding_flow_tool_steps": [ + "dalle", + "file_upload", + "canvas", + ], + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "871635014": { + "name": "871635014", + "value": { + "snowflake_composer_entry_point": false, + "use_broad_rate_limit_language": false, + }, + "group": "prestart", + "rule_id": "prestart", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [ + "snowflake_composer_entry_point", + ], + "allocated_experiment_name": "2533035210", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [], + }, + "1238742812": { + "name": "1238742812", + "value": { "should_enable_zh_tw": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1320801051": { + "name": "1320801051", + "value": { + "hide_new_at_workspace_section": false, + "hide_section_new_at_workspace": false, + "gpt_discovery_experiment_enabled": true, + "popular_at_my_workspace_enabled": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1346366956": { + "name": "1346366956", + "value": { + "use_email_otp": false, + "signup_cta_copy": "SIGN_UP", + "login_allow_phone": false, + "signup_allow_phone": false, + "forwardToAuthApi": false, + "use_new_phone_ui": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1358849452": { + "name": "1358849452", + "value": { "disable-ssr": false }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1547743984": { + "name": "1547743984", + "value": { + "should_simplify_modal": false, + "is_simplified_sharing_modal_enabled": false, + "is_social_share_options_enabled": false, + "is_update_shared_links_enabled": false, + "is_discoverability_toggle_enabled": false, + "show_copylink_state_if_no_updates": false, + "is_continue_enabled": false, + "show_share_button_text": true, + "is_meta_improvements_enabled": false, + "show_share_button_inline": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4038001028", + "gateValue": "false", + "ruleID": "7sPHYLu6jyA6IUXgqQQsJW", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "4038001028", + "gateValue": "false", + "ruleID": "7sPHYLu6jyA6IUXgqQQsJW", + }], + }, + "1578749296": { + "name": "1578749296", + "value": { "is_sticky_toggle_off": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1630876919": { + "name": "1630876919", + "value": { + "enable_indexing": false, + "backfill_completed": false, + "enable_local_indexing": false, + "enable_ux": false, + "enable_us_rollout": false, + "enable_ux_rollout": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "2372319800", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "enable_local_indexing", + "backfill_completed", + "enable_ux", + "enable_ux_rollout", + ], + "allocated_experiment_name": "1028722647", + "is_experiment_active": true, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [], + }, + "1696863369": { + "name": "1696863369", + "value": { + "has_sidekick_access": false, + "show_nux_banner": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1938289220", + "gateValue": "false", + "ruleID": "79O8DQPDmTKxnLdAH9loVk", + }, { + "gate": "2033872549", + "gateValue": "false", + "ruleID": "7dScmNU0bu2UQuzCNtva50", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "1938289220", + "gateValue": "false", + "ruleID": "79O8DQPDmTKxnLdAH9loVk", + }, { + "gate": "2033872549", + "gateValue": "false", + "ruleID": "7dScmNU0bu2UQuzCNtva50", + }], + }, + "1846737571": { + "name": "1846737571", + "value": { "is_upgrade_button_blue": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "1914829685": { + "name": "1914829685", + "value": { "forward_to_authapi": false }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "14938527", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "14938527", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2670443078": { + "name": "2670443078", + "value": { "is_gating_fix_enabled": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "2723963139": { + "name": "2723963139", + "value": { + "is_dynamic_model_enabled": false, + "show_message_model_info": false, + "show_message_regenerate_model_selector": + true, + "is_conversation_model_switching_allowed": + true, + "show_rate_limit_downgrade_banner": true, + "config": {}, + "show_message_regenerate_model_selector_on_every_message": + true, + "is_AG8PqS2q_enabled": false, + "is_chive_enabled": false, + "sahara_model_id_override": "", + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "2840731323": { + "name": "2840731323", + "value": { + "is_continue_enabled": true, + "is_direct_continue_enabled": true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [{ + "gate": "1860647109", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "1860647109", + "gateValue": "false", + "ruleID": "default", + }], + }, + "2888142241": { + "name": "2888142241", + "value": { "is_upgrade_in_settings": true }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "2932223118": { + "name": "2932223118", + "value": { + "is_mobile_web_toggle_enabled": true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3048336830": { + "name": "3048336830", + "value": { + "is-enabled": true, + "use-rtl-layout": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3922145230", + "gateValue": "false", + "ruleID": "14DZA2LumaPqAdCo52CrUB", + }, { + "gate": "3700615661", + "gateValue": "false", + "ruleID": "66covmutTYx82FWVUlZAqF", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "3922145230", + "gateValue": "false", + "ruleID": "14DZA2LumaPqAdCo52CrUB", + }, { + "gate": "3700615661", + "gateValue": "false", + "ruleID": "66covmutTYx82FWVUlZAqF", + }], + }, + "3119715334": { + "name": "3119715334", + "value": { "should-enable-hojicha": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3436367576": { + "name": "3436367576", + "value": { + "enable_indexing": false, + "wave": 0, + "enable_ux": false, + "enable_local_indexing": false, + "backfill_completed": false, + }, + "group": "targetingGate", + "rule_id": "targetingGate", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1221279314", + "gateValue": "false", + "ruleID": "default", + }], + "explicit_parameters": [ + "enable_indexing", + "wave", + "backfill_completed", + "enable_ux", + "enable_local_indexing", + ], + "allocated_experiment_name": "938456440", + "is_experiment_active": true, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [], + }, + "3517133692": { + "name": "3517133692", + "value": { "is_memory_undo_enabled": false }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3533083032": { + "name": "3533083032", + "value": { + "enable_new_homepage_anon": true, + "filter_prompt_by_model": true, + "headline_option": "HELP_WITH", + "disclaimer_color_adjust": false, + "show_composer_header": false, + "enable_new_mobile": true, + "enable_cached_response": false, + "show_dalle_starter_prompts": false, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3637408529": { + "name": "3637408529", + "value": { + "is_anon_chat_enabled": true, + "is_anon_chat_enabled_for_new_users_only": + false, + "is_try_it_first_on_login_page_enabled": + false, + "is_no_auth_welcome_modal_enabled": false, + "no_auth_soft_rate_limit": 10, + "no_auth_hard_rate_limit": 1200, + "should_show_no_auth_signup_banner": true, + "is_no_auth_welcome_back_modal_enabled": true, + "is_no_auth_soft_rate_limit_modal_enabled": + true, + "is_no_auth_gpt4o_modal_enabled": false, + "is_login_primary_button": true, + "is_desktop_primary_auth_button_on_right": + false, + "is_primary_btn_blue": false, + "should_show_disclaimer_only_once_per_device": + true, + "is_secondary_banner_button_enabled": false, + "is_secondary_auth_banner_button_enabled": + true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "3238165271", + "gateValue": "false", + "ruleID": "disabled", + }, { + "gate": "2983591614", + "gateValue": "false", + "ruleID": "disabled", + }], + }, + "3768341700": { + "name": "3768341700", + "value": { + "is_produce_text": false, + "remove_early_access_upsell": false, + "is_produce_text_design": false, + "is_produce_design": true, + "is_country_selector_enabled": false, + }, + "group": "2l2l6SrM9hXnjCjkr95Ibl:override", + "rule_id": "2l2l6SrM9hXnjCjkr95Ibl:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": ["is_produce_design"], + "allocated_experiment_name": "2818189709", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + "3927927759": { + "name": "3927927759", + "value": { + "enable_mobile_app_upsell_banner": true, + }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "3972089454": { + "name": "3972089454", + "value": { + "search_scoring_dyconfig_name": + "gizmo_search_score_config", + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "4031588851": { + "name": "4031588851", + "value": { + "enable_hardcoded_vision_prompts": true, + "enable_hardcoded_file_document_prompts": + true, + "enable_hardcoded_data_vis_prompts": true, + "enable_hardcoded_browse_prompts": true, + "is_two_line": false, + "enable_new_homepage": false, + "autocomplete_mode": "INDEX", + "starter_prompt_ranking_algorithm": + "homepage_v2", + "autocomplete_min_char": 4, + "autocomplete_max_char": 32, + "headline_option": "HELP_WITH", + "use_memory_in_model_autocomplete": false, + "filter_starter_prompt_by_model": false, + "use_modapi_in_autocomplete": false, + "autocomplete_qualified_start_date": + "2000-10-11T00:00:00Z", + "enable_new_autocomplete_homepage": true, + "model_talks_option": "", + "enable_new_mobile": true, + "search_autocomplete_mode": "BING", + "enable_hardcoded_onboarding_prompt": false, + "autocomplete_fetch_interval": 200, + "num_completions_to_fetch_from_index": 8, + }, + "group": "default", + "rule_id": "default", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + "explicit_parameters": [], + "undelegated_secondary_exposures": [{ + "gate": "4273941502", + "gateValue": "false", + "ruleID": "1nGrz4l6GM0LgZvm0pDCtp:2.00:1", + }], + }, + "4211831761": { + "name": "4211831761", + "value": { "enabled": false }, + "group": "default", + "rule_id": "default", + "is_device_based": true, + "secondary_exposures": [], + "explicit_parameters": [], + "undelegated_secondary_exposures": [], + }, + "4250072504": { + "name": "4250072504", + "value": { + "is_enterprise_enabled": true, + "is_enterprise_desktop_enabled": false, + "is_desktop_enterprise_enabled": false, + }, + "group": "3HX7vpdJsUkuyCUEL4V9cE:override", + "rule_id": "3HX7vpdJsUkuyCUEL4V9cE:override", + "is_device_based": false, + "secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + "explicit_parameters": [ + "is_enterprise_enabled", + ], + "allocated_experiment_name": "3311396813", + "is_experiment_active": false, + "is_user_in_experiment": false, + "undelegated_secondary_exposures": [{ + "gate": "1028682714", + "gateValue": "true", + "ruleID": "735n03snBvba4AEhd2Qwqu:100.00:3", + }], + }, + }, + "sdkParams": {}, + "has_updates": true, + "generator": "statsig-node-sdk", + "sdkInfo": { + "sdkType": "statsig-node", + "sdkVersion": "5.26.0", + }, + "time": 1733904874794, + "evaluated_keys": { + "customIDs": { + "WebAnonymousCookieID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "DeviceId": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "stableID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + }, + }, + "hash_used": "djb2", + "user": { + "country": "CA", + "custom": { + "has_logged_in_before": false, + "user_agent": "Wget/1.25.0", + "is_punch_out_user": false, + "auth_status": "logged_out", + }, + "customIDs": { + "WebAnonymousCookieID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "DeviceId": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "stableID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + }, + "ip": "2001:df0:a280:1001::6:1", + "userAgent": "Wget/1.25.0", + "statsigEnvironment": { "tier": "production" }, + }, + }, + "user": { + "country": "CA", + "custom": { + "has_logged_in_before": false, + "user_agent": "Wget/1.25.0", + "is_punch_out_user": false, + "auth_status": "logged_out", + }, + "customIDs": { + "WebAnonymousCookieID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "DeviceId": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + "stableID": + "d76fbeb9-aa53-4dcb-b666-8358452854be", + }, + "ip": "2001:df0:a280:1001::6:1", + "privateAttributes": {}, + "userAgent": "Wget/1.25.0", + }, + }, + "accountsStatus": null, + }, + "meta": { + "title": "Git as Supply Chain", + "description": "Shared via ChatGPT", + "imageSrc": + "https://cdn.oaistatic.com/assets/chatgpt-share-og-u7j5uyao.webp", + "isIndexable": false, + "isBetterMetatagsEnabled": true, + }, + }, + }, + "actionData": null, + "errors": null, + }, + }; + __remixContext.p = function (v, e, p, x) { + if (typeof e !== "undefined") { + x = new Error("Unexpected Server Error"); + x.stack = undefined; + p = Promise.reject(x); + } else { + p = Promise.resolve(v); + } + return p; + }; + __remixContext.n = function (i, k) { + __remixContext.t = __remixContext.t || {}; + __remixContext.t[i] = __remixContext.t[i] || {}; + let p = new Promise((r, e) => { + __remixContext.t[i][k] = { + r: (v) => { + r(v); + }, + e: (v) => { + e(v); + }, + }; + }); + setTimeout(() => { + if ( + typeof p._error !== "undefined" || + typeof p._data !== "undefined" + ) return; + __remixContext.t[i][k].e(new Error("Server timeout.")); + }, 5000); + return p; + }; + __remixContext.r = function (i, k, v, e, p, x) { + p = __remixContext.t[i][k]; + if (typeof e !== "undefined") { + x = new Error("Unexpected Server Error"); + x.stack = undefined; + p.e(x); + } else { + p.r(v); + } + }; + Object.assign(__remixContext.state.loaderData["root"], { + 'rq:["account-status"]': __remixContext.p({ + "__type": "AccountState", + "accountItems": [{ + "data": { + "id": null, + "residencyRegion": "no_constraint", + "accountUserId": null, + "name": null, + "profilePictureId": null, + "profilePictureUrl": null, + "structure": "personal", + "role": "account-owner", + "organizationId": null, + "promoData": {}, + "deactivated": false, + "subscriptionStatus": { + "billingPeriod": null, + "hasPaidSubscription": false, + "isActiveSubscriptionGratis": false, + "subscriptionPlan": "chatgptfreeplan", + "planType": "free", + "wasPaidCustomer": false, + "hasCustomerObject": false, + "lastActiveSubscription": { + "subscription_id": null, + "purchase_origin_platform": "chatgpt_not_purchased", + "will_renew": false, + }, + "isResellerHosted": false, + }, + "features": [ + "bizmo_settings", + "breeze_available", + "canvas", + "canvas_code_execution", + "canvas_opt_in", + "chat_preferences_available", + "chatgpt_ios_attest", + "chatgpt_team_plan", + "d3_controls", + "d3_editor_gpts", + "gizmo_canvas_toggle", + "gizmo_support_emails", + "mfa", + "model_ab_use_v2", + "new_plugin_oauth_endpoint", + "no_auth_training_enabled_by_default", + "paragen_mainline_alternative", + "privacy_policy_nov_2023", + "sentinel_enabled_for_subscription", + "share_multimodal_links", + "shareable_links", + "starter_prompts", + "user_settings_announcements", + "voice_advanced_ga", + ], + "canAccessWithCurrentSession": true, + "ssoConnectionName": null, + }, + }], + "currentAccountId": null, + }), + }); + </script> + <script nonce="52bc4634-5139-448c-b73b-89e4eef4691c" type="module" async=""> + import "https://cdn.oaistatic.com/assets/manifest-7d43a138.js"; + import * as route0 from "https://cdn.oaistatic.com/assets/ihbvqeoqhbkuefft.js"; + import * as route1 from "https://cdn.oaistatic.com/assets/e8aqfi7hhkvyz94l.js"; + + window.__remixRouteModules = { + "root": route0, + "routes/share.$shareId.($action)": route1, + }; + + import("https://cdn.oaistatic.com/assets/mxkyxjre6muko6z4.js"); + </script> + <script> + (function () { + function c() { + var b = a.contentDocument || a.contentWindow.document; + if (b) { + var d = b.createElement("script"); + d.innerHTML = + "window.__CF$cv$params={r:'8f04137a8c19cdfe',t:'MTczMzkwNTQ0NC4wMDAwMDA='};var a=document.createElement('script');a.nonce='';a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);"; + b.getElementsByTagName("head")[0].appendChild(d); + } + } + if (document.body) { + var a = document.createElement("iframe"); + a.height = 1; + a.width = 1; + a.style.position = "absolute"; + a.style.top = 0; + a.style.left = 0; + a.style.border = "none"; + a.style.visibility = "hidden"; + document.body.appendChild(a); + if ("loading" !== document.readyState) c(); + else if (window.addEventListener) { + document.addEventListener("DOMContentLoaded", c); + } else { + var e = document.onreadystatechange || function () {}; + document.onreadystatechange = function (b) { + e(b); + "loading" !== document.readyState && + (document.onreadystatechange = e, c()); + }; + } + } + })(); + </script> + </body> +</html> diff --git a/utils/chat/extract.test.ts b/utils/chat/extract.test.ts new file mode 100644 index 000000000000..724129643d11 --- /dev/null +++ b/utils/chat/extract.test.ts @@ -0,0 +1,34 @@ +import { expect } from '@std/expect/expect' +import { extract } from './extract.ts' + +const start = `# Conversation + +### Message ID: 2ae52d22-dba5-457d-8147-faade76900dd +**Author**: system + +**Content:** + + + +---` + +const id = '6758f810-286c-800b-8e2c-490e634ca002' +Deno.test('extractConversation from example.html', async () => { + const html = await Deno.readTextFile('./example.html') + const [creationTime, conversation] = await extract(html, id) + expect(conversation.startsWith(start)).toBeTruthy() + expect(creationTime.startsWith('2024-12-11')).toBeTruthy() +}) + +Deno.test('extractConversation handles empty HTML', async () => { + await expect(extract('', id)).rejects.toThrow( + 'Could not find a <script> tag', + ) +}) + +Deno.test('extractConversation handles HTML with no conversation', async () => { + const html = '<html><body><p>No conversation here</p></body></html>' + await expect(extract(html, id)).rejects.toThrow( + 'Could not find a <script> tag', + ) +}) diff --git a/utils/chat/extract.ts b/utils/chat/extract.ts new file mode 100644 index 000000000000..96a313948072 --- /dev/null +++ b/utils/chat/extract.ts @@ -0,0 +1,222 @@ +import { parse } from '@babel/parser' +import { DOMParser, initParser } from '@b-fuze/deno-dom/wasm-noinit' +import * as t from '@babel/types' + +interface ConversationMessage { + id: string + authorRole?: string + content?: string +} + +interface LinearConvNode { + id: string + message?: { + content?: { + parts?: string[] + } + author?: { + role?: string + } + } +} + +export const extract = async ( + htmlContent: string, + sharedConversationId: string, +): Promise<[string, string]> => { + await initParser() + const parser = new DOMParser() + const doc = parser.parseFromString(htmlContent, 'text/html') + if (!doc) { + throw new Error('Could not parse HTML as DOM.') + } + + let targetScriptContent: string | null = null + const scripts = doc.querySelectorAll('script') + for (const script of scripts) { + const text = script.textContent + if (text && text.includes(sharedConversationId)) { + targetScriptContent = text + break + } + } + + if (!targetScriptContent) { + throw new Error( + `Could not find a <script> tag containing sharedConversationId ${sharedConversationId}.`, + ) + } + + const ast = parse(targetScriptContent, { + sourceType: 'script', + plugins: ['jsx', 'typescript', 'objectRestSpread'], + }) + + function evaluateObjectExpression( + node: t.ObjectExpression, + ): Record<string, unknown> { + const obj: Record<string, unknown> = {} + for (const prop of node.properties) { + if (t.isObjectProperty(prop)) { + let key: string | undefined + if (t.isIdentifier(prop.key)) { + key = prop.key.name + } else if (t.isStringLiteral(prop.key)) { + key = prop.key.value + } + + if (key) { + let value: unknown + if (t.isObjectExpression(prop.value)) { + value = evaluateObjectExpression(prop.value) + } else if (t.isArrayExpression(prop.value)) { + value = evaluateArrayExpression(prop.value) + } else if (t.isStringLiteral(prop.value)) { + value = prop.value.value + } else if (t.isNumericLiteral(prop.value)) { + value = prop.value.value + } else if (t.isBooleanLiteral(prop.value)) { + value = prop.value.value + } else if (t.isNullLiteral(prop.value)) { + value = null + } else { + value = undefined + } + obj[key] = value + } + } + } + return obj + } + + function evaluateArrayExpression(node: t.ArrayExpression): unknown[] { + return node.elements.map((elem) => { + if (!elem) return undefined + if (t.isObjectExpression(elem)) { + return evaluateObjectExpression(elem) + } else if (t.isArrayExpression(elem)) { + return evaluateArrayExpression(elem) + } else if (t.isStringLiteral(elem)) { + return elem.value + } else if (t.isNumericLiteral(elem)) { + return elem.value + } else if (t.isBooleanLiteral(elem)) { + return elem.value + } else if (t.isNullLiteral(elem)) { + return null + } else { + return undefined + } + }) + } + + let remixContext: Record<string, unknown> | null = null + for (const statement of ast.program.body) { + if ( + t.isExpressionStatement(statement) && + t.isAssignmentExpression(statement.expression) + ) { + const expr = statement.expression + if ( + t.isMemberExpression(expr.left) && + t.isIdentifier(expr.left.object, { name: 'window' }) && + t.isIdentifier(expr.left.property, { name: '__remixContext' }) && + t.isObjectExpression(expr.right) + ) { + remixContext = evaluateObjectExpression(expr.right) + break + } + } + } + + if (!remixContext) { + throw new Error('Could not find window.__remixContext in the script.') + } + + const loaderData = (remixContext.state as Record<string, unknown>) + ?.loaderData as Record<string, unknown> + if (!loaderData) { + throw new Error('Could not find loaderData in __remixContext.') + } + + const routeDataKey = Object.keys(loaderData).find((k) => + k.includes('routes/share.$shareId.($action)') + ) + if (!routeDataKey) { + throw new Error( + 'Could not find route data key containing share.$shareId.($action).', + ) + } + + const routeData = loaderData[routeDataKey] as Record<string, unknown> + const serverResponse = routeData?.serverResponse as Record<string, unknown> + if (!serverResponse || !serverResponse.data) { + throw new Error( + 'Could not find serverResponse.data in the extracted object.', + ) + } + + const data = serverResponse.data as Record<string, unknown> + if (data.conversation_id !== sharedConversationId) { + throw new Error('sharedConversationId does not match expected value.') + } + + const linearConversation = data.linear_conversation as unknown[] + if (!Array.isArray(linearConversation)) { + throw new Error('linear_conversation not found or is not an array.') + } + + function extractConversationMessages( + linearConv: unknown[], + ): ConversationMessage[] { + const messages: ConversationMessage[] = [] + for (const node of linearConv) { + const n = node as LinearConvNode + if ( + n.message && + n.message.content && + Array.isArray(n.message.content.parts) + ) { + const content = n.message.content.parts.join('\n') + const authorRole = n.message.author?.role + if (!authorRole) { + throw new Error('Author role not found.') + } + messages.push({ + id: n.id, + authorRole, + content, + }) + } + } + return messages + } + + const conversationMessages = extractConversationMessages(linearConversation) + + // Convert create_time to ISO 8601 + let creationTime = '' + if (typeof data.create_time === 'number') { + creationTime = new Date(data.create_time * 1000).toISOString() + } else if (typeof data.create_time === 'string') { + const parsed = parseFloat(data.create_time) + if (!isNaN(parsed)) { + creationTime = new Date(parsed * 1000).toISOString() + } + } + if (!creationTime) { + // fallback: just return the raw value if conversion fails + creationTime = String(data.create_time) + } + + // Markdown formatting + let output = + `# Conversation\nURL: https://chatgpt.com/share/${sharedConversationId}\n\n` + for (const msg of conversationMessages) { + output += `### Message ID: ${msg.id}\n` + output += `**Author**: ${msg.authorRole}\n\n` + output += `**Content:**\n\n${msg.content}\n\n---\n\n` + } + + return [creationTime, output.trim()] +} diff --git a/utils/chat/issues.md b/utils/chat/issues.md new file mode 100644 index 000000000000..a84b9396b8a6 --- /dev/null +++ b/utils/chat/issues.md @@ -0,0 +1,12 @@ +how would it know if the format was wrong, or if the mappings of the chats did +not match up ? + +having forked conversations should be handled by presenting different versions +of the conversation in different files. + +should handle the url being a little bit messy, like extra spaces or a trailing +slash. + +in chats, concat output used in context should be stripped ? + +help is not in the readme and does not output anything diff --git a/utils/chat/mod.ts b/utils/chat/mod.ts new file mode 100644 index 000000000000..9083273610d4 --- /dev/null +++ b/utils/chat/mod.ts @@ -0,0 +1,35 @@ +import { extract } from './extract.ts' + +export const main = async (url: string, output?: string): Promise<void> => { + const sharedConversationId = url.split('/').pop() + if (!sharedConversationId) { + throw new Error( + 'Could not determine sharedConversationId from the provided URL.', + ) + } + + const resp = await fetch(url) + if (!resp.ok) { + throw new Error(`Failed to fetch URL: ${url}, status: ${resp.status}`) + } + + const html = await resp.text() + const [creationTime, conversationMarkdown] = await extract( + html, + sharedConversationId, + ) + + // creationTime is ISO 8601; we only need YYYY-MM-DD + const date = creationTime.split('T')[0] + // shorten the guid (e.g. last 7 chars) + const shortGuid = sharedConversationId.slice(-7) + + if (!output) { + output = `chat_${date}_${shortGuid}.md` + } + + await Deno.writeTextFile(output, conversationMarkdown) + + // deno-lint-ignore no-console + console.log(`Conversation saved to ${output}`) +} diff --git a/utils/chat/short-example.html b/utils/chat/short-example.html new file mode 100644 index 000000000000..a29acb43ec97 --- /dev/null +++ b/utils/chat/short-example.html @@ -0,0 +1,855 @@ +<!DOCTYPE html> +<html + data-build="prod-11173c28d7974347784902fe1d57aae3a20e661b" + dir="ltr" + class="" +> + <head> + <meta charSet="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <link + rel="stylesheet" + href="https://cdn.oaistatic.com/assets/conversation-small-kq10986g.css" + /> + </head> + <body class=""> + <script> + !function () { + try { + var d = document.documentElement, c = d.classList + c.remove('light', 'dark') + var e = localStorage.getItem('theme') + if ('system' === e || (!e && true)) { + var t = '(prefers-color-scheme: dark)', + m = window.matchMedia(t) + if (m.media !== t || m.matches) { + d.style.colorScheme = 'dark' + c.add('dark') + } else { + d.style.colorScheme = 'light' + c.add('light') + } + } else if (e) c.add(e || '') + if (e === 'light' || e === 'dark') d.style.colorScheme = e + } catch (e) {} + }() + </script> + <div + class="relative flex h-full w-full overflow-hidden transition-colors z-0" + > + </div> + <div aria-live="assertive" aria-atomic="true" class="sr-only"></div> + <div aria-live="polite" aria-atomic="true" class="sr-only"></div> + <audio + class="fixed bottom-0 left-0 hidden h-0 w-0" + autoPlay="" + crossorigin="anonymous" + > + </audio> + <script nonce="52bc4634-5139-448c-b73b-89e4eef4691c"> + window.__remixContext = { + 'basename': '/', + 'future': { + 'v3_fetcherPersist': false, + 'v3_relativeSplatPath': false, + 'v3_throwAbortReason': false, + 'unstable_singleFetch': false, + 'unstable_lazyRouteDiscovery': false, + 'unstable_optimizeDeps': true, + }, + 'isSpaMode': false, + 'state': { + 'loaderData': { + 'root': { + 'rq:["session"]': { + 'data': null, + 'dataUpdateCount': 1, + 'dataUpdatedAt': 1733905443121, + 'error': null, + 'errorUpdateCount': 0, + 'errorUpdatedAt': 0, + 'fetchFailureCount': 0, + 'fetchFailureReason': null, + 'fetchMeta': null, + 'isInvalidated': false, + 'status': 'success', + 'fetchStatus': 'idle', + }, + 'rq:["account-status"]': {}, + 'clientBootstrap': { + 'session': {}, + 'user': { + 'email': '', + 'groups': [], + 'id': 'ua-d76fbeb9-aa53-4dcb-b666-8358452854be', + 'name': '', + 'intercom_hash': + 'ua-d76fbeb9-aa53-4dcb-b666-8358452854be', + 'image': '', + 'picture': '', + 'mfa': false, + }, + 'cluster': 'unified-21', + 'userCountry': 'CA', + 'userRegion': null, + 'userRegionCode': null, + 'cfConnectingIp': '2001:df0:a280:1001::6:1', + 'cfIpLatitude': '43.63190', + 'cfIpLongitude': '-79.37160', + 'cfTimezone': 'America/Toronto', + 'cfIpCity': null, + 'userLocale': null, + 'authStatus': 'logged_out', + 'statsig': { + 'user': { + 'country': 'CA', + 'custom': { + 'has_logged_in_before': false, + 'user_agent': 'Wget/1.25.0', + 'is_punch_out_user': false, + 'auth_status': 'logged_out', + }, + 'customIDs': { + 'WebAnonymousCookieID': + 'd76fbeb9-aa53-4dcb-b666-8358452854be', + 'DeviceId': + 'd76fbeb9-aa53-4dcb-b666-8358452854be', + 'stableID': + 'd76fbeb9-aa53-4dcb-b666-8358452854be', + }, + 'ip': '2001:df0:a280:1001::6:1', + 'privateAttributes': {}, + 'userAgent': 'Wget/1.25.0', + }, + }, + 'accountsStatus': null, + }, + 'isIos': false, + 'isAndroidChrome': false, + 'isElectron': false, + 'cspScriptNonce': '52bc4634-5139-448c-b73b-89e4eef4691c', + }, + 'routes/share.$shareId.($action)': { + 'sharedConversationId': + '6758f810-286c-800b-8e2c-490e634ca002', + 'serverResponse': { + 'type': 'data', + 'data': { + 'title': 'Git as Supply Chain', + 'create_time': 1733883920.159243, + 'update_time': 1733905184.927294, + 'mapping': { + 'aaa1615c-1f40-439e-8b5d-a779ee9a1192': { + 'id': 'aaa1615c-1f40-439e-8b5d-a779ee9a1192', + 'children': [ + '2ae52d22-dba5-457d-8147-faade76900dd', + ], + }, + '2ae52d22-dba5-457d-8147-faade76900dd': { + 'id': '2ae52d22-dba5-457d-8147-faade76900dd', + 'message': { + 'id': '2ae52d22-dba5-457d-8147-faade76900dd', + 'author': { 'role': 'system', 'metadata': {} }, + 'content': { + 'content_type': 'text', + 'parts': [''], + }, + 'status': 'finished_successfully', + 'end_turn': true, + 'weight': 0, + 'metadata': { + 'is_visually_hidden_from_conversation': true, + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaa1615c-1f40-439e-8b5d-a779ee9a1192', + 'children': [ + 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + ], + }, + 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074': { + 'id': 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'message': { + 'id': 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'author': { 'role': 'user', 'metadata': {} }, + 'create_time': 1733880310.052759, + 'content': { + 'content_type': 'text', + 'parts': [ + "here's a chat with my friend. what concepts might we be trying to hint at ? we need clarity of thought since we are poking around at things we don't fully understand. Give us a few options. Clarify our thinking. Be creative. Be bold. Be profound.\n\n", + ], + }, + 'status': 'finished_successfully', + 'weight': 1, + 'metadata': { + 'request_id': '8f01adde8942d9be-AKL', + 'message_source': null, + 'timestamp_': 'absolute', + 'message_type': null, + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': '2ae52d22-dba5-457d-8147-faade76900dd', + 'children': [ + 'e9410778-39b9-464a-a0f5-cc461cb28e85', + ], + }, + 'e9410778-39b9-464a-a0f5-cc461cb28e85': { + 'id': 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'message': { + 'id': 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'author': { + 'role': 'tool', + 'name': 'a8km123', + 'metadata': {}, + }, + 'create_time': 1733880397.691115, + 'content': { + 'content_type': 'text', + 'parts': [ + '**Exploring AI integrations**\n\nI’m examining how Git, npm modules, software supply chain, AI, and blockchains intersect, focusing on knowledge, reproducible model training, and traceability in AI development.', + ], + }, + 'status': 'finished_successfully', + 'weight': 0, + 'metadata': { + 'finished_text': 'Thought for 1m 26s', + 'initial_text': 'Thinking', + 'summarization_headline': + 'Git, Blockchain, and AI Integration', + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'request_id': '8f01adde8942d9be-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'children': [ + '46cc8de2-2860-4c13-9083-32117c314a3a', + ], + }, + '46cc8de2-2860-4c13-9083-32117c314a3a': { + 'id': '46cc8de2-2860-4c13-9083-32117c314a3a', + 'message': { + 'id': '46cc8de2-2860-4c13-9083-32117c314a3a', + 'author': { + 'role': 'assistant', + 'metadata': {}, + }, + 'create_time': 1733880397.691219, + 'content': { + 'content_type': 'text', + 'parts': [ + 'Below are a few conceptual frameworks and themes that might help clarify the directions you’re exploring. These are just options—bold thought experiments and profound re-interpretations of what you’ve discussed. Each tries to distill the underlying "essence" of what you\'re circling around:\n\n1. **Git as a Cognitive Substrate** \n *Concept*: Treat Git not just as a version-control system, but as a universal medium for knowledge representation—a canonical substrate into which intelligence can “think” its outputs directly. \n *Idea*: Instead of thinking of Git as infrastructure hiding behind the scenes, imagine it as the actual “mindspace” of an AI. Commits become “thoughts,” branches become “possible worlds,” and merges become negotiated alignments of reality. This would mean that interacting with systems wouldn’t require separately building APIs or wrapping logic in adapters; the “thought process” would already be in the universal language of code and data tracked by Git.\n\n2. **Provenance-Aware Execution and Training** \n *Concept*: Embrace full provenance tracking of models—every piece of training data, every parameter update, every intermediate artifact—being anchored in a Git-like or blockchain-like structure. \n *Idea*: On-chain (or on-git) training ensures that models, their lineage, and their transformations are fully transparent and repeatable. This is “Proof of Useful Work” done right: no contrived token games, just a history of refinement. Imagine a Hugging Face that merges seamlessly with GitHub: training datasets as commits, parameter updates as commits, final models as tagged releases. The model’s “mind” isn’t some opaque black box, but an ever-growing, fully versioned garden of knowledge and code.\n\n3. **Frictionless Integration With the Wider Software Supply Chain** \n *Concept*: The AI doesn’t produce code that must then be adapted or compiled into external ecosystems; it directly “commits” code into existing supply chains (e.g., npm modules) as a natural extension of its reasoning. \n *Idea*: By removing the “build step” or “adapter step” entirely, the AI’s output is not something that must be transformed—its output is directly usable. Code changes flow seamlessly into the world’s largest ecosystems of shared code, enabling the AI not only to contribute but also to continuously learn from incoming commits by humans and other agents.\n\n4. **Time, Context, and Political Awareness Through Commit Histories** \n *Concept*: Commit histories are fundamentally stories about human and machine collaboration, negotiation, conflict, and resolution. They are a record of political dynamics, group decisions, priorities shifting over time. \n *Idea*: By letting the AI “live” in this world of commits, it can learn to interpret the subtle signals encoded in version histories—understanding governance, consequence, and social context. Training on commit histories gives AI a vantage point to reason about time, intention, and human organizational structures. In other words, by modeling itself as a participant in Git’s political tapestry, the AI begins to understand what it means to cooperate, to iterate, to govern.\n\n5. **Merging Knowledge Networks: From Fragmented Tools to a Single Universal Medium** \n *Concept*: Git, package managers, blockchains, AI models—these have historically been disparate systems, each with its own protocols and friction points. \n *Idea*: Your conversation hints at merging these concepts into a unified environment: a single substrate (like Git) where data, code, models, and their entire genealogy live side by side. Instead of separate infrastructures for AI training, code versioning, supply chain management, and trust enforcement, you get one substrate that can host them all. This reduces conceptual overhead and unleashes a new kind of “global system,” where knowledge flows freely and every increment of intelligence is recorded and trusted.\n\n---\n\nAll of these concepts circle around the core theme: using version-controlled code repositories, possibly combined with blockchain-style invariants, to eliminate friction, ensure trust, enable provenance, and integrate AI deeply into the world’s software and knowledge ecosystems—no artificial boundaries, no hidden build steps, no murky provenances. Instead, a continuous, transparent, and universal environment of incremental, queryable, and trustworthy “thought commits.”', + ], + }, + 'status': 'finished_successfully', + 'end_turn': true, + 'weight': 1, + 'metadata': { + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'request_id': '8f01adde8942d9be-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'children': [ + 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + ], + }, + 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3': { + 'id': 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'message': { + 'id': 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'author': { 'role': 'user', 'metadata': {} }, + 'create_time': 1733881250.145666, + 'content': { + 'content_type': 'text', + 'parts': [ + 'if that conversation was being turned into a set of comparisons to make against competing blockchains, name what concepts are missing from the below comparison, and say how you might improve this comparator:\n\n1. Token dependency:\n \n - Does the chain have a built-in token?\n - How tightly coupled is that token to the consensus?\n - How easy is it to remove, replace, or adjust that token?\n - Is the consensus mechanism itself token-agnostic or inherently token-bound?\n - is the token stable or fluctuatable\n\n1. Integration format:\n \n - Can the chain’s data and consensus be naturally represented in Git commits or NPM packages?\n - Are translations or bridging layers required to interface with standard tools, and if so, do these weaken the consensus?\n3. Portability:\n \n - Can the system run in isolated parts, completely on your own infrastructure?\n - Can these isolated instances later merge consensus without loss of trust?\n\nFor the demonstration of a sharded or “splintered” proof-of-work system, consider calling it “Shattered Proof-of-Work” (S-PoW). This name reflects how the workload can be broken into isolated segments and later merged based on verifiable computational work.', + ], + }, + 'status': 'finished_successfully', + 'weight': 1, + 'metadata': { + 'serialization_metadata': { + 'custom_symbol_offsets': [], + }, + 'request_id': '8f01c4d23b851c5e-AKL', + 'message_source': null, + 'timestamp_': 'absolute', + 'message_type': null, + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': '46cc8de2-2860-4c13-9083-32117c314a3a', + 'children': [ + 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + ], + }, + 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6': { + 'id': 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'message': { + 'id': 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'author': { + 'role': 'tool', + 'name': 'a8km123', + 'metadata': {}, + }, + 'create_time': 1733881301.561146, + 'content': { + 'content_type': 'text', + 'parts': [ + '**Gathering blockchain insights**\n\nI\'m compiling info on blockchain developments, pinpointing gaps like "Community involvement" and "Transparency and trust." Also, looking at concepts such as "Provenance and lineage" and "Natural language promptability," considering how they can enrich the comparison.', + ], + }, + 'status': 'finished_successfully', + 'weight': 0, + 'metadata': { + 'finished_text': 'Thought for 50 seconds', + 'initial_text': 'Thinking', + 'summarization_headline': + 'Blockchain Comparison Concepts', + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'request_id': '8f01c4d23b851c5e-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'children': [ + 'c132e044-dade-4946-95a7-b1c942539af4', + ], + }, + 'c132e044-dade-4946-95a7-b1c942539af4': { + 'id': 'c132e044-dade-4946-95a7-b1c942539af4', + 'message': { + 'id': 'c132e044-dade-4946-95a7-b1c942539af4', + 'author': { + 'role': 'assistant', + 'metadata': {}, + }, + 'create_time': 1733881301.56126, + 'content': { + 'content_type': 'text', + 'parts': [ + '**Missing Concepts and How to Improve the Comparison:**\n\n1. **Provenance and Lineage of Data/Models:** \n **What’s missing:** \n - How does the chain handle the version history of data, code, or model parameters? \n - Can it track the full lineage of a model’s training process or a software artifact’s evolution? \n **Improvement:** \n Add criteria examining whether each system can embed transparent, verifiable version histories—like Git commit graphs—directly in its consensus. For example: \n - *“Can the chain immutably record the origin and version history of data and code as a first-class citizen?”* \n - *“Does it allow for ‘on-chain training’ records or a taxonomy of artifacts that mirror Git’s branching, merging, and tagging semantics?”*\n\n2. **Deterministic Computation and Reproducibility:** \n **What’s missing:** \n - Are computations (e.g., model training, code transformations) guaranteed to be deterministic and reproducible from on-chain data? \n - Can the entire state (including the model’s parameters) be reconstructed from on-chain commits? \n **Improvement:** \n Add criteria around determinism and replayability: \n - *“Is the chain’s consensus logic and stored data sufficient to deterministically reproduce any result, such as model parameters after training?”* \n - *“Does the system enable ‘proof-of-useful-work’ where the work is fully re-executable and verifiable from the recorded state?”*\n\n3. **Interoperability With Existing Ecosystems (Beyond Git \u0026 NPM):** \n **What’s missing:** \n - Beyond just Git commits and npm packages, how well does the system integrate with other major ecosystems (e.g., Docker images, PyPI packages, Hugging Face model hubs)? \n **Improvement:** \n Add criteria that measure the breadth of ecosystem compatibility: \n - *“Can the chain’s artifacts be directly consumed by standard developer tools, CI/CD pipelines, model registries, or container registries without translation?”* \n - *“Is the system extensible to support new tooling integrations as standards evolve?”*\n\n4. **Governance and Social Dynamics:** \n **What’s missing:** \n - How does the chain model governance over code and data? \n - Can it represent the political dimension of commits—who proposed changes, who merged them, and the rationale behind decisions? \n **Improvement:** \n Add criteria related to governance and social aspects: \n - *“Does the chain encode and preserve metadata of contributors, allowing AIs and humans to understand political, social, and organizational contexts of commits?”* \n - *“Can we trace not just the code but also the discussions, proposals, and decision-making processes that led to a given state?”*\n\n5. **Economic Neutrality and Flexible Incentive Structures:** \n **What’s missing:** \n - If the chain employs incentives, can those be non-financial or proof-of-actual-work rather than bound to a volatile token economy? \n **Improvement:** \n Add criteria around incentive flexibility and neutrality: \n - *“Are consensus and network security achievable without relying on a native token’s economic incentives?”* \n - *“Is it possible to introduce alternate reward mechanisms, like reputation, stable credits, or proof-of-completion of useful tasks?”*\n\n6. **Upgradability and Evolution Over Time:** \n **What’s missing:** \n - How easy is it to evolve the protocol over time, add new features, or adapt to new technological standards without causing ecosystem turbulence? \n **Improvement:** \n Include questions about extensibility and protocol migration: \n - *“Can the system’s core protocols be upgraded easily and cleanly, preserving full historical integrity?”* \n - *“Does it support branching/forking processes similar to Git, allowing for experimental consensus changes that can later be merged back?”*\n\n7. **Data Availability and Scalability Constraints:** \n **What’s missing:** \n - How does the system ensure that the entire state is always available and scalable as the code/data base grows large? \n **Improvement:** \n Add criteria regarding scaling and data availability: \n - *“Can the chain scale with increasing amounts of data and computations without degrading verification or trust?”* \n - *“Does it support ‘shattering’ (sharding) into separate chains or segments that can be recombined, ensuring data availability and trust remain intact?”*\n\n---\n\n**Improved Comparator Example:**\n\n- **Token Dependency \u0026 Incentive Neutrality:** \n - Does consensus rely on an intrinsic token or can it operate without one? \n - Can incentives be reconfigured (e.g., non-monetary proofs of useful work)?\n\n- **Integration Format \u0026 Ecosystem Compatibility:** \n - Beyond Git and NPM, can the chain represent and serve artifacts to other ecosystems (Hugging Face models, Docker, PyPI)? \n - Are commits, model parameters, and artifact states directly usable by standard devops and AI tooling?\n\n- **Provenance, Governance \u0026 Social Dynamics:** \n - Does it record full artifact lineage, enabling complete reproducibility? \n - Can the platform encode and understand governance (commit authors, merging policies) and social context for changes?\n\n- **Deterministic \u0026 Reproducible Computation:** \n - Are computations fully deterministic, so any artifact (including AI models) can be re-derived from on-chain data? \n - Are training processes and model outputs repeatably verifiable as a form of proof-of-useful-work?\n\n- **Upgradability \u0026 Evolution:** \n - Can the system’s core mechanics be upgraded without loss of trust or data fidelity? \n - Does it support branching and merging of different consensus versions, akin to code versioning in Git?\n\n- **Portability, Data Availability \u0026 Scalability:** \n - Can it run in isolated modes and then later reconcile states without losing trust? \n - Does “Shattered Proof-of-Work” support scaling by splitting workloads and merging them without compromising determinism or provenance?\n\nBy including these additional angles—provenance, governance, determinism, ecosystem compatibility, incentive neutrality, and upgradability—you create a more comprehensive set of criteria for evaluating next-generation chains against conventional blockchains.', + ], + }, + 'status': 'finished_successfully', + 'end_turn': true, + 'weight': 1, + 'metadata': { + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'request_id': '8f01c4d23b851c5e-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'children': [], + }, + }, + 'moderation_results': [], + 'current_node': + 'c132e044-dade-4946-95a7-b1c942539af4', + 'conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + 'is_archived': false, + 'safe_urls': ['https://jsr.io/@dreamcatcher/concat'], + 'default_model_slug': 'o1-pro', + 'disabled_tool_ids': [], + 'is_public': true, + 'linear_conversation': [{ + 'id': 'aaa1615c-1f40-439e-8b5d-a779ee9a1192', + 'children': [ + '2ae52d22-dba5-457d-8147-faade76900dd', + ], + }, { + 'id': '2ae52d22-dba5-457d-8147-faade76900dd', + 'message': { + 'id': '2ae52d22-dba5-457d-8147-faade76900dd', + 'author': { 'role': 'system', 'metadata': {} }, + 'content': { + 'content_type': 'text', + 'parts': [''], + }, + 'status': 'finished_successfully', + 'end_turn': true, + 'weight': 0, + 'metadata': { + 'is_visually_hidden_from_conversation': true, + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaa1615c-1f40-439e-8b5d-a779ee9a1192', + 'children': [ + 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + ], + }, { + 'id': 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'message': { + 'id': 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'author': { 'role': 'user', 'metadata': {} }, + 'create_time': 1733880310.052759, + 'content': { + 'content_type': 'text', + 'parts': [ + "here's a chat with my friend. what concepts might we be trying to hint at ? we need clarity of thought since we are poking around at things we don't fully understand. Give us a few options. Clarify our thinking. Be creative. Be bold. Be profound.\n\n11/12/24, 11:51 - Ben Hoffman: Hey feel like the mood was a bit off when we talked. Wanted to say one more thing I think is really neat about git being the fabric in which the system natively thinks. There’s an unexplored interface between the system and the software supply chain. The system can “think” files into places that make its execution space consumable as an npm module hosted on GitHub for example \u003cThis message was edited\u003e\n11/12/24, 11:52 - Ben Hoffman: I’m sure that’s something you considered already, but I didn’t quite visualize it properly in my head until after that call\n11/12/24, 11:55 - Ben Hoffman: It’s possibly a weird way to couple into current systems, but it feels more direct than wrapping them all in APIs and then consuming those APIs from a chatbot adapter\n11/12/24, 12:45 - Tom Thompson 🧴: so you're talking about the benefit of publishing packages as units of work, rather than these live and mutable systems ?\n11/12/24, 12:45 - Tom Thompson 🧴: as well as having the bots be provenance aware, where they know how the software got there, and how to iterate on the history to produce the next version ?\n11/12/24, 12:46 - Tom Thompson 🧴: like a kind of next level introspection\n11/12/24, 12:47 - Tom Thompson 🧴: but I think the real thing you're getting at is if it stays in git, then it doesn't break that whole impure compilation problem\n11/12/24, 12:47 - Ben Hoffman: I guess what I mean is, if one of the bots decides to set up a remote on GitHub of its own “mind”, it can “think” code directly into software supply chains by just cherry-picking commits onto that remote that adhere to package format for whatever ecosystem it’s trying to supply into\n11/12/24, 12:49 - Tom Thompson 🧴: ok that's cool, but what is the benefit you're seeing there ?\n11/12/24, 12:49 - Ben Hoffman: Yeah I mean I think that’s more or less it, you don’t have to have a separate build step to make the outcome compatible with a foreign software system\n11/12/24, 12:49 - Tom Thompson 🧴: you effectively killed the build step completely\n11/12/24, 12:49 - Tom Thompson 🧴: there is no build, there is only commit\n11/12/24, 12:50 - Ben Hoffman: I’m not sure I’m thinking in terms of benefits yet, just an interface I never really considered before\n11/12/24, 12:50 - Tom Thompson 🧴: publishing to npm and jsr are just transmissions of something that could have stayed pure, if it wasn't for the need for backwards compatibility\n11/12/24, 12:50 - Ben Hoffman: Like it might be a lot easier to make this system plug in fairly directly into the software of the world without adapters and wrappers\n11/12/24, 12:52 - Tom Thompson 🧴: if the context of the LLM was git, then the context can be shared more readily, is what you're saying ?\n11/12/24, 12:52 - Ben Hoffman: Yeah\n11/12/24, 12:52 - Tom Thompson 🧴: as an anti example, if the context was all stored on Ethereum, then sharing anywhere is a copy paste rather than sharing a git commit, which is pretty universal, you mean ?\n11/12/24, 12:53 - Ben Hoffman: Yeah I think so\n11/12/24, 12:53 - Ben Hoffman: Let me sit with it a bit more and come up with a clearer expression of what I’m trying to communicate here\n11/12/24, 12:54 - Ben Hoffman: If the agent presented itself in such a way on GitHub, I guess ultimately what would need to be figured out is what does it think of commits coming in from other humans and agents onto those files\n11/12/24, 12:55 - Ben Hoffman: I’m using the word “think” a lot in an ambiguous way here\n11/12/24, 12:55 - Ben Hoffman: That’s what I need to fix\n11/12/24, 12:56 - Tom Thompson 🧴: so to conclude, there is value in having git be the blockchain, since it completely removes a transformation step, which adds impedance to flows, and hinders integration with the rest of the knowledge based economy\n11/12/24, 12:56 - Ben Hoffman: Yeah that’s part of it for sure\n11/12/24, 12:56 - Ben Hoffman: Most of it probably\n11/12/24, 12:56 - Tom Thompson 🧴: that means that, it is frictionless, or native, to transmit thoughts out from the model, and to receive thoughts in, using the most common format for knowledge transmission, which is git, with no translation whatsoever\n11/12/24, 12:57 - Tom Thompson 🧴: so its sort of like a git repo got up and started talking\n11/12/24, 12:57 - Tom Thompson 🧴: in a fanciful sort of metaphor\n11/12/24, 12:58 - Tom Thompson 🧴: so what I'm hearing then, is there is still value to pursue our own blockchain, purely for the sake of the compatibility a pure git model gives everyone\n11/12/24, 12:59 - Ben Hoffman: Yes this\n11/12/24, 12:59 - Ben Hoffman: Well look the world is not going to want to model time and synchrony less in the future than it does now\n11/12/24, 13:00 - Ben Hoffman: Since that’s probably the biggest source of WTF in big systems\n11/12/24, 13:00 - Ben Hoffman: There’s a reason the whole software industry jumped on git within a few years, make and sftp are kinda shit by comparison\n11/12/24, 13:05 - Ben Hoffman: Also I maintain that the most interesting links between git/blockchain/ai will not emerge until training is fully on chain and there’s a taxonomy of models cooked right into the source tree\n11/12/24, 13:06 - Ben Hoffman: That and AIs just simply won’t obviate the need for “basic plumbing” style software, so better to make them more aware of how the plumbing is maintained than less\n11/12/24, 13:07 - Ben Hoffman: I’m still not convinced you need to carry a blockchain around with you to build v1 of what you’re thinking, but I am still completely convinced of the long term utility of having one in mind that fits the outcome you’re aiming at\n11/12/24, 13:09 - Ben Hoffman: They just feel qualitatively super complementary to me. AIs obfuscate context, blockchains freeze context and keep it retrievable forever. You can trust an AI a lot more that you can see what it gobbled into its latent space \u003cThis message was edited\u003e\n11/12/24, 13:32 - Tom Thompson 🧴: so training being git native as a series of commits, and then calls being commits too, against a long term stable commit which is the training epoc ?\n\nSo that would mean that a model being trained like this could be consumed early, while it was still being trained, to encourage more resources to be put in it, whether you sponsor it with money, or with compute resources.\n11/12/24, 13:33 - Tom Thompson 🧴: so getting training on chain is something we should totally be going for, you're saying ?\n11/12/24, 13:33 - Tom Thompson 🧴: and then using ambient attribution to pay back to people who did training work\n11/12/24, 13:35 - Tom Thompson 🧴: yes, and further they should be the plumbing themselves, with actual software only used in cases where scale, speed, cost are an issue\n11/12/24, 13:38 - Tom Thompson 🧴: I think this is the key concept\n11/12/24, 13:39 - Tom Thompson 🧴: I think the only reason not to carry one with us is the build cost. After the last few days, I feel like the cost just fell thru the floor with o1-pro\n11/12/24, 13:39 - Ben Hoffman: Yes, training data being committed into a payload layer or git lfs for hash provenance followed by commit triggered compute that results in a new model being checked in. Training is deterministic I think, so it’s a natural fit for on-chain, and then you get a free taxonomy of models baked into the source tree structure by just cherry picking training data cross branch for new models\n11/12/24, 13:40 - Ben Hoffman: Very easy to create “cousin” models and have good training governance in such a system\n11/12/24, 13:40 - Tom Thompson 🧴: training and execution can be deterministic if configured to be so. I was hoping to use this as a proof of work system to fortify the network consensus\n11/12/24, 13:40 - Ben Hoffman: Hell yes\n11/12/24, 13:41 - Tom Thompson 🧴: rather than having consensus dorkily separate, and worse having its own economics like a token issuance that is all tangled up in there\n11/12/24, 13:41 - Ben Hoffman: It will happen eventually when people realize hugging face and GitHub could be the same thing\n11/12/24, 13:42 - Ben Hoffman: This is a god tier idea. Proof of actually useful work\n11/12/24, 13:42 - Tom Thompson 🧴: or rather, should be the same thing\n11/12/24, 13:42 - Ben Hoffman: Yes\n11/12/24, 13:42 - Tom Thompson 🧴: yeah well that's what I've been trying to fucking do this whole time 😅\n11/12/24, 13:42 - Tom Thompson 🧴: but baking ambient attribution in there too\n11/12/24, 13:42 - Ben Hoffman: Yes I think you’re seeing it\n11/12/24, 13:43 - Ben Hoffman: What I was seeing I mean\n11/12/24, 13:43 - Tom Thompson 🧴: so its part of the actual next tick of the global system, is attribution\n11/12/24, 13:43 - Ben Hoffman: No better way to make them “be the plumbing” than making them be the source repos\n11/12/24, 13:44 - Tom Thompson 🧴: so enabling the llms to be the plumbing workforce, by connecting up to git, npm, etc ?\n11/12/24, 13:44 - Ben Hoffman: Yeah build cost for sure\n11/12/24, 13:44 - Tom Thompson 🧴: so if a Napp is also the repo it lives in, then it is sort of a repo that is alive, and can commit to itself, as well as manage other repos\n11/12/24, 13:44 - Ben Hoffman: Yeah\n11/12/24, 13:46 - Ben Hoffman: And, crucially, other peoples’ commits can become part of its domain of analysis, which should eventually teach AIs to “understand” human commit streams, which are a type of political information\n11/12/24, 13:46 - Tom Thompson 🧴: ok\n11/12/24, 13:46 - Tom Thompson 🧴: I was thinking after our call too, and whilst we are seeing point solutions pop up for pieces of the dreamcatcher, there is still no unified solution, which I think is essential. Furthermore, the unified solution should have zero friction back into the established knowledge net, so I'm thinking there is a benefit to keep going with our own pure blockchain piece.\n\nYesterday I used o1 pro to produce a software module entirely from prompting. https://jsr.io/@dreamcatcher/concat\n\nThis gives me hope that actually, if we do things right, building artifact might not be nearly as hard as previously thought, since I think I can prompt large chunks of it. It's almost as tho not being able to prompt it is a fatal sign ? like code should be able to be generated from prompting a smart model, or else the design is bad.\n11/12/24, 13:47 - Ben Hoffman: I think learning how to understand commits and eventually incorporate them into the training data will give AIs a better shot at understanding consequence, time, politics, group dynamics\n11/12/24, 13:48 - Ben Hoffman: And if the AI processes via commit, then understanding commits gives it a type of introspection\n11/12/24, 13:50 - Tom Thompson 🧴: they could even begin to understand themselves, as they see their effect on that plane\n11/12/24, 13:51 - Ben Hoffman: That’s pretty exciting\n11/12/24, 13:51 - Ben Hoffman: Yes!\n11/12/24, 13:51 - Ben Hoffman: And yes, agree to “not being able to prompt is a fatal sign”\n11/12/24, 13:51 - Ben Hoffman: I really like that\n11/12/24, 13:52 - Ben Hoffman: Smart right from the outset, no bits that can’t be queried in natural language\n11/12/24, 13:52 - Ben Hoffman: It’s a type of transparency guarantee\n11/12/24, 13:52 - Ben Hoffman: Hopefully the ai doesn’t lie about stuff tho lol\n11/12/24, 13:53 - Tom Thompson 🧴: lol at least if it does, it will do so repeatably. \n\nI think this is the stongest case for on chain training - its not open ai unless the training process is repeatable too\n11/12/24, 13:54 - Tom Thompson 🧴: ok well I think you've given me the confidence to continue building artifact\n11/12/24, 13:54 - Tom Thompson 🧴: its VERY SUBTLE, what you say\n11/12/24, 13:54 - Tom Thompson 🧴: but it is a genuine\n11/12/24, 13:54 - Tom Thompson 🧴: advantage\n11/12/24, 13:55 - Tom Thompson 🧴: so the punchline is, an execution framework that is git native\n11/12/24, 13:55 - Tom Thompson 🧴: where the LLMs don't just do the commits, they ARE the commits\n11/12/24, 13:55 - Ben Hoffman: Yeah I think so. Just knowing how important git is, and how it’s really the structure that we all politically align on\n11/12/24, 13:56 - Ben Hoffman: Yes!\n11/12/24, 13:56 - Tom Thompson 🧴: yeah its so common its forgotten about hey\n11/12/24, 13:56 - Tom Thompson 🧴: its like air\n11/12/24, 13:56 - Ben Hoffman: Yeah\n11/12/24, 13:56 - Ben Hoffman: That’s a good way of thinking about it\n11/12/24, 13:56 - Tom Thompson 🧴: ok thank you 🙏\n11/12/24, 13:57 - Tom Thompson 🧴: such rough times these days - feels like sound barrier type of problems all the time\n11/12/24, 13:57 - Ben Hoffman: Yeah I think it’s the singularity\n11/12/24, 13:57 - Ben Hoffman: I guess I never thought of the singularity like a sound barrier\n11/12/24, 13:57 - Ben Hoffman: Have the right design or get crushed by the energy\n11/12/24, 13:58 - Ben Hoffman: But it seems obvious once you frame it that way\n11/12/24, 13:59 - Tom Thompson 🧴: lolololo\n11/12/24, 13:59 - Tom Thompson 🧴: that is actually 100% how I feel every day\n11/12/24, 13:59 - Tom Thompson 🧴: oscillating rapidly between crushing energy waves and serenity\n11/12/24, 13:59 - Ben Hoffman: Also I still get too attached to things so I hope I’m not pushing you in the wrong direction\n11/12/24, 13:59 - Tom Thompson 🧴: lol\n11/12/24, 13:59 - Tom Thompson 🧴: no I don't think you are\n11/12/24, 13:59 - Tom Thompson 🧴: the difference is clear\n11/12/24, 13:59 - Ben Hoffman: Ain’t it the truth\n11/12/24, 14:00 - Tom Thompson 🧴: and then when you remind me about the utility of genuine proof of computational work\n11/12/24, 14:00 - Tom Thompson 🧴: all the other blockchains have some dorky consensus with a token and they're always wedlocked to it\n11/12/24, 14:00 - Tom Thompson 🧴: like it is their whole identity,\n11/12/24, 14:00 - Tom Thompson 🧴: and they can never be upgraded to change that core thing\n11/12/24, 14:00 - Ben Hoffman: And they all create extremely toxic cultures because of it\n11/12/24, 14:01 - Tom Thompson 🧴: yeah its a totally unnecessary thing, too - like it is entirely fictitious - physics does not demand it in the least\n11/12/24, 14:01 - Ben Hoffman: Lots of people stressed by the big quantum breakthrough Google just posted about because the crypto in btc and eth are both not quantum proof lol\n11/12/24, 14:01 - Tom Thompson 🧴: I don't think they have currencies in heaven, for example\n11/12/24, 14:01 - Ben Hoffman: I will literally kill myself if I get there and that’s what it is\n11/12/24, 14:02 - Tom Thompson 🧴: well with artifact, giving it quantum computation services via an api is easy, might be a fun thing to tack on lol\n11/12/24, 14:03 - Ben Hoffman: Yeah it should be way less punishing to change the crypto when you don’t have an army of shitcoin ruggers trying to stop you\n11/12/24, 14:03 - Ben Hoffman: Totally", + ], + }, + 'status': 'finished_successfully', + 'weight': 1, + 'metadata': { + 'request_id': '8f01adde8942d9be-AKL', + 'message_source': null, + 'timestamp_': 'absolute', + 'message_type': null, + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': '2ae52d22-dba5-457d-8147-faade76900dd', + 'children': [ + 'e9410778-39b9-464a-a0f5-cc461cb28e85', + ], + }, { + 'id': 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'message': { + 'id': 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'author': { + 'role': 'tool', + 'name': 'a8km123', + 'metadata': {}, + }, + 'create_time': 1733880397.691115, + 'content': { + 'content_type': 'text', + 'parts': [ + '**Exploring AI integrations**\n\nI’m examining how Git, npm modules, software supply chain, AI, and blockchains intersect, focusing on knowledge, reproducible model training, and traceability in AI development.', + ], + }, + 'status': 'finished_successfully', + 'weight': 0, + 'metadata': { + 'finished_text': 'Thought for 1m 26s', + 'initial_text': 'Thinking', + 'summarization_headline': + 'Git, Blockchain, and AI Integration', + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'request_id': '8f01adde8942d9be-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaa2a53f-bbdd-4ab4-9b2e-9d6936a27074', + 'children': [ + '46cc8de2-2860-4c13-9083-32117c314a3a', + ], + }, { + 'id': '46cc8de2-2860-4c13-9083-32117c314a3a', + 'message': { + 'id': '46cc8de2-2860-4c13-9083-32117c314a3a', + 'author': { 'role': 'assistant', 'metadata': {} }, + 'create_time': 1733880397.691219, + 'content': { + 'content_type': 'text', + 'parts': [ + 'Below are a few conceptual frameworks and themes that might help clarify the directions you’re exploring. These are just options—bold thought experiments and profound re-interpretations of what you’ve discussed. Each tries to distill the underlying "essence" of what you\'re circling around:\n\n1. **Git as a Cognitive Substrate** \n *Concept*: Treat Git not just as a version-control system, but as a universal medium for knowledge representation—a canonical substrate into which intelligence can “think” its outputs directly. \n *Idea*: Instead of thinking of Git as infrastructure hiding behind the scenes, imagine it as the actual “mindspace” of an AI. Commits become “thoughts,” branches become “possible worlds,” and merges become negotiated alignments of reality. This would mean that interacting with systems wouldn’t require separately building APIs or wrapping logic in adapters; the “thought process” would already be in the universal language of code and data tracked by Git.\n\n2. **Provenance-Aware Execution and Training** \n *Concept*: Embrace full provenance tracking of models—every piece of training data, every parameter update, every intermediate artifact—being anchored in a Git-like or blockchain-like structure. \n *Idea*: On-chain (or on-git) training ensures that models, their lineage, and their transformations are fully transparent and repeatable. This is “Proof of Useful Work” done right: no contrived token games, just a history of refinement. Imagine a Hugging Face that merges seamlessly with GitHub: training datasets as commits, parameter updates as commits, final models as tagged releases. The model’s “mind” isn’t some opaque black box, but an ever-growing, fully versioned garden of knowledge and code.\n\n3. **Frictionless Integration With the Wider Software Supply Chain** \n *Concept*: The AI doesn’t produce code that must then be adapted or compiled into external ecosystems; it directly “commits” code into existing supply chains (e.g., npm modules) as a natural extension of its reasoning. \n *Idea*: By removing the “build step” or “adapter step” entirely, the AI’s output is not something that must be transformed—its output is directly usable. Code changes flow seamlessly into the world’s largest ecosystems of shared code, enabling the AI not only to contribute but also to continuously learn from incoming commits by humans and other agents.\n\n4. **Time, Context, and Political Awareness Through Commit Histories** \n *Concept*: Commit histories are fundamentally stories about human and machine collaboration, negotiation, conflict, and resolution. They are a record of political dynamics, group decisions, priorities shifting over time. \n *Idea*: By letting the AI “live” in this world of commits, it can learn to interpret the subtle signals encoded in version histories—understanding governance, consequence, and social context. Training on commit histories gives AI a vantage point to reason about time, intention, and human organizational structures. In other words, by modeling itself as a participant in Git’s political tapestry, the AI begins to understand what it means to cooperate, to iterate, to govern.\n\n5. **Merging Knowledge Networks: From Fragmented Tools to a Single Universal Medium** \n *Concept*: Git, package managers, blockchains, AI models—these have historically been disparate systems, each with its own protocols and friction points. \n *Idea*: Your conversation hints at merging these concepts into a unified environment: a single substrate (like Git) where data, code, models, and their entire genealogy live side by side. Instead of separate infrastructures for AI training, code versioning, supply chain management, and trust enforcement, you get one substrate that can host them all. This reduces conceptual overhead and unleashes a new kind of “global system,” where knowledge flows freely and every increment of intelligence is recorded and trusted.\n\n---\n\nAll of these concepts circle around the core theme: using version-controlled code repositories, possibly combined with blockchain-style invariants, to eliminate friction, ensure trust, enable provenance, and integrate AI deeply into the world’s software and knowledge ecosystems—no artificial boundaries, no hidden build steps, no murky provenances. Instead, a continuous, transparent, and universal environment of incremental, queryable, and trustworthy “thought commits.”', + ], + }, + 'status': 'finished_successfully', + 'end_turn': true, + 'weight': 1, + 'metadata': { + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'request_id': '8f01adde8942d9be-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'e9410778-39b9-464a-a0f5-cc461cb28e85', + 'children': [ + 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + ], + }, { + 'id': 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'message': { + 'id': 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'author': { 'role': 'user', 'metadata': {} }, + 'create_time': 1733881250.145666, + 'content': { + 'content_type': 'text', + 'parts': [ + 'if that conversation was being turned into a set of comparisons to make against competing blockchains, name what concepts are missing from the below comparison, and say how you might improve this comparator:\n\n1. Token dependency:\n \n - Does the chain have a built-in token?\n - How tightly coupled is that token to the consensus?\n - How easy is it to remove, replace, or adjust that token?\n - Is the consensus mechanism itself token-agnostic or inherently token-bound?\n - is the token stable or fluctuatable\n\n1. Integration format:\n \n - Can the chain’s data and consensus be naturally represented in Git commits or NPM packages?\n - Are translations or bridging layers required to interface with standard tools, and if so, do these weaken the consensus?\n3. Portability:\n \n - Can the system run in isolated parts, completely on your own infrastructure?\n - Can these isolated instances later merge consensus without loss of trust?\n\nFor the demonstration of a sharded or “splintered” proof-of-work system, consider calling it “Shattered Proof-of-Work” (S-PoW). This name reflects how the workload can be broken into isolated segments and later merged based on verifiable computational work.', + ], + }, + 'status': 'finished_successfully', + 'weight': 1, + 'metadata': { + 'serialization_metadata': { + 'custom_symbol_offsets': [], + }, + 'request_id': '8f01c4d23b851c5e-AKL', + 'message_source': null, + 'timestamp_': 'absolute', + 'message_type': null, + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': '46cc8de2-2860-4c13-9083-32117c314a3a', + 'children': [ + 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + ], + }, { + 'id': 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'message': { + 'id': 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'author': { + 'role': 'tool', + 'name': 'a8km123', + 'metadata': {}, + }, + 'create_time': 1733881301.561146, + 'content': { + 'content_type': 'text', + 'parts': [ + '**Gathering blockchain insights**\n\nI\'m compiling info on blockchain developments, pinpointing gaps like "Community involvement" and "Transparency and trust." Also, looking at concepts such as "Provenance and lineage" and "Natural language promptability," considering how they can enrich the comparison.', + ], + }, + 'status': 'finished_successfully', + 'weight': 0, + 'metadata': { + 'finished_text': 'Thought for 50 seconds', + 'initial_text': 'Thinking', + 'summarization_headline': + 'Blockchain Comparison Concepts', + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'request_id': '8f01c4d23b851c5e-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaa2979c-fd1f-4f49-bfd7-051c15a258b3', + 'children': [ + 'c132e044-dade-4946-95a7-b1c942539af4', + ], + }, { + 'id': 'c132e044-dade-4946-95a7-b1c942539af4', + 'message': { + 'id': 'c132e044-dade-4946-95a7-b1c942539af4', + 'author': { 'role': 'assistant', 'metadata': {} }, + 'create_time': 1733881301.56126, + 'content': { + 'content_type': 'text', + 'parts': [ + '**Missing Concepts and How to Improve the Comparison:**\n\n1. **Provenance and Lineage of Data/Models:** \n **What’s missing:** \n - How does the chain handle the version history of data, code, or model parameters? \n - Can it track the full lineage of a model’s training process or a software artifact’s evolution? \n **Improvement:** \n Add criteria examining whether each system can embed transparent, verifiable version histories—like Git commit graphs—directly in its consensus. For example: \n - *“Can the chain immutably record the origin and version history of data and code as a first-class citizen?”* \n - *“Does it allow for ‘on-chain training’ records or a taxonomy of artifacts that mirror Git’s branching, merging, and tagging semantics?”*\n\n2. **Deterministic Computation and Reproducibility:** \n **What’s missing:** \n - Are computations (e.g., model training, code transformations) guaranteed to be deterministic and reproducible from on-chain data? \n - Can the entire state (including the model’s parameters) be reconstructed from on-chain commits? \n **Improvement:** \n Add criteria around determinism and replayability: \n - *“Is the chain’s consensus logic and stored data sufficient to deterministically reproduce any result, such as model parameters after training?”* \n - *“Does the system enable ‘proof-of-useful-work’ where the work is fully re-executable and verifiable from the recorded state?”*\n\n3. **Interoperability With Existing Ecosystems (Beyond Git \u0026 NPM):** \n **What’s missing:** \n - Beyond just Git commits and npm packages, how well does the system integrate with other major ecosystems (e.g., Docker images, PyPI packages, Hugging Face model hubs)? \n **Improvement:** \n Add criteria that measure the breadth of ecosystem compatibility: \n - *“Can the chain’s artifacts be directly consumed by standard developer tools, CI/CD pipelines, model registries, or container registries without translation?”* \n - *“Is the system extensible to support new tooling integrations as standards evolve?”*\n\n4. **Governance and Social Dynamics:** \n **What’s missing:** \n - How does the chain model governance over code and data? \n - Can it represent the political dimension of commits—who proposed changes, who merged them, and the rationale behind decisions? \n **Improvement:** \n Add criteria related to governance and social aspects: \n - *“Does the chain encode and preserve metadata of contributors, allowing AIs and humans to understand political, social, and organizational contexts of commits?”* \n - *“Can we trace not just the code but also the discussions, proposals, and decision-making processes that led to a given state?”*\n\n5. **Economic Neutrality and Flexible Incentive Structures:** \n **What’s missing:** \n - If the chain employs incentives, can those be non-financial or proof-of-actual-work rather than bound to a volatile token economy? \n **Improvement:** \n Add criteria around incentive flexibility and neutrality: \n - *“Are consensus and network security achievable without relying on a native token’s economic incentives?”* \n - *“Is it possible to introduce alternate reward mechanisms, like reputation, stable credits, or proof-of-completion of useful tasks?”*\n\n6. **Upgradability and Evolution Over Time:** \n **What’s missing:** \n - How easy is it to evolve the protocol over time, add new features, or adapt to new technological standards without causing ecosystem turbulence? \n **Improvement:** \n Include questions about extensibility and protocol migration: \n - *“Can the system’s core protocols be upgraded easily and cleanly, preserving full historical integrity?”* \n - *“Does it support branching/forking processes similar to Git, allowing for experimental consensus changes that can later be merged back?”*\n\n7. **Data Availability and Scalability Constraints:** \n **What’s missing:** \n - How does the system ensure that the entire state is always available and scalable as the code/data base grows large? \n **Improvement:** \n Add criteria regarding scaling and data availability: \n - *“Can the chain scale with increasing amounts of data and computations without degrading verification or trust?”* \n - *“Does it support ‘shattering’ (sharding) into separate chains or segments that can be recombined, ensuring data availability and trust remain intact?”*\n\n---\n\n**Improved Comparator Example:**\n\n- **Token Dependency \u0026 Incentive Neutrality:** \n - Does consensus rely on an intrinsic token or can it operate without one? \n - Can incentives be reconfigured (e.g., non-monetary proofs of useful work)?\n\n- **Integration Format \u0026 Ecosystem Compatibility:** \n - Beyond Git and NPM, can the chain represent and serve artifacts to other ecosystems (Hugging Face models, Docker, PyPI)? \n - Are commits, model parameters, and artifact states directly usable by standard devops and AI tooling?\n\n- **Provenance, Governance \u0026 Social Dynamics:** \n - Does it record full artifact lineage, enabling complete reproducibility? \n - Can the platform encode and understand governance (commit authors, merging policies) and social context for changes?\n\n- **Deterministic \u0026 Reproducible Computation:** \n - Are computations fully deterministic, so any artifact (including AI models) can be re-derived from on-chain data? \n - Are training processes and model outputs repeatably verifiable as a form of proof-of-useful-work?\n\n- **Upgradability \u0026 Evolution:** \n - Can the system’s core mechanics be upgraded without loss of trust or data fidelity? \n - Does it support branching and merging of different consensus versions, akin to code versioning in Git?\n\n- **Portability, Data Availability \u0026 Scalability:** \n - Can it run in isolated modes and then later reconcile states without losing trust? \n - Does “Shattered Proof-of-Work” support scaling by splitting workloads and merging them without compromising determinism or provenance?\n\nBy including these additional angles—provenance, governance, determinism, ecosystem compatibility, incentive neutrality, and upgradability—you create a more comprehensive set of criteria for evaluating next-generation chains against conventional blockchains.', + ], + }, + 'status': 'finished_successfully', + 'end_turn': true, + 'weight': 1, + 'metadata': { + 'message_type': null, + 'model_slug': 'o1-pro', + 'default_model_slug': 'o1-pro', + 'parent_id': + 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'request_id': '8f01c4d23b851c5e-AKL', + 'timestamp_': 'absolute', + 'shared_conversation_id': + '6758f810-286c-800b-8e2c-490e634ca002', + }, + 'recipient': 'all', + }, + 'parent': 'aaea9e7c-bfa4-46e7-8a58-adb4a1e2e7c6', + 'children': [], + }], + 'has_user_editable_context': false, + 'continue_conversation_url': + 'https://chatgpt.com/share/6758f810-286c-800b-8e2c-490e634ca002/continue', + 'moderation_state': { + 'has_been_moderated': false, + 'has_been_blocked': false, + 'has_been_accepted': false, + 'has_been_auto_blocked': false, + 'has_been_auto_moderated': false, + }, + 'is_indexable': false, + 'is_better_metatags_enabled': true, + }, + }, + 'continueMode': false, + 'moderationMode': false, + 'chatPageProps': { + 'session': {}, + 'user': { + 'email': '', + 'groups': [], + 'id': 'ua-d76fbeb9-aa53-4dcb-b666-8358452854be', + 'name': '', + 'intercom_hash': + 'ua-d76fbeb9-aa53-4dcb-b666-8358452854be', + 'image': '', + 'picture': '', + 'mfa': false, + }, + 'cluster': 'unified-21', + 'userCountry': 'CA', + 'userRegion': null, + 'userRegionCode': null, + 'cfConnectingIp': '2001:df0:a280:1001::6:1', + 'cfIpLatitude': '43.63190', + 'cfIpLongitude': '-79.37160', + 'cfTimezone': 'America/Toronto', + 'cfIpCity': null, + 'userLocale': null, + 'authStatus': 'logged_out', + 'statsig': { + 'user': { + 'country': 'CA', + 'custom': { + 'has_logged_in_before': false, + 'user_agent': 'Wget/1.25.0', + 'is_punch_out_user': false, + 'auth_status': 'logged_out', + }, + 'customIDs': { + 'WebAnonymousCookieID': + 'd76fbeb9-aa53-4dcb-b666-8358452854be', + 'DeviceId': + 'd76fbeb9-aa53-4dcb-b666-8358452854be', + 'stableID': + 'd76fbeb9-aa53-4dcb-b666-8358452854be', + }, + 'ip': '2001:df0:a280:1001::6:1', + 'privateAttributes': {}, + 'userAgent': 'Wget/1.25.0', + }, + }, + 'accountsStatus': null, + }, + 'meta': { + 'title': 'Git as Supply Chain', + 'description': 'Shared via ChatGPT', + 'imageSrc': + 'https://cdn.oaistatic.com/assets/chatgpt-share-og-u7j5uyao.webp', + 'isIndexable': false, + 'isBetterMetatagsEnabled': true, + }, + }, + }, + 'actionData': null, + 'errors': null, + }, + } + __remixContext.p = function (v, e, p, x) { + if (typeof e !== 'undefined') { + x = new Error('Unexpected Server Error') + x.stack = undefined + p = Promise.reject(x) + } else { + p = Promise.resolve(v) + } + return p + } + __remixContext.n = function (i, k) { + __remixContext.t = __remixContext.t || {} + __remixContext.t[i] = __remixContext.t[i] || {} + let p = new Promise((r, e) => { + __remixContext.t[i][k] = { + r: (v) => { + r(v) + }, + e: (v) => { + e(v) + }, + } + }) + setTimeout(() => { + if ( + typeof p._error !== 'undefined' || + typeof p._data !== 'undefined' + ) return + __remixContext.t[i][k].e(new Error('Server timeout.')) + }, 5000) + return p + } + __remixContext.r = function (i, k, v, e, p, x) { + p = __remixContext.t[i][k] + if (typeof e !== 'undefined') { + x = new Error('Unexpected Server Error') + x.stack = undefined + p.e(x) + } else { + p.r(v) + } + } + Object.assign(__remixContext.state.loaderData['root'], { + 'rq:["account-status"]': __remixContext.p({ + '__type': 'AccountState', + 'accountItems': [{ + 'data': { + 'id': null, + 'residencyRegion': 'no_constraint', + 'accountUserId': null, + 'name': null, + 'profilePictureId': null, + 'profilePictureUrl': null, + 'structure': 'personal', + 'role': 'account-owner', + 'organizationId': null, + 'promoData': {}, + 'deactivated': false, + 'subscriptionStatus': { + 'billingPeriod': null, + 'hasPaidSubscription': false, + 'isActiveSubscriptionGratis': false, + 'subscriptionPlan': 'chatgptfreeplan', + 'planType': 'free', + 'wasPaidCustomer': false, + 'hasCustomerObject': false, + 'lastActiveSubscription': { + 'subscription_id': null, + 'purchase_origin_platform': 'chatgpt_not_purchased', + 'will_renew': false, + }, + 'isResellerHosted': false, + }, + 'features': [ + 'bizmo_settings', + 'breeze_available', + 'canvas', + 'canvas_code_execution', + 'canvas_opt_in', + 'chat_preferences_available', + 'chatgpt_ios_attest', + 'chatgpt_team_plan', + 'd3_controls', + 'd3_editor_gpts', + 'gizmo_canvas_toggle', + 'gizmo_support_emails', + 'mfa', + 'model_ab_use_v2', + 'new_plugin_oauth_endpoint', + 'no_auth_training_enabled_by_default', + 'paragen_mainline_alternative', + 'privacy_policy_nov_2023', + 'sentinel_enabled_for_subscription', + 'share_multimodal_links', + 'shareable_links', + 'starter_prompts', + 'user_settings_announcements', + 'voice_advanced_ga', + ], + 'canAccessWithCurrentSession': true, + 'ssoConnectionName': null, + }, + }], + 'currentAccountId': null, + }), + }) + </script> + <script nonce="52bc4634-5139-448c-b73b-89e4eef4691c" type="module" async=""> + import 'https://cdn.oaistatic.com/assets/manifest-7d43a138.js' + import * as route0 from 'https://cdn.oaistatic.com/assets/ihbvqeoqhbkuefft.js' + import * as route1 from 'https://cdn.oaistatic.com/assets/e8aqfi7hhkvyz94l.js' + + window.__remixRouteModules = { + 'root': route0, + 'routes/share.$shareId.($action)': route1, + } + + import('https://cdn.oaistatic.com/assets/mxkyxjre6muko6z4.js') + </script> + <script> + ;(function () { + function c() { + var b = a.contentDocument || a.contentWindow.document + if (b) { + var d = b.createElement('script') + d.innerHTML = + "window.__CF$cv$params={r:'8f04137a8c19cdfe',t:'MTczMzkwNTQ0NC4wMDAwMDA='};var a=document.createElement('script');a.nonce='';a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);" + b.getElementsByTagName('head')[0].appendChild(d) + } + } + if (document.body) { + var a = document.createElement('iframe') + a.height = 1 + a.width = 1 + a.style.position = 'absolute' + a.style.top = 0 + a.style.left = 0 + a.style.border = 'none' + a.style.visibility = 'hidden' + document.body.appendChild(a) + if ('loading' !== document.readyState) c() + else if (window.addEventListener) { + document.addEventListener('DOMContentLoaded', c) + } else { + var e = document.onreadystatechange || function () {} + document.onreadystatechange = function (b) { + e(b) + 'loading' !== document.readyState && + (document.onreadystatechange = e, c()) + } + } + } + })() + </script> + </body> +</html> diff --git a/utils/chat/spec.md b/utils/chat/spec.md new file mode 100644 index 000000000000..d99e94f8b760 --- /dev/null +++ b/utils/chat/spec.md @@ -0,0 +1,30 @@ +# Overview + +The module takes a shared conversation URL, extracts the conversation’s unique +ID, fetches and parses its HTML to obtain the conversation data and creation +time. It then formats the conversation into Markdown and saves it under a +filename derived from the conversation’s date and a shortened version of the ID. +In essence, it transforms a given URL into a clean Markdown file containing the +extracted chat. + +## Formal Representation + +We define the following functions: + +- $g(u)$: Extract the conversation ID from URL $u$. +- $h(u)$: Fetch HTML content from URL $u$. +- $i(\text{html}, \text{id})$: Extract the creation time $t$ and conversation + $C$ from the given HTML and ID. +- $d(t)$: Extract the date (YYYY-MM-DD) from timestamp $t$. +- $s(\text{id})$: Shorten the conversation ID. +- $f(t, \text{id})$: Construct the output filename using $d(t)$ and + $s(\text{id})$. +- $w(\text{filename}, C)$: Write the Markdown conversation $C$ to the specified + filename. + +Putting it together: + +$$ M(u) = w\bigl(f(t, g(u)), C\bigr) \text{ where } (t, C) = i(h(u), g(u)) $$ + +This equation expresses the entire module’s functionality as a composition of +these functions. diff --git a/utils/chat/vendor-docs/commander/README.md b/utils/chat/vendor-docs/commander/README.md new file mode 100644 index 000000000000..aefa28f75150 --- /dev/null +++ b/utils/chat/vendor-docs/commander/README.md @@ -0,0 +1,1353 @@ +# Commander.js + +[](https://github.com/tj/commander.js/actions?query=workflow%3A%22build%22) +[](https://www.npmjs.org/package/commander) +[](https://npmcharts.com/compare/commander?minimal=true) +[](https://packagephobia.now.sh/result?p=commander) + +The complete solution for [node.js](http://nodejs.org) command-line interfaces. + +Read this in other languages: English | [简体中文](./Readme_zh-CN.md) + +- [Commander.js](#commanderjs) + - [Installation](#installation) + - [Quick Start](#quick-start) + - [Declaring _program_ variable](#declaring-program-variable) + - [Options](#options) + - [Common option types, boolean and value](#common-option-types-boolean-and-value) + - [Default option value](#default-option-value) + - [Other option types, negatable boolean and boolean|value](#other-option-types-negatable-boolean-and-booleanvalue) + - [Required option](#required-option) + - [Variadic option](#variadic-option) + - [Version option](#version-option) + - [More configuration](#more-configuration) + - [Custom option processing](#custom-option-processing) + - [Commands](#commands) + - [Command-arguments](#command-arguments) + - [More configuration](#more-configuration-1) + - [Custom argument processing](#custom-argument-processing) + - [Action handler](#action-handler) + - [Stand-alone executable (sub)commands](#stand-alone-executable-subcommands) + - [Life cycle hooks](#life-cycle-hooks) + - [Automated help](#automated-help) + - [Custom help](#custom-help) + - [Display help after errors](#display-help-after-errors) + - [Display help from code](#display-help-from-code) + - [.name](#name) + - [.usage](#usage) + - [.description and .summary](#description-and-summary) + - [.helpOption(flags, description)](#helpoptionflags-description) + - [.helpCommand()](#helpcommand) + - [More configuration](#more-configuration-2) + - [Custom event listeners](#custom-event-listeners) + - [Bits and pieces](#bits-and-pieces) + - [.parse() and .parseAsync()](#parse-and-parseasync) + - [Parsing Configuration](#parsing-configuration) + - [Legacy options as properties](#legacy-options-as-properties) + - [TypeScript](#typescript) + - [createCommand()](#createcommand) + - [Node options such as `--harmony`](#node-options-such-as---harmony) + - [Debugging stand-alone executable subcommands](#debugging-stand-alone-executable-subcommands) + - [npm run-script](#npm-run-script) + - [Display error](#display-error) + - [Override exit and output handling](#override-exit-and-output-handling) + - [Additional documentation](#additional-documentation) + - [Support](#support) + - [Commander for enterprise](#commander-for-enterprise) + +For information about terms used in this document see: +[terminology](./docs/terminology.md) + +## Installation + +```sh +npm install commander +``` + +## Quick Start + +You write code to describe your command line interface. Commander looks after +parsing the arguments into options and command-arguments, displays usage errors +for problems, and implements a help system. + +Commander is strict and displays an error for unrecognised options. The two most +used option types are a boolean option, and an option which takes its value from +the following argument. + +Example file: [split.js](./examples/split.js) + +```js +const { program } = require('commander') + +program + .option('--first') + .option('-s, --separator <char>') + +program.parse() + +const options = program.opts() +const limit = options.first ? 1 : undefined +console.log(program.args[0].split(options.separator, limit)) +``` + +```console +$ node split.js -s / --fits a/b/c +error: unknown option '--fits' +(Did you mean --first?) +$ node split.js -s / --first a/b/c +[ 'a' ] +``` + +Here is a more complete program using a subcommand and with descriptions for the +help. In a multi-command program, you have an action handler for each command +(or stand-alone executables for the commands). + +Example file: [string-util.js](./examples/string-util.js) + +```js +const { Command } = require('commander') +const program = new Command() + +program + .name('string-util') + .description('CLI to some JavaScript string utilities') + .version('0.8.0') + +program.command('split') + .description('Split a string into substrings and display as an array') + .argument('<string>', 'string to split') + .option('--first', 'display just the first substring') + .option('-s, --separator <char>', 'separator character', ',') + .action((str, options) => { + const limit = options.first ? 1 : undefined + console.log(str.split(options.separator, limit)) + }) + +program.parse() +``` + +```console +$ node string-util.js help split +Usage: string-util split [options] <string> + +Split a string into substrings and display as an array. + +Arguments: + string string to split + +Options: + --first display just the first substring + -s, --separator <char> separator character (default: ",") + -h, --help display help for command + +$ node string-util.js split --separator=/ a/b/c +[ 'a', 'b', 'c' ] +``` + +More samples can be found in the +[examples](https://github.com/tj/commander.js/tree/master/examples) directory. + +## Declaring _program_ variable + +Commander exports a global object which is convenient for quick programs. This +is used in the examples in this README for brevity. + +```js +// CommonJS (.cjs) +const { program } = require('commander') +``` + +For larger programs which may use commander in multiple ways, including unit +testing, it is better to create a local Command object to use. + +```js +// CommonJS (.cjs) +const { Command } = require('commander') +const program = new Command() +``` + +```js +// ECMAScript (.mjs) +import { Command } from 'commander' +const program = new Command() +``` + +```ts +// TypeScript (.ts) +import { Command } from 'commander' +const program = new Command() +``` + +## Options + +Options are defined with the `.option()` method, also serving as documentation +for the options. Each option can have a short flag (single character) and a long +name, separated by a comma or space or vertical bar ('|'). + +The parsed options can be accessed by calling `.opts()` on a `Command` object, +and are passed to the action handler. + +Multi-word options such as "--template-engine" are camel-cased, becoming +`program.opts().templateEngine` etc. + +An option and its option-argument can be separated by a space, or combined into +the same argument. The option-argument can follow the short option directly or +follow an `=` for a long option. + +```sh +serve -p 80 +serve -p80 +serve --port 80 +serve --port=80 +``` + +You can use `--` to indicate the end of the options, and any remaining arguments +will be used without being interpreted. + +By default, options on the command line are not positional, and can be specified +before or after other arguments. + +There are additional related routines for when `.opts()` is not enough: + +- `.optsWithGlobals()` returns merged local and global option values +- `.getOptionValue()` and `.setOptionValue()` work with a single option value +- `.getOptionValueSource()` and `.setOptionValueWithSource()` include where the + option value came from + +### Common option types, boolean and value + +The two most used option types are a boolean option, and an option which takes +its value from the following argument (declared with angle brackets like +`--expect <value>`). Both are `undefined` unless specified on command line. + +Example file: [options-common.js](./examples/options-common.js) + +```js +program + .option('-d, --debug', 'output extra debugging') + .option('-s, --small', 'small pizza size') + .option('-p, --pizza-type <type>', 'flavour of pizza') + +program.parse(process.argv) + +const options = program.opts() +if (options.debug) console.log(options) +console.log('pizza details:') +if (options.small) console.log('- small pizza size') +if (options.pizzaType) console.log(`- ${options.pizzaType}`) +``` + +```console +$ pizza-options -p +error: option '-p, --pizza-type <type>' argument missing +$ pizza-options -d -s -p vegetarian +{ debug: true, small: true, pizzaType: 'vegetarian' } +pizza details: +- small pizza size +- vegetarian +$ pizza-options --pizza-type=cheese +pizza details: +- cheese +``` + +Multiple boolean short options may be combined following the dash, and may be +followed by a single short option taking a value. For example `-d -s -p cheese` +may be written as `-ds -p cheese` or even `-dsp cheese`. + +Options with an expected option-argument are greedy and will consume the +following argument whatever the value. So `--id -xyz` reads `-xyz` as the +option-argument. + +`program.parse(arguments)` processes the arguments, leaving any args not +consumed by the program options in the `program.args` array. The parameter is +optional and defaults to `process.argv`. + +### Default option value + +You can specify a default value for an option. + +Example file: [options-defaults.js](./examples/options-defaults.js) + +```js +program + .option('-c, --cheese <type>', 'add the specified type of cheese', 'blue') + +program.parse() + +console.log(`cheese: ${program.opts().cheese}`) +``` + +```console +$ pizza-options +cheese: blue +$ pizza-options --cheese stilton +cheese: stilton +``` + +### Other option types, negatable boolean and boolean|value + +You can define a boolean option long name with a leading `no-` to set the option +value to false when used. Defined alone this also makes the option true by +default. + +If you define `--foo` first, adding `--no-foo` does not change the default value +from what it would otherwise be. + +Example file: [options-negatable.js](./examples/options-negatable.js) + +```js +program + .option('--no-sauce', 'Remove sauce') + .option('--cheese <flavour>', 'cheese flavour', 'mozzarella') + .option('--no-cheese', 'plain with no cheese') + .parse() + +const options = program.opts() +const sauceStr = options.sauce ? 'sauce' : 'no sauce' +const cheeseStr = (options.cheese === false) + ? 'no cheese' + : `${options.cheese} cheese` +console.log(`You ordered a pizza with ${sauceStr} and ${cheeseStr}`) +``` + +```console +$ pizza-options +You ordered a pizza with sauce and mozzarella cheese +$ pizza-options --sauce +error: unknown option '--sauce' +$ pizza-options --cheese=blue +You ordered a pizza with sauce and blue cheese +$ pizza-options --no-sauce --no-cheese +You ordered a pizza with no sauce and no cheese +``` + +You can specify an option which may be used as a boolean option but may +optionally take an option-argument (declared with square brackets like +`--optional [value]`). + +Example file: +[options-boolean-or-value.js](./examples/options-boolean-or-value.js) + +```js +program + .option('-c, --cheese [type]', 'Add cheese with optional type') + +program.parse(process.argv) + +const options = program.opts() +if (options.cheese === undefined) console.log('no cheese') +else if (options.cheese === true) console.log('add cheese') +else console.log(`add cheese type ${options.cheese}`) +``` + +```console +$ pizza-options +no cheese +$ pizza-options --cheese +add cheese +$ pizza-options --cheese mozzarella +add cheese type mozzarella +``` + +Options with an optional option-argument are not greedy and will ignore +arguments starting with a dash. So `id` behaves as a boolean option for +`--id -5`, but you can use a combined form if needed like `--id=-5`. + +For information about possible ambiguous cases, see +[options taking varying arguments](./docs/options-in-depth.md). + +### Required option + +You may specify a required (mandatory) option using `.requiredOption()`. The +option must have a value after parsing, usually specified on the command line, +or perhaps from a default value (say from environment). The method is otherwise +the same as `.option()` in format, taking flags and description, and optional +default value or custom processing. + +Example file: [options-required.js](./examples/options-required.js) + +```js +program + .requiredOption('-c, --cheese <type>', 'pizza must have cheese') + +program.parse() +``` + +```console +$ pizza +error: required option '-c, --cheese <type>' not specified +``` + +### Variadic option + +You may make an option variadic by appending `...` to the value placeholder when +declaring the option. On the command line you can then specify multiple +option-arguments, and the parsed option value will be an array. The extra +arguments are read until the first argument starting with a dash. The special +argument `--` stops option processing entirely. If a value is specified in the +same argument as the option then no further values are read. + +Example file: [options-variadic.js](./examples/options-variadic.js) + +```js +program + .option('-n, --number <numbers...>', 'specify numbers') + .option('-l, --letter [letters...]', 'specify letters') + +program.parse() + +console.log('Options: ', program.opts()) +console.log('Remaining arguments: ', program.args) +``` + +```console +$ collect -n 1 2 3 --letter a b c +Options: { number: [ '1', '2', '3' ], letter: [ 'a', 'b', 'c' ] } +Remaining arguments: [] +$ collect --letter=A -n80 operand +Options: { number: [ '80' ], letter: [ 'A' ] } +Remaining arguments: [ 'operand' ] +$ collect --letter -n 1 -n 2 3 -- operand +Options: { number: [ '1', '2', '3' ], letter: true } +Remaining arguments: [ 'operand' ] +``` + +For information about possible ambiguous cases, see +[options taking varying arguments](./docs/options-in-depth.md). + +### Version option + +The optional `version` method adds handling for displaying the command version. +The default option flags are `-V` and `--version`, and when present the command +prints the version number and exits. + +```js +program.version('0.0.1') +``` + +```console +$ ./examples/pizza -V +0.0.1 +``` + +You may change the flags and description by passing additional parameters to the +`version` method, using the same syntax for flags as the `option` method. + +```js +program.version('0.0.1', '-v, --vers', 'output the current version') +``` + +### More configuration + +You can add most options using the `.option()` method, but there are some +additional features available by constructing an `Option` explicitly for less +common cases. + +Example files: [options-extra.js](./examples/options-extra.js), +[options-env.js](./examples/options-env.js), +[options-conflicts.js](./examples/options-conflicts.js), +[options-implies.js](./examples/options-implies.js) + +```js +program + .addOption(new Option('-s, --secret').hideHelp()) + .addOption( + new Option('-t, --timeout <delay>', 'timeout in seconds').default( + 60, + 'one minute', + ), + ) + .addOption( + new Option('-d, --drink <size>', 'drink size').choices([ + 'small', + 'medium', + 'large', + ]), + ) + .addOption(new Option('-p, --port <number>', 'port number').env('PORT')) + .addOption( + new Option('--donate [amount]', 'optional donation in dollars').preset('20') + .argParser(parseFloat), + ) + .addOption( + new Option('--disable-server', 'disables the server').conflicts('port'), + ) + .addOption( + new Option('--free-drink', 'small drink included free ').implies({ + drink: 'small', + }), + ) +``` + +```console +$ extra --help +Usage: help [options] + +Options: + -t, --timeout <delay> timeout in seconds (default: one minute) + -d, --drink <size> drink cup size (choices: "small", "medium", "large") + -p, --port <number> port number (env: PORT) + --donate [amount] optional donation in dollars (preset: "20") + --disable-server disables the server + --free-drink small drink included free + -h, --help display help for command + +$ extra --drink huge +error: option '-d, --drink <size>' argument 'huge' is invalid. Allowed choices are small, medium, large. + +$ PORT=80 extra --donate --free-drink +Options: { timeout: 60, donate: 20, port: '80', freeDrink: true, drink: 'small' } + +$ extra --disable-server --port 8000 +error: option '--disable-server' cannot be used with option '-p, --port <number>' +``` + +Specify a required (mandatory) option using the `Option` method +`.makeOptionMandatory()`. This matches the `Command` method +[.requiredOption()](#required-option). + +### Custom option processing + +You may specify a function to do custom processing of option-arguments. The +callback function receives two parameters, the user specified option-argument +and the previous value for the option. It returns the new value for the option. + +This allows you to coerce the option-argument to the desired type, or accumulate +values, or do entirely custom processing. + +You can optionally specify the default/starting value for the option after the +function parameter. + +Example file: +[options-custom-processing.js](./examples/options-custom-processing.js) + +```js +function myParseInt(value, dummyPrevious) { + // parseInt takes a string and a radix + const parsedValue = parseInt(value, 10) + if (isNaN(parsedValue)) { + throw new commander.InvalidArgumentError('Not a number.') + } + return parsedValue +} + +function increaseVerbosity(dummyValue, previous) { + return previous + 1 +} + +function collect(value, previous) { + return previous.concat([value]) +} + +function commaSeparatedList(value, dummyPrevious) { + return value.split(',') +} + +program + .option('-f, --float <number>', 'float argument', parseFloat) + .option('-i, --integer <number>', 'integer argument', myParseInt) + .option( + '-v, --verbose', + 'verbosity that can be increased', + increaseVerbosity, + 0, + ) + .option('-c, --collect <value>', 'repeatable value', collect, []) + .option('-l, --list <items>', 'comma separated list', commaSeparatedList) + +program.parse() + +const options = program.opts() +if (options.float !== undefined) console.log(`float: ${options.float}`) +if (options.integer !== undefined) console.log(`integer: ${options.integer}`) +if (options.verbose > 0) console.log(`verbosity: ${options.verbose}`) +if (options.collect.length > 0) console.log(options.collect) +if (options.list !== undefined) console.log(options.list) +``` + +```console +$ custom -f 1e2 +float: 100 +$ custom --integer 2 +integer: 2 +$ custom -v -v -v +verbose: 3 +$ custom -c a -c b -c c +[ 'a', 'b', 'c' ] +$ custom --list x,y,z +[ 'x', 'y', 'z' ] +``` + +## Commands + +You can specify (sub)commands using `.command()` or `.addCommand()`. There are +two ways these can be implemented: using an action handler attached to the +command, or as a stand-alone executable file (described in more detail later). +The subcommands may be nested ([example](./examples/nestedCommands.js)). + +In the first parameter to `.command()` you specify the command name. You may +append the command-arguments after the command name, or specify them separately +using `.argument()`. The arguments may be `<required>` or `[optional]`, and the +last argument may also be `variadic...`. + +You can use `.addCommand()` to add an already configured subcommand to the +program. + +For example: + +```js +// Command implemented using action handler (description is supplied separately to `.command`) +// Returns new command for configuring. +program + .command('clone <source> [destination]') + .description('clone a repository into a newly created directory') + .action((source, destination) => { + console.log('clone command called') + }) + +// Command implemented using stand-alone executable file, indicated by adding description as second parameter to `.command`. +// Returns `this` for adding more commands. +program + .command('start <service>', 'start named service') + .command('stop [service]', 'stop named service, or all if no name supplied') + +// Command prepared separately. +// Returns `this` for adding more commands. +program + .addCommand(build.makeBuildCommand()) +``` + +Configuration options can be passed with the call to `.command()` and +`.addCommand()`. Specifying `hidden: true` will remove the command from the +generated help output. Specifying `isDefault: true` will run the subcommand if +no other subcommand is specified ([example](./examples/defaultCommand.js)). + +You can add alternative names for a command with `.alias()`. +([example](./examples/alias.js)) + +`.command()` automatically copies the inherited settings from the parent command +to the newly created subcommand. This is only done during creation, any later +setting changes to the parent are not inherited. + +For safety, `.addCommand()` does not automatically copy the inherited settings +from the parent command. There is a helper routine `.copyInheritedSettings()` +for copying the settings when they are wanted. + +### Command-arguments + +For subcommands, you can specify the argument syntax in the call to `.command()` +(as shown above). This is the only method usable for subcommands implemented +using a stand-alone executable, but for other subcommands you can instead use +the following method. + +To configure a command, you can use `.argument()` to specify each expected +command-argument. You supply the argument name and an optional description. The +argument may be `<required>` or `[optional]`. You can specify a default value +for an optional command-argument. + +Example file: [argument.js](./examples/argument.js) + +```js +program + .version('0.1.0') + .argument('<username>', 'user to login') + .argument('[password]', 'password for user, if required', 'no password given') + .action((username, password) => { + console.log('username:', username) + console.log('password:', password) + }) +``` + +The last argument of a command can be variadic, and only the last argument. To +make an argument variadic you append `...` to the argument name. A variadic +argument is passed to the action handler as an array. For example: + +```js +program + .version('0.1.0') + .command('rmdir') + .argument('<dirs...>') + .action(function (dirs) { + dirs.forEach((dir) => { + console.log('rmdir %s', dir) + }) + }) +``` + +There is a convenience method to add multiple arguments at once, but without +descriptions: + +```js +program + .arguments('<username> <password>') +``` + +#### More configuration + +There are some additional features available by constructing an `Argument` +explicitly for less common cases. + +Example file: [arguments-extra.js](./examples/arguments-extra.js) + +```js +program + .addArgument( + new commander.Argument('<drink-size>', 'drink cup size').choices([ + 'small', + 'medium', + 'large', + ]), + ) + .addArgument( + new commander.Argument('[timeout]', 'timeout in seconds').default( + 60, + 'one minute', + ), + ) +``` + +#### Custom argument processing + +You may specify a function to do custom processing of command-arguments (like +for option-arguments). The callback function receives two parameters, the user +specified command-argument and the previous value for the argument. It returns +the new value for the argument. + +The processed argument values are passed to the action handler, and saved as +`.processedArgs`. + +You can optionally specify the default/starting value for the argument after the +function parameter. + +Example file: +[arguments-custom-processing.js](./examples/arguments-custom-processing.js) + +```js +program + .command('add') + .argument('<first>', 'integer argument', myParseInt) + .argument('[second]', 'integer argument', myParseInt, 1000) + .action((first, second) => { + console.log(`${first} + ${second} = ${first + second}`) + }) +``` + +### Action handler + +The action handler gets passed a parameter for each command-argument you +declared, and two additional parameters which are the parsed options and the +command object itself. + +Example file: [thank.js](./examples/thank.js) + +```js +program + .argument('<name>') + .option('-t, --title <honorific>', 'title to use before name') + .option('-d, --debug', 'display some debugging') + .action((name, options, command) => { + if (options.debug) { + console.error('Called %s with options %o', command.name(), options) + } + const title = options.title ? `${options.title} ` : '' + console.log(`Thank-you ${title}${name}`) + }) +``` + +If you prefer, you can work with the command directly and skip declaring the +parameters for the action handler. The `this` keyword is set to the running +command and can be used from a function expression (but not from an arrow +function). + +Example file: [action-this.js](./examples/action-this.js) + +```js +program + .command('serve') + .argument('<script>') + .option('-p, --port <number>', 'port number', 80) + .action(function () { + console.error('Run script %s on port %s', this.args[0], this.opts().port) + }) +``` + +You may supply an `async` action handler, in which case you call `.parseAsync` +rather than `.parse`. + +```js +async function run() {/* code goes here */} + +async function main() { + program + .command('run') + .action(run) + await program.parseAsync(process.argv) +} +``` + +A command's options and arguments on the command line are validated when the +command is used. Any unknown options or missing arguments will be reported as an +error. You can suppress the unknown option checks with `.allowUnknownOption()`. +By default, it is not an error to pass more arguments than declared, but you can +make this an error with `.allowExcessArguments(false)`. + +### Stand-alone executable (sub)commands + +When `.command()` is invoked with a description argument, this tells Commander +that you're going to use stand-alone executables for subcommands. Commander will +search the files in the directory of the entry script for a file with the name +combination `command-subcommand`, like `pm-install` or `pm-search` in the +example below. The search includes trying common file extensions, like `.js`. +You may specify a custom name (and path) with the `executableFile` configuration +option. You may specify a custom search directory for subcommands with +`.executableDir()`. + +You handle the options for an executable (sub)command in the executable, and +don't declare them at the top-level. + +Example file: [pm](./examples/pm) + +```js +program + .name('pm') + .version('0.1.0') + .command('install [name]', 'install one or more packages') + .command('search [query]', 'search with optional query') + .command('update', 'update installed packages', { + executableFile: 'myUpdateSubCommand', + }) + .command('list', 'list packages installed', { isDefault: true }) + +program.parse(process.argv) +``` + +If the program is designed to be installed globally, make sure the executables +have proper modes, like `755`. + +### Life cycle hooks + +You can add callback hooks to a command for life cycle events. + +Example file: [hook.js](./examples/hook.js) + +```js +program + .option('-t, --trace', 'display trace statements for commands') + .hook('preAction', (thisCommand, actionCommand) => { + if (thisCommand.opts().trace) { + console.log( + `About to call action handler for subcommand: ${actionCommand.name()}`, + ) + console.log('arguments: %O', actionCommand.args) + console.log('options: %o', actionCommand.opts()) + } + }) +``` + +The callback hook can be `async`, in which case you call `.parseAsync` rather +than `.parse`. You can add multiple hooks per event. + +The supported events are: + +| event name | when hook called | callback parameters | +| :------------------------ | :---------------------------------------------------------------------- | :----------------------------- | +| `preAction`, `postAction` | before/after action handler for this command and its nested subcommands | `(thisCommand, actionCommand)` | +| `preSubcommand` | before parsing direct subcommand | `(thisCommand, subcommand)` | + +For an overview of the life cycle events see +[parsing life cycle and hooks](./docs/parsing-and-hooks.md). + +## Automated help + +The help information is auto-generated based on the information commander +already knows about your program. The default help option is `-h,--help`. + +Example file: [pizza](./examples/pizza) + +```console +$ node ./examples/pizza --help +Usage: pizza [options] + +An application for pizza ordering + +Options: + -p, --peppers Add peppers + -c, --cheese <type> Add the specified type of cheese (default: "marble") + -C, --no-cheese You do not want any cheese + -h, --help display help for command +``` + +A `help` command is added by default if your command has subcommands. It can be +used alone, or with a subcommand name to show further help for the subcommand. +These are effectively the same if the `shell` program has implicit help: + +```sh +shell help +shell --help + +shell help spawn +shell spawn --help +``` + +Long descriptions are wrapped to fit the available width. (However, a +description that includes a line-break followed by whitespace is assumed to be +pre-formatted and not wrapped.) + +### Custom help + +You can add extra text to be displayed along with the built-in help. + +Example file: [custom-help](./examples/custom-help) + +```js +program + .option('-f, --foo', 'enable some foo') + +program.addHelpText( + 'after', + ` + +Example call: + $ custom-help --help`, +) +``` + +Yields the following help output: + +```Text +Usage: custom-help [options] + +Options: + -f, --foo enable some foo + -h, --help display help for command + +Example call: + $ custom-help --help +``` + +The positions in order displayed are: + +- `beforeAll`: add to the program for a global banner or header +- `before`: display extra information before built-in help +- `after`: display extra information after built-in help +- `afterAll`: add to the program for a global footer (epilog) + +The positions "beforeAll" and "afterAll" apply to the command and all its +subcommands. + +The second parameter can be a string, or a function returning a string. The +function is passed a context object for your convenience. The properties are: + +- error: a boolean for whether the help is being displayed due to a usage error +- command: the Command which is displaying the help + +### Display help after errors + +The default behaviour for usage errors is to just display a short error message. +You can change the behaviour to show the full help or a custom help message +after an error. + +```js +program.showHelpAfterError() +// or +program.showHelpAfterError('(add --help for additional information)') +``` + +```console +$ pizza --unknown +error: unknown option '--unknown' +(add --help for additional information) +``` + +The default behaviour is to suggest correct spelling after an error for an +unknown command or option. You can disable this. + +```js +program.showSuggestionAfterError(false) +``` + +```console +$ pizza --hepl +error: unknown option '--hepl' +(Did you mean --help?) +``` + +### Display help from code + +`.help()`: display help information and exit immediately. You can optionally +pass `{ error: true }` to display on stderr and exit with an error status. + +`.outputHelp()`: output help information without exiting. You can optionally +pass `{ error: true }` to display on stderr. + +`.helpInformation()`: get the built-in command help information as a string for +processing or displaying yourself. + +### .name + +The command name appears in the help, and is also used for locating stand-alone +executable subcommands. + +You may specify the program name using `.name()` or in the Command constructor. +For the program, Commander will fall back to using the script name from the full +arguments passed into `.parse()`. However, the script name varies depending on +how your program is launched, so you may wish to specify it explicitly. + +```js +program.name('pizza') +const pm = new Command('pm') +``` + +Subcommands get a name when specified using `.command()`. If you create the +subcommand yourself to use with `.addCommand()`, then set the name using +`.name()` or in the Command constructor. + +### .usage + +This allows you to customise the usage description in the first line of the +help. Given: + +```js +program + .name('my-command') + .usage('[global options] command') +``` + +The help will start with: + +```Text +Usage: my-command [global options] command +``` + +### .description and .summary + +The description appears in the help for the command. You can optionally supply a +shorter summary to use when listed as a subcommand of the program. + +```js +program + .command('duplicate') + .summary('make a copy') + .description(`Make a copy of the current project. +This may require additional disk space. + `) +``` + +### .helpOption(flags, description) + +By default, every command has a help option. You may change the default help +flags and description. Pass false to disable the built-in help option. + +```js +program + .helpOption('-e, --HELP', 'read more information') +``` + +(Or use `.addHelpOption()` to add an option you construct yourself.) + +### .helpCommand() + +A help command is added by default if your command has subcommands. You can +explicitly turn on or off the implicit help command with `.helpCommand(true)` +and `.helpCommand(false)`. + +You can both turn on and customise the help command by supplying the name and +description: + +```js +program.helpCommand('assist [command]', 'show assistance') +``` + +(Or use `.addHelpCommand()` to add a command you construct yourself.) + +### More configuration + +The built-in help is formatted using the Help class. You can configure the Help +behaviour by modifying data properties and methods using `.configureHelp()`, or +by subclassing using `.createHelp()` if you prefer. + +The data properties are: + +- `helpWidth`: specify the wrap width, useful for unit tests +- `sortSubcommands`: sort the subcommands alphabetically +- `sortOptions`: sort the options alphabetically +- `showGlobalOptions`: show a section with the global options from the parent + command(s) + +You can override any method on the [Help](./lib/help.js) class. There are +methods getting the visible lists of arguments, options, and subcommands. There +are methods for formatting the items in the lists, with each item having a +_term_ and _description_. Take a look at `.formatHelp()` to see how they are +used. + +Example file: [configure-help.js](./examples/configure-help.js) + +```js +program.configureHelp({ + sortSubcommands: true, + subcommandTerm: (cmd) => cmd.name(), // Just show the name, instead of short usage. +}) +``` + +## Custom event listeners + +You can execute custom actions by listening to command and option events. + +```js +program.on('option:verbose', function () { + process.env.VERBOSE = this.opts().verbose +}) +``` + +## Bits and pieces + +### .parse() and .parseAsync() + +Call with no parameters to parse `process.argv`. Detects Electron and special +node options like `node --eval`. Easy mode! + +Or call with an array of strings to parse, and optionally where the user +arguments start by specifying where the arguments are `from`: + +- `'node'`: default, `argv[0]` is the application and `argv[1]` is the script + being run, with user arguments after that +- `'electron'`: `argv[0]` is the application and `argv[1]` varies depending on + whether the electron application is packaged +- `'user'`: just user arguments + +For example: + +```js +program.parse() // parse process.argv and auto-detect electron and special node flags +program.parse(process.argv) // assume argv[0] is app and argv[1] is script +program.parse(['--port', '80'], { from: 'user' }) // just user supplied arguments, nothing special about argv[0] +``` + +Use parseAsync instead of parse if any of your action handlers are async. + +If you want to parse multiple times, create a new program each time. Calling +parse does not clear out any previous state. + +### Parsing Configuration + +If the default parsing does not suit your needs, there are some behaviours to +support other usage patterns. + +By default, program options are recognised before and after subcommands. To only +look for program options before subcommands, use `.enablePositionalOptions()`. +This lets you use an option for a different purpose in subcommands. + +Example file: [positional-options.js](./examples/positional-options.js) + +With positional options, the `-b` is a program option in the first line and a +subcommand option in the second line: + +```sh +program -b subcommand +program subcommand -b +``` + +By default, options are recognised before and after command-arguments. To only +process options that come before the command-arguments, use +`.passThroughOptions()`. This lets you pass the arguments and following options +through to another program without needing to use `--` to end the option +processing. To use pass through options in a subcommand, the program needs to +enable positional options. + +Example file: [pass-through-options.js](./examples/pass-through-options.js) + +With pass through options, the `--port=80` is a program option in the first line +and passed through as a command-argument in the second line: + +```sh +program --port=80 arg +program arg --port=80 +``` + +By default, the option processing shows an error for an unknown option. To have +an unknown option treated as an ordinary command-argument and continue looking +for options, use `.allowUnknownOption()`. This lets you mix known and unknown +options. + +By default, the argument processing does not display an error for more +command-arguments than expected. To display an error for excess arguments, +use`.allowExcessArguments(false)`. + +### Legacy options as properties + +Before Commander 7, the option values were stored as properties on the command. +This was convenient to code, but the downside was possible clashes with existing +properties of `Command`. You can revert to the old behaviour to run unmodified +legacy code by using `.storeOptionsAsProperties()`. + +```js +program + .storeOptionsAsProperties() + .option('-d, --debug') + .action((commandAndOptions) => { + if (commandAndOptions.debug) { + console.error(`Called ${commandAndOptions.name()}`) + } + }) +``` + +### TypeScript + +extra-typings: There is an optional project to infer extra type information from +the option and argument definitions. This adds strong typing to the options +returned by `.opts()` and the parameters to `.action()`. See +[commander-js/extra-typings](https://github.com/commander-js/extra-typings) for +more. + +``` +import { Command } from '@commander-js/extra-typings'; +``` + +ts-node: If you use `ts-node` and stand-alone executable subcommands written as +`.ts` files, you need to call your program through node to get the subcommands +called correctly. e.g. + +```sh +node -r ts-node/register pm.ts +``` + +### createCommand() + +This factory function creates a new command. It is exported and may be used +instead of using `new`, like: + +```js +const { createCommand } = require('commander') +const program = createCommand() +``` + +`createCommand` is also a method of the Command object, and creates a new +command rather than a subcommand. This gets used internally when creating +subcommands using `.command()`, and you may override it to customise the new +subcommand (example file +[custom-command-class.js](./examples/custom-command-class.js)). + +### Node options such as `--harmony` + +You can enable `--harmony` option in two ways: + +- Use `#! /usr/bin/env node --harmony` in the subcommands scripts. (Note Windows + does not support this pattern.) +- Use the `--harmony` option when call the command, like + `node --harmony examples/pm publish`. The `--harmony` option will be preserved + when spawning subcommand process. + +### Debugging stand-alone executable subcommands + +An executable subcommand is launched as a separate child process. + +If you are using the node inspector for +[debugging](https://nodejs.org/en/docs/guides/debugging-getting-started/) +executable subcommands using `node --inspect` et al., the inspector port is +incremented by 1 for the spawned subcommand. + +If you are using VSCode to debug executable subcommands you need to set the +`"autoAttachChildProcesses": true` flag in your launch.json configuration. + +### npm run-script + +By default, when you call your program using run-script, `npm` will parse any +options on the command-line and they will not reach your program. Use `--` to +stop the npm option parsing and pass through all the arguments. + +The synopsis for +[npm run-script](https://docs.npmjs.com/cli/v9/commands/npm-run-script) +explicitly shows the `--` for this reason: + +```console +npm run-script <command> [-- <args>] +``` + +### Display error + +This routine is available to invoke the Commander error handling for your own +error conditions. (See also the next section about exit handling.) + +As well as the error message, you can optionally specify the `exitCode` (used +with `process.exit`) and `code` (used with `CommanderError`). + +```js +program.error('Password must be longer than four characters') +program.error('Custom processing has failed', { + exitCode: 2, + code: 'my.custom.error', +}) +``` + +### Override exit and output handling + +By default, Commander calls `process.exit` when it detects errors, or after +displaying the help or version. You can override this behaviour and optionally +supply a callback. The default override throws a `CommanderError`. + +The override callback is passed a `CommanderError` with properties `exitCode` +number, `code` string, and `message`. Commander expects the callback to +terminate the normal program flow, and will call `process.exit` if the callback +returns. The normal display of error messages or version or help is not affected +by the override which is called after the display. + +```js +program.exitOverride() + +try { + program.parse(process.argv) +} catch (err) { + // custom processing... +} +``` + +By default, Commander is configured for a command-line application and writes to +stdout and stderr. You can modify this behaviour for custom applications. In +addition, you can modify the display of error messages. + +Example file: [configure-output.js](./examples/configure-output.js) + +```js +function errorColor(str) { + // Add ANSI escape codes to display text in red. + return `\x1b[31m${str}\x1b[0m` +} + +program + .configureOutput({ + // Visibly override write routines as example! + writeOut: (str) => process.stdout.write(`[OUT] ${str}`), + writeErr: (str) => process.stdout.write(`[ERR] ${str}`), + // Highlight errors in color. + outputError: (str, write) => write(errorColor(str)), + }) +``` + +### Additional documentation + +There is more information available about: + +- [deprecated](./docs/deprecated.md) features still supported for backwards + compatibility +- [options taking varying arguments](./docs/options-in-depth.md) +- [parsing life cycle and hooks](./docs/parsing-and-hooks.md) + +## Support + +The current version of Commander is fully supported on Long Term Support +versions of Node.js, and requires at least v18. (For older versions of Node.js, +use an older version of Commander.) + +The main forum for free and community support is the project +[Issues](https://github.com/tj/commander.js/issues) on GitHub. + +### Commander for enterprise + +Available as part of the Tidelift Subscription + +The maintainers of Commander and thousands of other packages are working with +Tidelift to deliver commercial support and maintenance for the open source +dependencies you use to build your applications. Save time, reduce risk, and +improve code health, while paying the maintainers of the exact dependencies you +use. +[Learn more.](https://tidelift.com/subscription/pkg/npm-commander?utm_source=npm-commander&utm_medium=referral&utm_campaign=enterprise&utm_term=repo) diff --git a/utils/concat/CHANGELOG.md b/utils/concat/CHANGELOG.md new file mode 100644 index 000000000000..4c877c6009ea --- /dev/null +++ b/utils/concat/CHANGELOG.md @@ -0,0 +1,22 @@ +# CHANGELOG + +## v0.0.6 + +- empty version bump to test the version check feature + +## v0.0.5 + +- Added version check feature that notifies users when a newer version is + available on JSR. + +## v0.0.4 + +- Changed default behavior to write output to `concat.txt` if `--output` is not + specified. +- Added `--stdout` mode to produce raw output without info messages. +- Expanded default ignore list to include + `.lock, .git, .gitignore, LICENSE, concat.txt`. +- By default, now only includes `*.ts, *.tsx, *.js, *.jsx, *.md, *.txt` files. +- Added `--ignore` option to specify additional globs to ignore. +- Improved token counting output and message indicating if the file was created + or updated. diff --git a/utils/concat/README.md b/utils/concat/README.md new file mode 100644 index 000000000000..ae88e60d6fd5 --- /dev/null +++ b/utils/concat/README.md @@ -0,0 +1,79 @@ +# Concat + +A CLI tool that concatenates the contents of specified files, directories, and +glob patterns into a single output. Files are wrapped in ASCII armor style +headers and footers to clearly delimit them. + +## Features + +- Accepts multiple files, directories, and glob patterns. +- Recursively includes directory contents if the directory path or glob includes + `**`. +- Outputs file contents with `-----BEGIN FILE ...-----` and + `-----END FILE ...-----` delimiters. +- Optional `--output FILE` argument to write output to a file; otherwise prints + to stdout. +- When `--output` is used, after processing, it prints a summary including: + 1. A list of processed files + 2. A count of the total files processed + 3. The token count in the `o1-preview` encoding + +## Usage + +```sh +concat [options] [file|folder|glob ...] + +Options: + --help + Show the help message and exit. + + --output FILE + Write output to FILE. If omitted, output goes to stdout. + +If directories are provided, their contents are included recursively. +Glob patterns are expanded to include matched files recursively. + +Note: When using glob patterns, you may need to quote them to prevent shell expansion: + concat "src/**/*.ts" # Correct: glob is passed to concat + concat src/**/*.ts # May fail: shell expands glob before concat runs +``` + +## Example + +```sh +# Without output file (prints to stdout) +concat src/**/*.ts + +# With output file (prints summary after processing) +concat --output out.txt ../reasoning/**/*.md +``` + +After running the second example, you'll see output like: + +- The list of processed files +- A count of how many files were processed +- A final message including the token count in out.txt. + +## Requirements + +- Deno (latest stable) +- Permissions: + - `--allow-read` to read files + - `--allow-write` if using `--output` option + +## Installing as a CLI command + +```sh +deno install --global --allow-read --allow-write --allow-net=jsr.io jsr:@dreamcatcher/concat +``` + +After this, `concat` will be available as a system-wide command. + +> Note: The `--allow-net` permission is required to check for package updates +> from the jsr.io registry. + +## Upgrading the CLI command + +```sh +deno install --global --reload --force --allow-read --allow-write --allow-net=jsr.io jsr:@dreamcatcher/concat +``` diff --git a/utils/concat/deno.json b/utils/concat/deno.json new file mode 100644 index 000000000000..02821dc8494e --- /dev/null +++ b/utils/concat/deno.json @@ -0,0 +1,13 @@ +{ + "name": "@dreamcatcher/concat", + "version": "0.0.6", + "tasks": { + "run": "deno run --allow-read --allow-write --allow-net=jsr.io main.ts" + }, + "bin": { + "concat": "main.ts" + }, + "exports": { + ".": "./main.ts" + } +} diff --git a/utils/concat/issues.md b/utils/concat/issues.md new file mode 100644 index 000000000000..80dfb83f148c --- /dev/null +++ b/utils/concat/issues.md @@ -0,0 +1,43 @@ +Does not ignore .pdf by default + +Default is not recursive, but maybe that is correct, and maybe it should have a +-r flag to make it recursive. + +Use https://www.npmjs.com/package/git-diff to pull up diffs. Allow arbitrarily +many diffs to be compared, with each other, or with a single commit. Used with +concat to ask the model to reason about what has changed between different +things. Takes advantage of the snapshots history that is part of the napp +format. + +concat **/so-what.md does not recurse into subdirectories unless you quote the +glob. The shell is actually our problem, and we should work around it. + +passing in a folder path does not recurse it - this might be the shell problem +tho. + +concat should show what its args were at the start, so we can see what the shell +processed. + +ignore concat.txt since this is the output of a previous run of this tool. + +allow --tokens to only count the tokens and not write an actual concat.txt file. + +should allow a preselection of purposes to include some base prompts with the +concat as well as the project contents. + +optionally pull in the import map if in a monorepo, and the base rules. If we +have a format for putting rules in somewhere known, it can navigate appropriate +files and pull them in. Be like .concat-rules which gives it some command line +options when it runs. + +.concat-rules should say what should specially be ignored as well ? + +honour .gitignore all the way up to the root of the repo. + +BEGIN FILE should maybe be a single name, and end in a colon, making it plain +that it is a key attribute, and separate from the file name. + +If I run `concat . ../../reasoner/PROJECT_MAP_INSTRUCTIONS.md` then it does not +pull in everything recursively from . however seems to be shell globbing again + +Should supply a file system overview in the concat, optionally diff --git a/utils/concat/main.ts b/utils/concat/main.ts new file mode 100644 index 000000000000..0e93bfb45119 --- /dev/null +++ b/utils/concat/main.ts @@ -0,0 +1,174 @@ +#!/usr/bin/env -S deno run --allow-read --allow-write +import { expandGlob } from "@std/fs"; +import type { Writer } from "@std/io"; +import { relative, globToRegExp } from "@std/path"; +import { encode } from "gpt-tokenizer/model/o1-preview"; +import humanize from "humanize-number"; +import denoData from "./deno.json" with { type: "json" }; + +const printHelp = async (): Promise<void> => { + console.log(`concat v${denoData.version} + +Usage: + concat [options] [glob ...] + +Options: + --help Show this help message and exit. + --output FILE Write output to FILE (default: concat.txt if not using --stdout). + --stdout Write output to stdout only, no info messages. + --ignore GLOB Add a glob to ignore. Can be used multiple times. + +If no arguments are provided, it's equivalent to "." which is equivalent to "**/*". +This means all files are included by default, except those ignored. + +By default, these are ignored: + .lock, .git, .git/**, .gitignore, LICENSE, concat.txt + +Additional ignores can be specified via --ignore. +`); +}; + +const compareSemver = (a: string, b: string): number => { + const pa = a.replace(/^v/, "").split(".").map(Number); + const pb = b.replace(/^v/, "").split(".").map(Number); + for (let i = 0; i < Math.max(pa.length, pb.length); i++) { + const diff = (pa[i] || 0) - (pb[i] || 0); + if (diff !== 0) return diff < 0 ? -1 : 1; + } + return 0; +}; + +export const main = async (args = [...Deno.args]) => { + if (args.includes("--help")) { + await printHelp(); + Deno.exit(0); + } + + let outputFile: string | undefined; + let toStdout = false; + const ignoreGlobs: string[] = []; + + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === "--output") { + if (i + 1 >= args.length) { + console.error("Error: --output specified but no file provided."); + Deno.exit(1); + } + outputFile = args[i + 1]; + args.splice(i, 2); + i--; + } else if (a === "--stdout") { + toStdout = true; + args.splice(i, 1); + i--; + } else if (a === "--ignore") { + if (i + 1 >= args.length) { + console.error("Error: --ignore specified but no glob provided."); + Deno.exit(1); + } + ignoreGlobs.push(args[i + 1]); + args.splice(i, 2); + i -= 2; + } + } + + let patterns: string[] = []; + if (args.length === 0) { + patterns = ["**/*"]; + } else if (args.length === 1 && args[0] === ".") { + patterns = ["**/*"]; + } else { + patterns = args; + } + + if (!outputFile && !toStdout) { + outputFile = "concat.txt"; + } + + const defaultIgnores = [ + "*.lock", + ".git", + ".git/**", + ".gitignore", + "LICENSE", + "concat.txt", + ]; + const allIgnoredGlobs = [...defaultIgnores, ...ignoreGlobs].map(g => globToRegExp(g, { globstar: true })); + + let fileExistedBefore = false; + if (outputFile && !toStdout) { + try { + await Deno.stat(outputFile); + fileExistedBefore = true; + } catch { + fileExistedBefore = false; + } + } + + let out: Writer & { close?: () => void }; + if (toStdout) { + out = Deno.stdout; + } else { + out = await Deno.open(outputFile as string, { write: true, create: true, truncate: true }); + } + + const enc = new TextEncoder(); + const processedFiles: string[] = []; + + for (const pattern of patterns) { + for await (const file of expandGlob(pattern, { globstar: true })) { + if (!file.isFile) continue; + const rel = relative(Deno.cwd(), file.path); + const ignored = allIgnoredGlobs.some((re) => re.test(rel)); + if (ignored) continue; + const data = await Deno.readTextFile(file.path); + processedFiles.push(rel); + await out.write(enc.encode(`-----BEGIN FILE ${rel}-----\n${data}\n-----END FILE ${rel}-----\n`)); + } + } + + if (out.close) out.close(); + + if (toStdout) return; + + console.log("Processed files:"); + for (const f of processedFiles) { + console.log(`- ${f}`); + } + console.log(`Total files processed: ${processedFiles.length}`); + + if (outputFile) { + const outputText = await Deno.readTextFile(outputFile); + const tokens = await encode(outputText); + const formattedTokenCount = humanize(tokens.length); + const fileStatus = fileExistedBefore ? "updated" : "created"; + console.log(`✅ Operation complete! ${fileStatus} ${outputFile} with ${formattedTokenCount} o1 tokens. 🎉`); + } + + // Check for newer version + const currentVersion = `v${denoData.version}`; + try { + const metaRes = await fetch("https://jsr.io/@dreamcatcher/concat/meta.json", { + headers: { "Accept": "application/json" }, + }); + if (metaRes.ok) { + const meta = await metaRes.json(); + const availableVersions = Object.keys(meta.versions).filter((v) => !meta.versions[v].yanked); + const latest = availableVersions.sort((a, b) => compareSemver(a, b))[availableVersions.length - 1]; + if (compareSemver(latest, currentVersion) > 0) { + console.log(` +A newer version (${latest}) is available. Run the following to upgrade: + +deno install --global --reload --force --allow-read --allow-write --allow-net=jsr.io jsr:@dreamcatcher/concat@${latest} +`); + } + } + } catch { + // ignore errors + } +}; + +if (import.meta.main) { + main(); +} diff --git a/utils/concat/vendor-docs/api-jsr.io.md b/utils/concat/vendor-docs/api-jsr.io.md new file mode 100644 index 000000000000..981b18088a3b --- /dev/null +++ b/utils/concat/vendor-docs/api-jsr.io.md @@ -0,0 +1,276 @@ +--- +title: API +description: JSR uses a simple HTTP + JSON API to interact with the registry and download packages. +--- + +JSR has three APIs that you can interact with: + +- The [JSR registry API](#jsr-registry-api), which is used to download modules + and package metadata. This is used when a runtime or tool supports native + `jsr:` specifiers. +- The [npm compatibility registry API](#npm-compatibility-registry-api), which + is an npm compatible registry endpoint for package managers using + [JSR's npm compatibility layer](/docs/npm-compatibility). +- The [management API](#management-api) which allows you to publish packages, + manage your scopes, and retrieve account information. + +## JSR registry API + +The JSR registry API is used to download modules (individual JS/TS files in a +package), package version metadata, and package metadata. + +> NOTE: All requests to the JSR registry API must be made with an `Accept` +> header that does **not** include `text/html`, and requests must not specify +> `Sec-Fetch-Dest: document`. When fetching with `Accept: text/html`, the +> registry may return an HTML page with a rendered version of the underlying +> data. + +### Modules + +Modules are served using standard HTTP GET requests as would be initiated by a +browser `import` statement. Modules are served from the `https://jsr.io` domain, +at the following URL: + +``` +https://jsr.io/@<scope>/<package-name>/<version>/<path> +``` + +For example, the `/main.ts` file of the `@luca/flag` package version `1.0.0` +would be served at the following URL: + +``` +https://jsr.io/@luca/flag/1.0.0/main.ts +``` + +Modules are immutable, so they can be cached indefinitely. The registry +understands conditional requests, so will appropriately respond with +`304 Not Modified` when `If-No-Match` or `If-Modified-Since` headers are +provided. + +### Package metadata + +Package metadata refers to the information about a package as a whole, such as +the package scope, name and versions. Package metadata is served as JSON at the +following URL: + +``` +https://jsr.io/@<scope>/<package-name>/meta.json +``` + +For example, the metadata for the `@luca/flag` package would be served at the +following URL: + +``` +https://jsr.io/@luca/flag/meta.json +``` + +This metadata contains a list of published versions for the package. This allows +a tool to discover the available versions of a package and perform semantic +versioning resolution to determine the best version to use. + +Each version in the metadata contains information about the version, such as the +yanked status, and the `exports` field for the package version. The `exports` +field is normalized to simple object form. + +For the above `@luca/flag` package, the metadata would look like this: + +```json +{ + "scope": "luca", + "name": "flag", + "versions": { + "1.0.0": { + "yanked": true + }, + "1.0.1": {} + } +} +``` + +### Package version metadata + +Package version metadata refers to the information about a specific version of a +package. + +Version metadata is served as JSON at the following URL: + +``` +https://jsr.io/@<scope>/<package-name>/<version>_meta.json +``` + +For example, the metadata for the `@luca/flag` package version `1.0.0` would be +served at the following URL: + +``` +https://jsr.io/@luca/flag/1.0.0_meta.json +``` + +This metadata contains information about the requested version, such as the list +of files in the package version, the `exports` field, and a `moduleGraph1` or +`moduleGraph2` field that contains information about the module graph to allow +for less waterfall when downloading modules. + +For the above `@luca/flag` package version `1.0.0`, the metadata would look like +this: + +```json +{ + "manifest": { + "/deno.json": { + "size": 75, + "checksum": "sha256-98719bf861369684be254b01f1427084dc6d16b506809719122890784542496b" + }, + "/LICENSE": { + "size": 1070, + "checksum": "sha256-c3f0644e8374585b209ea5206ab88055c1c503c202bff5d1f01bb29c07041fbb" + }, + "/README.md": { + "size": 279, + "checksum": "sha256-f544a1489e93e93957d6bd03f069e0db7a9bef4af6eeae46a86b4e3316e598c3" + }, + "/main.ts": { + "size": 2989, + "checksum": "sha256-a41796ceb0be1bca3aa446ddebebcd732492ccb2cdcb8912adbabef3375fafc8" + } + }, + "moduleGraph1": { + "/main.ts": {} + }, + "exports": { + ".": "./main.ts" + } +} +``` + +The version metadata field is immutable, so it can be cached indefinitely. +Because of this immutability, the `yanked` field is not included in the version +metadata. Instead, retrieve yanked status from the package metadata. + +## npm compatibility registry API + +The npm compatibility registry API is used to download npm compatible tarballs +for JSR packages, and to retrieve npm compatible package metadata. + +[Learn more about the npm compatibility layer.](/docs/npm-compatibility) + +The npm compatibility registry API is served from the `https://npm.jsr.io` +domain. + +The entrypoint to the npm compatibility registry API is the package metadata +endpoint. For example, the metadata for the `@luca/flag` package on JSR (with +compatibility name `@jsr/luca__flag`) is served at the following URL: + +``` +https://npm.jsr.io/@jsr/luca__flag +``` + +This returns an npm compatible package metadata object, with the following +fields: + +- `name`: The npm compatibility name of the package. +- `description`: The package description. +- `dist-tags`: The `latest` version of the package. +- `versions`: A map of versions to version metadata. +- `time`: A map of versions to publish timestamps. + +The `versions` field is a map of versions to version metadata. The version +metadata is an npm compatible version metadata object, with the following +fields: + +- `name`: The npm compatibility name of the package. +- `version`: The version of the package. +- `description`: The package description. +- `dist`: The `dist` field for the package version. This contains the tarball + URL for the package version, and the checksums / integrity hashes for the + tarball. +- `dependencies`: The `dependencies` of the package version. + +Yanked versions are not included in the `versions` field, and are never +referenced from the `latest` dist-tag. + +> Note: The data served from the npm compatibility registry API may not always +> be up to date or consistent with the data served from the JSR registry API. +> Versions may be missing, or the `latest` dist-tag may be out of date. +> Generally this resolves itself within a few minutes. + +## Management API + +The management API is used to publish packages, manage your scopes, and retrieve +account information. + +The management API is served from the `https://api.jsr.io` domain. It is a REST +API that uses JSON for serialization. + +Many requests to the management API require authentication. Authentication is +done using an `Authorization` header with a `Bearer` or `githuboidc` token. + +### Authentication tokens + +JSR supports authenticating with three types of tokens: + +- Short-lived device access tokens, which are used to authenticate as a user. + These tokens are authenticated by the user interactively, and are only valid + for one specific purpose for a short period of time. + +- Long-lived personal access tokens, which are used to authenticate as a user. + Personal access tokens may have an expiration date, and may grant only limited + permissions. Personal access tokens can be created on the JSR account settings + page in the "Tokens" tab. + +- GitHub Actions OIDC tokens, which are used to authenticate as a GitHub Actions + runner. These tokens are created from within GitHub Actions, and are only + valid for a short period of time. They can only be used to publish packages. + +Both device access tokens, and personal access tokens are passed in the +`Authorization` header with a `Bearer` prefix. GitHub Actions OIDC tokens are +passed in the `Authorization` header with a `githuboidc` prefix. + +```http +Authorization: Bearer jsrd_5gVEGU852nnRH2opZeP9uZ1UdNXog0fcvP8 +Authorization: Bearer jsrp_Kj0yFdcksJqHPm04l5tic2WXVtaLS2292b2 +Authorization: githuboidc eyJhbGciOiJSUzI1NiIsImtpZCI6IjEwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMCJ9.eyJyZXBvIjoibHVjYWNhc29uYXRvL2ZsYWciLCJkaWQiOiAieW91IHJlYWxseSB0aGluayBpIHdhcyBnb2luZyB0byBnaXZlIHlvdSBhIHZhbGlkIHRva2VuPyJ9.ZXlKeVpYQnZJam9pYkhWallXTmhjMjl1WVhSdkwyWnNZV2NpTENKa2FXUWlPaUFpZVc5MU +``` + +### Permissions + +The management API has various permissions that can be granted to tokens. These +permissions are: + +- `package/publish`: Only allows publishing a specific package, possibly at a + specific version. +- _all_: Allows all actions with the exception of token management. + +GitHub Actions OIDC tokens only support the `package/publish` permission, with a +specific package and version specified. + +### Endpoints + +An OpenAPI 3.0 specification for the management API is available at +https://api.jsr.io/.well-known/openapi. + +A rendered version of the OpenAPI specification is available at +[/docs/api-reference](/docs/api-reference). + +### Usage restrictions + +The management API should not be used during registry operations. You should not +retrieve the list of versions for a package, or the metadata for a package +version, from the management API. Instead, use the JSR registry API or the npm +compatibility registry API. + +The management API is intended for use by tools and services that need to +interact with the registry on behalf of a user. For example, a tool that +publishes packages to JSR on behalf of a user would use the management API to +publish packages. + +Tools should identify themselves using the `User-Agent` header. The `User-Agent` +header should be in the following format: + +``` +<tool-name>/<tool-version>; <tool-url> +``` + +If a tool is misbehaving, we may block it from using the management API. + +If you are not sure whether the management API is appropriate for your use case, +please contact us at help@jsr.io. diff --git a/utils/crawl/deno.json b/utils/crawl/deno.json new file mode 100644 index 000000000000..8d11f4f95478 --- /dev/null +++ b/utils/crawl/deno.json @@ -0,0 +1,8 @@ +{ + "tasks": { + "dev": "deno run -A --watch main.ts" + }, + "imports": { + "@std/assert": "jsr:@std/assert@1" + } +} diff --git a/utils/crawl/issues.md b/utils/crawl/issues.md new file mode 100644 index 000000000000..7e49bf156900 --- /dev/null +++ b/utils/crawl/issues.md @@ -0,0 +1,33 @@ +Want to be able to stay on the same subdomain, and crawl the entire site. This +should be the default behaviour. + +headers and footers should be ignored if possible, as well as sidebar +navigation. Basicaly ignore everything that is not the main content of the page. + +the best crawler might be to read the dom and then make an o1-mini call to +extract the text from, in a clear way, like markdown. + +Using o1-mini could be extended to remove duplicate content between pages. + +The text output should include some prompt instructions on the content so the +LLM knows how to interpret it + +Pages should be sepearated using the ascii armour style separation, or some +other notation to express the difference. + +Might have the option to write each link to a different file, so we have a +folder structure of heirarchy of the site, which makes it easier to exclude +content, and allow updates to be more granular. + +May store the original dom with the summary to do provenance. + +Should recognize some common urls, like if it is a github url, then we should +know how to access the raw file. + +Might make this pluggable, and so have things like the youtube crawler be in +here specially. Ideally an LLM would make the choice of what crawler to use and +how to parse the output, since it might be a complex multistep job. + +Must set its user agent informatively. + +Recognize when using a chatgpt chat, and switch to using the chatgpt crawler. diff --git a/utils/crawl/main.ts b/utils/crawl/main.ts new file mode 100644 index 000000000000..264f10924a5f --- /dev/null +++ b/utils/crawl/main.ts @@ -0,0 +1,77 @@ +// main.ts +import { DOMParser, initParser } from 'jsr:@b-fuze/deno-dom/wasm-noinit' +import { htmlToText } from 'npm:html-to-text' + +interface PageData { + url: string + links: string[] + text: string +} + +const crawlPage = async (url: string, startUrl: string): Promise<PageData> => { + const res = await fetch(url) + if (!res.ok) throw new Error(`Fetch fail ${url}: ${res.status}`) + const html = await res.text() + const doc = new DOMParser().parseFromString(html, 'text/html') + if (!doc) return { url, links: [], text: '' } + + const baseHost = new URL(startUrl).hostname + + const links = [...doc.querySelectorAll('a')] + .map((a) => a.getAttribute('href') || '') + .map((l) => { + try { + return new URL(l, url).href + } catch { + return '' + } + }) + .filter((l) => { + try { + return new URL(l).hostname === baseHost + } catch { + return false + } + }) + + const text = htmlToText(html, { + wordwrap: 0, + baseElements: { + selectors: ['main', 'article', 'body'], + returnDomByDefault: false, + }, + selectors: [ + { selector: 'header', format: 'skip' }, + { selector: 'footer', format: 'skip' }, + { selector: 'nav', format: 'skip' }, + { selector: 'aside', format: 'skip' }, + { selector: '.sidebar', format: 'skip' }, + ], + }) + + return { url, links, text } +} + +const crawl = async (startUrl: string, maxDepth = 0) => { + await initParser() + const visited = new Set<string>() + let queue = [{ url: startUrl, depth: 0 }] + await Deno.writeTextFile('crawl.txt', '') + while (queue.length > 0) { + const { url, depth } = queue.shift()! + if (visited.has(url) || depth > maxDepth) continue + visited.add(url) + const { links, text } = await crawlPage(url, startUrl) + console.log(`Links from ${url}:`, links) + await Deno.writeTextFile('crawl.txt', `URL: ${url}\n${text}\n\n`, { + append: true, + }) + for (const link of links) { + if (!visited.has(link)) queue.push({ url: link, depth: depth + 1 }) + } + } +} + +crawl('https://github.com/b-fuze/deno-dom/blob/master/README.md').catch( + console.error, +) diff --git a/utils/crawl/rules.md b/utils/crawl/rules.md new file mode 100644 index 000000000000..fa4391aadbfd --- /dev/null +++ b/utils/crawl/rules.md @@ -0,0 +1,14 @@ +# Rules + +These are the rules that must be followed when generating the software for this +project. + +1. Update the Deno DOM imports to use `wasm-noinit` and call `initParser()` + before parsing HTML. +2. Utilize `htmlToText` to convert HTML into a plain text format suitable for + LLM consumption. +3. Write extracted text to a file (`crawl.txt`) as the crawl progresses, rather + than collecting it at the end. +4. Print links immediately after visiting each page, instead of only returning + them at the end. +5. Reduce whitespace in the code for a more compact, inline style. diff --git a/utils/crawl/vendor-docs/html-to-text/README.md b/utils/crawl/vendor-docs/html-to-text/README.md new file mode 100644 index 000000000000..a217dfbc773b --- /dev/null +++ b/utils/crawl/vendor-docs/html-to-text/README.md @@ -0,0 +1,361 @@ +# html-to-text + +[](https://github.com/html-to-text/node-html-to-text/actions/workflows/lint.yml) +[](https://github.com/html-to-text/node-html-to-text/actions/workflows/test.yml) +[](https://github.com/html-to-text/node-html-to-text/blob/master/LICENSE) +[](https://www.npmjs.com/package/html-to-text) +[](https://www.npmjs.com/package/html-to-text) + +Advanced converter that parses HTML and returns beautiful text. + +## Features + +* Inline and block-level tags. +* Tables with colspans and rowspans. +* Links with both text and href. +* Word wrapping. +* Unicode support. +* Plenty of customization options. + +## Changelog + +Available here: [CHANGELOG.md](https://github.com/html-to-text/node-html-to-text/blob/master/packages/html-to-text/CHANGELOG.md) + +Version 6 contains a ton of changes, so it worth to take a look at the full changelog. + +Version 7 contains an important change for custom formatters. + +Version 8 brings the selectors support to greatly increase the flexibility but that also changes some things introduced in version 6. Base element(s) selection also got important changes. + +Version 9 drops a lot of previously deprecated options, introduces some new formatters and new capabilities for custom formatters. Now a dual-mode package (cjs and esm). CLI is moved to a [separate package](https://github.com/html-to-text/node-html-to-text/tree/master/packages/html-to-text-cli/). + +## Installation + +``` +npm install html-to-text +``` + +## Usage + +Convert a single document: + +```js +const { convert } = require('html-to-text'); +// There is also an alias to `convert` called `htmlToText`. + +const options = { + wordwrap: 130, + // ... +}; +const html = '<div>Hello World</div>'; +const text = convert(html, options); +console.log(text); // Hello World +``` + +Configure `html-to-text` once to convert many documents with the same options (recommended for [good performance](https://github.com/html-to-text/node-html-to-text/issues/265#issuecomment-1337470852) when processing big batches of documents): + +```js +const { compile } = require('html-to-text'); + +const options = { + wordwrap: 130, + // ... +}; +const compiledConvert = compile(options); // options passed here + +const htmls = [ + '<div>Hello World!</div>', + '<div>こんにちは世界!</div>', + '<div>Привіт Світ!</div>' +]; +const texts = htmls.map(compiledConvert); +console.log(texts.join('\n')); +// Hello World! +// こんにちは世界! +// Привіт Світ! +``` + +Both `convert` and `compiledConvert` can take one more optional argument - [metadata object](#custom-metadata) that can be used by formatters. + +### Options + +#### General options + +Option | Default | Description +----------------------- | ------------ | ----------- +`baseElements` | | Describes which parts of the input document have to be converted and present in the output text, and in what order. +`baseElements.selectors` | `['body']` | Elements matching any of provided selectors will be processed and included in the output text, with all inner content.<br/>Refer to [Supported selectors](#supported-selectors) section below. +`baseElements.orderBy` | `'selectors'` | `'selectors'` - arrange base elements in the same order as `baseElements.selectors` array;<br/>`'occurrence'` - arrange base elements in the order they are found in the input document. +`baseElements.returnDomByDefault` | `true` | Convert the entire document if none of provided selectors match. +`decodeEntities` | `true` | Decode HTML entities found in the input HTML if `true`. Otherwise preserve in output text. +`encodeCharacters` | `{}` | A dictionary with characters that should be replaced in the output text and corresponding escape sequences. +`formatters` | `{}` | An object with custom formatting functions for specific elements (see [Override formatting](#override-formatting) section below). +`limits` | | Describes how to limit the output text in case of large HTML documents. +`limits.ellipsis` | `'...'` | A string to insert in place of skipped content. +`limits.maxBaseElements` | `undefined` | Stop looking for more base elements after reaching this amount. Unlimited if undefined. +`limits.maxChildNodes` | `undefined` | Maximum number of child nodes of a single node to be added to the output. Unlimited if undefined. +`limits.maxDepth` | `undefined` | Stop looking for nodes to add to the output below this depth in the DOM tree. Unlimited if undefined. +`limits.maxInputLength` | `16_777_216` | If the input string is longer than this value - it will be truncated and a message will be sent to `stderr`. Ellipsis is not used in this case. Unlimited if undefined. +`longWordSplit` | | Describes how to wrap long words. +`longWordSplit.wrapCharacters` | `[]` | An array containing the characters that may be wrapped on. Checked in order, search stops once line length requirement can be met. +`longWordSplit.forceWrapOnLimit` | `false` | Break long words at the line length limit in case no better wrap opportunities found. +`preserveNewlines` | `false` | By default, any newlines `\n` from the input HTML are collapsed into space as any other HTML whitespace characters. If `true`, these newlines will be preserved in the output. This is only useful when input HTML carries some plain text formatting instead of proper tags. +`selectors` | `[]` | Describes how different HTML elements should be formatted. See [Selectors](#selectors) section below. +`whitespaceCharacters` | `' \t\r\n\f\u200b'` | A string of characters that are recognized as HTML whitespace. Default value uses the set of characters defined in [HTML4 standard](https://www.w3.org/TR/html4/struct/text.html#h-9.1). (It includes Zero-width space compared to [living standard](https://infra.spec.whatwg.org#ascii-whitespace).) +`wordwrap` | `80` | After how many chars a line break should follow.<br/>Set to `null` or `false` to disable word-wrapping. + +#### Deprecated or removed options + +Old option | Depr. | Rem. | Instead use +-------------------------- | --- | ----- | ----------------- +`baseElement` | 8.0 | | `baseElements: { selectors: [ 'body' ] }` +`decodeOptions` | | 9.0 | Entity decoding is now handled by [htmlparser2](https://github.com/fb55/htmlparser2) itself and [entities](https://github.com/fb55/entities) internally. No user-configurable parts compared to [he](https://github.com/mathiasbynens/he) besides boolean `decodeEntities`. +`format` | | 6.0 | The way formatters are written has changed completely. New formatters have to be added to the `formatters` option, old ones can not be reused without rewrite. See [new instructions](#override-formatting) below. +`hideLinkHrefIfSameAsText` | 6.0 | 9.0 | `selectors: [ { selector: 'a', options: { hideLinkHrefIfSameAsText: true } } ]` +`ignoreHref` | 6.0 | 9.0 | `selectors: [ { selector: 'a', options: { ignoreHref: true } } ]` +`ignoreImage` | 6.0 | 9.0 | `selectors: [ { selector: 'img', format: 'skip' } ]` +`linkHrefBaseUrl` | 6.0 | 9.0 | `selectors: [`<br/>`{ selector: 'a', options: { baseUrl: 'https://example.com' } },`<br/>`{ selector: 'img', options: { baseUrl: 'https://example.com' } }`<br/>`]` +`noAnchorUrl` | 6.0 | 9.0 | `selectors: [ { selector: 'a', options: { noAnchorUrl: true } } ]` +`noLinkBrackets` | 6.0 | 9.0 | `selectors: [ { selector: 'a', options: { linkBrackets: false } } ]` +`returnDomByDefault` | 8.0 | | `baseElements: { returnDomByDefault: true }` +`singleNewLineParagraphs` | 6.0 | 9.0 | `selectors: [`<br/>`{ selector: 'p', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },`<br/>`{ selector: 'pre', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } }`<br/>`]` +`tables` | 8.0 | | `selectors: [ { selector: 'table.class#id', format: 'dataTable' } ]` +`tags` | 8.0 | | See [Selectors](#selectors) section below. +`unorderedListItemPrefix` | 6.0 | 9.0 | `selectors: [ { selector: 'ul', options: { itemPrefix: ' * ' } } ]` +`uppercaseHeadings` | 6.0 | 9.0 | `selectors: [`<br/>`{ selector: 'h1', options: { uppercase: false } },`<br/>`...`<br/>`{ selector: 'table', options: { uppercaseHeaderCells: false } }`<br/>`]` + +Other things removed: + +* `fromString` method - use `convert` or `htmlToText` instead; +* positional arguments in `BlockTextBuilder` methods - pass option objects instead. + +#### Selectors + +Some example: + +```javascript +const { convert } = require('html-to-text'); + +const html = '<a href="/page.html">Page</a><a href="!#" class="button">Action</a>'; +const text = convert(html, { + selectors: [ + { selector: 'a', options: { baseUrl: 'https://example.com' } }, + { selector: 'a.button', format: 'skip' } + ] +}); +console.log(text); // Page [https://example.com/page.html] +``` + +Selectors array is our loose approximation of a stylesheet. + +* highest [specificity](https://www.w3.org/TR/selectors/#specificity) selector is used when there are multiple matches; +* the last selector is used when there are multiple matches of equal specificity; +* all entries with the same selector value are merged (recursively) at the compile stage, in such way so the last defined properties are kept and the relative order of unique selectors is kept; +* user-defined entries are appended after [predefined entries](#predefined-formatters); +* Every unique selector must have `format` value specified (at least once); +* unlike in CSS, values from different matched selectors are NOT merged at the convert stage. Single best match is used instead (that is the last one of those with highest specificity). + +To achieve the best performance when checking each DOM element against provided selectors, they are compiled into a decision tree. But it is also important how you choose selectors. For example, `div#id` is much better than `#id` - the former will only check divs for the id while the latter has to check every element in the DOM. + +##### Supported selectors + +`html-to-text` relies on [parseley](https://github.com/mxxii/parseley) and [selderee](https://github.com/mxxii/selderee) packages for selectors support. + +Following selectors can be used in any combinations: + +* `*` - universal selector; +* `div` - tag name; +* `.foo` - class name; +* `#bar` - id; +* `[baz]` - attribute presence; +* `[baz=buzz]` - attribute value (with any operators and also quotes and case sensitivity modifiers - [syntax](https://developer.mozilla.org/en-US/docs/Web/CSS/Attribute_selectors#syntax)); +* `+` and `>` combinators (other combinators are not supported). + +You can match `<p style="...; display:INLINE; ...">...</p>` with `p[style*="display:inline"i]` for example. + +##### Predefined formatters + +Following selectors have a formatter specified as a part of the default configuration. Everything can be overridden, but you don't have to repeat the `format` or options that you don't want to override. (But keep in mind this is only true for the same selector. There is no connection between different selectors.) + +Selector | Default format | Notes +------------- | ------------------- | ----- +`*` | `inline` | Universal selector. +`a` | `anchor` | +`article` | `block` | +`aside` | `block` | +`blockquote` | `blockquote` | +`br` | `lineBreak` | +`div` | `block` | +`footer` | `block` | +`form` | `block` | +`h1` | `heading` | +`h2` | `heading` | +`h3` | `heading` | +`h4` | `heading` | +`h5` | `heading` | +`h6` | `heading` | +`header` | `block` | +`hr` | `horizontalLine` | +`img` | `image` | +`main` | `block` | +`nav` | `block` | +`ol` | `orderedList` | +`p` | `paragraph` | +`pre` | `pre` | +`table` | `table` | Equivalent to `block`. Use `dataTable` instead for tabular data. +`ul` | `unorderedList` | +`wbr` | `wbr` | + +More formatters also available for use: + +Format | Description +---------------- | ----------- +`dataTable` | For visually-accurate tables. Note that this might be not search-friendly (output text will look like gibberish to a machine when there is any wrapped cell contents) and also better to be avoided for tables used as a page layout tool. +`skip` | Skips the given tag with it's contents without printing anything. +`blockString` | Insert a block with the given string literal (`formatOptions.string`) instead of the tag. +`blockTag` | Render an element as HTML block tag, convert it's contents to text. +`blockHtml` | Render an element with all it's children as HTML block. +`inlineString` | Insert the given string literal (`formatOptions.string`) inline instead of the tag. +`inlineSurround` | Render inline element wrapped with given strings (`formatOptions.prefix` and `formatOptions.suffix`). +`inlineTag` | Render an element as inline HTML tag, convert it's contents to text. +`inlineHtml` | Render an element with all it's children as inline HTML. + +##### Format options + +Following options are available for built-in formatters. + +Option | Default | Applies to | Description +------------------- | ----------- | ------------------ | ----------- +`leadingLineBreaks` | `1`, `2` or `3` | all block-level formatters | Number of line breaks to separate previous block from this one.<br/>Note that N+1 line breaks are needed to make N empty lines. +`trailingLineBreaks` | `1` or `2` | all block-level formatters | Number of line breaks to separate this block from the next one.<br/>Note that N+1 line breaks are needed to make N empty lines. +`baseUrl` | `null` | `anchor`, `image` | Server host for link `href` attributes and image `src` attributes relative to the root (the ones that start with `/`).<br/>For example, with `baseUrl = 'http://asdf.com'` and `<a href='/dir/subdir'>...</a>` the link in the text will be `http://asdf.com/dir/subdir`. +`linkBrackets` | `['[', ']']` | `anchor`, `image` | Surround links with these brackets.<br/>Set to `false` or `['', '']` to disable. +`pathRewrite` | `undefined` | `anchor`, `image` | A function to rewrite link `href` attributes and image `src` attributes. Optional second argument is the metadata object.<br/>Applied before `baseUrl`. +`hideLinkHrefIfSameAsText` | `false` | `anchor` | By default links are translated in the following way:<br/>`<a href='link'>text</a>` => becomes => `text [link]`.<br/>If this option is set to `true` and `link` and `text` are the same, `[link]` will be omitted and only `text` will be present. +`ignoreHref` | `false` | `anchor` | Ignore all links. Only process internal text of anchor tags. +`noAnchorUrl` | `true` | `anchor` | Ignore anchor links (where `href='#...'`). +`itemPrefix` | `' * '` | `unorderedList` | String prefix for each list item. +`uppercase` | `true` | `heading` | By default, headings (`<h1>`, `<h2>`, etc) are uppercased.<br/>Set this to `false` to leave headings as they are. +`length` | `undefined` | `horizontalLine` | Length of the line. If undefined then `wordwrap` value is used. Falls back to 40 if that's also disabled. +`trimEmptyLines` | `true` | `blockquote` | Trim empty lines from blockquote.<br/>While empty lines should be preserved in HTML, space-saving behavior is chosen as default for convenience. +`uppercaseHeaderCells` | `true` | `dataTable` | By default, heading cells (`<th>`) are uppercased.<br/>Set this to `false` to leave heading cells as they are. +`maxColumnWidth` | `60` | `dataTable` | Data table cell content will be wrapped to fit this width instead of global `wordwrap` limit.<br/>Set this to `undefined` in order to fall back to `wordwrap` limit. +`colSpacing` | `3` | `dataTable` | Number of spaces between data table columns. +`rowSpacing` | `0` | `dataTable` | Number of empty lines between data table rows. +`string` | `''` | `blockString`, `inlineString` | A string to be inserted in place of a tag. +`prefix` | `''` | `inlineSurround` | String prefix to be inserted before inline tag contents. +`suffix` | `''` | `inlineSurround` | String suffix to be inserted after inline tag contents. + +##### Deprecated format options + +Old option | Applies to | Depr. | Rem. | Instead use +------------------- | ------------------ | ----- | ---- | --------------------- +`noLinkBrackets` | `anchor` | 8.1 | | `linkBrackets: false` + +### Override formatting + +`formatters` option is an object that holds formatting functions. They can be assigned to format different elements in the `selectors` array. + +Each formatter is a function of four arguments that returns nothing. Arguments are: + +* `elem` - the HTML element to be processed by this formatter; +* `walk` - recursive function to process the children of this element. Called as `walk(elem.children, builder)`; +* `builder` - [BlockTextBuilder](https://github.com/html-to-text/node-html-to-text/blob/master/packages/base/src/block-text-builder.js) object. Manipulate this object state to build the output text; +* `formatOptions` - options that are specified for a tag, along with this formatter (Note: if you need general html-to-text [options](#general-options) - they are accessible via `builder.options`). + +Custom formatter example: + +```javascript +const { convert } = require('html-to-text'); + +const html = '<foo>Hello World</foo>'; +const text = convert(html, { + formatters: { + // Create a formatter. + 'fooBlockFormatter': function (elem, walk, builder, formatOptions) { + builder.openBlock({ leadingLineBreaks: formatOptions.leadingLineBreaks || 1 }); + walk(elem.children, builder); + builder.addInline('!'); + builder.closeBlock({ trailingLineBreaks: formatOptions.trailingLineBreaks || 1 }); + } + }, + selectors: [ + // Assign it to `foo` tags. + { + selector: 'foo', + format: 'fooBlockFormatter', + options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } + } + ] +}); +console.log(text); // Hello World! +``` + +New in version 9: metadata object can be provided as the last optional argument of the `convert` function (or the function returned by `compile` function). It can be accessed by formatters as `builder.metadata`. + +Refer to [generic formatters](https://github.com/html-to-text/node-html-to-text/blob/master/packages/base/src/generic-formatters.js) of the base package and [text formatters](https://github.com/html-to-text/node-html-to-text/blob/master/packages/html-to-text/src/text-formatters.js) of this package for more examples. The easiest way to write your own is to pick an existing one and customize. + +Refer to [BlockTextBuilder](https://github.com/html-to-text/node-html-to-text/blob/master/packages/base/src/block-text-builder.js) for available functions and arguments. + +#### Custom metadata + +If you need to supply extra information about your HTML documents to use in custom formatters - it can be done with the help of metadata object. + +It is supplied as an extra argument to the convert function: + +```javascript +import { compile, convert } from 'html-to-text'; + +// for batch use: +const compiledConvert = compile(options); +let text = compiledConvert(html, metadata); + +// for single use: +let text = convert(html, options, metadata); +``` + +And it can be accessed within formatter functions as `builder.metadata`. + +#### Call other formatters from a custom formatter + +Most of the times this is *not* what you actually need. Most practical problems can be solved with [selectors](#selectors). + +If you really need to inspect the node internals, not just attributes, then you can do it like this: + +```javascript +const options = { + // ... + formatters: { + filterBlockFormatter: function (elem, walk, builder, formatOptions) { + // all built-in and custom formatters available by name + const blockFormatter = builder.options.formatters['block']; + if (blockFormatter && elem.children.some(/* predicate */)) { + blockFormatter(elem, walk, builder, formatOptions); + } + } + }, + selectors: [ + { + selector: 'div.questionable', + format: 'filterBlockFormatter', + options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } + } + ], + // ... +} +``` + +## Example + +* Input text: [test.html](https://github.com/html-to-text/node-html-to-text/blob/master/packages/html-to-text/test/test.html) +* Output text: [test.txt](https://github.com/html-to-text/node-html-to-text/blob/master/packages/html-to-text/test/test.txt) + +## Contributors + +* [@mlegenhausen](https://github.com/mlegenhausen) - creator; +* [@KillyMXI](https://github.com/KillyMXI) - maintainer since 2020; +* Everyone else who [added something](https://github.com/html-to-text/node-html-to-text/graphs/contributors) to the tool or helped us shaping it via [issues](https://github.com/html-to-text/node-html-to-text/issues) and [PRs](https://github.com/html-to-text/node-html-to-text/pulls). + +## License + +[MIT License](https://github.com/html-to-text/node-html-to-text/blob/master/LICENSE) \ No newline at end of file diff --git a/utils/pdfs/.gitignore b/utils/pdfs/.gitignore new file mode 100644 index 000000000000..b0da9cafc918 --- /dev/null +++ b/utils/pdfs/.gitignore @@ -0,0 +1,2 @@ +pdf.json +pdf.txt \ No newline at end of file diff --git a/utils/pdfs/deno.json b/utils/pdfs/deno.json new file mode 100644 index 000000000000..47ac90c35d9b --- /dev/null +++ b/utils/pdfs/deno.json @@ -0,0 +1,20 @@ +{ + "name": "@dreamcatcher/pdfs", + "version": "0.0.1", + "description": "Processes PDFs into a text like form suitable for AI processing", + "tasks": { + "run": "deno run --allow-read --allow-write main.ts", + "dev": "deno run --watch --allow-read --allow-write main.ts" + }, + "imports": { + "@std/assert": "jsr:@std/assert@1", + "@std/cli": "jsr:@std/cli@^1.0.8", + "pdf.js-extract": "npm:pdf.js-extract@^0.2.1" + }, + "exports": { + ".": "./main.ts" + }, + "bin": { + "pdfs": "./main.ts" + } +} diff --git a/utils/pdfs/issues.md b/utils/pdfs/issues.md new file mode 100644 index 000000000000..1908383d7fa5 --- /dev/null +++ b/utils/pdfs/issues.md @@ -0,0 +1,14 @@ +text conversion must preserve the paragraph structure and page numbers for the +purpose of making accurate citations. + +A text reference might be mapped back to the json representation of the pdf so +that we can actually highlight the text in the pdf, given its geometric layout +as well as its content. + +The output text format from multiple formats, such as pdf, docs, html, should +always be the same, and so we need some kind of fidelity testing to ensure +things are the same. + +Conversion of an unknown doc should be attempt to be done programmatically, but +if not, start using an LLM to try to recognize what is going on and guide its +conversion. diff --git a/utils/pdfs/main.ts b/utils/pdfs/main.ts new file mode 100644 index 000000000000..9ff5a3d120d1 --- /dev/null +++ b/utils/pdfs/main.ts @@ -0,0 +1,87 @@ +#!/usr/bin/env -S deno run --allow-read --allow-write + +import { parseArgs } from '@std/cli/parse-args' +import { PDFExtract, type PDFExtractOptions } from 'pdf.js-extract' + +/** + * Convert PDFExtractOptions from CLI args to a typed object. + */ +function getPDFExtractOptions( + args: ReturnType<typeof parseArgs>, +): PDFExtractOptions { + const options: PDFExtractOptions = {} + + if (typeof args.firstPage === 'number') options.firstPage = args.firstPage + if (typeof args.lastPage === 'number') options.lastPage = args.lastPage + if (typeof args.password === 'string') options.password = args.password + if (typeof args.verbosity === 'number') options.verbosity = args.verbosity + if (typeof args.normalizeWhitespace === 'boolean') { + options.normalizeWhitespace = args.normalizeWhitespace + } + if (typeof args.disableCombineTextItems === 'boolean') { + options.disableCombineTextItems = args.disableCombineTextItems + } + + return options +} + +async function main() { + const args = parseArgs(Deno.args, { + string: ['out', 'password'], + boolean: [ + 'text', + 'normalizeWhitespace', + 'disableCombineTextItems', + ], + default: { + out: undefined, + text: false, + }, + }) + + const inputPath = args._[0] + if (typeof inputPath !== 'string' || !inputPath.endsWith('.pdf')) { + // deno-lint-ignore no-console + console.error( + 'Usage: pdfs input.pdf [--out=outputFile] [--text] [pdfExtractOptions...]', + ) + Deno.exit(1) + } + + // Determine output filename defaults + const isTextMode = args.text === true + let outputPath = args.out + if (!outputPath) { + outputPath = isTextMode ? 'pdf.txt' : 'pdf.json' + } + + const data = await Deno.readFile(inputPath) + const pdfExtract = new PDFExtract() + const extractOptions = getPDFExtractOptions(args) + const result = await pdfExtract.extractBuffer(data, extractOptions) + + if (isTextMode) { + // In text mode, we include some basic metadata (like number of pages) + // and then flatten all pages into a single text output separated by '---'. + const numPages = result.pages.length + let metadataText = `# PDF Metadata\n` + metadataText += `Filename: ${inputPath}\n` + metadataText += `Number of Pages: ${numPages}\n` + if (result.meta?.info?.Title) { + metadataText += `Title: ${result.meta.info.Title}\n` + } + metadataText += `\n\n` + + const pagesText = result.pages.map((page) => { + return page.content.map((item) => item.str.trim()).join(' ') + }).join('\n---\n') + const fullText = metadataText + pagesText + await Deno.writeTextFile(outputPath, fullText) + } else { + // Otherwise, output full JSON data + const jsonStr = JSON.stringify(result, null, 2) + await Deno.writeTextFile(outputPath, jsonStr) + } +} + +await main() diff --git a/utils/pdfs/sample.pdf b/utils/pdfs/sample.pdf new file mode 100644 index 000000000000..c01805e89c16 Binary files /dev/null and b/utils/pdfs/sample.pdf differ diff --git a/utils/pdfs/vendor-docs/@std/cli/parse_args.ts b/utils/pdfs/vendor-docs/@std/cli/parse_args.ts new file mode 100644 index 000000000000..b6001a645c15 --- /dev/null +++ b/utils/pdfs/vendor-docs/@std/cli/parse_args.ts @@ -0,0 +1,776 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +// This module is browser compatible. + +/** + * Command line arguments parser based on + * {@link https://github.com/minimistjs/minimist | minimist}. + * + * @example + * ```ts + * import { parseArgs } from "@std/cli/parse-args"; + * + * console.dir(parseArgs(Deno.args)); + * ``` + * + * @module + */ + +/** Combines recursively all intersection types and returns a new single type. + * @internal + */ +type Id<TRecord> = TRecord extends Record<string, unknown> + ? TRecord extends infer InferredRecord + ? { [Key in keyof InferredRecord]: Id<InferredRecord[Key]> } + : never + : TRecord; + +/** Converts a union type `A | B | C` into an intersection type `A & B & C`. + * @internal + */ +type UnionToIntersection<TValue> = + (TValue extends unknown ? (args: TValue) => unknown : never) extends + (args: infer R) => unknown ? R extends Record<string, unknown> ? R : never + : never; + +/** @internal */ +type BooleanType = boolean | string | undefined; +/** @internal */ +type StringType = string | undefined; +/** @internal */ +type ArgType = StringType | BooleanType; + +/** @internal */ +type Collectable = string | undefined; +/** @internal */ +type Negatable = string | undefined; + +type UseTypes< + TBooleans extends BooleanType, + TStrings extends StringType, + TCollectable extends Collectable, +> = undefined extends ( + & (false extends TBooleans ? undefined : TBooleans) + & TCollectable + & TStrings +) ? false + : true; + +/** + * Creates a record with all available flags with the corresponding type and + * default type. + * @internal + */ +type Values< + TBooleans extends BooleanType, + TStrings extends StringType, + TCollectable extends Collectable, + TNegatable extends Negatable, + TDefault extends Record<string, unknown> | undefined, + TAliases extends Aliases | undefined, +> = UseTypes<TBooleans, TStrings, TCollectable> extends true ? + & Record<string, unknown> + & AddAliases< + SpreadDefaults< + & CollectValues<TStrings, string, TCollectable, TNegatable> + & RecursiveRequired<CollectValues<TBooleans, boolean, TCollectable>> + & CollectUnknownValues< + TBooleans, + TStrings, + TCollectable, + TNegatable + >, + DedotRecord<TDefault> + >, + TAliases + > + // deno-lint-ignore no-explicit-any + : Record<string, any>; + +/** @internal */ +type Aliases<TArgNames = string, TAliasNames extends string = string> = Partial< + Record<Extract<TArgNames, string>, TAliasNames | ReadonlyArray<TAliasNames>> +>; + +type AddAliases< + TArgs, + TAliases extends Aliases | undefined, +> = { + [TArgName in keyof TArgs as AliasNames<TArgName, TAliases>]: TArgs[TArgName]; +}; + +type AliasNames< + TArgName, + TAliases extends Aliases | undefined, +> = TArgName extends keyof TAliases + ? string extends TAliases[TArgName] ? TArgName + : TAliases[TArgName] extends string ? TArgName | TAliases[TArgName] + : TAliases[TArgName] extends Array<string> + ? TArgName | TAliases[TArgName][number] + : TArgName + : TArgName; + +/** + * Spreads all default values of Record `TDefaults` into Record `TArgs` + * and makes default values required. + * + * **Example:** + * `SpreadValues<{ foo?: boolean, bar?: number }, { foo: number }>` + * + * **Result:** `{ foo: boolean | number, bar?: number }` + */ +type SpreadDefaults<TArgs, TDefaults> = TDefaults extends undefined ? TArgs + : TArgs extends Record<string, unknown> ? + & Omit<TArgs, keyof TDefaults> + & { + [Default in keyof TDefaults]: Default extends keyof TArgs + ? (TArgs[Default] & TDefaults[Default] | TDefaults[Default]) extends + Record<string, unknown> + ? NonNullable<SpreadDefaults<TArgs[Default], TDefaults[Default]>> + : TDefaults[Default] | NonNullable<TArgs[Default]> + : unknown; + } + : never; + +/** + * Defines the Record for the `default` option to add + * auto-suggestion support for IDE's. + * @internal + */ +type Defaults<TBooleans extends BooleanType, TStrings extends StringType> = Id< + UnionToIntersection< + & Record<string, unknown> + // Dedotted auto suggestions: { foo: { bar: unknown } } + & MapTypes<TStrings, unknown> + & MapTypes<TBooleans, unknown> + // Flat auto suggestions: { "foo.bar": unknown } + & MapDefaults<TBooleans> + & MapDefaults<TStrings> + > +>; + +type MapDefaults<TArgNames extends ArgType> = Partial< + Record<TArgNames extends string ? TArgNames : string, unknown> +>; + +type RecursiveRequired<TRecord> = TRecord extends Record<string, unknown> ? { + [Key in keyof TRecord]-?: RecursiveRequired<TRecord[Key]>; + } + : TRecord; + +/** Same as `MapTypes` but also supports collectable options. */ +type CollectValues< + TArgNames extends ArgType, + TType, + TCollectable extends Collectable, + TNegatable extends Negatable = undefined, +> = UnionToIntersection< + Extract<TArgNames, TCollectable> extends string ? + & (Exclude<TArgNames, TCollectable> extends never ? Record<never, never> + : MapTypes<Exclude<TArgNames, TCollectable>, TType, TNegatable>) + & (Extract<TArgNames, TCollectable> extends never ? Record<never, never> + : RecursiveRequired< + MapTypes<Extract<TArgNames, TCollectable>, Array<TType>, TNegatable> + >) + : MapTypes<TArgNames, TType, TNegatable> +>; + +/** Same as `Record` but also supports dotted and negatable options. */ +type MapTypes< + TArgNames extends ArgType, + TType, + TNegatable extends Negatable = undefined, +> = undefined extends TArgNames ? Record<never, never> + : TArgNames extends `${infer Name}.${infer Rest}` ? { + [Key in Name]?: MapTypes< + Rest, + TType, + TNegatable extends `${Name}.${infer Negate}` ? Negate : undefined + >; + } + : TArgNames extends string ? Partial< + Record<TArgNames, TNegatable extends TArgNames ? TType | false : TType> + > + : Record<never, never>; + +type CollectUnknownValues< + TBooleans extends BooleanType, + TStrings extends StringType, + TCollectable extends Collectable, + TNegatable extends Negatable, +> = UnionToIntersection< + TCollectable extends TBooleans & TStrings ? Record<never, never> + : DedotRecord< + // Unknown collectable & non-negatable args. + & Record< + Exclude< + Extract<Exclude<TCollectable, TNegatable>, string>, + Extract<TStrings | TBooleans, string> + >, + Array<unknown> + > + // Unknown collectable & negatable args. + & Record< + Exclude< + Extract<Extract<TCollectable, TNegatable>, string>, + Extract<TStrings | TBooleans, string> + >, + Array<unknown> | false + > + > +>; + +/** Converts `{ "foo.bar.baz": unknown }` into `{ foo: { bar: { baz: unknown } } }`. */ +type DedotRecord<TRecord> = Record<string, unknown> extends TRecord ? TRecord + : TRecord extends Record<string, unknown> ? UnionToIntersection< + ValueOf< + { + [Key in keyof TRecord]: Key extends string ? Dedot<Key, TRecord[Key]> + : never; + } + > + > + : TRecord; + +type Dedot<TKey extends string, TValue> = TKey extends + `${infer Name}.${infer Rest}` ? { [Key in Name]: Dedot<Rest, TValue> } + : { [Key in TKey]: TValue }; + +type ValueOf<TValue> = TValue[keyof TValue]; + +/** The value returned from {@linkcode parseArgs}. */ +export type Args< + // deno-lint-ignore no-explicit-any + TArgs extends Record<string, unknown> = Record<string, any>, + TDoubleDash extends boolean | undefined = undefined, +> = Id< + & TArgs + & { + /** Contains all the arguments that didn't have an option associated with + * them. */ + _: Array<string | number>; + } + & (boolean extends TDoubleDash ? DoubleDash + : true extends TDoubleDash ? Required<DoubleDash> + : Record<never, never>) +>; + +/** @internal */ +type DoubleDash = { + /** Contains all the arguments that appear after the double dash: "--". */ + "--"?: Array<string>; +}; + +/** Options for {@linkcode parseArgs}. */ +export interface ParseOptions< + TBooleans extends BooleanType = BooleanType, + TStrings extends StringType = StringType, + TCollectable extends Collectable = Collectable, + TNegatable extends Negatable = Negatable, + TDefault extends Record<string, unknown> | undefined = + | Record<string, unknown> + | undefined, + TAliases extends Aliases | undefined = Aliases | undefined, + TDoubleDash extends boolean | undefined = boolean | undefined, +> { + /** + * When `true`, populate the result `_` with everything before the `--` and + * the result `['--']` with everything after the `--`. + * + * @default {false} + * + * @example + * ```ts + * // $ deno run example.ts -- a arg1 + * import { parseArgs } from "@std/cli/parse-args"; + * console.dir(parseArgs(Deno.args, { "--": false })); + * // output: { _: [ "a", "arg1" ] } + * console.dir(parseArgs(Deno.args, { "--": true })); + * // output: { _: [], --: [ "a", "arg1" ] } + * ``` + */ + "--"?: TDoubleDash; + + /** + * An object mapping string names to strings or arrays of string argument + * names to use as aliases. + * + * @default {{}} + */ + alias?: TAliases; + + /** + * A boolean, string or array of strings to always treat as booleans. If + * `true` will treat all double hyphenated arguments without equal signs as + * `boolean` (e.g. affects `--foo`, not `-f` or `--foo=bar`). + * All `boolean` arguments will be set to `false` by default. + * + * @default {false} + */ + boolean?: TBooleans | ReadonlyArray<Extract<TBooleans, string>>; + + /** + * An object mapping string argument names to default values. + * + * @default {{}} + */ + default?: TDefault & Defaults<TBooleans, TStrings>; + + /** + * When `true`, populate the result `_` with everything after the first + * non-option. + * + * @default {false} + */ + stopEarly?: boolean; + + /** + * A string or array of strings argument names to always treat as strings. + * + * @default {[]} + */ + string?: TStrings | ReadonlyArray<Extract<TStrings, string>>; + + /** + * A string or array of strings argument names to always treat as arrays. + * Collectable options can be used multiple times. All values will be + * collected into one array. If a non-collectable option is used multiple + * times, the last value is used. + * + * @default {[]} + */ + collect?: TCollectable | ReadonlyArray<Extract<TCollectable, string>>; + + /** + * A string or array of strings argument names which can be negated + * by prefixing them with `--no-`, like `--no-config`. + * + * @default {[]} + */ + negatable?: TNegatable | ReadonlyArray<Extract<TNegatable, string>>; + + /** + * A function which is invoked with a command line parameter not defined in + * the `options` configuration object. If the function returns `false`, the + * unknown option is not added to `parsedArgs`. + * + * @default {unknown} + */ + unknown?: (arg: string, key?: string, value?: unknown) => unknown; +} + +interface NestedMapping { + [key: string]: NestedMapping | unknown; +} + +const FLAG_REGEXP = + /^(?:-(?:(?<doubleDash>-)(?<negated>no-)?)?)(?<key>.+?)(?:=(?<value>.+?))?$/s; +const LETTER_REGEXP = /[A-Za-z]/; +const NUMBER_REGEXP = /-?\d+(\.\d*)?(e-?\d+)?$/; +const HYPHEN_REGEXP = /^(-|--)[^-]/; +const VALUE_REGEXP = /=(?<value>.+)/; +const FLAG_NAME_REGEXP = /^--[^=]+$/; +const SPECIAL_CHAR_REGEXP = /\W/; + +const NON_WHITESPACE_REGEXP = /\S/; + +function isNumber(string: string): boolean { + return NON_WHITESPACE_REGEXP.test(string) && Number.isFinite(Number(string)); +} + +function setNested( + object: NestedMapping, + keys: string[], + value: unknown, + collect = false, +) { + keys = [...keys]; + const key = keys.pop()!; + + keys.forEach((key) => object = (object[key] ??= {}) as NestedMapping); + + if (collect) { + const v = object[key]; + if (Array.isArray(v)) { + v.push(value); + return; + } + + value = v ? [v, value] : [value]; + } + + object[key] = value; +} + +function hasNested(object: NestedMapping, keys: string[]): boolean { + for (const key of keys) { + const value = object[key]; + if (!Object.hasOwn(object, key)) return false; + object = value as NestedMapping; + } + return true; +} + +function aliasIsBoolean( + aliasMap: Map<string, Set<string>>, + booleanSet: Set<string>, + key: string, +): boolean { + const set = aliasMap.get(key); + if (set === undefined) return false; + for (const alias of set) if (booleanSet.has(alias)) return true; + return false; +} + +function isBooleanString(value: string) { + return value === "true" || value === "false"; +} + +function parseBooleanString(value: unknown) { + return value !== "false"; +} + +/** + * Take a set of command line arguments, optionally with a set of options, and + * return an object representing the flags found in the passed arguments. + * + * By default, any arguments starting with `-` or `--` are considered boolean + * flags. If the argument name is followed by an equal sign (`=`) it is + * considered a key-value pair. Any arguments which could not be parsed are + * available in the `_` property of the returned object. + * + * By default, this module tries to determine the type of all arguments + * automatically and the return type of this function will have an index + * signature with `any` as value (`{ [x: string]: any }`). + * + * If the `string`, `boolean` or `collect` option is set, the return value of + * this function will be fully typed and the index signature of the return + * type will change to `{ [x: string]: unknown }`. + * + * Any arguments after `'--'` will not be parsed and will end up in `parsedArgs._`. + * + * Numeric-looking arguments will be returned as numbers unless `options.string` + * or `options.boolean` is set for that argument name. + * + * @param args An array of command line arguments. + * @param options Options for the parse function. + * + * @typeParam TArgs Type of result. + * @typeParam TDoubleDash Used by `TArgs` for the result. + * @typeParam TBooleans Used by `TArgs` for the result. + * @typeParam TStrings Used by `TArgs` for the result. + * @typeParam TCollectable Used by `TArgs` for the result. + * @typeParam TNegatable Used by `TArgs` for the result. + * @typeParam TDefaults Used by `TArgs` for the result. + * @typeParam TAliases Used by `TArgs` for the result. + * @typeParam TAliasArgNames Used by `TArgs` for the result. + * @typeParam TAliasNames Used by `TArgs` for the result. + * + * @return The parsed arguments. + * + * @example Usage + * ```ts + * import { parseArgs } from "@std/cli/parse-args"; + * import { assertEquals } from "@std/assert"; + * + * // For proper use, one should use `parseArgs(Deno.args)` + * assertEquals(parseArgs(["--foo", "--bar=baz", "./quux.txt"]), { + * foo: true, + * bar: "baz", + * _: ["./quux.txt"], + * }); + * ``` + */ +export function parseArgs< + TArgs extends Values< + TBooleans, + TStrings, + TCollectable, + TNegatable, + TDefaults, + TAliases + >, + TDoubleDash extends boolean | undefined = undefined, + TBooleans extends BooleanType = undefined, + TStrings extends StringType = undefined, + TCollectable extends Collectable = undefined, + TNegatable extends Negatable = undefined, + TDefaults extends Record<string, unknown> | undefined = undefined, + TAliases extends Aliases<TAliasArgNames, TAliasNames> | undefined = undefined, + TAliasArgNames extends string = string, + TAliasNames extends string = string, +>( + args: string[], + options?: ParseOptions< + TBooleans, + TStrings, + TCollectable, + TNegatable, + TDefaults, + TAliases, + TDoubleDash + >, +): Args<TArgs, TDoubleDash> { + const { + "--": doubleDash = false, + alias = {} as NonNullable<TAliases>, + boolean = false, + default: defaults = {} as TDefaults & Defaults<TBooleans, TStrings>, + stopEarly = false, + string = [], + collect = [], + negatable = [], + unknown: unknownFn = (i: string): unknown => i, + } = options ?? {}; + const aliasMap: Map<string, Set<string>> = new Map(); + const booleanSet = new Set<string>(); + const stringSet = new Set<string>(); + const collectSet = new Set<string>(); + const negatableSet = new Set<string>(); + + let allBools = false; + + if (alias) { + for (const [key, value] of Object.entries(alias)) { + if (value === undefined) { + throw new TypeError("Alias value must be defined"); + } + const aliases = Array.isArray(value) ? value : [value]; + aliasMap.set(key, new Set(aliases)); + aliases.forEach((alias) => + aliasMap.set( + alias, + new Set([key, ...aliases.filter((it) => it !== alias)]), + ) + ); + } + } + + if (boolean) { + if (typeof boolean === "boolean") { + allBools = boolean; + } else { + const booleanArgs = Array.isArray(boolean) ? boolean : [boolean]; + for (const key of booleanArgs.filter(Boolean)) { + booleanSet.add(key); + aliasMap.get(key)?.forEach((al) => { + booleanSet.add(al); + }); + } + } + } + + if (string) { + const stringArgs = Array.isArray(string) ? string : [string]; + for (const key of stringArgs.filter(Boolean)) { + stringSet.add(key); + aliasMap.get(key)?.forEach((al) => stringSet.add(al)); + } + } + + if (collect) { + const collectArgs = Array.isArray(collect) ? collect : [collect]; + for (const key of collectArgs.filter(Boolean)) { + collectSet.add(key); + aliasMap.get(key)?.forEach((al) => collectSet.add(al)); + } + } + + if (negatable) { + const negatableArgs = Array.isArray(negatable) ? negatable : [negatable]; + for (const key of negatableArgs.filter(Boolean)) { + negatableSet.add(key); + aliasMap.get(key)?.forEach((alias) => negatableSet.add(alias)); + } + } + + const argv: Args = { _: [] }; + + function setArgument( + key: string, + value: string | number | boolean, + arg: string, + collect: boolean, + ) { + if ( + !booleanSet.has(key) && + !stringSet.has(key) && + !aliasMap.has(key) && + !(allBools && FLAG_NAME_REGEXP.test(arg)) && + unknownFn?.(arg, key, value) === false + ) { + return; + } + + if (typeof value === "string" && !stringSet.has(key)) { + value = isNumber(value) ? Number(value) : value; + } + + const collectable = collect && collectSet.has(key); + setNested(argv, key.split("."), value, collectable); + aliasMap.get(key)?.forEach((key) => { + setNested(argv, key.split("."), value, collectable); + }); + } + + let notFlags: string[] = []; + + // all args after "--" are not parsed + const index = args.indexOf("--"); + if (index !== -1) { + notFlags = args.slice(index + 1); + args = args.slice(0, index); + } + + argsLoop: + for (let i = 0; i < args.length; i++) { + const arg = args[i]!; + + const groups = arg.match(FLAG_REGEXP)?.groups; + + if (groups) { + const { doubleDash, negated } = groups; + let key = groups.key!; + let value: string | number | boolean | undefined = groups.value; + + if (doubleDash) { + if (value) { + if (booleanSet.has(key)) value = parseBooleanString(value); + setArgument(key, value, arg, true); + continue; + } + + if (negated) { + if (negatableSet.has(key)) { + setArgument(key, false, arg, false); + continue; + } + key = `no-${key}`; + } + + const next = args[i + 1]; + + if (next) { + if ( + !booleanSet.has(key) && + !allBools && + !next.startsWith("-") && + (!aliasMap.has(key) || !aliasIsBoolean(aliasMap, booleanSet, key)) + ) { + value = next; + i++; + setArgument(key, value, arg, true); + continue; + } + + if (isBooleanString(next)) { + value = parseBooleanString(next); + i++; + setArgument(key, value, arg, true); + continue; + } + } + + value = stringSet.has(key) ? "" : true; + setArgument(key, value, arg, true); + continue; + } + const letters = arg.slice(1, -1).split(""); + + for (const [j, letter] of letters.entries()) { + const next = arg.slice(j + 2); + + if (next === "-") { + setArgument(letter, next, arg, true); + continue; + } + + if (LETTER_REGEXP.test(letter)) { + const groups = VALUE_REGEXP.exec(next)?.groups; + if (groups) { + setArgument(letter, groups.value!, arg, true); + continue argsLoop; + } + if (NUMBER_REGEXP.test(next)) { + setArgument(letter, next, arg, true); + continue argsLoop; + } + } + + if (letters[j + 1]?.match(SPECIAL_CHAR_REGEXP)) { + setArgument(letter, arg.slice(j + 2), arg, true); + continue argsLoop; + } + setArgument(letter, stringSet.has(letter) ? "" : true, arg, true); + } + + key = arg.slice(-1); + if (key === "-") continue; + + const nextArg = args[i + 1]; + + if (nextArg) { + if ( + !HYPHEN_REGEXP.test(nextArg) && + !booleanSet.has(key) && + (!aliasMap.has(key) || !aliasIsBoolean(aliasMap, booleanSet, key)) + ) { + setArgument(key, nextArg, arg, true); + i++; + continue; + } + if (isBooleanString(nextArg)) { + const value = parseBooleanString(nextArg); + setArgument(key, value, arg, true); + i++; + continue; + } + } + setArgument(key, stringSet.has(key) ? "" : true, arg, true); + continue; + } + + if (unknownFn?.(arg) !== false) { + argv._.push( + stringSet.has("_") || !isNumber(arg) ? arg : Number(arg), + ); + } + + if (stopEarly) { + argv._.push(...args.slice(i + 1)); + break; + } + } + + for (const [key, value] of Object.entries(defaults)) { + const keys = key.split("."); + if (!hasNested(argv, keys)) { + setNested(argv, keys, value); + aliasMap.get(key)?.forEach((key) => + setNested(argv, key.split("."), value) + ); + } + } + + for (const key of booleanSet.keys()) { + const keys = key.split("."); + if (!hasNested(argv, keys)) { + const value = collectSet.has(key) ? [] : false; + setNested(argv, keys, value); + } + } + + for (const key of stringSet.keys()) { + const keys = key.split("."); + if (!hasNested(argv, keys) && collectSet.has(key)) { + setNested(argv, keys, []); + } + } + + if (doubleDash) { + argv["--"] = notFlags; + } else { + argv._.push(...notFlags); + } + + return argv as Args<TArgs, TDoubleDash>; +} diff --git a/utils/pdfs/vendor-docs/pdf.js-extract/README.md b/utils/pdfs/vendor-docs/pdf.js-extract/README.md new file mode 100644 index 000000000000..065976bf28e3 --- /dev/null +++ b/utils/pdfs/vendor-docs/pdf.js-extract/README.md @@ -0,0 +1,134 @@ +# pdf.js-extract + +extracts text from PDF files + +This is just a library packaged out of the examples for usage of pdf.js with +nodejs. + +It reads a pdf file and exports all pages & texts with coordinates. This can be +e.g. used to extract structured table data. + +This package includes a build of [pdf.js](https://github.com/mozilla/pdf.js). +why? [pdfs-dist](https://github.com/mozilla/pdfjs-dist) installs not needed +dependencies into production deployment. + +Note: NO OCR! + +## Install + +[](https://nodei.co/npm/pdf.js-extract/) + + +[](http://opensource.org/licenses/MIT) + +## Example Usage + +javascript async with callback + +```javascript +const PDFExtract = require('pdf.js-extract').PDFExtract +const pdfExtract = new PDFExtract() +const options = {} /* see below */ +pdfExtract.extract('test.pdf', options, (err, data) => { + if (err) return console.log(err) + console.log(data) +}) +``` + +javascript async with callback using buffer + +```javascript +const PDFExtract = require('pdf.js-extract').PDFExtract +const pdfExtract = new PDFExtract() +const fs = require('fs') +const buffer = fs.readFileSync('./example.pdf') +const options = {} /* see below */ +pdfExtract.extractBuffer(buffer, options, (err, data) => { + if (err) return console.log(err) + console.log(data) +}) +``` + +typescript async with promise + +```typescript +import { PDFExtract, PDFExtractOptions } from 'pdf.js-extract' +const pdfExtract = new PDFExtract() +const options: PDFExtractOptions = {} /* see below */ +pdfExtract.extract('test.pdf', options) + .then((data) => console.log(data)) + .catch((err) => console.log(err)) +``` + +## Options + +```typescript +export interface PDFExtractOptions { + firstPage?: number // default:`1` - start extract at page nr + lastPage?: number // stop extract at page nr, no default value + password?: string // for decrypting password-protected PDFs., no default value + verbosity?: number // default:`-1` - log level of pdf.js + normalizeWhitespace?: boolean // default:`false` - replaces all occurrences of whitespace with standard spaces (0x20). + disableCombineTextItems?: boolean // default:`false` - do not attempt to combine same line {@link TextItem}'s. +} +``` + +Example Output + +```json +{ + "filename": "helloworld.pdf", + "meta": { + "info": { + "PDFFormatVersion": "1.7", + "IsAcroFormPresent": false, + "IsCollectionPresent": false, + "IsLinearized": true, + "IsXFAPresent": false + }, + "metadata": { + "dc:format": "application/pdf", + "dc:creator": "someone", + "dc:title": "This is a hello world PDF file", + "xmp:createdate": "2000-06-29T10:21:08+11:00", + "xmp:creatortool": "Microsoft Word 8.0", + "xmp:modifydate": "2013-10-28T15:24:13-04:00", + "xmp:metadatadate": "2013-10-28T15:24:13-04:00", + "pdf:producer": "Acrobat Distiller 4.0 for Windows", + "xmpmm:documentid": "uuid:0205e221-80a8-459e-a522-635ed5c1e2e6", + "xmpmm:instanceid": "uuid:68d6ae6d-43c4-472d-9b28-7c4add8f9e46" + } + }, + "pages": [ + { + "pageInfo": { + "num": 1, + "scale": 1, + "rotation": 0, + "offsetX": 0, + "offsetY": 0, + "width": 200, + "height": 200 + }, + "links": [ + "https://github.com" + ], + "content": [ + { + "x": 70, + "y": 150, + "str": "Hello, world!", + "dir": "ltr", + "width": 64.656, + "height": 12, + "fontName": "Times" + } + ] + } + ], + "pdfInfo": { + "numPages": 1, + "fingerprint": "1ee9219eb9eaa49acbfc20155ac359c3" + } +} +``` diff --git a/utils/youtube/deno.json b/utils/youtube/deno.json new file mode 100644 index 000000000000..dd5332a9d71a --- /dev/null +++ b/utils/youtube/deno.json @@ -0,0 +1,9 @@ +{ + "tasks": { + "dev": "deno run --watch main.ts" + }, + "imports": { + "@distube/ytdl-core": "npm:@distube/ytdl-core@^4.15.1", + "@std/assert": "jsr:@std/assert@1" + } +} diff --git a/utils/youtube/main.ts b/utils/youtube/main.ts new file mode 100644 index 000000000000..d2e72f12a151 --- /dev/null +++ b/utils/youtube/main.ts @@ -0,0 +1,7 @@ +import ytdl from '@distube/ytdl-core' + +const url = 'https://www.youtube.com/watch?v=YOUR_VIDEO_ID' +const audioStream = ytdl(url, { quality: 'highestaudio', filter: 'audioonly' }) + +const file = await Deno.open('audio.m4a', { create: true, write: true }) +await audioStream.pipeTo(file.writable) diff --git a/utils/youtube/vendor-docs/@distube/ytdl-core/README.md b/utils/youtube/vendor-docs/@distube/ytdl-core/README.md new file mode 100644 index 000000000000..572a050dd81b --- /dev/null +++ b/utils/youtube/vendor-docs/@distube/ytdl-core/README.md @@ -0,0 +1,260 @@ +# @distube/ytdl-core + +DisTube fork of `ytdl-core`. This fork is dedicated to fixing bugs and adding +features that are not merged into the original repo as soon as possible. + +<a href='https://ko-fi.com/skick' target='_blank'><img height='48' src='https://storage.ko-fi.com/cdn/kofi3.png' alt='Buy Me a Coffee at ko-fi.com' /></a> + +## Installation + +```bash +npm install @distube/ytdl-core@latest +``` + +Make sure you're installing the latest version of `@distube/ytdl-core` to keep +up with the latest fixes. + +## Usage + +```js +const ytdl = require('@distube/ytdl-core') +// TypeScript: import ytdl from '@distube/ytdl-core'; with --esModuleInterop +// TypeScript: import * as ytdl from '@distube/ytdl-core'; with --allowSyntheticDefaultImports +// TypeScript: import ytdl = require('@distube/ytdl-core'); with neither of the above + +// Download a video +ytdl('http://www.youtube.com/watch?v=aqz-KE-bpKQ').pipe( + require('fs').createWriteStream('video.mp4'), +) + +// Get video info +ytdl.getBasicInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ').then((info) => { + console.log(info.videoDetails.title) +}) + +// Get video info with download formats +ytdl.getInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ').then((info) => { + console.log(info.formats) +}) +``` + +### Cookies Support + +```js +const ytdl = require('@distube/ytdl-core') + +// (Optional) Below are examples, NOT the recommended options +const cookies = [ + { name: 'cookie1', value: 'COOKIE1_HERE' }, + { name: 'cookie2', value: 'COOKIE2_HERE' }, +] + +// (Optional) http-cookie-agent / undici agent options +// Below are examples, NOT the recommended options +const agentOptions = { + pipelining: 5, + maxRedirections: 0, + localAddress: '127.0.0.1', +} + +// agent should be created once if you don't want to change your cookie +const agent = ytdl.createAgent(cookies, agentOptions) + +ytdl.getBasicInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { agent }) +ytdl.getInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { agent }) +``` + +#### How to get cookies + +- Install [EditThisCookie](http://www.editthiscookie.com/) extension for your + browser. +- Go to [YouTube](https://www.youtube.com/). +- Log in to your account. (You should use a new account for this purpose) +- Click on the extension icon and click "Export" icon. +- Your cookies will be added to your clipboard and paste it into your code. + +> [!WARNING] +> Don't logout it by clicking logout button on youtube/google account manager, +> it will expire your cookies. You can delete your browser's cookies to log it +> out on your browser. Or use incognito mode to get your cookies then close it. + +> [!WARNING] +> Paste all the cookies array from clipboard into `createAgent` function. Don't +> remove/edit any cookies if you don't know what you're doing. + +> [!WARNING] +> Make sure your account, which logged in when you getting your cookies, use 1 +> IP at the same time only. It will make your cookies alive longer. + +```js +const ytdl = require('@distube/ytdl-core') +const agent = ytdl.createAgent([ + { + domain: '.youtube.com', + expirationDate: 1234567890, + hostOnly: false, + httpOnly: true, + name: '---xxx---', + path: '/', + sameSite: 'no_restriction', + secure: true, + session: false, + value: '---xxx---', + }, + { + '...': '...', + }, +]) +``` + +- Or you can paste your cookies array into a file and use `fs.readFileSync` to + read it. + +```js +const ytdl = require('@distube/ytdl-core') +const fs = require('fs') +const agent = ytdl.createAgent(JSON.parse(fs.readFileSync('cookies.json'))) +``` + +### Proxy Support + +```js +const ytdl = require('@distube/ytdl-core') + +const agent = ytdl.createProxyAgent({ uri: 'my.proxy.server' }) + +ytdl.getBasicInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { agent }) +ytdl.getInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { agent }) +``` + +Use both proxy and cookies: + +```js +const ytdl = require('@distube/ytdl-core') + +const agent = ytdl.createProxyAgent({ uri: 'my.proxy.server' }, [{ + name: 'cookie', + value: 'COOKIE_HERE', +}]) + +ytdl.getBasicInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { agent }) +ytdl.getInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { agent }) +``` + +### IP Rotation + +_Built-in ip rotation (`getRandomIPv6`) won't be updated and will be removed in +the future, create your own ip rotation instead._ + +To implement IP rotation, you need to assign the desired IP address to the +`localAddress` property within `undici.Agent.Options`. Therefore, you'll need to +use a different `ytdl.Agent` for each IP address you want to use. + +```js +const ytdl = require('@distube/ytdl-core') +const { getRandomIPv6 } = require('@distube/ytdl-core/lib/utils') + +const agentForARandomIP = ytdl.createAgent(undefined, { + localAddress: getRandomIPv6('2001:2::/48'), +}) + +ytdl.getBasicInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { + agent: agentForARandomIP, +}) + +const agentForAnotherRandomIP = ytdl.createAgent(undefined, { + localAddress: getRandomIPv6('2001:2::/48'), +}) + +ytdl.getInfo('http://www.youtube.com/watch?v=aqz-KE-bpKQ', { + agent: agentForAnotherRandomIP, +}) +``` + +## API + +You can find the API documentation in the +[original repo](https://github.com/fent/node-ytdl-core#api). Except a few +changes: + +### `ytdl.getInfoOptions` + +- `requestOptions` is now `undici`'s + [`RequestOptions`](https://github.com/nodejs/undici#undicirequesturl-options-promise). +- `agent`: + [`ytdl.Agent`](https://github.com/distubejs/ytdl-core/blob/master/typings/index.d.ts#L10-L14) +- `playerClients`: An array of player clients to use. Accepts `WEB`, + `WEB_CREATOR`, `IOS`, and `ANDROID`. Defaults to `["WEB_CREATOR", "IOS"]`. + +### `ytdl.createAgent([cookies]): ytdl.Agent` + +`cookies`: an array of json cookies exported with +[EditThisCookie](http://www.editthiscookie.com/). + +### `ytdl.createProxyAgent(proxy[, cookies]): ytdl.Agent` + +`proxy`: +[`ProxyAgentOptions`](https://github.com/nodejs/undici/blob/main/docs/api/ProxyAgent.md#parameter-proxyagentoptions) +contains your proxy server information. + +#### How to implement `ytdl.Agent` with your own Dispatcher + +You can find the example +[here](https://github.com/distubejs/ytdl-core/blob/master/lib/cookie.js#L73-L86) + +## Limitations + +ytdl cannot download videos that fall into the following + +- Regionally restricted (requires a [proxy](#proxy-support)) +- Private (if you have access, requires [cookies](#cookies-support)) +- Rentals (if you have access, requires [cookies](#cookies-support)) +- YouTube Premium content (if you have access, requires + [cookies](#cookies-support)) +- Only [HLS Livestreams](https://en.wikipedia.org/wiki/HTTP_Live_Streaming) are + currently supported. Other formats will get filtered out in ytdl.chooseFormats + +Generated download links are valid for 6 hours, and may only be downloadable +from the same IP address. + +## Rate Limiting + +When doing too many requests YouTube might block. This will result in your +requests getting denied with HTTP-StatusCode 429. The following steps might help +you: + +- Update `@distube/ytdl-core` to the latest version +- Use proxies (you can find an example [here](#proxy-support)) +- Extend the Proxy Idea by rotating (IPv6-)Addresses + - read + [this](https://github.com/fent/node-ytdl-core#how-does-using-an-ipv6-block-help) + for more information about this +- Use cookies (you can find an example [here](#cookies-support)) + - for this to take effect you have to FIRST wait for the current rate limit to + expire +- Wait it out (it usually goes away within a few days) + +## Update Checks + +The issue of using an outdated version of ytdl-core became so prevalent, that +ytdl-core now checks for updates at run time, and every 12 hours. If it finds an +update, it will print a warning to the console advising you to update. Due to +the nature of this library, it is important to always use the latest version as +YouTube continues to update. + +If you'd like to disable this update check, you can do so by providing the +`YTDL_NO_UPDATE` env variable. + +``` +env YTDL_NO_UPDATE=1 node myapp.js +``` + +## Related Projects + +- [DisTube](https://github.com/skick1234/DisTube) - A Discord.js module to + simplify your music commands and play songs with audio filters on Discord + without any API key. +- [@distube/ytsr](https://github.com/distubejs/ytsr) - DisTube fork of + [ytsr](https://github.com/TimeForANinja/node-ytsr). +- [@distube/ytpl](https://github.com/distubejs/ytpl) - DisTube fork of + [ytpl](https://github.com/TimeForANinja/node-ytpl).