From d0fc6c52bc38b1d5d8c955b42ea8af0abcbfe46c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Jan 2024 10:09:13 +0100 Subject: [PATCH 001/101] add basic scaffolding to use QuartoNotebookRunner for running Julia qmd files --- src/execute/engine.ts | 2 + src/execute/julia.ts | 204 ++++++++++++++++++++ src/execute/types.ts | 1 + src/resources/julia/quartonotebookrunner.jl | 8 + 4 files changed, 215 insertions(+) create mode 100644 src/execute/julia.ts create mode 100644 src/resources/julia/quartonotebookrunner.jl diff --git a/src/execute/engine.ts b/src/execute/engine.ts index 338b7fe2c0..61de67e42f 100644 --- a/src/execute/engine.ts +++ b/src/execute/engine.ts @@ -29,9 +29,11 @@ import { mergeConfigs } from "../core/config.ts"; import { ProjectContext } from "../project/types.ts"; import { pandocBuiltInFormats } from "../core/pandoc/pandoc-formats.ts"; import { gitignoreEntries } from "../project/project-gitignore.ts"; +import { juliaEngine } from "./julia.ts"; const kEngines: ExecutionEngine[] = [ knitrEngine, + juliaEngine, jupyterEngine, markdownEngine, ]; diff --git a/src/execute/julia.ts b/src/execute/julia.ts new file mode 100644 index 0000000000..b1f5bee420 --- /dev/null +++ b/src/execute/julia.ts @@ -0,0 +1,204 @@ +import { error } from "log/mod.ts"; +import { join } from "path/mod.ts"; +import { MappedString, mappedStringFromFile } from "../core/mapped-text.ts"; +import { partitionMarkdown } from "../core/pandoc/pandoc-partition.ts"; +import { execProcess } from "../core/process.ts"; +import { readYamlFromMarkdown } from "../core/yaml.ts"; +import { ProjectContext } from "../project/types.ts"; +import { + DependenciesOptions, + ExecuteOptions, + ExecuteResult, + ExecutionEngine, + ExecutionTarget, + kJuliaEngine, + PandocIncludes, + PostProcessOptions, +} from "./types.ts"; +import { + jupyterAssets, + jupyterFromFile, + jupyterToMarkdown, +} from "../core/jupyter/jupyter.ts"; +import { + kFigDpi, + kFigFormat, + kFigPos, + kIpynbProduceSourceNotebook, + kKeepHidden, +} from "../config/constants.ts"; +import { + isHtmlCompatible, + isIpynbOutput, + isLatexOutput, + isMarkdownOutput, + isPresentationOutput, +} from "../config/format.ts"; +import { resourcePath } from "../core/resources.ts"; + +export const juliaEngine: ExecutionEngine = { + name: kJuliaEngine, + + defaultExt: ".qmd", + + defaultYaml: () => [], + + defaultContent: () => [ + "```{r}", + "1 + 1", + "```", + ], + + validExtensions: () => [], + + claimsFile: (file: string, ext: string) => false, + + claimsLanguage: (language: string) => { + return language.toLowerCase() === "julia"; + }, + + partitionedMarkdown: async (file: string) => { + return partitionMarkdown(Deno.readTextFileSync(file)); + }, + + // TODO: ask dragonstyle what to do here + executeTargetSkipped: () => false, + + // TODO: just return dependencies from execute and this can do nothing + dependencies: (_options: DependenciesOptions) => { + const includes: PandocIncludes = {}; + return Promise.resolve({ + includes, + }); + }, + + // TODO: this can also probably do nothing + postprocess: (_options: PostProcessOptions) => { + return Promise.resolve(); + }, + + canFreeze: true, + + generatesFigures: true, + + ignoreDirs: () => { + return []; + }, + + canKeepSource: (_target: ExecutionTarget) => { + return true; + }, + + execute: async (options: ExecuteOptions): Promise => { + console.log("running execute method of the julia engine"); + console.log(options); + options.target.source; + + console.log("trying to run QuartoNotebookRunner"); + const outputIpynbPath = options.tempDir + "output.ipynb"; + const processResult = await execProcess( + { + cmd: [ + "julia", + "--project=@quarto", + resourcePath("julia/quartonotebookrunner.jl"), + options.target.source, + outputIpynbPath, + ], + }, + ); + console.log(processResult); + + if (!processResult.success) { + error("Running QuartoNotebookRunner failed"); + } + + const nb = jupyterFromFile(outputIpynbPath); + + // TODO: jupyterFromFile sets python as the default kernelspec for the files we get from QuartoNotebookRunner, + // maybe the correct "kernel" needs to be set there instead (there isn't really a kernel needed as we don't execute via Jupyter + // but this seems to be needed later to assign the correct language markers to code cells etc.) + nb.metadata.kernelspec = { + display_name: "Julia", + name: "julia", + language: "julia", + }; + + const assets = jupyterAssets( + options.target.input, + options.format.pandoc.to, + ); + + // NOTE: for perforance reasons the 'nb' is mutated in place + // by jupyterToMarkdown (we don't want to make a copy of a + // potentially very large notebook) so should not be relied + // on subseuqent to this call + console.log(nb.metadata); + + const result = await jupyterToMarkdown( + nb, + { + executeOptions: options, + language: nb.metadata.kernelspec.language.toLowerCase(), + assets, + execute: options.format.execute, + keepHidden: options.format.render[kKeepHidden], + toHtml: isHtmlCompatible(options.format), + toLatex: isLatexOutput(options.format.pandoc), + toMarkdown: isMarkdownOutput(options.format), + toIpynb: isIpynbOutput(options.format.pandoc), + toPresentation: isPresentationOutput(options.format.pandoc), + figFormat: options.format.execute[kFigFormat], + figDpi: options.format.execute[kFigDpi], + figPos: options.format.render[kFigPos], + // preserveCellMetadata, + preserveCodeCellYaml: + options.format.render[kIpynbProduceSourceNotebook] === true, + }, + ); + + // Create markdown from the result + const outputs = result.cellOutputs.map((output) => output.markdown); + if (result.notebookOutputs) { + if (result.notebookOutputs.prefix) { + outputs.unshift(result.notebookOutputs.prefix); + } + if (result.notebookOutputs.suffix) { + outputs.push(result.notebookOutputs.suffix); + } + } + const markdown = outputs.join(""); + + // return results + return { + engine: kJuliaEngine, + markdown: markdown, + supporting: [join(assets.base_dir, assets.supporting_dir)], + filters: [], + pandoc: result.pandoc, + // includes, + // engineDependencies, + preserve: result.htmlPreserve, + postProcess: result.htmlPreserve && + (Object.keys(result.htmlPreserve).length > 0), + }; + }, + + target: async ( + file: string, + _quiet?: boolean, + markdown?: MappedString, + _project?: ProjectContext, + ): Promise => { + if (markdown === undefined) { + markdown = mappedStringFromFile(file); + } + const target: ExecutionTarget = { + source: file, + input: file, + markdown, + metadata: readYamlFromMarkdown(markdown.value), + }; + return Promise.resolve(target); + }, +}; diff --git a/src/execute/types.ts b/src/execute/types.ts index 09e4af0350..a55ffa6bdc 100644 --- a/src/execute/types.ts +++ b/src/execute/types.ts @@ -21,6 +21,7 @@ export const kQmdExtensions = [".qmd"]; export const kMarkdownEngine = "markdown"; export const kKnitrEngine = "knitr"; export const kJupyterEngine = "jupyter"; +export const kJuliaEngine = "julia"; export interface ExecutionEngine { name: string; diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl new file mode 100644 index 0000000000..e43a4f62da --- /dev/null +++ b/src/resources/julia/quartonotebookrunner.jl @@ -0,0 +1,8 @@ +using QuartoNotebookRunner + +function execute_qmd(sourcefile::String, output_ipynb::String) + server = QuartoNotebookRunner.Server() + run!(server, sourcefile, output=output_ipynb) +end + +execute_qmd(ARGS[1], ARGS[2]) From b8d8771014b8656094921c495c8d30146fab0c62 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Jan 2024 10:11:06 +0100 Subject: [PATCH 002/101] change default content to julia --- src/execute/julia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index b1f5bee420..58f9c13721 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -44,7 +44,7 @@ export const juliaEngine: ExecutionEngine = { defaultYaml: () => [], defaultContent: () => [ - "```{r}", + "```{julia}", "1 + 1", "```", ], From 1c78e04155095dd6ce777e53a7bd02d317308b6e Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Jan 2024 10:12:24 +0100 Subject: [PATCH 003/101] add note --- src/execute/julia.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 58f9c13721..ca866b203d 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -113,6 +113,8 @@ export const juliaEngine: ExecutionEngine = { error("Running QuartoNotebookRunner failed"); } + // NOTE: the following is all mostly copied from the jupyter kernel file + const nb = jupyterFromFile(outputIpynbPath); // TODO: jupyterFromFile sets python as the default kernelspec for the files we get from QuartoNotebookRunner, From ca10f511d188d8f9c4a87ab0b0d2596415888063 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 23 Jan 2024 10:30:11 +0100 Subject: [PATCH 004/101] simply julia code --- src/resources/julia/quartonotebookrunner.jl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index e43a4f62da..4ee32dd3fe 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -1,8 +1,3 @@ using QuartoNotebookRunner -function execute_qmd(sourcefile::String, output_ipynb::String) - server = QuartoNotebookRunner.Server() - run!(server, sourcefile, output=output_ipynb) -end - -execute_qmd(ARGS[1], ARGS[2]) +QuartoNotebookRunner.render(ARGS[1]; output = ARGS[2]) From 1bb0f14dc8831dd99128d1b7f682be43f0a076b1 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 24 Jan 2024 19:09:24 +0100 Subject: [PATCH 005/101] first working version with transport file --- src/execute/julia.ts | 257 ++++++++++++++++++-- src/resources/julia/quartonotebookrunner.jl | 39 ++- 2 files changed, 274 insertions(+), 22 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index ca866b203d..be20386353 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -1,4 +1,4 @@ -import { error } from "log/mod.ts"; +import { error, info } from "log/mod.ts"; import { join } from "path/mod.ts"; import { MappedString, mappedStringFromFile } from "../core/mapped-text.ts"; import { partitionMarkdown } from "../core/pandoc/pandoc-partition.ts"; @@ -17,10 +17,11 @@ import { } from "./types.ts"; import { jupyterAssets, - jupyterFromFile, + jupyterFromJSON, jupyterToMarkdown, } from "../core/jupyter/jupyter.ts"; import { + kExecuteDaemon, kFigDpi, kFigFormat, kFigPos, @@ -35,6 +36,22 @@ import { isPresentationOutput, } from "../config/format.ts"; import { resourcePath } from "../core/resources.ts"; +import { quartoRuntimeDir } from "../core/appdirs.ts"; +import { normalizePath } from "../core/path.ts"; +import { md5Hash } from "../core/hash.ts"; +import { isInteractiveSession } from "../core/platform.ts"; +import { runningInCI } from "../core/ci-info.ts"; +import { ProcessResult } from "../core/process-types.ts"; +import { sleep } from "../core/async.ts"; +import { JupyterNotebook } from "../core/jupyter/types.ts"; +import { existsSync } from "fs/mod.ts"; +import { readTextFileSync } from "../core/qualified-path.ts"; +import { number } from "https://deno.land/x/cliffy@v0.25.4/flags/types/number.ts"; + +export interface JuliaExecuteOptions extends ExecuteOptions { + julia_cmd: string[]; + supervisor_pid?: number; +} export const juliaEngine: ExecutionEngine = { name: kJuliaEngine, @@ -91,31 +108,64 @@ export const juliaEngine: ExecutionEngine = { execute: async (options: ExecuteOptions): Promise => { console.log("running execute method of the julia engine"); - console.log(options); options.target.source; - console.log("trying to run QuartoNotebookRunner"); - const outputIpynbPath = options.tempDir + "output.ipynb"; - const processResult = await execProcess( - { - cmd: [ - "julia", - "--project=@quarto", - resourcePath("julia/quartonotebookrunner.jl"), - options.target.source, - outputIpynbPath, - ], + // use daemon by default if we are in an interactive session (terminal + // or rstudio) and not running in a CI system. + let executeDaemon = options.format.execute[kExecuteDaemon]; + if (executeDaemon === null || executeDaemon === undefined) { + // if (await disableDaemonForNotebook(options.target)) { + // executeDaemon = false; + // } else { + executeDaemon = isInteractiveSession() && !runningInCI(); + // } + } + + // julia back end requires full path to input (to ensure that + // keepalive kernels are never re-used across multiple inputs + // that happen to share a hash) + const execOptions = { + ...options, + target: { + ...options.target, + input: normalizePath(options.target.input), }, - ); - console.log(processResult); + }; - if (!processResult.success) { - error("Running QuartoNotebookRunner failed"); - } + const juliaExecOptions: JuliaExecuteOptions = { + julia_cmd: ["julia"], + supervisor_pid: options.previewServer ? Deno.pid : undefined, + ...execOptions, + }; + + // if (executeDaemon === false || executeDaemon === 0) { + const nb = await executeJuliaOneshot(juliaExecOptions); + // } else { + // // await executeJuliaKeepalive(juliaExecOptions); + // } + + // console.log("trying to run QuartoNotebookRunner"); + // const outputIpynbPath = options.tempDir + "output.ipynb"; + // const processResult = await execProcess( + // { + // cmd: [ + // "julia", + // "--project=@quarto", + // resourcePath("julia/quartonotebookrunner.jl"), + // options.target.source, + // outputIpynbPath, + // ], + // }, + // ); + // console.log(processResult); + + // if (!processResult.success) { + // error("Running QuartoNotebookRunner failed"); + // } // NOTE: the following is all mostly copied from the jupyter kernel file - const nb = jupyterFromFile(outputIpynbPath); + // const nb = jupyterFromJSON(notebookJSON); // TODO: jupyterFromFile sets python as the default kernelspec for the files we get from QuartoNotebookRunner, // maybe the correct "kernel" needs to be set there instead (there isn't really a kernel needed as we don't execute via Jupyter @@ -135,7 +185,6 @@ export const juliaEngine: ExecutionEngine = { // by jupyterToMarkdown (we don't want to make a copy of a // potentially very large notebook) so should not be relied // on subseuqent to this call - console.log(nb.metadata); const result = await jupyterToMarkdown( nb, @@ -204,3 +253,169 @@ export const juliaEngine: ExecutionEngine = { return Promise.resolve(target); }, }; + +async function startJuliaServer() { + const transportFile = juliaTransportFile(); + console.log("Transport file: ", transportFile); + if (!existsSync(transportFile)) { + console.log("Transport file doesn't exist, starting server"); + execProcess( + { + cmd: [ + "julia", + "--project=@quarto", + resourcePath("julia/quartonotebookrunner.jl"), + transportFile, + ], + }, + ).then((result) => { + if (!result.success) { + console.log( + `Julia server returned with exit code ${result.code}`, + result.stderr, + ); + } + }); + } + return Promise.resolve(); +} + +interface JuliaTransportFile { + port: number; + pid: number; +} + +async function pollTransportFile(): Promise { + const transportFile = juliaTransportFile(); + + for (let i = 0; i < 20; i++) { + await sleep(i * 100); + if (existsSync(transportFile)) { + const content = Deno.readTextFileSync(transportFile); + return JSON.parse(content) as JuliaTransportFile; + } + } + return Promise.reject(); +} + +async function establishServerConnection(port: number): Promise { + let conn = null; + for (let i = 0; i < 20; i++) { + await sleep(i * 100); + conn = await Deno.connect({ + port: port, + }).catch((reason) => { + console.log(`Connecting to julia server failed on try ${i}`, reason); + return null; + }); + if (conn !== null) { + console.log("Connection successfully established"); + return conn; + } + } + + return Promise.reject(); +} + +async function executeJuliaOneshot( + options: JuliaExecuteOptions, +): Promise { + await startJuliaServer(); + const transportOptions = await pollTransportFile(); + console.log(transportOptions); + const conn = await establishServerConnection(transportOptions.port); + + await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + const response = await writeJuliaCommand( + conn, + "run", + "TODOsomesecret", + options, + ); + await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + + return response.notebook as JupyterNotebook; +} + +async function writeJuliaCommand( + conn: Deno.Conn, + command: "run" | "close" | "stop", + secret: string, + options: JuliaExecuteOptions, +) { + // TODO: no secret used, yet + let messageBytes = new TextEncoder().encode( + JSON.stringify({ + type: command, + content: options.target.input, + }) + "\n", + ); + + // // don't send the message if it's big. + // // Instead, write it to a file and send the file path + // // This is disappointing, but something is deeply wrong with Deno.Conn: + // // https://github.com/quarto-dev/quarto-cli/issues/7737#issuecomment-1830665357 + // if (messageBytes.length > 1024) { + // const tempFile = Deno.makeTempFileSync(); + // Deno.writeFileSync(tempFile, messageBytes); + // const msg = kernelCommand("file", secret, { file: tempFile }) + "\n"; + // messageBytes = new TextEncoder().encode(msg); + // } + + console.log("write command ", command, " to socket server"); + const bytesWritten = await conn.write(messageBytes); + if (bytesWritten !== messageBytes.length) { + throw new Error("Internal Error"); + } + + let response = ""; + while (true) { + const buffer = new Uint8Array(512); + + console.log("trying to read bytes into buffer"); + const bytesRead = await conn.read(buffer); + if (bytesRead === null) { + break; + } + + if (bytesRead > 0) { + const payload = new TextDecoder().decode( + buffer.slice(0, bytesRead), + ); + response += payload; + if (payload.includes("\n")) { + break; + } + } + } + // one command should be sent, ended by a newline, currently just throwing away anything else because we don't + // expect multiple commmands + const json = response.split("\n")[0]; + const data = JSON.parse(json); + + console.log("Received data from server"); + // console.log(data); + + return data; +} + +function juliaTransportFile() { + let transportsDir: string; + + try { + transportsDir = quartoRuntimeDir("julia"); + } catch (e) { + console.error("Could create runtime directory for the julia pidfile."); + console.error( + "This is possibly a permission issue in the environment Quarto is running in.", + ); + console.error( + "Please consult the following documentation for more information:", + ); + console.error( + "https://github.com/quarto-dev/quarto-cli/issues/4594#issuecomment-1619177667", + ); + throw e; + } + return join(transportsDir, "julia_transport.txt"); +} diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index 4ee32dd3fe..cc7599afd4 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -1,3 +1,40 @@ using QuartoNotebookRunner +using Sockets -QuartoNotebookRunner.render(ARGS[1]; output = ARGS[2]) + +transport_file = ARGS[1] +transport_dir = dirname(transport_file) + +atexit() do + rm(transport_file; force=true) +end + +# redirect_stdio(; stderr = joinpath(transport_dir, "stderr.txt")) do + server, port = let + server = nothing + port = nothing + + for i in 1:20 + # find an open port by creating a server there and immediately closing it + port, _server = Sockets.listenany(8000) + close(_server) + try + server = QuartoNotebookRunner.serve(; port) + break + catch e + error("Opening server on port $port failed.") + end + end + server, port + end + + if server === nothing + error("Giving up.") + end + + open(transport_file, "w") do io + println(io, """{"port": $port, "pid": $(Base.Libc.getpid())}""") + end + + wait(server) +# end From 8137ce83826c15595c34ff33091c7edf8efa2091 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 24 Jan 2024 19:15:05 +0100 Subject: [PATCH 006/101] fix julia server bug by using Deno.Command --- src/execute/julia.ts | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index be20386353..54c46a0894 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -259,23 +259,20 @@ async function startJuliaServer() { console.log("Transport file: ", transportFile); if (!existsSync(transportFile)) { console.log("Transport file doesn't exist, starting server"); - execProcess( - { - cmd: [ - "julia", - "--project=@quarto", - resourcePath("julia/quartonotebookrunner.jl"), - transportFile, - ], - }, - ).then((result) => { - if (!result.success) { - console.log( - `Julia server returned with exit code ${result.code}`, - result.stderr, - ); - } + // when quarto's execProc function is used here, there is a strange bug. + // The first time render is called on a file, the julia server is started correctly. + // The second time it is called, however, the socket server hangs if during the first + // run anything was written to stderr. This goes away when redirecting stderr to + // a file on the julia side, but also when using Deno.Command which is recommended + // as a replacement for the old Deno.run anyway. + const command = new Deno.Command("julia", { + args: [ + "--project=@quarto", + resourcePath("julia/quartonotebookrunner.jl"), + transportFile, + ], }); + command.spawn(); } return Promise.resolve(); } From 7ca08c261c1ee126ef4d8504b86b6ddd1e4e00e4 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 24 Jan 2024 19:27:44 +0100 Subject: [PATCH 007/101] add oneShot option to exec options --- src/execute/julia.ts | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 54c46a0894..dd6ddb1eaf 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -50,6 +50,7 @@ import { number } from "https://deno.land/x/cliffy@v0.25.4/flags/types/number.ts export interface JuliaExecuteOptions extends ExecuteOptions { julia_cmd: string[]; + oneShot: boolean; // if true, the file's worker process is closed before and after running supervisor_pid?: number; } @@ -134,35 +135,17 @@ export const juliaEngine: ExecutionEngine = { const juliaExecOptions: JuliaExecuteOptions = { julia_cmd: ["julia"], + oneShot: !executeDaemon, supervisor_pid: options.previewServer ? Deno.pid : undefined, ...execOptions, }; // if (executeDaemon === false || executeDaemon === 0) { - const nb = await executeJuliaOneshot(juliaExecOptions); + const nb = await executeJulia(juliaExecOptions); // } else { // // await executeJuliaKeepalive(juliaExecOptions); // } - // console.log("trying to run QuartoNotebookRunner"); - // const outputIpynbPath = options.tempDir + "output.ipynb"; - // const processResult = await execProcess( - // { - // cmd: [ - // "julia", - // "--project=@quarto", - // resourcePath("julia/quartonotebookrunner.jl"), - // options.target.source, - // outputIpynbPath, - // ], - // }, - // ); - // console.log(processResult); - - // if (!processResult.success) { - // error("Running QuartoNotebookRunner failed"); - // } - // NOTE: the following is all mostly copied from the jupyter kernel file // const nb = jupyterFromJSON(notebookJSON); @@ -254,7 +237,7 @@ export const juliaEngine: ExecutionEngine = { }, }; -async function startJuliaServer() { +async function startJuliaServer(options: JuliaExecuteOptions) { const transportFile = juliaTransportFile(); console.log("Transport file: ", transportFile); if (!existsSync(transportFile)) { @@ -265,13 +248,16 @@ async function startJuliaServer() { // run anything was written to stderr. This goes away when redirecting stderr to // a file on the julia side, but also when using Deno.Command which is recommended // as a replacement for the old Deno.run anyway. - const command = new Deno.Command("julia", { + const command = new Deno.Command(options.julia_cmd[0], { args: [ + ...(options.julia_cmd.slice(1)), "--project=@quarto", resourcePath("julia/quartonotebookrunner.jl"), transportFile, ], }); + // this process is supposed to outlive the quarto process, because + // in it, the references to the cached julia worker processes live command.spawn(); } return Promise.resolve(); @@ -314,22 +300,26 @@ async function establishServerConnection(port: number): Promise { return Promise.reject(); } -async function executeJuliaOneshot( +async function executeJulia( options: JuliaExecuteOptions, ): Promise { - await startJuliaServer(); + await startJuliaServer(options); const transportOptions = await pollTransportFile(); console.log(transportOptions); const conn = await establishServerConnection(transportOptions.port); - await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + if (options.oneShot) { + await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + } const response = await writeJuliaCommand( conn, "run", "TODOsomesecret", options, ); - await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + if (options.oneShot) { + await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + } return response.notebook as JupyterNotebook; } From 9d9e60cf6ebf23da7b736e78e18a6cbd7d8e52ab Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 24 Jan 2024 20:06:51 +0100 Subject: [PATCH 008/101] replace console.log with trace --- src/execute/julia.ts | 52 ++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index dd6ddb1eaf..8c21094b87 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -2,7 +2,6 @@ import { error, info } from "log/mod.ts"; import { join } from "path/mod.ts"; import { MappedString, mappedStringFromFile } from "../core/mapped-text.ts"; import { partitionMarkdown } from "../core/pandoc/pandoc-partition.ts"; -import { execProcess } from "../core/process.ts"; import { readYamlFromMarkdown } from "../core/yaml.ts"; import { ProjectContext } from "../project/types.ts"; import { @@ -22,6 +21,7 @@ import { } from "../core/jupyter/jupyter.ts"; import { kExecuteDaemon, + kExecuteDebug, kFigDpi, kFigFormat, kFigPos, @@ -108,7 +108,6 @@ export const juliaEngine: ExecutionEngine = { }, execute: async (options: ExecuteOptions): Promise => { - console.log("running execute method of the julia engine"); options.target.source; // use daemon by default if we are in an interactive session (terminal @@ -239,9 +238,11 @@ export const juliaEngine: ExecutionEngine = { async function startJuliaServer(options: JuliaExecuteOptions) { const transportFile = juliaTransportFile(); - console.log("Transport file: ", transportFile); if (!existsSync(transportFile)) { - console.log("Transport file doesn't exist, starting server"); + trace( + options, + `Transport file ${transportFile} doesn't exist, starting server.`, + ); // when quarto's execProc function is used here, there is a strange bug. // The first time render is called on a file, the julia server is started correctly. // The second time it is called, however, the socket server hangs if during the first @@ -259,6 +260,11 @@ async function startJuliaServer(options: JuliaExecuteOptions) { // this process is supposed to outlive the quarto process, because // in it, the references to the cached julia worker processes live command.spawn(); + } else { + trace( + options, + `Transport file ${transportFile} exists, reusing server.`, + ); } return Promise.resolve(); } @@ -281,18 +287,21 @@ async function pollTransportFile(): Promise { return Promise.reject(); } -async function establishServerConnection(port: number): Promise { +async function establishServerConnection( + options: ExecuteOptions, + port: number, +): Promise { let conn = null; for (let i = 0; i < 20; i++) { await sleep(i * 100); conn = await Deno.connect({ port: port, }).catch((reason) => { - console.log(`Connecting to julia server failed on try ${i}`, reason); + trace(options, `Connecting to julia server failed on try ${i}`); return null; }); if (conn !== null) { - console.log("Connection successfully established"); + trace(options, "Connection successfully established"); return conn; } } @@ -305,8 +314,11 @@ async function executeJulia( ): Promise { await startJuliaServer(options); const transportOptions = await pollTransportFile(); - console.log(transportOptions); - const conn = await establishServerConnection(transportOptions.port); + trace( + options, + `Connecting to server at port ${transportOptions.port}, pid ${transportOptions.pid}`, + ); + const conn = await establishServerConnection(options, transportOptions.port); if (options.oneShot) { await writeJuliaCommand(conn, "close", "TODOsomesecret", options); @@ -349,7 +361,7 @@ async function writeJuliaCommand( // messageBytes = new TextEncoder().encode(msg); // } - console.log("write command ", command, " to socket server"); + trace(options, `write command "${command}" to socket server`); const bytesWritten = await conn.write(messageBytes); if (bytesWritten !== messageBytes.length) { throw new Error("Internal Error"); @@ -358,8 +370,6 @@ async function writeJuliaCommand( let response = ""; while (true) { const buffer = new Uint8Array(512); - - console.log("trying to read bytes into buffer"); const bytesRead = await conn.read(buffer); if (bytesRead === null) { break; @@ -375,14 +385,12 @@ async function writeJuliaCommand( } } } + trace(options, "received server response"); // one command should be sent, ended by a newline, currently just throwing away anything else because we don't // expect multiple commmands const json = response.split("\n")[0]; const data = JSON.parse(json); - console.log("Received data from server"); - // console.log(data); - return data; } @@ -392,17 +400,23 @@ function juliaTransportFile() { try { transportsDir = quartoRuntimeDir("julia"); } catch (e) { - console.error("Could create runtime directory for the julia pidfile."); - console.error( + error("Could create runtime directory for the julia pidfile."); + error( "This is possibly a permission issue in the environment Quarto is running in.", ); - console.error( + error( "Please consult the following documentation for more information:", ); - console.error( + error( "https://github.com/quarto-dev/quarto-cli/issues/4594#issuecomment-1619177667", ); throw e; } return join(transportsDir, "julia_transport.txt"); } + +function trace(options: ExecuteOptions, msg: string) { + if (options.format.execute[kExecuteDebug]) { + info("- " + msg, { bold: true }); + } +} From 93b24755248df7aee3cc7c419d350317989f482e Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 25 Jan 2024 11:27:24 +0100 Subject: [PATCH 009/101] cleanups --- src/execute/julia.ts | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 8c21094b87..088228640f 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -14,11 +14,7 @@ import { PandocIncludes, PostProcessOptions, } from "./types.ts"; -import { - jupyterAssets, - jupyterFromJSON, - jupyterToMarkdown, -} from "../core/jupyter/jupyter.ts"; +import { jupyterAssets, jupyterToMarkdown } from "../core/jupyter/jupyter.ts"; import { kExecuteDaemon, kExecuteDebug, @@ -38,15 +34,11 @@ import { import { resourcePath } from "../core/resources.ts"; import { quartoRuntimeDir } from "../core/appdirs.ts"; import { normalizePath } from "../core/path.ts"; -import { md5Hash } from "../core/hash.ts"; import { isInteractiveSession } from "../core/platform.ts"; import { runningInCI } from "../core/ci-info.ts"; -import { ProcessResult } from "../core/process-types.ts"; import { sleep } from "../core/async.ts"; import { JupyterNotebook } from "../core/jupyter/types.ts"; import { existsSync } from "fs/mod.ts"; -import { readTextFileSync } from "../core/qualified-path.ts"; -import { number } from "https://deno.land/x/cliffy@v0.25.4/flags/types/number.ts"; export interface JuliaExecuteOptions extends ExecuteOptions { julia_cmd: string[]; @@ -114,16 +106,9 @@ export const juliaEngine: ExecutionEngine = { // or rstudio) and not running in a CI system. let executeDaemon = options.format.execute[kExecuteDaemon]; if (executeDaemon === null || executeDaemon === undefined) { - // if (await disableDaemonForNotebook(options.target)) { - // executeDaemon = false; - // } else { executeDaemon = isInteractiveSession() && !runningInCI(); - // } } - // julia back end requires full path to input (to ensure that - // keepalive kernels are never re-used across multiple inputs - // that happen to share a hash) const execOptions = { ...options, target: { @@ -139,18 +124,13 @@ export const juliaEngine: ExecutionEngine = { ...execOptions, }; - // if (executeDaemon === false || executeDaemon === 0) { + // TODO: executeDaemon can take a number for timeout of kernels, but + // QuartoNotebookRunner currently doesn't support that const nb = await executeJulia(juliaExecOptions); - // } else { - // // await executeJuliaKeepalive(juliaExecOptions); - // } // NOTE: the following is all mostly copied from the jupyter kernel file - // const nb = jupyterFromJSON(notebookJSON); - - // TODO: jupyterFromFile sets python as the default kernelspec for the files we get from QuartoNotebookRunner, - // maybe the correct "kernel" needs to be set there instead (there isn't really a kernel needed as we don't execute via Jupyter + // there isn't really a "kernel" as we don't execute via Jupyter // but this seems to be needed later to assign the correct language markers to code cells etc.) nb.metadata.kernelspec = { display_name: "Julia", @@ -343,7 +323,7 @@ async function writeJuliaCommand( options: JuliaExecuteOptions, ) { // TODO: no secret used, yet - let messageBytes = new TextEncoder().encode( + const messageBytes = new TextEncoder().encode( JSON.stringify({ type: command, content: options.target.input, From 7974a36dc9de69866947f7666eb8829117b898c8 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 25 Jan 2024 11:39:48 +0100 Subject: [PATCH 010/101] restart daemon if requested --- src/execute/julia.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 088228640f..0a3d85ddac 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -17,6 +17,7 @@ import { import { jupyterAssets, jupyterToMarkdown } from "../core/jupyter/jupyter.ts"; import { kExecuteDaemon, + kExecuteDaemonRestart, kExecuteDebug, kFigDpi, kFigFormat, @@ -300,7 +301,7 @@ async function executeJulia( ); const conn = await establishServerConnection(options, transportOptions.port); - if (options.oneShot) { + if (options.oneShot || options.format.execute[kExecuteDaemonRestart]) { await writeJuliaCommand(conn, "close", "TODOsomesecret", options); } const response = await writeJuliaCommand( From ba02f1c36ba0094e856e70f9f6cc114c6e5e792c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 25 Jan 2024 11:43:56 +0100 Subject: [PATCH 011/101] move sleeps --- src/execute/julia.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 0a3d85ddac..e3c78a1622 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -259,11 +259,11 @@ async function pollTransportFile(): Promise { const transportFile = juliaTransportFile(); for (let i = 0; i < 20; i++) { - await sleep(i * 100); if (existsSync(transportFile)) { const content = Deno.readTextFileSync(transportFile); return JSON.parse(content) as JuliaTransportFile; } + await sleep(i * 100); } return Promise.reject(); } @@ -274,11 +274,11 @@ async function establishServerConnection( ): Promise { let conn = null; for (let i = 0; i < 20; i++) { - await sleep(i * 100); conn = await Deno.connect({ port: port, - }).catch((reason) => { + }).catch(async (_) => { trace(options, `Connecting to julia server failed on try ${i}`); + await sleep(i * 100); return null; }); if (conn !== null) { From 12b8a2bab8e22ed1601d8bd29aaea4c526021b0c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 25 Jan 2024 18:24:13 +0100 Subject: [PATCH 012/101] use custom env for server --- src/execute/julia.ts | 104 ++++++++++++++++------ src/resources/julia/Project.toml | 5 ++ src/resources/julia/ensure_environment.jl | 13 +++ 3 files changed, 93 insertions(+), 29 deletions(-) create mode 100644 src/resources/julia/Project.toml create mode 100644 src/resources/julia/ensure_environment.jl diff --git a/src/execute/julia.ts b/src/execute/julia.ts index e3c78a1622..03a2c15312 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -217,13 +217,18 @@ export const juliaEngine: ExecutionEngine = { }, }; -async function startJuliaServer(options: JuliaExecuteOptions) { +async function startOrReuseJuliaServer( + options: JuliaExecuteOptions, +): Promise<{ reused: boolean }> { const transportFile = juliaTransportFile(); if (!existsSync(transportFile)) { trace( options, `Transport file ${transportFile} doesn't exist, starting server.`, ); + + await ensureQuartoNotebookRunnerEnvironment(options); + // when quarto's execProc function is used here, there is a strange bug. // The first time render is called on a file, the julia server is started correctly. // The second time it is called, however, the socket server hangs if during the first @@ -233,7 +238,7 @@ async function startJuliaServer(options: JuliaExecuteOptions) { const command = new Deno.Command(options.julia_cmd[0], { args: [ ...(options.julia_cmd.slice(1)), - "--project=@quarto", + `--project=${juliaRuntimeDir()}`, resourcePath("julia/quartonotebookrunner.jl"), transportFile, ], @@ -246,10 +251,41 @@ async function startJuliaServer(options: JuliaExecuteOptions) { options, `Transport file ${transportFile} exists, reusing server.`, ); + return { reused: true }; + } + return { reused: false }; +} + +async function ensureQuartoNotebookRunnerEnvironment( + options: JuliaExecuteOptions, +) { + const projectTomlTemplate = juliaResourcePath("Project.toml"); + const projectToml = join(juliaRuntimeDir(), "Project.toml"); + Deno.copyFileSync(projectTomlTemplate, projectToml); + const command = new Deno.Command(options.julia_cmd[0], { + args: [ + ...(options.julia_cmd.slice(1)), + `--project=${juliaRuntimeDir()}`, + juliaResourcePath("ensure_environment.jl"), + ], + }); + const { success, stderr } = await command.output(); + if (!success) { + error( + `Ensuring an updated julia server environment failed: ${ + stderr && new TextDecoder().decode(stderr) + }`, + ); + return Promise.reject(); } + trace(options, "The julia server environment is correctly instantiated."); return Promise.resolve(); } +function juliaResourcePath(...parts: string[]) { + return join(resourcePath("julia"), ...parts); +} + interface JuliaTransportFile { port: number; pid: number; @@ -269,38 +305,47 @@ async function pollTransportFile(): Promise { } async function establishServerConnection( - options: ExecuteOptions, port: number, ): Promise { - let conn = null; - for (let i = 0; i < 20; i++) { - conn = await Deno.connect({ - port: port, - }).catch(async (_) => { - trace(options, `Connecting to julia server failed on try ${i}`); - await sleep(i * 100); - return null; - }); - if (conn !== null) { - trace(options, "Connection successfully established"); - return conn; - } - } - - return Promise.reject(); + // Because the transport file is written after the server goes live, + // the connection should succeed immediately. + return await Deno.connect({ + port: port, + }); } -async function executeJulia( +async function getJuliaServerConnection( options: JuliaExecuteOptions, -): Promise { - await startJuliaServer(options); +): Promise { + const { reused } = await startOrReuseJuliaServer(options); const transportOptions = await pollTransportFile(); trace( options, `Connecting to server at port ${transportOptions.port}, pid ${transportOptions.pid}`, ); - const conn = await establishServerConnection(options, transportOptions.port); + try { + return await establishServerConnection(transportOptions.port); + } catch { + if (reused) { + trace( + options, + "Connecting to server failed, a transport file was reused so it might be stale. Delete transport file and retry.", + ); + Deno.removeSync(juliaTransportFile()); + return await getJuliaServerConnection(options); + } else { + error( + "Connecting to server failed. A transport file was successfully created by the server process, so something in the server process might be broken.", + ); + return Promise.reject(); + } + } +} +async function executeJulia( + options: JuliaExecuteOptions, +): Promise { + const conn = await getJuliaServerConnection(options); if (options.oneShot || options.format.execute[kExecuteDaemonRestart]) { await writeJuliaCommand(conn, "close", "TODOsomesecret", options); } @@ -375,13 +420,11 @@ async function writeJuliaCommand( return data; } -function juliaTransportFile() { - let transportsDir: string; - +function juliaRuntimeDir(): string { try { - transportsDir = quartoRuntimeDir("julia"); + return quartoRuntimeDir("julia"); } catch (e) { - error("Could create runtime directory for the julia pidfile."); + error("Could not create julia runtime directory."); error( "This is possibly a permission issue in the environment Quarto is running in.", ); @@ -393,7 +436,10 @@ function juliaTransportFile() { ); throw e; } - return join(transportsDir, "julia_transport.txt"); +} + +function juliaTransportFile() { + return join(juliaRuntimeDir(), "julia_transport.txt"); } function trace(options: ExecuteOptions, msg: string) { diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml new file mode 100644 index 0000000000..cd89b8e5c6 --- /dev/null +++ b/src/resources/julia/Project.toml @@ -0,0 +1,5 @@ +[deps] +QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" + +[compat] +QuartoNotebookRunner = "0.1.3" diff --git a/src/resources/julia/ensure_environment.jl b/src/resources/julia/ensure_environment.jl new file mode 100644 index 0000000000..421e7786ab --- /dev/null +++ b/src/resources/julia/ensure_environment.jl @@ -0,0 +1,13 @@ +using Pkg + +# If the manifest was resolved with the exact Project.toml that we have copied +# over from quarto's resource folder, then we just ensure that it is instantiated, +# all packages are downloaded correctly, etc. +# Otherwise, we update to get the newest packages fulfilling the compat bounds set in +# the Project.toml. + +if Pkg.is_manifest_current() === true # this returns nothing if there's no manifest + Pkg.instantiate() +else + Pkg.update() +end From 558fe3775b6b825e531b3c3b46c3552255cf79fd Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 25 Jan 2024 19:09:49 +0100 Subject: [PATCH 013/101] add notice when server starts --- src/execute/julia.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 03a2c15312..e42c1d1219 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -227,6 +227,9 @@ async function startOrReuseJuliaServer( `Transport file ${transportFile} doesn't exist, starting server.`, ); + info( + "Initializing julia control server. This might take a while...", + ); await ensureQuartoNotebookRunnerEnvironment(options); // when quarto's execProc function is used here, there is a strange bug. From c79e5b20af4c3f1d4c2f12002d1e2269192e1573 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 25 Jan 2024 19:35:48 +0100 Subject: [PATCH 014/101] also check that julia version in manifest matches --- src/resources/julia/ensure_environment.jl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/resources/julia/ensure_environment.jl b/src/resources/julia/ensure_environment.jl index 421e7786ab..7ca9898465 100644 --- a/src/resources/julia/ensure_environment.jl +++ b/src/resources/julia/ensure_environment.jl @@ -1,4 +1,5 @@ using Pkg +using TOML # If the manifest was resolved with the exact Project.toml that we have copied # over from quarto's resource folder, then we just ensure that it is instantiated, @@ -6,7 +7,16 @@ using Pkg # Otherwise, we update to get the newest packages fulfilling the compat bounds set in # the Project.toml. -if Pkg.is_manifest_current() === true # this returns nothing if there's no manifest +function manifest_has_correct_julia_version() + project_file = Base.active_project() + manifest_file = joinpath(dirname(project_file), "Manifest.toml") + version = VersionNumber(TOML.parsefile(manifest_file)["julia_version"]) + return version.major == VERSION.major && version.minor == VERSION.minor +end + +manifest_matches_project_toml = Pkg.is_manifest_current() === true # this returns nothing if there's no manifest + +if manifest_matches_project_toml && manifest_has_correct_julia_version() Pkg.instantiate() else Pkg.update() From 9a544c1637f1131edda663f85e4632d99c8b2b52 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 1 Feb 2024 11:13:16 +0100 Subject: [PATCH 015/101] allow override of julia binary via QUARTO_JULIA env var --- src/execute/julia.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index e42c1d1219..920a0be054 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -42,7 +42,7 @@ import { JupyterNotebook } from "../core/jupyter/types.ts"; import { existsSync } from "fs/mod.ts"; export interface JuliaExecuteOptions extends ExecuteOptions { - julia_cmd: string[]; + julia_cmd: string; oneShot: boolean; // if true, the file's worker process is closed before and after running supervisor_pid?: number; } @@ -119,7 +119,7 @@ export const juliaEngine: ExecutionEngine = { }; const juliaExecOptions: JuliaExecuteOptions = { - julia_cmd: ["julia"], + julia_cmd: Deno.env.get("QUARTO_JULIA") ?? "julia", oneShot: !executeDaemon, supervisor_pid: options.previewServer ? Deno.pid : undefined, ...execOptions, @@ -238,9 +238,8 @@ async function startOrReuseJuliaServer( // run anything was written to stderr. This goes away when redirecting stderr to // a file on the julia side, but also when using Deno.Command which is recommended // as a replacement for the old Deno.run anyway. - const command = new Deno.Command(options.julia_cmd[0], { + const command = new Deno.Command(options.julia_cmd, { args: [ - ...(options.julia_cmd.slice(1)), `--project=${juliaRuntimeDir()}`, resourcePath("julia/quartonotebookrunner.jl"), transportFile, @@ -265,9 +264,8 @@ async function ensureQuartoNotebookRunnerEnvironment( const projectTomlTemplate = juliaResourcePath("Project.toml"); const projectToml = join(juliaRuntimeDir(), "Project.toml"); Deno.copyFileSync(projectTomlTemplate, projectToml); - const command = new Deno.Command(options.julia_cmd[0], { + const command = new Deno.Command(options.julia_cmd, { args: [ - ...(options.julia_cmd.slice(1)), `--project=${juliaRuntimeDir()}`, juliaResourcePath("ensure_environment.jl"), ], From 11347f1953a9cbeb3152d0c4aa82e7d5fcab3837 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 5 Feb 2024 16:08:07 +0100 Subject: [PATCH 016/101] bump quartonotebookrunner --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index cd89b8e5c6..c6ff3b65d3 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "0.1.3" +QuartoNotebookRunner = "0.2.0" From 503230f9f61033e5de5d1163802220fbf5154930 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 5 Feb 2024 16:08:26 +0100 Subject: [PATCH 017/101] send options along with "run" command --- src/execute/julia.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 920a0be054..8f631e0aae 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -369,11 +369,16 @@ async function writeJuliaCommand( secret: string, options: JuliaExecuteOptions, ) { + // send the options along with the "run" command + const content = command === "run" + ? { file: options.target.input, options } + : options.target.input; + // TODO: no secret used, yet const messageBytes = new TextEncoder().encode( JSON.stringify({ type: command, - content: options.target.input, + content, }) + "\n", ); From 2153023c591c9970d9f5c9f48d2c84a9e8b0d8fb Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 26 Feb 2024 15:04:11 +0100 Subject: [PATCH 018/101] register julia engine again --- src/execute/engine.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/execute/engine.ts b/src/execute/engine.ts index ab80d9bccf..8edc00ce74 100644 --- a/src/execute/engine.ts +++ b/src/execute/engine.ts @@ -41,7 +41,9 @@ export function executionEngine(name: string) { return kEngines.get(name); } -for (const engine of [knitrEngine, jupyterEngine, markdownEngine]) { +for ( + const engine of [knitrEngine, jupyterEngine, markdownEngine, juliaEngine] +) { registerExecutionEngine(engine); } From c2c20123e2b0769d429d83704b80789ad9517edd Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 26 Feb 2024 15:04:39 +0100 Subject: [PATCH 019/101] reject if notebook exec returns undefined --- src/execute/julia.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 8f631e0aae..1c083928ea 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -129,6 +129,11 @@ export const juliaEngine: ExecutionEngine = { // QuartoNotebookRunner currently doesn't support that const nb = await executeJulia(juliaExecOptions); + if (!nb) { + error("Execution of notebook returned undefined"); + return Promise.reject(); + } + // NOTE: the following is all mostly copied from the jupyter kernel file // there isn't really a "kernel" as we don't execute via Jupyter From c8ac7f9f78b5242c1f113e73f19f28d635309e05 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 26 Feb 2024 15:10:24 +0100 Subject: [PATCH 020/101] bump QuartoNotebookRunner --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index c6ff3b65d3..24d48cf6cf 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "0.2.0" +QuartoNotebookRunner = "0.3.1" From 4de287c75deb8d8ca3ce268ccce8aeedbf40a012 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 09:02:15 +0100 Subject: [PATCH 021/101] remove commented out code --- src/resources/julia/quartonotebookrunner.jl | 46 ++++++++++----------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index cc7599afd4..5ec51a8fe2 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -9,32 +9,30 @@ atexit() do rm(transport_file; force=true) end -# redirect_stdio(; stderr = joinpath(transport_dir, "stderr.txt")) do - server, port = let - server = nothing - port = nothing - - for i in 1:20 - # find an open port by creating a server there and immediately closing it - port, _server = Sockets.listenany(8000) - close(_server) - try - server = QuartoNotebookRunner.serve(; port) - break - catch e - error("Opening server on port $port failed.") - end +server, port = let + server = nothing + port = nothing + + for i in 1:20 + # find an open port by creating a server there and immediately closing it + port, _server = Sockets.listenany(8000) + close(_server) + try + server = QuartoNotebookRunner.serve(; port) + break + catch e + error("Opening server on port $port failed.") end - server, port end + server, port +end - if server === nothing - error("Giving up.") - end +if server === nothing + error("Giving up.") +end - open(transport_file, "w") do io - println(io, """{"port": $port, "pid": $(Base.Libc.getpid())}""") - end +open(transport_file, "w") do io + println(io, """{"port": $port, "pid": $(Base.Libc.getpid())}""") +end - wait(server) -# end +wait(server) From 9f11e35bd8145d79bf58514edb623007d3198201 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 09:02:42 +0100 Subject: [PATCH 022/101] bump QuartoNotebookRunner to 0.3.2 --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index 24d48cf6cf..2c6ec84327 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "0.3.1" +QuartoNotebookRunner = "0.3.2" From 9f8cb5b4f4b806a99b85ed3dd6e8710af68d2058 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 10:21:06 +0100 Subject: [PATCH 023/101] let connection timeout if `isready` isn't returned quickly enough --- src/execute/julia.ts | 50 ++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 1c083928ea..ac98ea7ade 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -310,27 +310,43 @@ async function pollTransportFile(): Promise { return Promise.reject(); } -async function establishServerConnection( - port: number, -): Promise { - // Because the transport file is written after the server goes live, - // the connection should succeed immediately. - return await Deno.connect({ - port: port, - }); -} - async function getJuliaServerConnection( options: JuliaExecuteOptions, ): Promise { const { reused } = await startOrReuseJuliaServer(options); const transportOptions = await pollTransportFile(); + trace( options, `Connecting to server at port ${transportOptions.port}, pid ${transportOptions.pid}`, ); + try { - return await establishServerConnection(transportOptions.port); + const conn = await Deno.connect({ + port: transportOptions.port, + }); + const isready = writeJuliaCommand( + conn, + "isready", + "TODOsomesecret", + options, + ) as Promise; + const timeoutMilliseconds = 5000; + const timeout = new Promise((_, reject) => + setTimeout(() => { + trace(options, `Timed out after ${timeoutMilliseconds} milliseconds.`); + reject("Timeout"); + }, timeoutMilliseconds) + ); + const result = await Promise.race([isready, timeout]); + if (result !== true) { + error( + `Expected isready command to return true, returned ${isready} instead. Closing connection.`, + ); + conn.close(); + return Promise.reject(); + } + return conn; } catch { if (reused) { trace( @@ -353,7 +369,15 @@ async function executeJulia( ): Promise { const conn = await getJuliaServerConnection(options); if (options.oneShot || options.format.execute[kExecuteDaemonRestart]) { - await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + const isopen = await writeJuliaCommand( + conn, + "isopen", + "TODOsomesecret", + options, + ) as boolean; + if (isopen) { + await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + } } const response = await writeJuliaCommand( conn, @@ -370,7 +394,7 @@ async function executeJulia( async function writeJuliaCommand( conn: Deno.Conn, - command: "run" | "close" | "stop", + command: "run" | "close" | "stop" | "isready" | "isopen", secret: string, options: JuliaExecuteOptions, ) { From 8bb791b9b3b724a883d81014e9b42604f68eae2a Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 10:36:35 +0100 Subject: [PATCH 024/101] formatting --- src/execute/julia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index ac98ea7ade..6931a77446 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -278,7 +278,7 @@ async function ensureQuartoNotebookRunnerEnvironment( const { success, stderr } = await command.output(); if (!success) { error( - `Ensuring an updated julia server environment failed: ${ + `Ensuring an updated julia server environment failed:\n${ stderr && new TextDecoder().decode(stderr) }`, ); From 2e185031e5ffc354e1117018ab5dbe2de56c3965 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 10:36:48 +0100 Subject: [PATCH 025/101] don't claim julia automatically --- src/execute/julia.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 6931a77446..8a465ab078 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -65,7 +65,9 @@ export const juliaEngine: ExecutionEngine = { claimsFile: (file: string, ext: string) => false, claimsLanguage: (language: string) => { - return language.toLowerCase() === "julia"; + // we don't claim `julia` so the old behavior of using the jupyter + // backend by default stays intact + return false; // language.toLowerCase() === "julia"; }, partitionedMarkdown: async (file: string) => { From dcc21b3c05c0ca639403ad764b7381d5b6a1d03a Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 10:40:39 +0100 Subject: [PATCH 026/101] add julia engine test --- tests/docs/markdown/commonmark-julia-jupyter.qmd | 8 ++++++++ tests/docs/markdown/commonmark-julia.qmd | 1 + 2 files changed, 9 insertions(+) create mode 100644 tests/docs/markdown/commonmark-julia-jupyter.qmd diff --git a/tests/docs/markdown/commonmark-julia-jupyter.qmd b/tests/docs/markdown/commonmark-julia-jupyter.qmd new file mode 100644 index 0000000000..eb48d2247e --- /dev/null +++ b/tests/docs/markdown/commonmark-julia-jupyter.qmd @@ -0,0 +1,8 @@ +--- +title: test +format: commonmark +--- + +```{julia} +1 + 1 +``` diff --git a/tests/docs/markdown/commonmark-julia.qmd b/tests/docs/markdown/commonmark-julia.qmd index eb48d2247e..0e7247d7a7 100644 --- a/tests/docs/markdown/commonmark-julia.qmd +++ b/tests/docs/markdown/commonmark-julia.qmd @@ -1,6 +1,7 @@ --- title: test format: commonmark +engine: julia --- ```{julia} From 61c0c1b7d4feb8244427e9569e0a6f6ac0d7d11b Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 10:41:06 +0100 Subject: [PATCH 027/101] timeout after 3000 ms --- src/execute/julia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 8a465ab078..37bfb361db 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -333,7 +333,7 @@ async function getJuliaServerConnection( "TODOsomesecret", options, ) as Promise; - const timeoutMilliseconds = 5000; + const timeoutMilliseconds = 3000; const timeout = new Promise((_, reject) => setTimeout(() => { trace(options, `Timed out after ${timeoutMilliseconds} milliseconds.`); From af007175b0dd5b69dc3a345f3d272ef0e7da0c1a Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 11:38:57 +0100 Subject: [PATCH 028/101] take out `info` as tests expect no output --- src/execute/julia.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 37bfb361db..a9907c4e75 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -231,11 +231,7 @@ async function startOrReuseJuliaServer( if (!existsSync(transportFile)) { trace( options, - `Transport file ${transportFile} doesn't exist, starting server.`, - ); - - info( - "Initializing julia control server. This might take a while...", + `Transport file ${transportFile} doesn't exist, starting server. This might take a while.`, ); await ensureQuartoNotebookRunnerEnvironment(options); From 32509fd15a50ef973344a530c17d0deb4cbed99f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 13:29:37 +0100 Subject: [PATCH 029/101] use jupyter file for old test --- tests/smoke/render/render-commonmark.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smoke/render/render-commonmark.test.ts b/tests/smoke/render/render-commonmark.test.ts index a95206b649..ebc5265ccb 100644 --- a/tests/smoke/render/render-commonmark.test.ts +++ b/tests/smoke/render/render-commonmark.test.ts @@ -13,7 +13,7 @@ const tests = [ { file: "commonmark-plain.qmd", python: false }, { file: "commonmark-r.qmd", python: false }, { file: "commonmark-python.qmd", python: true }, - { file: "commonmark-julia.qmd", python: false }, + { file: "commonmark-julia-jupyter.qmd", python: false }, ]; tests.forEach((test) => { const input = docs(join("markdown", test.file)); From 3adf782d45ed141c6e3b7eec985b021e3d78e9e7 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 27 Feb 2024 13:58:27 +0100 Subject: [PATCH 030/101] don't always trace timeout message --- src/execute/julia.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index a9907c4e75..24d44aa390 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -332,8 +332,7 @@ async function getJuliaServerConnection( const timeoutMilliseconds = 3000; const timeout = new Promise((_, reject) => setTimeout(() => { - trace(options, `Timed out after ${timeoutMilliseconds} milliseconds.`); - reject("Timeout"); + reject(`Timed out after ${timeoutMilliseconds} milliseconds.`); }, timeoutMilliseconds) ); const result = await Promise.race([isready, timeout]); From 762c22cbdd7ccd262beba7d5cd1d882a0a8e201e Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 28 Feb 2024 12:33:10 +0100 Subject: [PATCH 031/101] re-add native julia commonmark smoke test --- tests/smoke/render/render-commonmark.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/smoke/render/render-commonmark.test.ts b/tests/smoke/render/render-commonmark.test.ts index ebc5265ccb..a3465dab23 100644 --- a/tests/smoke/render/render-commonmark.test.ts +++ b/tests/smoke/render/render-commonmark.test.ts @@ -10,10 +10,11 @@ import { ensureFileRegexMatches } from "../../verify.ts"; import { testRender } from "./render.ts"; const tests = [ - { file: "commonmark-plain.qmd", python: false }, - { file: "commonmark-r.qmd", python: false }, - { file: "commonmark-python.qmd", python: true }, - { file: "commonmark-julia-jupyter.qmd", python: false }, + { file: "commonmark-plain.qmd" }, + { file: "commonmark-r.qmd" }, + { file: "commonmark-python.qmd"}, + { file: "commonmark-julia-jupyter.qmd" }, + { file: "commonmark-julia.qmd" }, ]; tests.forEach((test) => { const input = docs(join("markdown", test.file)); From e96a3415a8d136f3db85a12d1478bdaf5ae8cd23 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 28 Feb 2024 12:47:30 +0100 Subject: [PATCH 032/101] add native julia crossref test --- tests/docs/crossrefs/julia-native.qmd | 22 +++++++++++++++++++++ tests/smoke/crossref/figures.test.ts | 28 ++++++++++++++------------- 2 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 tests/docs/crossrefs/julia-native.qmd diff --git a/tests/docs/crossrefs/julia-native.qmd b/tests/docs/crossrefs/julia-native.qmd new file mode 100644 index 0000000000..2ae6bbfb9a --- /dev/null +++ b/tests/docs/crossrefs/julia-native.qmd @@ -0,0 +1,22 @@ +--- +title: Julia Crossref Test +engine: julia +--- + +```{julia} +Pkg.activate(temp = true) +Pkg.add(name = "Plots", version = "1.40.1") +``` + +## Julia Crossref Figure + +```{julia} +#| label: fig-plot +#| fig-cap: "Plot" + +using Plots +Plots.gr(format="png") +plot([1,23,2,4]) +``` + +For example, see @fig-plot. diff --git a/tests/smoke/crossref/figures.test.ts b/tests/smoke/crossref/figures.test.ts index c5dcfe80ec..dad3213075 100644 --- a/tests/smoke/crossref/figures.test.ts +++ b/tests/smoke/crossref/figures.test.ts @@ -62,19 +62,21 @@ testRender(pythonSubfigQmd.input, "html", false, [ ]), ]); -const juliaQmd = crossref("julia.qmd", "html"); -testRender(juliaQmd.input, "html", false, [ - ensureHtmlElements(juliaQmd.output.outputPath, [ - "section#julia-crossref-figure div#fig-plot > figure img.figure-img", - "section#julia-crossref-figure div#fig-plot > figure > figcaption", - ]), - ensureFileRegexMatches(juliaQmd.output.outputPath, [ - /Figure 1: Plot/, - /Figure 1/, - ], [ - /\?@fig-/, - ]), -]); +for (const file of ["julia.qmd", "julia-native.qmd"]) { + const juliaQmd = crossref(file, "html"); + testRender(juliaQmd.input, "html", false, [ + ensureHtmlElements(juliaQmd.output.outputPath, [ + "section#julia-crossref-figure div#fig-plot > figure img.figure-img", + "section#julia-crossref-figure div#fig-plot > figure > figcaption", + ]), + ensureFileRegexMatches(juliaQmd.output.outputPath, [ + /Figure 1: Plot/, + /Figure 1/, + ], [ + /\?@fig-/, + ]), + ]); +} const juliaSubfigQmd = crossref("julia-subfig.qmd", "html"); testRender(juliaSubfigQmd.input, "html", false, [ From 906d6c18e723c5b062dff2e8fa893ade421efb6e Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 28 Feb 2024 15:15:42 +0100 Subject: [PATCH 033/101] rethrow error if server connection fails to establish --- src/execute/julia.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 24d44aa390..9eef1e9835 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -344,7 +344,7 @@ async function getJuliaServerConnection( return Promise.reject(); } return conn; - } catch { + } catch (e) { if (reused) { trace( options, @@ -356,7 +356,7 @@ async function getJuliaServerConnection( error( "Connecting to server failed. A transport file was successfully created by the server process, so something in the server process might be broken.", ); - return Promise.reject(); + throw e; } } } From 468583be17959d803fd964ef8015fafcc7c79b5d Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 28 Feb 2024 15:46:13 +0100 Subject: [PATCH 034/101] try if 30 seconds timeout removes the timeout errors --- src/execute/julia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 9eef1e9835..dcb6764f75 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -329,7 +329,7 @@ async function getJuliaServerConnection( "TODOsomesecret", options, ) as Promise; - const timeoutMilliseconds = 3000; + const timeoutMilliseconds = 30000; const timeout = new Promise((_, reject) => setTimeout(() => { reject(`Timed out after ${timeoutMilliseconds} milliseconds.`); From 654633d5e9703b8a27415592dca8247a454049a5 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 28 Feb 2024 16:03:04 +0100 Subject: [PATCH 035/101] try if specifying `localhost` makes a connection difference --- src/execute/julia.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index dcb6764f75..758f676e3f 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -321,6 +321,7 @@ async function getJuliaServerConnection( try { const conn = await Deno.connect({ + hostname: "localhost", port: transportOptions.port, }); const isready = writeJuliaCommand( From 334c15a2e47f7e7e9bd0cf59b0cd53b06b9bf4f7 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 29 Feb 2024 11:04:31 +0100 Subject: [PATCH 036/101] remove localhost again --- src/execute/julia.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 758f676e3f..dcb6764f75 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -321,7 +321,6 @@ async function getJuliaServerConnection( try { const conn = await Deno.connect({ - hostname: "localhost", port: transportOptions.port, }); const isready = writeJuliaCommand( From f34d2c8cbd3d7e475a96858d20779fba10b29ad9 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 29 Feb 2024 12:13:00 +0100 Subject: [PATCH 037/101] change timeout back to 3 seconds --- src/execute/julia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index dcb6764f75..9eef1e9835 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -329,7 +329,7 @@ async function getJuliaServerConnection( "TODOsomesecret", options, ) as Promise; - const timeoutMilliseconds = 30000; + const timeoutMilliseconds = 3000; const timeout = new Promise((_, reject) => setTimeout(() => { reject(`Timed out after ${timeoutMilliseconds} milliseconds.`); From be684ed5ef7bcf8ace56935ac35c7414481a1875 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 29 Feb 2024 12:25:54 +0100 Subject: [PATCH 038/101] clarify command creation code and add console.log for CI check --- src/execute/julia.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 9eef1e9835..b7c3406799 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -398,15 +398,21 @@ async function writeJuliaCommand( // send the options along with the "run" command const content = command === "run" ? { file: options.target.input, options } + : command === "stop" || command === "isready" + ? {} : options.target.input; + const commandData = { + type: command, + content, + }; + + console.log(commandData); + + const message = JSON.stringify(commandData) + "\n"; + // TODO: no secret used, yet - const messageBytes = new TextEncoder().encode( - JSON.stringify({ - type: command, - content, - }) + "\n", - ); + const messageBytes = new TextEncoder().encode(message); // // don't send the message if it's big. // // Instead, write it to a file and send the file path From 171e688ab0350b9fa159424939f7de368ee76a3d Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 29 Feb 2024 12:35:27 +0100 Subject: [PATCH 039/101] remove logging, not visible in CI --- src/execute/julia.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index b7c3406799..0f0e6d72f7 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -407,8 +407,6 @@ async function writeJuliaCommand( content, }; - console.log(commandData); - const message = JSON.stringify(commandData) + "\n"; // TODO: no secret used, yet From f2c23f2e98ca722d80e77af72774ec22d0ff4f2d Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 29 Feb 2024 13:33:16 +0100 Subject: [PATCH 040/101] make server process detached through julia indirection --- src/execute/julia.ts | 10 ++++++---- .../julia/start_quartonotebookrunner_detached.jl | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 src/resources/julia/start_quartonotebookrunner_detached.jl diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 0f0e6d72f7..0332a541a2 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -243,14 +243,16 @@ async function startOrReuseJuliaServer( // as a replacement for the old Deno.run anyway. const command = new Deno.Command(options.julia_cmd, { args: [ - `--project=${juliaRuntimeDir()}`, + resourcePath("julia/start_quartonotebookrunner_detached.jl"), + options.julia_cmd, + juliaRuntimeDir(), resourcePath("julia/quartonotebookrunner.jl"), transportFile, ], }); - // this process is supposed to outlive the quarto process, because - // in it, the references to the cached julia worker processes live - command.spawn(); + // when this process finishes, a detached julia process with the quartonotebookrunner server will have been started + trace(options, "Spawning detached julia server through julia"); + command.outputSync(); } else { trace( options, diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl new file mode 100644 index 0000000000..d8111caf58 --- /dev/null +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -0,0 +1,15 @@ +# it appears that deno cannot launch detached processes https://github.com/denoland/deno/issues/5501 +# so we use an indirection where we start the detached julia process using julia itself +julia_bin = ARGS[1] +project = ARGS[2] +julia_file = ARGS[3] +transport_file = ARGS[4] + +if length(ARGS) > 4 + error("Too many arguments") +end + +cmd = `$julia_bin --project=$project $julia_file $transport_file` +@info cmd +run(detach(cmd), wait = false) +@info "ran that" From 761fc8aa65b6d73efe76fc7a1cd2fd93d24f4a98 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 1 Mar 2024 11:00:47 +0100 Subject: [PATCH 041/101] look at `lsof` output in CI --- src/execute/julia.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 0332a541a2..8712742b7e 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -322,6 +322,10 @@ async function getJuliaServerConnection( ); try { + const comm = new Deno.Command("lsof", { + args: ["-i", `:${transportOptions.port}`], + }); + comm.spawn(); const conn = await Deno.connect({ port: transportOptions.port, }); From e38b03a448bbcea6a52a6d3e0d654480cb763898 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 08:57:47 +0100 Subject: [PATCH 042/101] add debug log level --- .github/workflows/test-smokes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index f16be2925f..cda0b9c264 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -201,6 +201,7 @@ jobs: env: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + QUARTO_LOG_LEVEL: DEBUG run: | haserror=0 readarray -t my_array < <(echo '${{ inputs.buckets }}' | jq -rc '.[]') From f77bdbb27574199eb6b5bdd9987f7a22b5492b40 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 09:12:20 +0100 Subject: [PATCH 043/101] run only linux --- .github/workflows/test-smokes.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index cda0b9c264..79db674083 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -38,7 +38,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + # os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] time-test: - ${{ inputs.time-test }} exclude: From 75e5f3bb7dacd0dea41bb7d72536f11153c82681 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 11:11:33 +0100 Subject: [PATCH 044/101] try tmate session for debugging --- .github/workflows/test-smokes.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 79db674083..26b74958c5 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -242,6 +242,9 @@ jobs: working-directory: tests shell: pwsh + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Create Pull Request for new tests timing if: matrix.time-test id: cpr From 9ebba87623e06d94f233d3e7a03350f254a563f6 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 11:18:25 +0100 Subject: [PATCH 045/101] move session before failure --- .github/workflows/test-smokes.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 26b74958c5..b008bb555b 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -179,6 +179,9 @@ jobs: run: | echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Run all Smoke Tests Windows if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) == '' && matrix.time-test == false }} env: @@ -242,9 +245,6 @@ jobs: working-directory: tests shell: pwsh - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Create Pull Request for new tests timing if: matrix.time-test id: cpr From 60c537f1a7bb413b5c667e124d8900d1b07b2522 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 12:32:46 +0100 Subject: [PATCH 046/101] increase timeout to 10 seconds --- src/execute/julia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 8712742b7e..16d6a71499 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -335,7 +335,7 @@ async function getJuliaServerConnection( "TODOsomesecret", options, ) as Promise; - const timeoutMilliseconds = 3000; + const timeoutMilliseconds = 10000; const timeout = new Promise((_, reject) => setTimeout(() => { reject(`Timed out after ${timeoutMilliseconds} milliseconds.`); From e280ee8d08e92d778f7215c17ef2bc3799621e57 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 13:39:46 +0100 Subject: [PATCH 047/101] try warming up the julia native server before tests run --- .github/workflows/test-smokes.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index b008bb555b..8d47c05dec 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -179,8 +179,15 @@ jobs: run: | echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + + # FIXME: for some reason on CI, it seems that communicating with the child process julia (even if it's detached) + # from the parent deno process fails. But if we run this step beforehand, that one server should be able to handle + # the remaining connections coming from new quarto processes + - name: Warmup Julia native server + run: | + quarto render tests/docs/markdown/commonmark-julia.qmd --execute-debug - name: Run all Smoke Tests Windows if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) == '' && matrix.time-test == false }} From 8c4c0279fd21d99dcead541371aef4a2ffb19dca Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 13:39:58 +0100 Subject: [PATCH 048/101] continue on error --- .github/workflows/test-smokes.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 8d47c05dec..a4e146b6ba 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -186,6 +186,7 @@ jobs: # from the parent deno process fails. But if we run this step beforehand, that one server should be able to handle # the remaining connections coming from new quarto processes - name: Warmup Julia native server + continue-on-error: true # this will fail as long as the bug with deno exists run: | quarto render tests/docs/markdown/commonmark-julia.qmd --execute-debug From 3ef741e7ef80f052dcd7eca276139981e942647e Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 13:52:36 +0100 Subject: [PATCH 049/101] remove `lsof` used for debugging --- src/execute/julia.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 16d6a71499..f5a7df9303 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -322,10 +322,6 @@ async function getJuliaServerConnection( ); try { - const comm = new Deno.Command("lsof", { - args: ["-i", `:${transportOptions.port}`], - }); - comm.spawn(); const conn = await Deno.connect({ port: transportOptions.port, }); From 50517835ec51d2cf4d1e93fc23f9718fab4489f9 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 14:17:48 +0100 Subject: [PATCH 050/101] format timeout error better --- src/execute/julia.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index f5a7df9303..23a7023731 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -332,13 +332,18 @@ async function getJuliaServerConnection( options, ) as Promise; const timeoutMilliseconds = 10000; - const timeout = new Promise((_, reject) => + const timeout = new Promise((accept, _) => setTimeout(() => { - reject(`Timed out after ${timeoutMilliseconds} milliseconds.`); + accept( + `Timed out after getting no response for ${timeoutMilliseconds} milliseconds.`, + ); }, timeoutMilliseconds) ); const result = await Promise.race([isready, timeout]); - if (result !== true) { + if (typeof result === "string") { + // timed out + throw new Error(result); + } else if (result !== true) { error( `Expected isready command to return true, returned ${isready} instead. Closing connection.`, ); From 0ca91d2ba163db859aac8d690989accf4b680a8c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 14:28:39 +0100 Subject: [PATCH 051/101] bump quartonotebookrunner version to 0.4 --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index 2c6ec84327..7d9dad7241 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "0.3.2" +QuartoNotebookRunner = "0.4.0" From a7319af16f36eb4680d956511105bde2ed6fdd3f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 14:33:38 +0100 Subject: [PATCH 052/101] add julianative subfig test --- tests/docs/crossrefs/julianative-subfig.qmd | 21 +++++++++++ .../{julia-native.qmd => julianative.qmd} | 0 tests/smoke/crossref/figures.test.ts | 36 ++++++++++--------- 3 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 tests/docs/crossrefs/julianative-subfig.qmd rename tests/docs/crossrefs/{julia-native.qmd => julianative.qmd} (100%) diff --git a/tests/docs/crossrefs/julianative-subfig.qmd b/tests/docs/crossrefs/julianative-subfig.qmd new file mode 100644 index 0000000000..29a45eb91c --- /dev/null +++ b/tests/docs/crossrefs/julianative-subfig.qmd @@ -0,0 +1,21 @@ +--- +title: Julia Subfig Test +--- + +## Julia Crossref Figure + +```{julia} +#| label: fig-plots +#| fig-cap: "Plots" +#| fig-subcap: +#| - "Plot 1" +#| - "Plot 2" +#| layout-ncol: 2 + +using Plots +plot([1,23,2,4]) |> display + +plot([8,65,23,90]) |> display +``` + +See @fig-plots for examples. In particular, @fig-plots-2. diff --git a/tests/docs/crossrefs/julia-native.qmd b/tests/docs/crossrefs/julianative.qmd similarity index 100% rename from tests/docs/crossrefs/julia-native.qmd rename to tests/docs/crossrefs/julianative.qmd diff --git a/tests/smoke/crossref/figures.test.ts b/tests/smoke/crossref/figures.test.ts index dad3213075..5a23b41b7c 100644 --- a/tests/smoke/crossref/figures.test.ts +++ b/tests/smoke/crossref/figures.test.ts @@ -62,7 +62,7 @@ testRender(pythonSubfigQmd.input, "html", false, [ ]), ]); -for (const file of ["julia.qmd", "julia-native.qmd"]) { +for (const file of ["julia.qmd", "julianative.qmd"]) { const juliaQmd = crossref(file, "html"); testRender(juliaQmd.input, "html", false, [ ensureHtmlElements(juliaQmd.output.outputPath, [ @@ -78,22 +78,24 @@ for (const file of ["julia.qmd", "julia-native.qmd"]) { ]); } -const juliaSubfigQmd = crossref("julia-subfig.qmd", "html"); -testRender(juliaSubfigQmd.input, "html", false, [ - ensureHtmlElements(juliaSubfigQmd.output.outputPath, [ - "section#julia-crossref-figure div.quarto-layout-panel > figure div.quarto-layout-row", - "section#julia-crossref-figure div.quarto-layout-panel > figure > figcaption.quarto-float-fig.quarto-float-caption", - ]), - ensureFileRegexMatches(juliaSubfigQmd.output.outputPath, [ - /Figure 1: Plots/, - /Figure 1/, - /Figure 1 \(b\)/, - /\(a\) Plot 1/, - /\(b\) Plot 2/, - ], [ - /\?@fig-/, - ]), -]); +for (const file of ["julia-subfig.qmd", "julianative-subfig.qmd"]) { + const juliaSubfigQmd = crossref(file, "html"); + testRender(juliaSubfigQmd.input, "html", false, [ + ensureHtmlElements(juliaSubfigQmd.output.outputPath, [ + "section#julia-crossref-figure div.quarto-layout-panel > figure div.quarto-layout-row", + "section#julia-crossref-figure div.quarto-layout-panel > figure > figcaption.quarto-float-fig.quarto-float-caption", + ]), + ensureFileRegexMatches(juliaSubfigQmd.output.outputPath, [ + /Figure 1: Plots/, + /Figure 1/, + /Figure 1 \(b\)/, + /\(a\) Plot 1/, + /\(b\) Plot 2/, + ], [ + /\?@fig-/, + ]), + ]); +} const knitrQmd = crossref("knitr.qmd", "html"); testRender(knitrQmd.input, "html", false, [ From 8c22e07c5be7e0ee102b290c4aafb2dd2a9232c2 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 16:11:27 +0100 Subject: [PATCH 053/101] rename files consistently --- .github/workflows/test-smokes.yml | 2 +- tests/docs/markdown/commonmark-julia.qmd | 1 - ...{commonmark-julia-jupyter.qmd => commonmark-julianative.qmd} | 1 + tests/smoke/render/render-commonmark.test.ts | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename tests/docs/markdown/{commonmark-julia-jupyter.qmd => commonmark-julianative.qmd} (81%) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index a4e146b6ba..a7acab198b 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -188,7 +188,7 @@ jobs: - name: Warmup Julia native server continue-on-error: true # this will fail as long as the bug with deno exists run: | - quarto render tests/docs/markdown/commonmark-julia.qmd --execute-debug + quarto render tests/docs/markdown/commonmark-julianative.qmd --execute-debug - name: Run all Smoke Tests Windows if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) == '' && matrix.time-test == false }} diff --git a/tests/docs/markdown/commonmark-julia.qmd b/tests/docs/markdown/commonmark-julia.qmd index 0e7247d7a7..eb48d2247e 100644 --- a/tests/docs/markdown/commonmark-julia.qmd +++ b/tests/docs/markdown/commonmark-julia.qmd @@ -1,7 +1,6 @@ --- title: test format: commonmark -engine: julia --- ```{julia} diff --git a/tests/docs/markdown/commonmark-julia-jupyter.qmd b/tests/docs/markdown/commonmark-julianative.qmd similarity index 81% rename from tests/docs/markdown/commonmark-julia-jupyter.qmd rename to tests/docs/markdown/commonmark-julianative.qmd index eb48d2247e..0e7247d7a7 100644 --- a/tests/docs/markdown/commonmark-julia-jupyter.qmd +++ b/tests/docs/markdown/commonmark-julianative.qmd @@ -1,6 +1,7 @@ --- title: test format: commonmark +engine: julia --- ```{julia} diff --git a/tests/smoke/render/render-commonmark.test.ts b/tests/smoke/render/render-commonmark.test.ts index a3465dab23..a388407a46 100644 --- a/tests/smoke/render/render-commonmark.test.ts +++ b/tests/smoke/render/render-commonmark.test.ts @@ -13,8 +13,8 @@ const tests = [ { file: "commonmark-plain.qmd" }, { file: "commonmark-r.qmd" }, { file: "commonmark-python.qmd"}, - { file: "commonmark-julia-jupyter.qmd" }, { file: "commonmark-julia.qmd" }, + { file: "commonmark-julianative.qmd" }, ]; tests.forEach((test) => { const input = docs(join("markdown", test.file)); From 4efeb61d59e7d2689c49d82e1a650598bae276ec Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 19:05:59 +0100 Subject: [PATCH 054/101] take out warmup again --- .github/workflows/test-smokes.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index a7acab198b..e817a65aba 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -185,10 +185,10 @@ jobs: # FIXME: for some reason on CI, it seems that communicating with the child process julia (even if it's detached) # from the parent deno process fails. But if we run this step beforehand, that one server should be able to handle # the remaining connections coming from new quarto processes - - name: Warmup Julia native server - continue-on-error: true # this will fail as long as the bug with deno exists - run: | - quarto render tests/docs/markdown/commonmark-julianative.qmd --execute-debug + # - name: Warmup Julia native server + # continue-on-error: true # this will fail as long as the bug with deno exists + # run: | + # quarto render tests/docs/markdown/commonmark-julianative.qmd --execute-debug - name: Run all Smoke Tests Windows if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) == '' && matrix.time-test == false }} From e822d46e9389c3ce40db44b81c4aa6c49c3e072f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 19:06:48 +0100 Subject: [PATCH 055/101] add test connection before writing transport file --- src/resources/julia/quartonotebookrunner.jl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index 5ec51a8fe2..9028c843c2 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -31,6 +31,25 @@ if server === nothing error("Giving up.") end +@info "trying to establish test connection to port $port" +let + connected = false + for i in 1:20 + try + sock = Sockets.connect(port) + println(sock, """{"type": "isready", "content": {}}""") + @assert readline(sock) == "true" + close(sock) + connected = true + break + catch + sleep(0.1) + end + end + connected || error("Test connection could not be established.") +end +@info "successful test connection" + open(transport_file, "w") do io println(io, """{"port": $port, "pid": $(Base.Libc.getpid())}""") end From 1419e87ab12868491ac6d580453505efdd014f0c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 4 Mar 2024 19:27:56 +0100 Subject: [PATCH 056/101] put windows back --- .github/workflows/test-smokes.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index e817a65aba..6d0695b6fc 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -38,8 +38,7 @@ jobs: strategy: fail-fast: false matrix: - # os: [ubuntu-latest, windows-latest] - os: [ubuntu-latest] + os: [ubuntu-latest, windows-latest] time-test: - ${{ inputs.time-test }} exclude: From ebadf3c4a135e083832995abf892627f2a15b9b7 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 10:02:07 +0100 Subject: [PATCH 057/101] add tmate action and only test windows --- .github/workflows/test-smokes.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 6d0695b6fc..8ad851613c 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -38,7 +38,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [windows-latest] + # os: [ubuntu-latest, windows-latest] time-test: - ${{ inputs.time-test }} exclude: @@ -178,8 +179,8 @@ jobs: run: | echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 # FIXME: for some reason on CI, it seems that communicating with the child process julia (even if it's detached) # from the parent deno process fails. But if we run this step beforehand, that one server should be able to handle From b4f803bafa8bc788724cfc75c6159700193f677e Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 12:44:58 +0100 Subject: [PATCH 058/101] use command.spawn and log transport file accesses --- src/execute/julia.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 23a7023731..1fe5a1ffcd 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -251,8 +251,11 @@ async function startOrReuseJuliaServer( ], }); // when this process finishes, a detached julia process with the quartonotebookrunner server will have been started - trace(options, "Spawning detached julia server through julia"); - command.outputSync(); + trace( + options, + "Spawning detached julia server through julia, once transport file exists, server should be running.", + ); + command.spawn(); } else { trace( options, @@ -297,14 +300,18 @@ interface JuliaTransportFile { pid: number; } -async function pollTransportFile(): Promise { +async function pollTransportFile( + options: JuliaExecuteOptions, +): Promise { const transportFile = juliaTransportFile(); for (let i = 0; i < 20; i++) { if (existsSync(transportFile)) { const content = Deno.readTextFileSync(transportFile); + trace(options, "Transport file read successfully."); return JSON.parse(content) as JuliaTransportFile; } + trace(options, "Transport file did not exist, yet."); await sleep(i * 100); } return Promise.reject(); @@ -314,7 +321,7 @@ async function getJuliaServerConnection( options: JuliaExecuteOptions, ): Promise { const { reused } = await startOrReuseJuliaServer(options); - const transportOptions = await pollTransportFile(); + const transportOptions = await pollTransportFile(options); trace( options, From a3e190ab371d16c1b8c5db507594baa3bad28618 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 13:03:09 +0100 Subject: [PATCH 059/101] remove tmate session --- .github/workflows/test-smokes.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 8ad851613c..2b117c4774 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -179,8 +179,8 @@ jobs: run: | echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 # FIXME: for some reason on CI, it seems that communicating with the child process julia (even if it's detached) # from the parent deno process fails. But if we run this step beforehand, that one server should be able to handle From 20957d4ae68f95d8acbba2e52cf7b279b3d6f31f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 15:12:09 +0100 Subject: [PATCH 060/101] call exit manually to see if that truly stops the spawned process --- src/resources/julia/start_quartonotebookrunner_detached.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index d8111caf58..49911da133 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -13,3 +13,4 @@ cmd = `$julia_bin --project=$project $julia_file $transport_file` @info cmd run(detach(cmd), wait = false) @info "ran that" +exit() \ No newline at end of file From d0e3ea15915abc7b33fad606c7fd9589facec236 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 15:52:28 +0100 Subject: [PATCH 061/101] try starting the detached julia the way that detached python is started, too --- src/execute/julia.ts | 15 ++++++-- .../start_quartonotebookrunner_detached.jl | 1 - .../start_quartonotebookrunner_detached.py | 35 +++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 src/resources/julia/start_quartonotebookrunner_detached.py diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 1fe5a1ffcd..8ee484f2ba 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -40,6 +40,7 @@ import { runningInCI } from "../core/ci-info.ts"; import { sleep } from "../core/async.ts"; import { JupyterNotebook } from "../core/jupyter/types.ts"; import { existsSync } from "fs/mod.ts"; +import { pythonExecForCaps } from "../core/jupyter/exec.ts"; export interface JuliaExecuteOptions extends ExecuteOptions { julia_cmd: string; @@ -241,9 +242,19 @@ async function startOrReuseJuliaServer( // run anything was written to stderr. This goes away when redirecting stderr to // a file on the julia side, but also when using Deno.Command which is recommended // as a replacement for the old Deno.run anyway. - const command = new Deno.Command(options.julia_cmd, { + // const command = new Deno.Command(options.julia_cmd, { + // args: [ + // resourcePath("julia/start_quartonotebookrunner_detached.jl"), + // options.julia_cmd, + // juliaRuntimeDir(), + // resourcePath("julia/quartonotebookrunner.jl"), + // transportFile, + // ], + // }); + const pybinary = pythonExecForCaps(undefined)[0]; + const command = new Deno.Command(pybinary, { args: [ - resourcePath("julia/start_quartonotebookrunner_detached.jl"), + resourcePath("julia/start_quartonotebookrunner_detached.py"), options.julia_cmd, juliaRuntimeDir(), resourcePath("julia/quartonotebookrunner.jl"), diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index 49911da133..d8111caf58 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -13,4 +13,3 @@ cmd = `$julia_bin --project=$project $julia_file $transport_file` @info cmd run(detach(cmd), wait = false) @info "ran that" -exit() \ No newline at end of file diff --git a/src/resources/julia/start_quartonotebookrunner_detached.py b/src/resources/julia/start_quartonotebookrunner_detached.py new file mode 100644 index 0000000000..483bd0f658 --- /dev/null +++ b/src/resources/julia/start_quartonotebookrunner_detached.py @@ -0,0 +1,35 @@ +import subprocess +import sys + + +julia_bin = sys.argv[1] +project = sys.argv[2] +julia_file = sys.argv[3] +transport_file = sys.argv[4] + +if len(sys.argv) > 5: + raise ValueError("Too many arguments") + +cmd = [julia_bin, f"--project={project}", julia_file, transport_file] +print("cmd:", cmd) +subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +print("ran that") + +# detached process flags for windows +flags = 0 +if sys.platform == 'win32': + python_exe = re.sub('python\\.exe$', 'pythonw.exe', python_exe) + flags |= 0x00000008 # DETACHED_PROCESS + flags |= 0x00000200 # CREATE_NEW_PROCESS_GROUP + flags |= 0x08000000 # CREATE_NO_WINDOW + flags |= 0x01000000 # CREATE_BREAKAWAY_FROM_JOB + +# create subprocess +subprocess.Popen(cmd, + stdin = subprocess.DEVNULL, + stdout = subprocess.DEVNULL, + stderr = subprocess.DEVNULL, + creationflags = flags, + close_fds = True, + start_new_session = True +) \ No newline at end of file From ac3d2f2810b1289aff07333c28821eb8be11977c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 16:13:23 +0100 Subject: [PATCH 062/101] import re --- src/resources/julia/start_quartonotebookrunner_detached.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.py b/src/resources/julia/start_quartonotebookrunner_detached.py index 483bd0f658..61d4ec081c 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.py +++ b/src/resources/julia/start_quartonotebookrunner_detached.py @@ -1,5 +1,6 @@ import subprocess import sys +import re julia_bin = sys.argv[1] From 0bdea7e77152cba63ebe49d5688d64e464eddb68 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 16:34:35 +0100 Subject: [PATCH 063/101] delete unnecessary lines --- src/resources/julia/start_quartonotebookrunner_detached.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.py b/src/resources/julia/start_quartonotebookrunner_detached.py index 61d4ec081c..30bd2011ed 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.py +++ b/src/resources/julia/start_quartonotebookrunner_detached.py @@ -1,6 +1,5 @@ import subprocess import sys -import re julia_bin = sys.argv[1] @@ -19,7 +18,6 @@ # detached process flags for windows flags = 0 if sys.platform == 'win32': - python_exe = re.sub('python\\.exe$', 'pythonw.exe', python_exe) flags |= 0x00000008 # DETACHED_PROCESS flags |= 0x00000200 # CREATE_NEW_PROCESS_GROUP flags |= 0x08000000 # CREATE_NO_WINDOW From 159b7e7e57c2529be3e1d20d75cb61cc57790079 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 18:14:45 +0100 Subject: [PATCH 064/101] delete accidental double Popen --- src/resources/julia/start_quartonotebookrunner_detached.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.py b/src/resources/julia/start_quartonotebookrunner_detached.py index 30bd2011ed..48a94a8266 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.py +++ b/src/resources/julia/start_quartonotebookrunner_detached.py @@ -11,9 +11,6 @@ raise ValueError("Too many arguments") cmd = [julia_bin, f"--project={project}", julia_file, transport_file] -print("cmd:", cmd) -subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) -print("ran that") # detached process flags for windows flags = 0 From b956433f5ebc8033158f08aa28cd33cee0a30a09 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 18:36:10 +0100 Subject: [PATCH 065/101] try detached julia with devnull'ed file descriptors --- src/execute/julia.ts | 24 +++++++++---------- .../start_quartonotebookrunner_detached.jl | 10 +++++++- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 8ee484f2ba..54e46c0429 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -242,25 +242,25 @@ async function startOrReuseJuliaServer( // run anything was written to stderr. This goes away when redirecting stderr to // a file on the julia side, but also when using Deno.Command which is recommended // as a replacement for the old Deno.run anyway. - // const command = new Deno.Command(options.julia_cmd, { - // args: [ - // resourcePath("julia/start_quartonotebookrunner_detached.jl"), - // options.julia_cmd, - // juliaRuntimeDir(), - // resourcePath("julia/quartonotebookrunner.jl"), - // transportFile, - // ], - // }); - const pybinary = pythonExecForCaps(undefined)[0]; - const command = new Deno.Command(pybinary, { + const command = new Deno.Command(options.julia_cmd, { args: [ - resourcePath("julia/start_quartonotebookrunner_detached.py"), + resourcePath("julia/start_quartonotebookrunner_detached.jl"), options.julia_cmd, juliaRuntimeDir(), resourcePath("julia/quartonotebookrunner.jl"), transportFile, ], }); + // const pybinary = pythonExecForCaps(undefined)[0]; + // const command = new Deno.Command(pybinary, { + // args: [ + // resourcePath("julia/start_quartonotebookrunner_detached.py"), + // options.julia_cmd, + // juliaRuntimeDir(), + // resourcePath("julia/quartonotebookrunner.jl"), + // transportFile, + // ], + // }); // when this process finishes, a detached julia process with the quartonotebookrunner server will have been started trace( options, diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index d8111caf58..e7fd419745 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -11,5 +11,13 @@ end cmd = `$julia_bin --project=$project $julia_file $transport_file` @info cmd -run(detach(cmd), wait = false) +run( + pipeline( + detach(cmd); + stderr = devnull, + stdin = devnull, + stdout = devnull + ), + wait = false +) @info "ran that" From ee65d599701b21e3225b4fe933927aad8e774e18 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 5 Mar 2024 19:22:34 +0100 Subject: [PATCH 066/101] use `uv_unref` on the process handle --- .../julia/start_quartonotebookrunner_detached.jl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index e7fd419745..fef7e9b99f 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -11,13 +11,7 @@ end cmd = `$julia_bin --project=$project $julia_file $transport_file` @info cmd -run( - pipeline( - detach(cmd); - stderr = devnull, - stdin = devnull, - stdout = devnull - ), - wait = false -) + +proc = run(detach(cmd), wait = false) +Base.uv_unref(proc.handle) @info "ran that" From c458113494e7937381ac6f84f15556cb6fdbd4cc Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 09:37:30 +0100 Subject: [PATCH 067/101] remove uv_unref again, didn't work --- src/resources/julia/start_quartonotebookrunner_detached.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index fef7e9b99f..e0323ef41f 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -12,6 +12,5 @@ end cmd = `$julia_bin --project=$project $julia_file $transport_file` @info cmd -proc = run(detach(cmd), wait = false) -Base.uv_unref(proc.handle) +run(detach(cmd), wait = false) @info "ran that" From 3e8a2eb88b1b9936b2f1136c00b9bbc75478284f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 09:37:48 +0100 Subject: [PATCH 068/101] try spawning julia server through powershell on windows --- src/execute/julia.ts | 66 +++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 54e46c0429..cdfe55e193 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -242,31 +242,47 @@ async function startOrReuseJuliaServer( // run anything was written to stderr. This goes away when redirecting stderr to // a file on the julia side, but also when using Deno.Command which is recommended // as a replacement for the old Deno.run anyway. - const command = new Deno.Command(options.julia_cmd, { - args: [ - resourcePath("julia/start_quartonotebookrunner_detached.jl"), - options.julia_cmd, - juliaRuntimeDir(), - resourcePath("julia/quartonotebookrunner.jl"), - transportFile, - ], - }); - // const pybinary = pythonExecForCaps(undefined)[0]; - // const command = new Deno.Command(pybinary, { - // args: [ - // resourcePath("julia/start_quartonotebookrunner_detached.py"), - // options.julia_cmd, - // juliaRuntimeDir(), - // resourcePath("julia/quartonotebookrunner.jl"), - // transportFile, - // ], - // }); - // when this process finishes, a detached julia process with the quartonotebookrunner server will have been started - trace( - options, - "Spawning detached julia server through julia, once transport file exists, server should be running.", - ); - command.spawn(); + + if (Deno.build.os === "windows") { + const command = new Deno.Command( + "PowerShell", + { + args: [ + "-Command", + "Start-Process", + options.julia_cmd, + "-ArgumentList", + `--project=${juliaRuntimeDir()}`, + ",", + resourcePath("julia/quartonotebookrunner.jl"), + ",", + transportFile, + "-WindowStyle", + "-Hidden", + ], + }, + ); + trace( + options, + "Spawning detached julia server through powershell, once transport file exists, server should be running.", + ); + command.spawn(); + } else { + const command = new Deno.Command(options.julia_cmd, { + args: [ + resourcePath("julia/start_quartonotebookrunner_detached.jl"), + options.julia_cmd, + juliaRuntimeDir(), + resourcePath("julia/quartonotebookrunner.jl"), + transportFile, + ], + }); + trace( + options, + "Spawning detached julia server through julia, once transport file exists, server should be running.", + ); + command.spawn(); + } } else { trace( options, From 8c220007e14562ded56a14f1dcba46e5124d6262 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 09:56:24 +0100 Subject: [PATCH 069/101] fix window style --- src/execute/julia.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index cdfe55e193..78ed418eb8 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -258,7 +258,7 @@ async function startOrReuseJuliaServer( ",", transportFile, "-WindowStyle", - "-Hidden", + "Hidden", ], }, ); From 729a5727ddc02e82ee13d881530504659fd7b1ca Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 10:22:37 +0100 Subject: [PATCH 070/101] add explanatory comment --- src/execute/julia.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 78ed418eb8..32a38096b5 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -236,13 +236,13 @@ async function startOrReuseJuliaServer( ); await ensureQuartoNotebookRunnerEnvironment(options); - // when quarto's execProc function is used here, there is a strange bug. - // The first time render is called on a file, the julia server is started correctly. - // The second time it is called, however, the socket server hangs if during the first - // run anything was written to stderr. This goes away when redirecting stderr to - // a file on the julia side, but also when using Deno.Command which is recommended - // as a replacement for the old Deno.run anyway. - + // We need to spawn the julia server in its own process that can outlive quarto. + // Apparently, `run(detach(cmd))` in julia does not do that reliably on Windows, + // at least deno never seems to recognize that the spawning julia process has finished, + // presumably because it waits for the active child process to exit. This makes the + // tests on windows hang forever if we use the same launching mechanism as for Unix systems. + // So we utilize powershell instead which can start completely detached processes with + // the Start-Process commandlet. if (Deno.build.os === "windows") { const command = new Deno.Command( "PowerShell", From 6e3abe17da245ffe63343eee9f7d93b81816172f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 10:23:05 +0100 Subject: [PATCH 071/101] remove printouts --- src/resources/julia/start_quartonotebookrunner_detached.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index e0323ef41f..d560954a39 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -10,7 +10,4 @@ if length(ARGS) > 4 end cmd = `$julia_bin --project=$project $julia_file $transport_file` -@info cmd - run(detach(cmd), wait = false) -@info "ran that" From cdfbef89e727b911134f5137807ba4daef451ac6 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 10:25:03 +0100 Subject: [PATCH 072/101] use outputSync to wait until spawning the detached process has succeeded --- src/execute/julia.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 32a38096b5..c6bff51123 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -264,9 +264,9 @@ async function startOrReuseJuliaServer( ); trace( options, - "Spawning detached julia server through powershell, once transport file exists, server should be running.", + "Starting detached julia server through powershell, once transport file exists, server should be running.", ); - command.spawn(); + command.outputSync(); } else { const command = new Deno.Command(options.julia_cmd, { args: [ @@ -279,9 +279,9 @@ async function startOrReuseJuliaServer( }); trace( options, - "Spawning detached julia server through julia, once transport file exists, server should be running.", + "Starting detached julia server through julia, once transport file exists, server should be running.", ); - command.spawn(); + command.outputSync(); } } else { trace( From 1df135f8a9be698d5a62eba3220ac5de63804e78 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 12:14:48 +0100 Subject: [PATCH 073/101] put ubuntu runners back in --- .github/workflows/test-smokes.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 2b117c4774..6d0695b6fc 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -38,8 +38,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest] - # os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest] time-test: - ${{ inputs.time-test }} exclude: From c43c136e13cda736a87e0695c0be956f2477abe2 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 15:50:29 +0100 Subject: [PATCH 074/101] add markdownForFile method --- src/execute/julia.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index c6bff51123..aa8dbf1729 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -103,6 +103,10 @@ export const juliaEngine: ExecutionEngine = { return true; }, + markdownForFile(file: string): Promise { + return Promise.resolve(mappedStringFromFile(file)); + }, + execute: async (options: ExecuteOptions): Promise => { options.target.source; @@ -252,6 +256,7 @@ async function startOrReuseJuliaServer( "Start-Process", options.julia_cmd, "-ArgumentList", + "--startup-file=no", `--project=${juliaRuntimeDir()}`, ",", resourcePath("julia/quartonotebookrunner.jl"), @@ -270,6 +275,7 @@ async function startOrReuseJuliaServer( } else { const command = new Deno.Command(options.julia_cmd, { args: [ + "--startup-file=no", resourcePath("julia/start_quartonotebookrunner_detached.jl"), options.julia_cmd, juliaRuntimeDir(), @@ -427,6 +433,10 @@ async function executeJulia( await writeJuliaCommand(conn, "close", "TODOsomesecret", options); } + if (response.error !== undefined) { + throw new Error("Running notebook failed:\n" + response.juliaError); + } + return response.notebook as JupyterNotebook; } From dc72ecd98010bc02fe0da5f3bf6a770d4282590f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 15:50:51 +0100 Subject: [PATCH 075/101] remove unused python script --- .../start_quartonotebookrunner_detached.py | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 src/resources/julia/start_quartonotebookrunner_detached.py diff --git a/src/resources/julia/start_quartonotebookrunner_detached.py b/src/resources/julia/start_quartonotebookrunner_detached.py deleted file mode 100644 index 48a94a8266..0000000000 --- a/src/resources/julia/start_quartonotebookrunner_detached.py +++ /dev/null @@ -1,31 +0,0 @@ -import subprocess -import sys - - -julia_bin = sys.argv[1] -project = sys.argv[2] -julia_file = sys.argv[3] -transport_file = sys.argv[4] - -if len(sys.argv) > 5: - raise ValueError("Too many arguments") - -cmd = [julia_bin, f"--project={project}", julia_file, transport_file] - -# detached process flags for windows -flags = 0 -if sys.platform == 'win32': - flags |= 0x00000008 # DETACHED_PROCESS - flags |= 0x00000200 # CREATE_NEW_PROCESS_GROUP - flags |= 0x08000000 # CREATE_NO_WINDOW - flags |= 0x01000000 # CREATE_BREAKAWAY_FROM_JOB - -# create subprocess -subprocess.Popen(cmd, - stdin = subprocess.DEVNULL, - stdout = subprocess.DEVNULL, - stderr = subprocess.DEVNULL, - creationflags = flags, - close_fds = True, - start_new_session = True -) \ No newline at end of file From caa7250774a4d8d3d3a04d60312edff97a5a17a4 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Wed, 6 Mar 2024 22:36:49 +0100 Subject: [PATCH 076/101] actually make julianative test use julia engine --- tests/docs/crossrefs/julianative-subfig.qmd | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/docs/crossrefs/julianative-subfig.qmd b/tests/docs/crossrefs/julianative-subfig.qmd index 29a45eb91c..85f024853d 100644 --- a/tests/docs/crossrefs/julianative-subfig.qmd +++ b/tests/docs/crossrefs/julianative-subfig.qmd @@ -1,5 +1,6 @@ --- title: Julia Subfig Test +engine: julia --- ## Julia Crossref Figure From c81c6245e106a45e533b5b12febe6e19905a0fe3 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 09:05:20 +0100 Subject: [PATCH 077/101] bump quartonotebookrunner --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index 7d9dad7241..e42a58e001 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "0.4.0" +QuartoNotebookRunner = "0.4.2" From 268d389e9d8d0195d8fb29236af6704bcc5e2c9c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 09:05:46 +0100 Subject: [PATCH 078/101] startup file no --- src/resources/julia/start_quartonotebookrunner_detached.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/start_quartonotebookrunner_detached.jl b/src/resources/julia/start_quartonotebookrunner_detached.jl index d560954a39..ce371b0e71 100644 --- a/src/resources/julia/start_quartonotebookrunner_detached.jl +++ b/src/resources/julia/start_quartonotebookrunner_detached.jl @@ -9,5 +9,5 @@ if length(ARGS) > 4 error("Too many arguments") end -cmd = `$julia_bin --project=$project $julia_file $transport_file` +cmd = `$julia_bin --startup-file=no --project=$project $julia_file $transport_file` run(detach(cmd), wait = false) From 1279112a7b4ca68ae0ea84a92af694d80b373fcc Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 09:07:23 +0100 Subject: [PATCH 079/101] use existing project for faster CI --- tests/docs/crossrefs/julianative-subfig.qmd | 6 ++++++ tests/docs/crossrefs/julianative.qmd | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/docs/crossrefs/julianative-subfig.qmd b/tests/docs/crossrefs/julianative-subfig.qmd index 85f024853d..52d2964d61 100644 --- a/tests/docs/crossrefs/julianative-subfig.qmd +++ b/tests/docs/crossrefs/julianative-subfig.qmd @@ -1,8 +1,14 @@ --- title: Julia Subfig Test engine: julia +julia: + exeflags: ["--project=../.."] --- +```{julia} +Pkg.instantiate() +``` + ## Julia Crossref Figure ```{julia} diff --git a/tests/docs/crossrefs/julianative.qmd b/tests/docs/crossrefs/julianative.qmd index 2ae6bbfb9a..9c0a6e0038 100644 --- a/tests/docs/crossrefs/julianative.qmd +++ b/tests/docs/crossrefs/julianative.qmd @@ -1,11 +1,12 @@ --- title: Julia Crossref Test engine: julia +julia: + exeflags: ["--project=../.."] --- ```{julia} -Pkg.activate(temp = true) -Pkg.add(name = "Plots", version = "1.40.1") +Pkg.instantiate() ``` ## Julia Crossref Figure From 1f1b46fa0a0ae0a54c04c1b17464e221a3df42aa Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 09:33:04 +0100 Subject: [PATCH 080/101] comment out most of CI, reenable windows tmate --- .github/workflows/test-smokes.yml | 143 +++++++++++++++--------------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 4a1fe89488..240d53f37a 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -34,7 +34,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + # os: [ubuntu-latest, windows-latest] + os: [windows-latest] time-test: - ${{ inputs.time-test }} exclude: @@ -54,53 +55,53 @@ jobs: echo "TEMP=${{ runner.temp }}" >> $GITHUB_ENV shell: bash - - name: Set up R - uses: r-lib/actions/setup-r@v2 - with: - r-version: "4.3.2" - use-public-rspm: true - # required to avoid rtools bin in path - windows-path-include-rtools: false + # - name: Set up R + # uses: r-lib/actions/setup-r@v2 + # with: + # r-version: "4.3.2" + # use-public-rspm: true + # # required to avoid rtools bin in path + # windows-path-include-rtools: false - - name: Install node (for Playwright, MECA) - uses: actions/setup-node@v3 - with: - node-version: 16 + # - name: Install node (for Playwright, MECA) + # uses: actions/setup-node@v3 + # with: + # node-version: 16 - - name: Install node dependencies - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} - run: yarn - working-directory: ./tests/integration/playwright - shell: bash + # - name: Install node dependencies + # if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + # run: yarn + # working-directory: ./tests/integration/playwright + # shell: bash - - name: Install Playwright Browsers - if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} - run: npx playwright install --with-deps - working-directory: ./tests/integration/playwright + # - name: Install Playwright Browsers + # if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + # run: npx playwright install --with-deps + # working-directory: ./tests/integration/playwright - - name: Install MECA validator - if: ${{ runner.os != 'Windows' }} - run: npm install -g meca + # - name: Install MECA validator + # if: ${{ runner.os != 'Windows' }} + # run: npm install -g meca - - name: Set RENV_PATHS_ROOT - shell: bash - run: | - echo "RENV_PATHS_ROOT=${{ runner.temp }}/renv" >> $GITHUB_ENV + # - name: Set RENV_PATHS_ROOT + # shell: bash + # run: | + # echo "RENV_PATHS_ROOT=${{ runner.temp }}/renv" >> $GITHUB_ENV - - name: Get R and OS version - id: get-version - run: | - cat("os-version=", sessionInfo()$running, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) - cat("r-version=", R.Version()$version.string, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) - shell: Rscript {0} + # - name: Get R and OS version + # id: get-version + # run: | + # cat("os-version=", sessionInfo()$running, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) + # cat("r-version=", R.Version()$version.string, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) + # shell: Rscript {0} - - name: Cache R packages - uses: actions/cache@v3 - with: - path: ${{ env.RENV_PATHS_ROOT }} - key: ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2-${{ hashFiles('tests/renv.lock') }} - restore-keys: | - ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2- + # - name: Cache R packages + # uses: actions/cache@v3 + # with: + # path: ${{ env.RENV_PATHS_ROOT }} + # key: ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2-${{ hashFiles('tests/renv.lock') }} + # restore-keys: | + # ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2- - name: Install missing system deps if: runner.os == 'Linux' @@ -110,39 +111,39 @@ jobs: sudo apt-get install -y libxml2-utils sudo apt-get install -y libharfbuzz-dev libfribidi-dev - - name: Restore R packages - working-directory: tests - run: | - if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv') - renv::restore() - # Install dev versions for our testing - # Use r-universe to avoid github api calls - try(install.packages('knitr', repos = 'https://yihui.r-universe.dev')) - try(install.packages('rmarkdown', repos = 'https://rstudio.r-universe.dev')) - shell: Rscript {0} - env: - GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + # - name: Restore R packages + # working-directory: tests + # run: | + # if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv') + # renv::restore() + # # Install dev versions for our testing + # # Use r-universe to avoid github api calls + # try(install.packages('knitr', repos = 'https://yihui.r-universe.dev')) + # try(install.packages('rmarkdown', repos = 'https://rstudio.r-universe.dev')) + # shell: Rscript {0} + # env: + # GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.12" - cache: "pipenv" - cache-dependency-path: "./tests/Pipfile.lock" + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.12" + # cache: "pipenv" + # cache-dependency-path: "./tests/Pipfile.lock" - - name: Restore Python Dependencies - working-directory: tests - run: | - python -m pip install pipenv - pipenv install + # - name: Restore Python Dependencies + # working-directory: tests + # run: | + # python -m pip install pipenv + # pipenv install - uses: ./.github/workflows/actions/quarto-dev - - name: Install Tinytex - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - quarto install tinytex + # - name: Install Tinytex + # env: + # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # run: | + # quarto install tinytex - name: Cache Typst packages uses: ./.github/actions/cache-typst @@ -174,8 +175,8 @@ jobs: run: | echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 # FIXME: for some reason on CI, it seems that communicating with the child process julia (even if it's detached) # from the parent deno process fails. But if we run this step beforehand, that one server should be able to handle From fbbb9f81980c3ccf1c1d5c95aed8d52873ab2b2c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 09:37:48 +0100 Subject: [PATCH 081/101] comment out more --- .github/workflows/test-smokes.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 240d53f37a..1fd20f510f 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -159,21 +159,21 @@ jobs: - name: Cache Julia Packages uses: julia-actions/cache@v1 - - name: Restore Julia Packages - working-directory: tests - shell: bash - run: | - # Setup IJulia with the jupyter from the Python environment - # https://julialang.github.io/IJulia.jl/stable/manual/installation/ - export JUPYTER=$(find $(dirname $(pipenv run which jupyter))/ -type f -name "jupyter.exe" -o -name "jupyter") - pipenv run julia --color=yes --project=. -e "import Pkg; Pkg.instantiate(); Pkg.build(\"IJulia\"); Pkg.precompile()" - echo "Julia Jupyter:" - julia --project=. -e "import IJulia;println(IJulia.JUPYTER);println(IJulia.find_jupyter_subcommand(\"notebook\"))" + # - name: Restore Julia Packages + # working-directory: tests + # shell: bash + # run: | + # # Setup IJulia with the jupyter from the Python environment + # # https://julialang.github.io/IJulia.jl/stable/manual/installation/ + # export JUPYTER=$(find $(dirname $(pipenv run which jupyter))/ -type f -name "jupyter.exe" -o -name "jupyter") + # pipenv run julia --color=yes --project=. -e "import Pkg; Pkg.instantiate(); Pkg.build(\"IJulia\"); Pkg.precompile()" + # echo "Julia Jupyter:" + # julia --project=. -e "import IJulia;println(IJulia.JUPYTER);println(IJulia.find_jupyter_subcommand(\"notebook\"))" - - name: Setup timing file for timed test - if: ${{ matrix.time-test == true }} - run: | - echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" + # - name: Setup timing file for timed test + # if: ${{ matrix.time-test == true }} + # run: | + # echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" - name: Setup tmate session uses: mxschmitt/action-tmate@v3 From 6cbbe4e084be43ec169abbf954c2d2d88d868625 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 10:32:20 +0100 Subject: [PATCH 082/101] fix bug and actually throw if spawning fails --- src/execute/julia.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index aa8dbf1729..cb85d07c72 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -256,12 +256,15 @@ async function startOrReuseJuliaServer( "Start-Process", options.julia_cmd, "-ArgumentList", + // string array argument list, each element but the last must have a "," element after "--startup-file=no", + ",", `--project=${juliaRuntimeDir()}`, ",", resourcePath("julia/quartonotebookrunner.jl"), ",", transportFile, + // end of string array "-WindowStyle", "Hidden", ], @@ -271,7 +274,10 @@ async function startOrReuseJuliaServer( options, "Starting detached julia server through powershell, once transport file exists, server should be running.", ); - command.outputSync(); + const result = command.outputSync(); + if (!result.success) { + throw new Error(new TextDecoder().decode(result.stderr)); + } } else { const command = new Deno.Command(options.julia_cmd, { args: [ @@ -287,7 +293,10 @@ async function startOrReuseJuliaServer( options, "Starting detached julia server through julia, once transport file exists, server should be running.", ); - command.outputSync(); + const result = command.outputSync(); + if (!result.success) { + throw new Error(new TextDecoder().decode(result.stderr)); + } } } else { trace( From 5d92d70d1482f106e1acc088f2c70d3b3b7117e9 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 10:33:32 +0100 Subject: [PATCH 083/101] restore CI components --- .github/workflows/test-smokes.yml | 179 ++++++++++++++---------------- 1 file changed, 85 insertions(+), 94 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 1fd20f510f..02f3b7dceb 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -34,8 +34,7 @@ jobs: strategy: fail-fast: false matrix: - # os: [ubuntu-latest, windows-latest] - os: [windows-latest] + os: [ubuntu-latest, windows-latest] time-test: - ${{ inputs.time-test }} exclude: @@ -55,53 +54,53 @@ jobs: echo "TEMP=${{ runner.temp }}" >> $GITHUB_ENV shell: bash - # - name: Set up R - # uses: r-lib/actions/setup-r@v2 - # with: - # r-version: "4.3.2" - # use-public-rspm: true - # # required to avoid rtools bin in path - # windows-path-include-rtools: false + - name: Set up R + uses: r-lib/actions/setup-r@v2 + with: + r-version: "4.3.2" + use-public-rspm: true + # required to avoid rtools bin in path + windows-path-include-rtools: false - # - name: Install node (for Playwright, MECA) - # uses: actions/setup-node@v3 - # with: - # node-version: 16 + - name: Install node (for Playwright, MECA) + uses: actions/setup-node@v3 + with: + node-version: 16 - # - name: Install node dependencies - # if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} - # run: yarn - # working-directory: ./tests/integration/playwright - # shell: bash + - name: Install node dependencies + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + run: yarn + working-directory: ./tests/integration/playwright + shell: bash - # - name: Install Playwright Browsers - # if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} - # run: npx playwright install --with-deps - # working-directory: ./tests/integration/playwright + - name: Install Playwright Browsers + if: ${{ runner.os != 'Windows' || github.event_name == 'schedule' }} + run: npx playwright install --with-deps + working-directory: ./tests/integration/playwright - # - name: Install MECA validator - # if: ${{ runner.os != 'Windows' }} - # run: npm install -g meca + - name: Install MECA validator + if: ${{ runner.os != 'Windows' }} + run: npm install -g meca - # - name: Set RENV_PATHS_ROOT - # shell: bash - # run: | - # echo "RENV_PATHS_ROOT=${{ runner.temp }}/renv" >> $GITHUB_ENV + - name: Set RENV_PATHS_ROOT + shell: bash + run: | + echo "RENV_PATHS_ROOT=${{ runner.temp }}/renv" >> $GITHUB_ENV - # - name: Get R and OS version - # id: get-version - # run: | - # cat("os-version=", sessionInfo()$running, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) - # cat("r-version=", R.Version()$version.string, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) - # shell: Rscript {0} + - name: Get R and OS version + id: get-version + run: | + cat("os-version=", sessionInfo()$running, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) + cat("r-version=", R.Version()$version.string, "\n", file = Sys.getenv("GITHUB_OUTPUT"), sep = "", append = TRUE) + shell: Rscript {0} - # - name: Cache R packages - # uses: actions/cache@v3 - # with: - # path: ${{ env.RENV_PATHS_ROOT }} - # key: ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2-${{ hashFiles('tests/renv.lock') }} - # restore-keys: | - # ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2- + - name: Cache R packages + uses: actions/cache@v3 + with: + path: ${{ env.RENV_PATHS_ROOT }} + key: ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2-${{ hashFiles('tests/renv.lock') }} + restore-keys: | + ${{ steps.get-version.outputs.os-version }}-${{ steps.get-version.outputs.r-version }}-renv-2- - name: Install missing system deps if: runner.os == 'Linux' @@ -111,39 +110,39 @@ jobs: sudo apt-get install -y libxml2-utils sudo apt-get install -y libharfbuzz-dev libfribidi-dev - # - name: Restore R packages - # working-directory: tests - # run: | - # if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv') - # renv::restore() - # # Install dev versions for our testing - # # Use r-universe to avoid github api calls - # try(install.packages('knitr', repos = 'https://yihui.r-universe.dev')) - # try(install.packages('rmarkdown', repos = 'https://rstudio.r-universe.dev')) - # shell: Rscript {0} - # env: - # GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + - name: Restore R packages + working-directory: tests + run: | + if (!requireNamespace('renv', quietly = TRUE)) install.packages('renv') + renv::restore() + # Install dev versions for our testing + # Use r-universe to avoid github api calls + try(install.packages('knitr', repos = 'https://yihui.r-universe.dev')) + try(install.packages('rmarkdown', repos = 'https://rstudio.r-universe.dev')) + shell: Rscript {0} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: "3.12" - # cache: "pipenv" - # cache-dependency-path: "./tests/Pipfile.lock" + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + cache: "pipenv" + cache-dependency-path: "./tests/Pipfile.lock" - # - name: Restore Python Dependencies - # working-directory: tests - # run: | - # python -m pip install pipenv - # pipenv install + - name: Restore Python Dependencies + working-directory: tests + run: | + python -m pip install pipenv + pipenv install - uses: ./.github/workflows/actions/quarto-dev - # - name: Install Tinytex - # env: - # GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # run: | - # quarto install tinytex + - name: Install Tinytex + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + quarto install tinytex - name: Cache Typst packages uses: ./.github/actions/cache-typst @@ -159,32 +158,24 @@ jobs: - name: Cache Julia Packages uses: julia-actions/cache@v1 - # - name: Restore Julia Packages - # working-directory: tests - # shell: bash - # run: | - # # Setup IJulia with the jupyter from the Python environment - # # https://julialang.github.io/IJulia.jl/stable/manual/installation/ - # export JUPYTER=$(find $(dirname $(pipenv run which jupyter))/ -type f -name "jupyter.exe" -o -name "jupyter") - # pipenv run julia --color=yes --project=. -e "import Pkg; Pkg.instantiate(); Pkg.build(\"IJulia\"); Pkg.precompile()" - # echo "Julia Jupyter:" - # julia --project=. -e "import IJulia;println(IJulia.JUPYTER);println(IJulia.find_jupyter_subcommand(\"notebook\"))" - - # - name: Setup timing file for timed test - # if: ${{ matrix.time-test == true }} - # run: | - # echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" + - name: Restore Julia Packages + working-directory: tests + shell: bash + run: | + # Setup IJulia with the jupyter from the Python environment + # https://julialang.github.io/IJulia.jl/stable/manual/installation/ + export JUPYTER=$(find $(dirname $(pipenv run which jupyter))/ -type f -name "jupyter.exe" -o -name "jupyter") + pipenv run julia --color=yes --project=. -e "import Pkg; Pkg.instantiate(); Pkg.build(\"IJulia\"); Pkg.precompile()" + echo "Julia Jupyter:" + julia --project=. -e "import IJulia;println(IJulia.JUPYTER);println(IJulia.find_jupyter_subcommand(\"notebook\"))" - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 + - name: Setup timing file for timed test + if: ${{ matrix.time-test == true }} + run: | + echo "QUARTO_TEST_TIMING=timing-for-ci.txt" >> "$GITHUB_ENV" - # FIXME: for some reason on CI, it seems that communicating with the child process julia (even if it's detached) - # from the parent deno process fails. But if we run this step beforehand, that one server should be able to handle - # the remaining connections coming from new quarto processes - # - name: Warmup Julia native server - # continue-on-error: true # this will fail as long as the bug with deno exists - # run: | - # quarto render tests/docs/markdown/commonmark-julianative.qmd --execute-debug + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 - name: Run all Smoke Tests Windows if: ${{ runner.os == 'Windows' && format('{0}', inputs.buckets) == '' && matrix.time-test == false }} From 2457ec0c9007583fb35f0ec7f79fa8950ec387a7 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 11:40:41 +0100 Subject: [PATCH 084/101] one more startup file no --- src/execute/julia.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index cb85d07c72..61c80f1534 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -316,6 +316,7 @@ async function ensureQuartoNotebookRunnerEnvironment( Deno.copyFileSync(projectTomlTemplate, projectToml); const command = new Deno.Command(options.julia_cmd, { args: [ + "--startup-file=no", `--project=${juliaRuntimeDir()}`, juliaResourcePath("ensure_environment.jl"), ], From f6a96579f9e53521c077e90228c27c8f6ab20cd0 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 7 Mar 2024 12:11:16 +0100 Subject: [PATCH 085/101] print useful julia internal server errors --- src/execute/julia.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 61c80f1534..aec95bdfc4 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -514,6 +514,19 @@ async function writeJuliaCommand( const json = response.split("\n")[0]; const data = JSON.parse(json); + const err = data.error; + if (err !== undefined) { + const juliaError = data.juliaError ?? "No julia error message available."; + error( + `Julia server returned error after receiving "${command}" command:\n` + + err, + ); + if (!juliaError !== undefined) { + error(juliaError); + } + throw new Error("Internal julia server error"); + } + return data; } From 9a90e7efc536edbcd91c01ae49459fe56b88fa9c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 8 Mar 2024 11:53:40 +0100 Subject: [PATCH 086/101] move to quartonotebookrunner 0.5 --- src/resources/julia/Project.toml | 2 +- src/resources/julia/quartonotebookrunner.jl | 23 ++------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index e42a58e001..a277c2d435 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "0.4.2" +QuartoNotebookRunner = "0.5.0" diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index 9028c843c2..2f3f68dea9 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -9,27 +9,8 @@ atexit() do rm(transport_file; force=true) end -server, port = let - server = nothing - port = nothing - - for i in 1:20 - # find an open port by creating a server there and immediately closing it - port, _server = Sockets.listenany(8000) - close(_server) - try - server = QuartoNotebookRunner.serve(; port) - break - catch e - error("Opening server on port $port failed.") - end - end - server, port -end - -if server === nothing - error("Giving up.") -end +server = QuartoNotebookRunner.serve() +port = server.port @info "trying to establish test connection to port $port" let From 333ed0019acea6c213734d924198507ab427b6af Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 8 Mar 2024 15:06:35 +0100 Subject: [PATCH 087/101] give visible notice about starting the julia server --- src/execute/julia.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index aec95bdfc4..c3f84b24b2 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -236,8 +236,9 @@ async function startOrReuseJuliaServer( if (!existsSync(transportFile)) { trace( options, - `Transport file ${transportFile} doesn't exist, starting server. This might take a while.`, + `Transport file ${transportFile} doesn't exist`, ); + info("Installing and starting julia server. This might take a while."); await ensureQuartoNotebookRunnerEnvironment(options); // We need to spawn the julia server in its own process that can outlive quarto. @@ -330,7 +331,7 @@ async function ensureQuartoNotebookRunnerEnvironment( ); return Promise.reject(); } - trace(options, "The julia server environment is correctly instantiated."); + info("The julia server environment is correctly instantiated."); return Promise.resolve(); } From cb166dfbc2a2e405a5fdb2f2d2ba5817de6e02e0 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 8 Mar 2024 15:39:15 +0100 Subject: [PATCH 088/101] fix patch version --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index a277c2d435..a47937aca3 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "0.5.0" +QuartoNotebookRunner = "=0.5.0" From 6699a91fda40a2440c2557c50878e40457a6f1f1 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 14 Mar 2024 14:37:34 +0100 Subject: [PATCH 089/101] implement hmac --- src/execute/julia.ts | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index c3f84b24b2..cd3375ccef 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -40,7 +40,7 @@ import { runningInCI } from "../core/ci-info.ts"; import { sleep } from "../core/async.ts"; import { JupyterNotebook } from "../core/jupyter/types.ts"; import { existsSync } from "fs/mod.ts"; -import { pythonExecForCaps } from "../core/jupyter/exec.ts"; +import { encodeBase64 } from "encoding/base64.ts"; export interface JuliaExecuteOptions extends ExecuteOptions { julia_cmd: string; @@ -342,6 +342,7 @@ function juliaResourcePath(...parts: string[]) { interface JuliaTransportFile { port: number; pid: number; + key: string; } async function pollTransportFile( @@ -379,7 +380,7 @@ async function getJuliaServerConnection( const isready = writeJuliaCommand( conn, "isready", - "TODOsomesecret", + transportOptions.key, options, ) as Promise; const timeoutMilliseconds = 10000; @@ -423,25 +424,26 @@ async function executeJulia( options: JuliaExecuteOptions, ): Promise { const conn = await getJuliaServerConnection(options); + const transportOptions = await pollTransportFile(options); if (options.oneShot || options.format.execute[kExecuteDaemonRestart]) { const isopen = await writeJuliaCommand( conn, "isopen", - "TODOsomesecret", + transportOptions.key, options, ) as boolean; if (isopen) { - await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + await writeJuliaCommand(conn, "close", transportOptions.key, options); } } const response = await writeJuliaCommand( conn, "run", - "TODOsomesecret", + transportOptions.key, options, ); if (options.oneShot) { - await writeJuliaCommand(conn, "close", "TODOsomesecret", options); + await writeJuliaCommand(conn, "close", transportOptions.key, options); } if (response.error !== undefined) { @@ -469,9 +471,27 @@ async function writeJuliaCommand( content, }; - const message = JSON.stringify(commandData) + "\n"; + const payload = JSON.stringify(commandData); + const key = await crypto.subtle.importKey( + "raw", + new TextEncoder().encode(secret), + { name: "HMAC", hash: "SHA-256" }, + true, + ["sign"], + ); + const canonicalRequestBytes = new TextEncoder().encode( + JSON.stringify(commandData), + ); + const signatureArrayBuffer = await crypto.subtle.sign( + "HMAC", + key, + canonicalRequestBytes, + ); + const signatureBytes = new Uint8Array(signatureArrayBuffer); + const hmac = encodeBase64(signatureBytes); + + const message = JSON.stringify({ hmac, payload }) + "\n"; - // TODO: no secret used, yet const messageBytes = new TextEncoder().encode(message); // // don't send the message if it's big. From 4ec4c7097564a8609846a72d5c72cf7074f3049f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 14 Mar 2024 16:07:39 +0100 Subject: [PATCH 090/101] bump to quartonotebookrunner 0.7 --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index a47937aca3..8d5c094372 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "=0.5.0" +QuartoNotebookRunner = "=0.7.0" From 1c70fe6458fca7278b128208716e6a6e6f153336 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 14 Mar 2024 16:08:01 +0100 Subject: [PATCH 091/101] simplify init script, server should already be `listen`ing --- src/resources/julia/quartonotebookrunner.jl | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index 2f3f68dea9..a2464b4a30 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -12,27 +12,8 @@ end server = QuartoNotebookRunner.serve() port = server.port -@info "trying to establish test connection to port $port" -let - connected = false - for i in 1:20 - try - sock = Sockets.connect(port) - println(sock, """{"type": "isready", "content": {}}""") - @assert readline(sock) == "true" - close(sock) - connected = true - break - catch - sleep(0.1) - end - end - connected || error("Test connection could not be established.") -end -@info "successful test connection" - open(transport_file, "w") do io - println(io, """{"port": $port, "pid": $(Base.Libc.getpid())}""") + println(io, """{"port": $port, "pid": $(Base.Libc.getpid()), "key": "$(server.key)"}""") end wait(server) From d5b10a931006a4c184397d32f6ace5db30fffaa2 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 14 Mar 2024 18:38:57 +0100 Subject: [PATCH 092/101] improve messages and show server init process output --- src/execute/julia.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index cd3375ccef..dccf2a8fd5 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -238,7 +238,7 @@ async function startOrReuseJuliaServer( options, `Transport file ${transportFile} doesn't exist`, ); - info("Installing and starting julia server. This might take a while."); + info("Starting julia control server process. This might take a while..."); await ensureQuartoNotebookRunnerEnvironment(options); // We need to spawn the julia server in its own process that can outlive quarto. @@ -322,16 +322,11 @@ async function ensureQuartoNotebookRunnerEnvironment( juliaResourcePath("ensure_environment.jl"), ], }); - const { success, stderr } = await command.output(); + const proc = command.spawn(); + const { success } = await proc.output(); if (!success) { - error( - `Ensuring an updated julia server environment failed:\n${ - stderr && new TextDecoder().decode(stderr) - }`, - ); - return Promise.reject(); + throw (new Error("Ensuring an updated julia server environment failed")); } - info("The julia server environment is correctly instantiated."); return Promise.resolve(); } @@ -368,6 +363,10 @@ async function getJuliaServerConnection( const { reused } = await startOrReuseJuliaServer(options); const transportOptions = await pollTransportFile(options); + if (!reused) { + info("Julia server process started."); + } + trace( options, `Connecting to server at port ${transportOptions.port}, pid ${transportOptions.pid}`, From 5776be80901597420c03bbf7d6b8b2d2c862b079 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 14 Mar 2024 18:39:28 +0100 Subject: [PATCH 093/101] time control process out after 5 minutes of inactivity --- src/resources/julia/quartonotebookrunner.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/quartonotebookrunner.jl b/src/resources/julia/quartonotebookrunner.jl index a2464b4a30..ee36de4138 100644 --- a/src/resources/julia/quartonotebookrunner.jl +++ b/src/resources/julia/quartonotebookrunner.jl @@ -9,7 +9,7 @@ atexit() do rm(transport_file; force=true) end -server = QuartoNotebookRunner.serve() +server = QuartoNotebookRunner.serve(; timeout = 300) port = server.port open(transport_file, "w") do io From c51ffc67e5e77f2a5a7b3fa94fe83720f1eb52e4 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 14 Mar 2024 18:46:56 +0100 Subject: [PATCH 094/101] fix error printing --- src/execute/julia.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index dccf2a8fd5..40eb807430 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -541,9 +541,7 @@ async function writeJuliaCommand( `Julia server returned error after receiving "${command}" command:\n` + err, ); - if (!juliaError !== undefined) { - error(juliaError); - } + error(juliaError); throw new Error("Internal julia server error"); } From f4f0d8a8487ceb437681566d089e4a869da54d79 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 15 Mar 2024 13:08:37 +0100 Subject: [PATCH 095/101] QuartoNotebookRunner 0.7.1 --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index 8d5c094372..1a36a4c10b 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "=0.7.0" +QuartoNotebookRunner = "=0.7.1" From 31c8b9e59a1d8066afc7f3535d24307b8606e729 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Fri, 15 Mar 2024 13:08:37 +0100 Subject: [PATCH 096/101] remove windows special behavior by using `spawn` and `unref` --- src/execute/julia.ts | 75 ++++++++++---------------------------------- 1 file changed, 17 insertions(+), 58 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 40eb807430..3512abc10e 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -241,64 +241,23 @@ async function startOrReuseJuliaServer( info("Starting julia control server process. This might take a while..."); await ensureQuartoNotebookRunnerEnvironment(options); - // We need to spawn the julia server in its own process that can outlive quarto. - // Apparently, `run(detach(cmd))` in julia does not do that reliably on Windows, - // at least deno never seems to recognize that the spawning julia process has finished, - // presumably because it waits for the active child process to exit. This makes the - // tests on windows hang forever if we use the same launching mechanism as for Unix systems. - // So we utilize powershell instead which can start completely detached processes with - // the Start-Process commandlet. - if (Deno.build.os === "windows") { - const command = new Deno.Command( - "PowerShell", - { - args: [ - "-Command", - "Start-Process", - options.julia_cmd, - "-ArgumentList", - // string array argument list, each element but the last must have a "," element after - "--startup-file=no", - ",", - `--project=${juliaRuntimeDir()}`, - ",", - resourcePath("julia/quartonotebookrunner.jl"), - ",", - transportFile, - // end of string array - "-WindowStyle", - "Hidden", - ], - }, - ); - trace( - options, - "Starting detached julia server through powershell, once transport file exists, server should be running.", - ); - const result = command.outputSync(); - if (!result.success) { - throw new Error(new TextDecoder().decode(result.stderr)); - } - } else { - const command = new Deno.Command(options.julia_cmd, { - args: [ - "--startup-file=no", - resourcePath("julia/start_quartonotebookrunner_detached.jl"), - options.julia_cmd, - juliaRuntimeDir(), - resourcePath("julia/quartonotebookrunner.jl"), - transportFile, - ], - }); - trace( - options, - "Starting detached julia server through julia, once transport file exists, server should be running.", - ); - const result = command.outputSync(); - if (!result.success) { - throw new Error(new TextDecoder().decode(result.stderr)); - } - } + const command = new Deno.Command(options.julia_cmd, { + args: [ + "--startup-file=no", + resourcePath("julia/start_quartonotebookrunner_detached.jl"), + options.julia_cmd, + juliaRuntimeDir(), + resourcePath("julia/quartonotebookrunner.jl"), + transportFile, + ], + }); + trace( + options, + "Starting detached julia server through julia, once transport file exists, server should be running.", + ); + // unref seems to be needed to let deno exit in the tests after having spawned the julia process + // that creates the detached child process + command.spawn().unref(); } else { trace( options, From 61bc4b4a414eb9694ea33e011bc672d9698f6e72 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Thu, 21 Mar 2024 13:37:15 +0100 Subject: [PATCH 097/101] bump QuartoNotebookRunner --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index 1a36a4c10b..ef7311e98c 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "=0.7.1" +QuartoNotebookRunner = "=0.8.0" From f71a88b9126e2ad93d85fe2b9a83408d19a105e3 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 25 Mar 2024 09:14:45 +0100 Subject: [PATCH 098/101] Revert "remove windows special behavior by using `spawn` and `unref`" This reverts commit 31c8b9e59a1d8066afc7f3535d24307b8606e729. --- src/execute/julia.ts | 75 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 3512abc10e..40eb807430 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -241,23 +241,64 @@ async function startOrReuseJuliaServer( info("Starting julia control server process. This might take a while..."); await ensureQuartoNotebookRunnerEnvironment(options); - const command = new Deno.Command(options.julia_cmd, { - args: [ - "--startup-file=no", - resourcePath("julia/start_quartonotebookrunner_detached.jl"), - options.julia_cmd, - juliaRuntimeDir(), - resourcePath("julia/quartonotebookrunner.jl"), - transportFile, - ], - }); - trace( - options, - "Starting detached julia server through julia, once transport file exists, server should be running.", - ); - // unref seems to be needed to let deno exit in the tests after having spawned the julia process - // that creates the detached child process - command.spawn().unref(); + // We need to spawn the julia server in its own process that can outlive quarto. + // Apparently, `run(detach(cmd))` in julia does not do that reliably on Windows, + // at least deno never seems to recognize that the spawning julia process has finished, + // presumably because it waits for the active child process to exit. This makes the + // tests on windows hang forever if we use the same launching mechanism as for Unix systems. + // So we utilize powershell instead which can start completely detached processes with + // the Start-Process commandlet. + if (Deno.build.os === "windows") { + const command = new Deno.Command( + "PowerShell", + { + args: [ + "-Command", + "Start-Process", + options.julia_cmd, + "-ArgumentList", + // string array argument list, each element but the last must have a "," element after + "--startup-file=no", + ",", + `--project=${juliaRuntimeDir()}`, + ",", + resourcePath("julia/quartonotebookrunner.jl"), + ",", + transportFile, + // end of string array + "-WindowStyle", + "Hidden", + ], + }, + ); + trace( + options, + "Starting detached julia server through powershell, once transport file exists, server should be running.", + ); + const result = command.outputSync(); + if (!result.success) { + throw new Error(new TextDecoder().decode(result.stderr)); + } + } else { + const command = new Deno.Command(options.julia_cmd, { + args: [ + "--startup-file=no", + resourcePath("julia/start_quartonotebookrunner_detached.jl"), + options.julia_cmd, + juliaRuntimeDir(), + resourcePath("julia/quartonotebookrunner.jl"), + transportFile, + ], + }); + trace( + options, + "Starting detached julia server through julia, once transport file exists, server should be running.", + ); + const result = command.outputSync(); + if (!result.success) { + throw new Error(new TextDecoder().decode(result.stderr)); + } + } } else { trace( options, From 33e6ad92e70e9e7253de6388d9323ab859bfc69c Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Mon, 25 Mar 2024 13:46:50 +0100 Subject: [PATCH 099/101] QuartoNotebookRunner 0.8.1 --- src/resources/julia/Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index ef7311e98c..c1fffefb97 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "=0.8.0" +QuartoNotebookRunner = "=0.8.1" From cfd6f208fffe76b0741a8f30f645b6c61680663f Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 26 Mar 2024 10:36:08 +0100 Subject: [PATCH 100/101] explicitly `precompile` to surface potential errors in the console --- src/resources/julia/ensure_environment.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/resources/julia/ensure_environment.jl b/src/resources/julia/ensure_environment.jl index 7ca9898465..11a431a22d 100644 --- a/src/resources/julia/ensure_environment.jl +++ b/src/resources/julia/ensure_environment.jl @@ -21,3 +21,6 @@ if manifest_matches_project_toml && manifest_has_correct_julia_version() else Pkg.update() end +# not strictly necessary, but in case of precompilation errors this will +# actually print them out explicitly +Pkg.precompile() From 192f0f4b91c02ccbca2b84539a44fc17912e2742 Mon Sep 17 00:00:00 2001 From: Julius Krumbiegel Date: Tue, 2 Apr 2024 10:57:26 +0200 Subject: [PATCH 101/101] bump to QNR 0.9 for render progress updates --- src/execute/julia.ts | 157 +++++++++++++++++++++++-------- src/resources/julia/Project.toml | 2 +- 2 files changed, 118 insertions(+), 41 deletions(-) diff --git a/src/execute/julia.ts b/src/execute/julia.ts index 40eb807430..9e8b63cf89 100644 --- a/src/execute/julia.ts +++ b/src/execute/julia.ts @@ -419,6 +419,37 @@ async function getJuliaServerConnection( } } +function firstSignificantLine(str: string, n: number): string { + const lines = str.split("\n"); + + for (const line of lines) { + const trimmedLine = line.trim(); + // Check if the line is significant + if (!trimmedLine.startsWith("#|") && trimmedLine !== "") { + // Return line as is if its length is less than or equal to n + if (trimmedLine.length <= n) { + return trimmedLine; + } else { + // Return substring up to n characters with an ellipsis + return trimmedLine.substring(0, n - 1) + "…"; + } + } + } + + // Return empty string if no significant line found + return ""; +} + +// from cliffy, MIT licensed +function getConsoleColumns(): number | null { + try { + // Catch error in none tty mode: Inappropriate ioctl for device (os error 25) + return Deno.consoleSize().columns ?? null; + } catch (_error) { + return null; + } +} + async function executeJulia( options: JuliaExecuteOptions, ): Promise { @@ -440,7 +471,21 @@ async function executeJulia( "run", transportOptions.key, options, + (update: ProgressUpdate) => { + const n = update.nChunks.toString(); + const i = update.chunkIndex.toString(); + const i_padded = `${" ".repeat(n.length - i.length)}${i}`; + const ncols = getConsoleColumns() ?? 80; + const firstPart = `Running [${i_padded}/${n}] at line ${update.line}: `; + const firstPartLength = firstPart.length; + const sigLine = firstSignificantLine( + update.source, + Math.max(0, ncols - firstPartLength), + ); + info(`${firstPart}${sigLine}`); + }, ); + if (options.oneShot) { await writeJuliaCommand(conn, "close", transportOptions.key, options); } @@ -452,11 +497,20 @@ async function executeJulia( return response.notebook as JupyterNotebook; } +interface ProgressUpdate { + type: "progress_update"; + chunkIndex: number; + nChunks: number; + source: string; + line: number; +} + async function writeJuliaCommand( conn: Deno.Conn, command: "run" | "close" | "stop" | "isready" | "isopen", secret: string, options: JuliaExecuteOptions, + onProgressUpdate?: (update: ProgressUpdate) => void, ) { // send the options along with the "run" command const content = command === "run" @@ -493,59 +547,82 @@ async function writeJuliaCommand( const messageBytes = new TextEncoder().encode(message); - // // don't send the message if it's big. - // // Instead, write it to a file and send the file path - // // This is disappointing, but something is deeply wrong with Deno.Conn: - // // https://github.com/quarto-dev/quarto-cli/issues/7737#issuecomment-1830665357 - // if (messageBytes.length > 1024) { - // const tempFile = Deno.makeTempFileSync(); - // Deno.writeFileSync(tempFile, messageBytes); - // const msg = kernelCommand("file", secret, { file: tempFile }) + "\n"; - // messageBytes = new TextEncoder().encode(msg); - // } - trace(options, `write command "${command}" to socket server`); const bytesWritten = await conn.write(messageBytes); if (bytesWritten !== messageBytes.length) { throw new Error("Internal Error"); } - let response = ""; + // a string of bytes received from the server could start with a + // partial message, contain multiple complete messages (separated by newlines) after that + // but they could also end in a partial one. + // so to read and process them all correctly, we read in a fixed number of bytes, if there's a newline, we process + // the string up to that part and save the rest for the next round. + let restOfPreviousResponse = ""; while (true) { - const buffer = new Uint8Array(512); - const bytesRead = await conn.read(buffer); - if (bytesRead === null) { - break; + let response = restOfPreviousResponse; + const newlineAt = response.indexOf("\n"); + // if we already have a newline, we don't need to read from conn + if (newlineAt !== -1) { + restOfPreviousResponse = response.substring(newlineAt + 1); + response = response.substring(0, newlineAt); + } // but if we don't have a newline, we read in more until we get one + else { + while (true) { + const buffer = new Uint8Array(512); + const bytesRead = await conn.read(buffer); + if (bytesRead === null) { + break; + } + + if (bytesRead > 0) { + const payload = new TextDecoder().decode( + buffer.slice(0, bytesRead), + ); + const payloadNewlineAt = payload.indexOf("\n"); + if (payloadNewlineAt === -1) { + response += payload; + restOfPreviousResponse = ""; + } else { + response += payload.substring(0, payloadNewlineAt); + restOfPreviousResponse = payload.substring(payloadNewlineAt + 1); + // when we have found a newline in a payload, we can stop reading in more data and continue + // with the response first + break; + } + } + } } + trace(options, "received server response"); + // one command should be sent, ended by a newline, currently just throwing away anything else because we don't + // expect multiple commmands at once + const json = response.split("\n")[0]; + const data = JSON.parse(json); - if (bytesRead > 0) { - const payload = new TextDecoder().decode( - buffer.slice(0, bytesRead), + if (data.type === "progress_update") { + trace( + options, + "received progress update response, listening for further responses", ); - response += payload; - if (payload.includes("\n")) { - break; + if (onProgressUpdate !== undefined) { + onProgressUpdate(data as ProgressUpdate); } + continue; // wait for the next message + } + + const err = data.error; + if (err !== undefined) { + const juliaError = data.juliaError ?? "No julia error message available."; + error( + `Julia server returned error after receiving "${command}" command:\n` + + err, + ); + error(juliaError); + throw new Error("Internal julia server error"); } - } - trace(options, "received server response"); - // one command should be sent, ended by a newline, currently just throwing away anything else because we don't - // expect multiple commmands - const json = response.split("\n")[0]; - const data = JSON.parse(json); - - const err = data.error; - if (err !== undefined) { - const juliaError = data.juliaError ?? "No julia error message available."; - error( - `Julia server returned error after receiving "${command}" command:\n` + - err, - ); - error(juliaError); - throw new Error("Internal julia server error"); - } - return data; + return data; + } } function juliaRuntimeDir(): string { diff --git a/src/resources/julia/Project.toml b/src/resources/julia/Project.toml index c1fffefb97..2b021f07ce 100644 --- a/src/resources/julia/Project.toml +++ b/src/resources/julia/Project.toml @@ -2,4 +2,4 @@ QuartoNotebookRunner = "4c0109c6-14e9-4c88-93f0-2b974d3468f4" [compat] -QuartoNotebookRunner = "=0.8.1" +QuartoNotebookRunner = "=0.9.0"