From 3c36a963eed81b413602b7f9861eec54bd4738c7 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 4 Oct 2023 15:30:48 -0700 Subject: [PATCH 01/53] first take at eliminating preemptive fork in runtime, replaced fork/exec with posix_spawn, still need to make it much more robust --- runtime/driver.c | 142 +++++++++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 60 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 999b902b..13721262 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -10,6 +10,7 @@ #include #include #endif +#include #include #include #include @@ -869,72 +870,93 @@ static void launcher_main (FILE* lin, FILE* lout){ if(pipe(exec_error) < 0) exit_with_error(); //Fork a new child - stz_long pid = (stz_long)fork(); - if(pid < 0) exit_with_error(); + //stz_long pid = (stz_long)fork(); + // Setup + posix_spawnattr_t attr; + posix_spawn_file_actions_t actions; + posix_spawnattr_init(&attr); + posix_spawn_file_actions_init(&actions); + // File actions + //TODO: handle errors here + int inpipe = posix_spawn_file_actions_addopen(&actions, 0, earg->in_pipe, O_RDONLY, 0); + int outpipe = posix_spawn_file_actions_addopen(&actions, 1, earg->out_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644); + int errpipe = posix_spawn_file_actions_addopen(&actions, 2, earg->err_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (earg->working_dir) { + int wdir = posix_spawn_file_actions_addchdir_np(&actions, earg->working_dir); + } + // Spawn child + pid_t child_pid; + stz_long pid = (stz_long)posix_spawn(&child_pid, "/path/to/your/executable", &actions, &attr, earg, environ); + //if(pid < 0) exit_with_error(); - if(pid > 0){ + if(pid == 0){ //Read from error-code pipe - int exec_code; - close(exec_error[WRITE]); - int exec_r = read(exec_error[READ], &exec_code, sizeof(int)); - close(exec_error[READ]); - - if(exec_r == 0){ - //Exec evaluated successfully - //Return new process id - write_long(lout, pid); - fflush(lout); - } - else if(exec_r == sizeof(int)){ - //Exec evaluated unsuccessfully - //Return error code as negative long - write_long(lout, (stz_long)-exec_code); - fflush(lout); - } - else{ - fprintf(stderr, "Unreachable code."); - exit(-1); - } + //int exec_code; + //close(exec_error[WRITE]); + //int exec_r = read(exec_error[READ], &exec_code, sizeof(int)); + //close(exec_error[READ]); + int exec_r; + waitpid(pid, &exec_r, 0); + //if(exec_r == 0){ + // //Exec evaluated successfully + // //Return new process id + // write_long(lout, pid); + // fflush(lout); + //} + //else if(exec_r == sizeof(int)){ + // //Exec evaluated unsuccessfully + // //Return error code as negative long + // write_long(lout, (stz_long)-exec_code); + // fflush(lout); + //} + //else{ + // fprintf(stderr, "Unreachable code."); + // exit(-1); + //} }else{ + exit_with_error(); //Close exec pipe read, and close write end on successful exec close(exec_error[READ]); fcntl(exec_error[WRITE], F_SETFD, FD_CLOEXEC); //Open named pipes - if(earg->in_pipe != NULL){ - int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->in_pipe), O_RDONLY); - if(fd < 0) write_error_and_exit(exec_error[WRITE]); - dup2(fd, 0); - } - if(earg->out_pipe != NULL){ - int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->out_pipe), O_WRONLY); - if(fd < 0) write_error_and_exit(exec_error[WRITE]); - dup2(fd, 1); - } - if(earg->err_pipe != NULL){ - int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->err_pipe), O_WRONLY); - if(fd < 0) write_error_and_exit(exec_error[WRITE]); - dup2(fd, 2); - } + //if(earg->in_pipe != NULL){ + // int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->in_pipe), O_RDONLY); + // if(fd < 0) write_error_and_exit(exec_error[WRITE]); + // dup2(fd, 0); + //} + //if(earg->out_pipe != NULL){ + // int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->out_pipe), O_WRONLY); + // if(fd < 0) write_error_and_exit(exec_error[WRITE]); + // dup2(fd, 1); + //} + //if(earg->err_pipe != NULL){ + // int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->err_pipe), O_WRONLY); + // if(fd < 0) write_error_and_exit(exec_error[WRITE]); + // dup2(fd, 2); + //} //Set the working directory of the child process if explicitly //requested. - if (earg->working_dir) { - if (chdir(C_CSTR(earg->working_dir)) == -1) { - write_error_and_exit(exec_error[WRITE]); - } - } + //if (earg->working_dir) { + // if (chdir(C_CSTR(earg->working_dir)) == -1) { + // write_error_and_exit(exec_error[WRITE]); + // } + //} //Launch child process. //If an environment is supplied then call execvpe, otherwise call execvp. - if(earg->env_vars == NULL) - execvp(C_CSTR(earg->file), (char**)earg->argvs); - else - execvpe(C_CSTR(earg->file), (char**)earg->argvs, (char**)earg->env_vars); + //if(earg->env_vars == NULL) + // execvp(C_CSTR(earg->file), (char**)earg->argvs); + //else + // execvpe(C_CSTR(earg->file), (char**)earg->argvs, (char**)earg->env_vars); //Unsuccessful exec, write error number write_error_and_exit(exec_error[WRITE]); } + // TODO: clean up spawn resources + posix_spawn_file_actions_destroy(&actions); + posix_spawnattr_destroy(&attr); } //Interpret state retrieval command else if(comm == STATE_COMMAND || comm == WAIT_COMMAND){ @@ -971,17 +993,17 @@ void initialize_launcher_process (){ stz_long pid = (stz_long)fork(); if(pid < 0) exit_with_error(); - if(pid > 0){ - //Parent - launcher_pid = pid; - close(lin[READ]); - close(lout[WRITE]); - launcher_in = fdopen(lin[WRITE], "w"); - if(launcher_in == NULL) exit_with_error(); - launcher_out = fdopen(lout[READ], "r"); - if(launcher_out == NULL) exit_with_error(); - } - else{ + //if(pid > 0){ + // //Parent + // launcher_pid = pid; + // close(lin[READ]); + // close(lout[WRITE]); + // launcher_in = fdopen(lin[WRITE], "w"); + // if(launcher_in == NULL) exit_with_error(); + // launcher_out = fdopen(lout[READ], "r"); + // if(launcher_out == NULL) exit_with_error(); + // } + // else{ //Child close(lin[WRITE]); close(lout[READ]); @@ -990,7 +1012,7 @@ void initialize_launcher_process (){ FILE* fout = fdopen(lout[WRITE], "w"); if(fout == NULL) exit_with_error(); launcher_main(fin, fout); - } + // } } } From e11c7a2a56ee7c34e7051c7464adfe34df5a72fc Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 23 Oct 2023 15:13:13 -0700 Subject: [PATCH 02/53] working commit --- runtime/driver.c | 56 +++++++++++++++++++++++++++-------------------- scripts/finish.sh | 2 +- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 13721262..88d2c87d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -855,6 +855,7 @@ static void launcher_main (FILE* lin, FILE* lout){ while(1){ //Read in command int comm = fgetc(lin); + printf("comm = %d", comm); if(feof(lin)) exit(0); //Interpret launch process command @@ -877,16 +878,21 @@ static void launcher_main (FILE* lin, FILE* lout){ posix_spawnattr_init(&attr); posix_spawn_file_actions_init(&actions); // File actions - //TODO: handle errors here - int inpipe = posix_spawn_file_actions_addopen(&actions, 0, earg->in_pipe, O_RDONLY, 0); - int outpipe = posix_spawn_file_actions_addopen(&actions, 1, earg->out_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644); - int errpipe = posix_spawn_file_actions_addopen(&actions, 2, earg->err_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644); + // TODO: fix all the warnings + int posix_ret; + if (posix_ret = posix_spawn_file_actions_addopen(&actions, 0, earg->in_pipe, O_RDONLY, 0)) + exit_with_error(); + if (posix_ret = posix_spawn_file_actions_addopen(&actions, 1, earg->out_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644)) + exit_with_error(); + if (posix_ret = posix_spawn_file_actions_addopen(&actions, 2, earg->err_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644)) + exit_with_error(); if (earg->working_dir) { - int wdir = posix_spawn_file_actions_addchdir_np(&actions, earg->working_dir); + if (posix_ret = posix_spawn_file_actions_addchdir_np(&actions, earg->working_dir)) + exit_with_error(); } // Spawn child pid_t child_pid; - stz_long pid = (stz_long)posix_spawn(&child_pid, "/path/to/your/executable", &actions, &attr, earg, environ); + stz_long pid = (stz_long)posix_spawn(&child_pid, earg->file, &actions, &attr, earg->argvs, earg->env_vars); //if(pid < 0) exit_with_error(); if(pid == 0){ @@ -914,10 +920,10 @@ static void launcher_main (FILE* lin, FILE* lout){ // exit(-1); //} }else{ - exit_with_error(); - //Close exec pipe read, and close write end on successful exec close(exec_error[READ]); fcntl(exec_error[WRITE], F_SETFD, FD_CLOEXEC); + exit_with_error(); + //Close exec pipe read, and close write end on successful exec //Open named pipes //if(earg->in_pipe != NULL){ @@ -952,7 +958,7 @@ static void launcher_main (FILE* lin, FILE* lout){ // execvpe(C_CSTR(earg->file), (char**)earg->argvs, (char**)earg->env_vars); //Unsuccessful exec, write error number - write_error_and_exit(exec_error[WRITE]); + //write_error_and_exit(exec_error[WRITE]); } // TODO: clean up spawn resources posix_spawn_file_actions_destroy(&actions); @@ -990,28 +996,29 @@ void initialize_launcher_process (){ if(pipe(lout) < 0) exit_with_error(); //Fork - stz_long pid = (stz_long)fork(); - if(pid < 0) exit_with_error(); + //stz_long pid = (stz_long)fork(); + //if(pid < 0) exit_with_error(); //if(pid > 0){ // //Parent // launcher_pid = pid; - // close(lin[READ]); - // close(lout[WRITE]); - // launcher_in = fdopen(lin[WRITE], "w"); - // if(launcher_in == NULL) exit_with_error(); - // launcher_out = fdopen(lout[READ], "r"); - // if(launcher_out == NULL) exit_with_error(); + //close(lin[READ]); + //close(lout[WRITE]); + launcher_in = fdopen(lin[WRITE], "w"); + if(launcher_in == NULL) exit_with_error(); + launcher_out = fdopen(lout[READ], "r"); + if(launcher_out == NULL) exit_with_error(); // } // else{ //Child - close(lin[WRITE]); - close(lout[READ]); - FILE* fin = fdopen(lin[READ], "r"); - if(fin == NULL) exit_with_error(); - FILE* fout = fdopen(lout[WRITE], "w"); - if(fout == NULL) exit_with_error(); - launcher_main(fin, fout); + //close(lin[WRITE]); + //close(lout[READ]); + FILE* fin = fdopen(lin[READ], "r"); + if(fin == NULL) exit_with_error(); + FILE* fout = fdopen(lout[WRITE], "w"); + if(fout == NULL) exit_with_error(); + printf("launching\n"); + launcher_main(fin, fout); // } } } @@ -1077,6 +1084,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, if(output == PROCESS_ERR) earg.out_pipe = STZ_STR("_err"); if(error == PROCESS_OUT) earg.err_pipe = STZ_STR("_out"); if(error == PROCESS_ERR) earg.err_pipe = STZ_STR("_err"); + printf("Writing command\n"); int r = fputc(LAUNCH_COMMAND, launcher_in); if(r == EOF) return -1; write_earg(launcher_in, &earg); diff --git a/scripts/finish.sh b/scripts/finish.sh index 8ee98466..597e28f0 100755 --- a/scripts/finish.sh +++ b/scripts/finish.sh @@ -1,4 +1,4 @@ -scripts/make-asmjit.sh os-x +# scripts/make-asmjit.sh os-x gcc -std=gnu99 -c core/sha256.c -O3 -o build/sha256.o -I include gcc -std=gnu99 -c compiler/cvm.c -O3 -o build/cvm.o -I include gcc -c runtime/linenoise-ng/linenoise.cpp -O3 -o build/linenoise.o -I include -I runtime/linenoise-ng From 0bc5dfd123374ecc8c1c4a20ea14bac9f79cf957 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 15 Nov 2023 09:55:34 -0800 Subject: [PATCH 03/53] working commit, doesnt work --- compiler/config.stanza | 6 +- compiler/main.stanza | 2 +- core/core.stanza | 14 +- runtime/driver-old.c | 1248 ++++++++++++++++++++++++++++++++++++++++ runtime/driver.c | 414 ++++++------- 5 files changed, 1440 insertions(+), 244 deletions(-) create mode 100644 runtime/driver-old.c diff --git a/compiler/config.stanza b/compiler/config.stanza index c939eead..4934cb50 100644 --- a/compiler/config.stanza +++ b/compiler/config.stanza @@ -260,7 +260,11 @@ defn verify-installation () : val v = try : ;Retrieve the version of the executable. - call-system-and-get-output(exe-file, [exe-file, "version", "-terse"]) + println("Calling %_" % [exe-file]) + val o = call-system-and-get-output(exe-file, [exe-file, "version", "-terse"]) + println("Called %_" % [exe-file]) + o + catch (e:Exception) : ;Could not retrieve version of the executable. val msg = "Stanza install directory is set to %_, \ diff --git a/compiler/main.stanza b/compiler/main.stanza index 49eff1de..ea448d0f 100644 --- a/compiler/main.stanza +++ b/compiler/main.stanza @@ -159,6 +159,7 @@ defn build-system (verbose?:True|False) : ;First directly try calling the cc-prog driver. try : ;Call system + println("Calling %_" % [to-tuple(args)]) val return-code = call-system(to-tuple(args)) ;Return true if successful return-code == 0 @@ -1210,7 +1211,6 @@ add-stanza-command(defs-db-command()) ;============================================================ public defn stanza-main (commands:Collection, default-command:String|False) : - initialize-process-launcher() set-max-heap-size(STANZA-MAX-COMPILER-HEAP-SIZE) simple-command-line-cli(version-message(), to-tuple(commands), default-command, true) diff --git a/core/core.stanza b/core/core.stanza index e91d10b1..9f6a085c 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -87,7 +87,7 @@ protected extern stz_memory_resize: (ptr, long, long) -> int #else: protected extern launch_process: (ptr, ptr>, int, int, int, int, ptr, ptr>, ptr) -> int protected extern delete_process_pipes: (ptr, ptr, ptr, int) -> int - protected extern initialize_launcher_process: () -> int + ;protected extern initialize_launcher_process: () -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int @@ -10238,10 +10238,10 @@ public lostanza defn error-stream (p:ref) -> ref : ; Initialization ; ============== -public lostanza defn initialize-process-launcher () -> ref : - #if-not-defined(PLATFORM-WINDOWS): - call-c clib/initialize_launcher_process() - return false +;public lostanza defn initialize-process-launcher () -> ref : +; #if-not-defined(PLATFORM-WINDOWS): +; call-c clib/initialize_launcher_process() +; return false ; State API ; ========= @@ -10341,14 +10341,18 @@ public defn call-system-and-get-output (file:String, working-dir:String|False, env-vars:Tuple>|False) -> String : val buffer = StringBuffer() + println("Making process %_" % [file]) val proc = Process(file, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, working-dir, env-vars) + println("Made process %_" % [file]) val proc-out = output-stream(proc) let loop () : val c = get-char(proc-out) match(c:Char) : add(buffer,c) loop() + println("Waiting for %_" % [file]) wait(proc) + println("%_ done" % [file]) to-string(buffer) public defn call-system-and-get-output (file:String, args:Seqable) -> String : diff --git a/runtime/driver-old.c b/runtime/driver-old.c new file mode 100644 index 00000000..f2e6dcd2 --- /dev/null +++ b/runtime/driver-old.c @@ -0,0 +1,1248 @@ +#ifdef PLATFORM_WINDOWS + //This define is necessary as a workaround for accessing CreateSymbolicLink + //function. This #define is added automatically by the MSVC compiler, but + //must be added manually when using gcc. + //Must be defined before including windows.h + #define _WIN32_WINNT 0x600 + + #include +#else + #include + #include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "process.h" + +// Forward Declarations +// ==================== + +void* stz_malloc (stz_long size); +void stz_free (void* ptr); + +#ifdef PLATFORM_WINDOWS +char* get_windows_api_error() { + char* lpMsgBuf; + char* ret; + + FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + GetLastError(), + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (char*)&lpMsgBuf, + 0, NULL ); + + ret = strdup(lpMsgBuf); + LocalFree(lpMsgBuf); + + return ret; +} +#endif + +#ifdef PLATFORM_WINDOWS +static void exit_with_error_line_and_func (const char* file, int line) { + fprintf(stderr, "[%s:%d] %s", file, line, get_windows_api_error()); + exit(-1); +} +#endif + +#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) +static void exit_with_error_line_and_func (const char* file, int line){ + fprintf(stderr, "[%s:%d] %s\n", file, line, strerror(errno)); + exit(-1); +} +#endif + +#define exit_with_error() exit_with_error_line_and_func(__FILE__, __LINE__) + +// Stanza Defined Entities +// ======================= +typedef struct{ + stz_long returnpc; + stz_long liveness_map; + stz_long slots[]; +} StackFrame; + +typedef struct Stack{ + stz_long size; + StackFrame* frames; + StackFrame* stack_pointer; + stz_long pc; + struct Stack* tail; +} Stack; + +typedef struct{ + stz_long current_stack; + stz_long system_stack; + stz_byte* heap_top; + stz_byte* heap_limit; + stz_byte* heap_start; + stz_byte* heap_old_objects_end; + stz_byte* heap_bitset; + stz_byte* heap_bitset_base; + stz_long heap_size; + stz_long heap_size_limit; + stz_long heap_max_size; + Stack* stacks; + void* trackers; + stz_byte* marking_stack_start; + stz_byte* marking_stack_bottom; + stz_byte* marking_stack_top; +} VMInit; + +// Macro Readers +// ============= +FILE* get_stdout () {return stdout;} +FILE* get_stderr () {return stderr;} +FILE* get_stdin () {return stdin;} +stz_int get_eof () {return (stz_int)EOF;} +stz_int get_errno () {return (stz_int)errno;} + +// Time of Day +// =========== +stz_long current_time_us (void) { + struct timeval tv; + gettimeofday(&tv, NULL); + return (stz_long)tv.tv_sec * 1000 * 1000 + (stz_long)tv.tv_usec; +} + +stz_long current_time_ms (void) { + struct timeval tv; + gettimeofday(&tv, NULL); + return (stz_long)tv.tv_sec * 1000 + (stz_long)tv.tv_usec / 1000; +} + +// Random Access Files +// =================== +stz_long get_file_size (FILE* f) { + int64_t cur_pos = ftell(f); + fseek(f, 0, SEEK_END); + stz_long size = (stz_long)ftell(f); + fseek(f, cur_pos, SEEK_SET); + return size; +} + +stz_int file_seek (FILE* f, stz_long pos) { + return (stz_int)fseek(f, pos, SEEK_SET); +} + +stz_int file_skip (FILE* f, stz_long num) { + return (stz_int)fseek(f, num, SEEK_CUR); +} + +stz_int file_set_length (FILE* f, stz_long size) { + return (stz_int)ftruncate(fileno(f), size); +} + +stz_long file_read_block (FILE* f, char* data, stz_long len) { + return (stz_long)fread(data, 1, len, f); +} + +stz_long file_write_block (FILE* f, char* data, stz_long len) { + return (stz_long)fwrite(data, 1, len, f); +} + + +// Path Resolution +// =============== +#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) + stz_byte* resolve_path (const stz_byte* filename){ + //Call the Linux realpath function. + return STZ_STR(realpath(C_CSTR(filename), 0)); + } +#endif + +#if defined(PLATFORM_WINDOWS) + // Return a bitmask that represents which of the 26 letters correspond + // to valid drive letters. + stz_int windows_logical_drives_bitmask (){ + return GetLogicalDrives(); + } + + // Resolve a given file path to its fully-resolved ("final") path name. + // This function tries to return an absolute path with symbolic links + // resolved. Sometimes it returns an UNC path, which is not usable. + stz_byte* windows_final_path_name (stz_byte* path){ + // First, open the file (to get a handle to it) + HANDLE hFile = CreateFile( + /* lpFileName */ (LPCSTR)path, + /* dwDesiredAccess */ 0, + /* dwShareMode */ FILE_SHARE_READ | FILE_SHARE_WRITE, + /* lpSecurityAttributes */ NULL, + /* dwCreationDisposition */ OPEN_EXISTING, + // necessary to open directories + /* dwFlagsAndAttributes */ FILE_FLAG_BACKUP_SEMANTICS, + /* hTemplateFile */ NULL); + + // Return -1 if a handle cannot be created. + if (hFile == INVALID_HANDLE_VALUE) return NULL; + + // Then resolve it into its fully-resolved ("final") path name + LPSTR ret = stz_malloc(sizeof(CHAR) * MAX_PATH); + int numchars = GetFinalPathNameByHandle(hFile, ret, MAX_PATH, FILE_NAME_OPENED); + + // Close handle now that we no longer need it (important to do so!) + CloseHandle(hFile); + + // Return null if GetFinalPath fails. + if(numchars == 0){ + stz_free(ret); + return NULL; + } + + // Return the path. + return STZ_STR(ret); + } + + // Resolve a given file path using its "full" path name. + // This function tries to return an absolute path. Symbolic + // links are not resolved. + stz_byte* windows_full_path_name (stz_byte* filename){ + char* fileext; + char* path = (char*)stz_malloc(2048); + int numchars = GetFullPathName((LPCSTR)filename, 2048, path, &fileext); + + // Return null if GetFullPath fails. + if(numchars == 0){ + stz_free(path); + return NULL; + } + + //Return the path + return path; + } +#endif + +#ifdef PLATFORM_WINDOWS +stz_int symlink(const stz_byte* target, const stz_byte* linkpath) { + DWORD attributes, flags; + + attributes = GetFileAttributes((LPCSTR)target); + flags = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0 ? + SYMBOLIC_LINK_FLAG_DIRECTORY : 0; + + if (!CreateSymbolicLink((LPCSTR)linkpath, (LPCSTR)target, flags)) { + return -1; + } + + return 0; +} + +//This function does not follow symbolic links. If we need +//to follow symbolic links, the caller should call this +//call this function with the result of resolve-path. +stz_int get_file_type (const stz_byte* filename0) { + WIN32_FILE_ATTRIBUTE_DATA attributes; + LPCSTR filename = C_CSTR(filename0); + bool is_directory = false, + is_symlink = false; + + // First grab the file's attributes + if (!GetFileAttributesEx(filename, GetFileExInfoStandard, &attributes)) { + return -1; // Non-existent or inaccessible file + } + + // Check if it's a directory + if (attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + is_directory = true; + } + + // Check for possible symlink (reparse point *may* be a symlink) + if (attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + // To know for sure, find the file and check its reparse tags + WIN32_FIND_DATA find_data; + HANDLE find_handle = FindFirstFile(filename, &find_data); + + if (find_handle == INVALID_HANDLE_VALUE) { + return -1; + } + + if (// Mount point a.k.a Junction (should be treated as a symlink) + find_data.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT || + // Actual symlinks (like those created by symlink()) + find_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { + is_symlink = true; + } + + FindClose(find_handle); + } + + // Now we can determine what kind of file it is + if (!is_directory && !is_symlink) { + return 0; // Regular file + } + else if (is_directory && !is_symlink) { + return 1; // Directory (non-symlink) + } + else if (is_symlink) { + return 2; // Symlink + } + else { + return 3; // Unknown (other) + } +} +#endif + +#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) +stz_int get_file_type (const stz_byte* filename, stz_int follow_sym_links) { + struct stat filestat; + int result; + if(follow_sym_links) result = stat(C_CSTR(filename), &filestat); + else result = lstat(C_CSTR(filename), &filestat); + + if(result == 0){ + if(S_ISREG(filestat.st_mode)) + return 0; + else if(S_ISDIR(filestat.st_mode)) + return 1; + else if(S_ISLNK(filestat.st_mode)) + return 2; + else + return 3; + } + else{ + return -1; + } +} + +#endif + +// Environment Variable Setting +// ============================ +#ifdef PLATFORM_WINDOWS + + //Retrieve all environment variables as a list. + extern char** _environ; + char** get_env_vars (){ + return _environ; + } + + stz_int setenv (const stz_byte* name, const stz_byte* value, stz_int overwrite) { + //If we don't want to overwrite previous value, then check whether it exists. + //If it does, then just return 0. + if(!overwrite){ + if(getenv(C_CSTR(name)) == 0) + return 0; + } + //(Over)write the environment variable. + char* buffer = (char*)stz_malloc(strlen(C_CSTR(name)) + strlen(C_CSTR(value)) + 10); + sprintf(buffer, "%s=%s", C_CSTR(name), C_CSTR(value)); + int r = _putenv(buffer); + stz_free(buffer); + return (stz_int)r; + } + + stz_int unsetenv (const stz_byte* name){ + char* buffer = (char*)stz_malloc(strlen(C_CSTR(name)) + 10); + sprintf(buffer, "%s=", C_CSTR(name)); + int r = _putenv(buffer); + stz_free(buffer); + return (stz_int)r; + } +#else + + //Retrieve all environment variables as a list. + extern char** environ; + char** get_env_vars (){ + return environ; + } + +#endif + +// Time Modified +// ============= + +stz_long file_time_modified (const stz_byte* filename){ + struct stat attrib; + if(stat(C_CSTR(filename), &attrib) == 0) + return (stz_long)attrib.st_mtime; + return 0; +} + +//============================================================ +//===================== String List ========================== +//============================================================ + +typedef struct { + stz_int n; + stz_int capacity; + stz_byte** strings; +} StringList; + +StringList* make_stringlist (stz_int capacity){ + StringList* list = (StringList*)malloc(sizeof(StringList)); + list->n = 0; + list->capacity = capacity; + list->strings = (stz_byte**)malloc(capacity * sizeof(stz_byte*)); + return list; +} + +static void ensure_stringlist_capacity (StringList* list, stz_int c) { + if(list->capacity < c){ + stz_int new_capacity = list->capacity; + while(new_capacity < c) new_capacity *= 2; + stz_byte** new_strings = (stz_byte**)malloc(new_capacity * sizeof(stz_byte*)); + memcpy(new_strings, list->strings, list->n * sizeof(stz_byte*)); + list->capacity = new_capacity; + free(list->strings); + list->strings = new_strings; + } +} + +void free_stringlist (StringList* list){ + for(int i=0; in; i++) + free(list->strings[i]); + free(list->strings); + free(list); +} + +void stringlist_add (StringList* list, const stz_byte* string){ + ensure_stringlist_capacity(list, list->n + 1); + char* copy = malloc(strlen(C_CSTR(string)) + 1); + strcpy(copy, C_CSTR(string)); + list->strings[list->n] = STZ_STR(copy); + list->n++; +} + +//============================================================ +//================== Directory Handling ====================== +//============================================================ + +StringList* list_dir (const stz_byte* filename){ + //Open directory + DIR* dir = opendir(C_CSTR(filename)); + if(dir == NULL) return 0; + + //Allocate memory for strings + StringList* list = make_stringlist(10); + //Loop through directory entries + while(1){ + //Read next entry + struct dirent* entry = readdir(dir); + if(entry == NULL){ + closedir(dir); + return list; + } + //Notify + stringlist_add(list, STZ_CSTR(entry->d_name)); + } + + free(list); + return 0; +} + +//============================================================ +//===================== Sleeping ============================= +//============================================================ + +stz_int sleep_us (stz_long us){ + struct timespec t1, t2; + t1.tv_sec = us / 1000000L; + t1.tv_nsec = (us % 1000000L) * 1000L; + return (stz_int)nanosleep(&t1, &t2); +} + +stz_int sleep_ms (stz_long ms){ + struct timespec t1, t2; + t1.tv_sec = ms / 1000L; + t1.tv_nsec = (ms % 1000L) * 1000000L; + return (stz_int)nanosleep(&t1, &t2); +} + +//============================================================ +//================= Stanza Memory Allocator ================== +//============================================================ + +void* stz_malloc (stz_long size){ + return malloc(size); +} + +void stz_free (void* ptr){ + free(ptr); +} + +//============================================================ +//============= Stanza Memory Mapping on POSIX =============== +//============================================================ +#if defined(PLATFORM_LINUX) | defined(PLATFORM_OS_X) + +//Set protection bits on address range p (inclusive) to p + size (exclusive). +//Fatal error if size > 0 and mprotect fails. +static void protect(void* p, stz_long size, stz_int prot) { + if (size && mprotect(p, (size_t)size, prot)) exit_with_error(); +} + +//Allocates a segment of memory that is min_size allocated, and can be +//resized up to max_size. +//This function is called from within Stanza, and min_size and max_size +//are assumed to be multiples of the system page size. +void* stz_memory_map (stz_long min_size, stz_long max_size) { + void* p = mmap(NULL, (size_t)max_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) exit_with_error(); + + protect(p, min_size, PROT_READ | PROT_WRITE | PROT_EXEC); + return p; +} + +//Unmaps the region of memory. +//This function is called from within Stanza, and size is +//assumed to be a multiple of the system page size. +void stz_memory_unmap (void* p, stz_long size) { + if (p && munmap(p, (size_t)size)) exit_with_error(); +} + +//Resizes the given segment. +//old_size is assumed to be the size that is already allocated. +//new_size is the size that we desired to be allocated, and +//must be a multiple of the system page size. +void stz_memory_resize (void* p, stz_long old_size, stz_long new_size) { + stz_long min_size = old_size; + stz_long max_size = new_size; + int prot = PROT_READ | PROT_WRITE | PROT_EXEC; + + if (min_size > max_size) { + min_size = new_size; + max_size = old_size; + prot = PROT_NONE; + } + + protect((char*)p + min_size, max_size - min_size, prot); +} + +#endif + +//============================================================ +//============= Stanza Memory Mapping on Windows ============= +//============================================================ +#ifdef PLATFORM_WINDOWS + +//Allocates a segment of memory that is min_size allocated, and can be +//resized up to max_size. +//This function is called from within Stanza, and min_size and max_size +//are assumed to be multiples of the system page size. +void* stz_memory_map (stz_long min_size, stz_long max_size) { + // Reserve the max size with no access + void* p = VirtualAlloc(NULL, (SIZE_T)max_size, MEM_RESERVE, PAGE_NOACCESS); + if (p == NULL) exit_with_error(); + + // Commit the min size with RWX access. + p = VirtualAlloc(p, (SIZE_T)min_size, MEM_COMMIT, PAGE_EXECUTE_READWRITE); + if (p == NULL) exit_with_error(); + + // Return the reserved and committed pointer. + return p; +} + +//Unmaps given segment of memory. +//This function is called from within Stanza, and size is +//assumed to be a multiple of the system page size. +void stz_memory_unmap (void* p, stz_long size) { + // End doing nothing if p is null. + if (p == NULL) return; + + // Release the memory and fatal if it fails. + if (!VirtualFree(p, 0, MEM_RELEASE)) + exit_with_error(); +} + +//Resizes the given segment. +//old_size is assumed to be the size that is already allocated. +//new_size is the size that we desired to be allocated, and +//must be a multiple of the system page size. +void stz_memory_resize (void* p, stz_long old_size, stz_long new_size) { + //Case: if growing the allocated size. + if (new_size > old_size) { + // Growing the allocation: commit all memory pages from the old limit to the new limit. + if (!VirtualAlloc((char*)p + old_size, (SIZE_T)(new_size - old_size), MEM_COMMIT, PAGE_EXECUTE_READWRITE)) + exit_with_error(); + } + //Case: if shrinking the allocated size. + else if(new_size < old_size) { + // Shrinking the allocation: decommit all memory pages from the new limit to the old limit. + if (!VirtualFree((char*)p + new_size, (SIZE_T)(old_size - new_size), MEM_DECOMMIT)) + exit_with_error(); + } +} + +#endif + +//============================================================ +//================= Process Runtime ========================== +//============================================================ +#if defined(PLATFORM_OS_X) || defined(PLATFORM_LINUX) + +//------------------------------------------------------------ +//------------------- Structures ----------------------------- +//------------------------------------------------------------ + +//- working_dir: If not null, the working directory of the +// new process. +//- env_vars: If not null, the environment variables to set in the +// child process. Each string has format "MYVAR=MYVALUE" and is null-terminated. +//- argvs: The string arguments that are passed to the C main function. +typedef struct { + stz_byte* pipe; + stz_byte* in_pipe; + stz_byte* out_pipe; + stz_byte* err_pipe; + stz_byte* file; + stz_byte* working_dir; + stz_byte** env_vars; + stz_byte** argvs; +} EvalArg; + +#define RETURN_NEG(x) {int r=(x); if(r < 0) return -1;} + +//------------------------------------------------------------ +//-------------------- Utilities ----------------------------- +//------------------------------------------------------------ + +static int count_non_null (void** xs){ + int n=0; + while(xs[n] != NULL) + n++; + return n; +} + +static char* string_join (const char* a, const char* b){ + int len = strlen(a) + strlen(b); + char* buffer = (char*)stz_malloc(len + 1); + sprintf(buffer, "%s%s", a, b); + return buffer; +} + +//Opening a named pipe +static int open_pipe (const char* prefix, const char* suffix, int options){ + char* name = string_join(prefix, suffix); + int fd = open(name, options); + stz_free(name); + return fd; +} + +//Creating a named pipe +static int make_pipe (char* prefix, char* suffix){ + char* name = string_join(prefix, suffix); + return mkfifo(name, S_IRUSR|S_IWUSR); +} +#endif + +//------------------------------------------------------------ +//----------------------- Serialization ---------------------- +//------------------------------------------------------------ +#if defined(PLATFORM_LINUX) | defined(PLATFORM_OS_X) + +// ===== Serialization ===== +static void write_int (FILE* f, stz_int x){ + fwrite(&x, sizeof(stz_int), 1, f); +} +static void write_long (FILE* f, stz_long x){ + fwrite(&x, sizeof(stz_long), 1, f); +} +static void write_string (FILE* f, stz_byte* s){ + if(s == NULL) + write_int(f, -1); + else{ + size_t n = strlen(C_CSTR(s)); + write_int(f, (stz_int)n); + fwrite(s, 1, n, f); + } +} +static void write_strings (FILE* f, stz_byte** s){ + int n = count_non_null((void**)s); + write_int(f, (stz_int)n); + for(int i=0; ipipe); + write_string(f, earg->in_pipe); + write_string(f, earg->out_pipe); + write_string(f, earg->err_pipe); + write_string(f, earg->file); + write_string(f, earg->working_dir); + write_optional_strings(f, earg->env_vars); + write_strings(f, earg->argvs); +} + +static void write_process_state (FILE* f, ProcessState* s){ + write_int(f, s->state); + write_int(f, s->code); +} + +// ===== Deserialization ===== +static void bread (void* xs0, int size, int n0, FILE* f){ + char* xs = xs0; + int n = n0; + while(n > 0){ + int c = fread(xs, size, n, f); + if(c < n){ + if(ferror(f)) exit_with_error(); + if(feof(f)) return; + } + n = n - c; + xs = xs + size*c; + } +} +static stz_int read_int (FILE* f){ + stz_int n; + bread(&n, sizeof(stz_int), 1, f); + return n; +} +static stz_long read_long (FILE* f){ + stz_long n; + bread(&n, sizeof(stz_long), 1, f); + return n; +} +static stz_byte* read_string (FILE* f){ + stz_int n = read_int(f); + if(n < 0) + return NULL; + else{ + stz_byte* s = (stz_byte*)stz_malloc(n + 1); + bread(s, 1, (int)n, f); + s[n] = '\0'; + return s; + } +} +static stz_byte** read_strings (FILE* f){ + stz_int n = read_int(f); + stz_byte** xs = (stz_byte**)stz_malloc(sizeof(stz_byte*)*(n + 1)); + for(int i=0; ipipe = read_string(f); + earg->in_pipe = read_string(f); + earg->out_pipe = read_string(f); + earg->err_pipe = read_string(f); + earg->file = read_string(f); + earg->working_dir = read_string(f); + earg->env_vars = read_optional_strings(f); + earg->argvs = read_strings(f); + return earg; +} +static void read_process_state (FILE* f, ProcessState* s){ + s->state = read_int(f); + s->code = read_int(f); +} + +//===== Free ===== +static void free_strings (stz_byte** ss){ + for(int i=0; ss[i] != NULL; i++) + stz_free(ss[i]); + stz_free(ss); +} + +static void free_earg (EvalArg* arg){ + stz_free(arg->pipe); + if(arg->in_pipe != NULL) stz_free(arg->in_pipe); + if(arg->out_pipe != NULL) stz_free(arg->out_pipe); + if(arg->err_pipe != NULL) stz_free(arg->err_pipe); + stz_free(arg->file); + if(arg->working_dir != NULL) stz_free(arg->working_dir); + if(arg->env_vars != NULL) free_strings(arg->env_vars); + free_strings(arg->argvs); + stz_free(arg); +} + +//------------------------------------------------------------ +//-------------------- Process Queries ----------------------- +//------------------------------------------------------------ + +static void get_process_state (stz_long pid, ProcessState* s, int wait_for_termination){ + int status; + int ret = waitpid((pid_t)pid, &status, wait_for_termination? 0 : WNOHANG); + + if(ret == 0) + *s = (ProcessState){PROCESS_RUNNING, 0}; + else if(WIFEXITED(status)) + *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; + else if(WIFSIGNALED(status)) + *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; + else if(WIFSTOPPED(status)) + *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; + else + *s = (ProcessState){PROCESS_RUNNING, 0}; +} + +//------------------------------------------------------------ +//----------------------- Execvpe ---------------------------- +//------------------------------------------------------------ +//The 'execvpe' frontend to 'execvp' does not exist in OS-X. +//So implement a quick version of it. + +#ifdef PLATFORM_OS_X +extern char **environ; +int execvpe(const char *program, char **argv, char **envp){ + char **saved = environ; + environ = envp; + int rc = execvp(program, argv); + environ = saved; + return rc; +} +#endif + +//------------------------------------------------------------ +//---------------------- Launcher Main ----------------------- +//------------------------------------------------------------ + +#define LAUNCH_COMMAND 0 +#define STATE_COMMAND 1 +#define WAIT_COMMAND 2 + +static void write_error_and_exit (int fd){ + int code = errno; + write(fd, &code, sizeof(int)); + close(fd); + exit(-1); +} + +static void disable_ctrl_c () { + #ifdef PLATFORM_WINDOWS + SetConsoleCtrlHandler(NULL, TRUE); + #else + signal(SIGINT, SIG_IGN); + #endif +} + +static void launcher_main (FILE* lin, FILE* lout){ + disable_ctrl_c(); + while(1){ + //Read in command + int comm = fgetc(lin); + if(feof(lin)) exit(0); + + //Interpret launch process command + if(comm == LAUNCH_COMMAND){ + //Read in evaluation arguments + EvalArg* earg = read_earg(lin); + if(feof(lin)) exit(0); + + //Create error-code pipe + int READ = 0; + int WRITE = 1; + int exec_error[2]; + if(pipe(exec_error) < 0) exit_with_error(); + + //Fork a new child + stz_long pid = (stz_long)fork(); + if(pid < 0) exit_with_error(); + + if(pid > 0){ + //Read from error-code pipe + int exec_code; + close(exec_error[WRITE]); + int exec_r = read(exec_error[READ], &exec_code, sizeof(int)); + close(exec_error[READ]); + + if(exec_r == 0){ + //Exec evaluated successfully + //Return new process id + write_long(lout, pid); + fflush(lout); + } + else if(exec_r == sizeof(int)){ + //Exec evaluated unsuccessfully + //Return error code as negative long + write_long(lout, (stz_long)-exec_code); + fflush(lout); + } + else{ + fprintf(stderr, "Unreachable code."); + exit(-1); + } + }else{ + //Close exec pipe read, and close write end on successful exec + close(exec_error[READ]); + fcntl(exec_error[WRITE], F_SETFD, FD_CLOEXEC); + + //Open named pipes + if(earg->in_pipe != NULL){ + int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->in_pipe), O_RDONLY); + if(fd < 0) write_error_and_exit(exec_error[WRITE]); + dup2(fd, 0); + } + if(earg->out_pipe != NULL){ + int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->out_pipe), O_WRONLY); + if(fd < 0) write_error_and_exit(exec_error[WRITE]); + dup2(fd, 1); + } + if(earg->err_pipe != NULL){ + int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->err_pipe), O_WRONLY); + if(fd < 0) write_error_and_exit(exec_error[WRITE]); + dup2(fd, 2); + } + + //Set the working directory of the child process if explicitly + //requested. + if (earg->working_dir) { + if (chdir(C_CSTR(earg->working_dir)) == -1) { + write_error_and_exit(exec_error[WRITE]); + } + } + + //Launch child process. + //If an environment is supplied then call execvpe, otherwise call execvp. + if(earg->env_vars == NULL) + execvp(C_CSTR(earg->file), (char**)earg->argvs); + else + execvpe(C_CSTR(earg->file), (char**)earg->argvs, (char**)earg->env_vars); + + //Unsuccessful exec, write error number + write_error_and_exit(exec_error[WRITE]); + } + } + //Interpret state retrieval command + else if(comm == STATE_COMMAND || comm == WAIT_COMMAND){ + //Read in process id + stz_long pid = read_long(lin); + + //Retrieve state + ProcessState s; get_process_state(pid, &s, comm == WAIT_COMMAND); + write_process_state(lout, &s); + fflush(lout); + } + //Unrecognized command + else{ + fprintf(stderr, "Illegal command: %d\n", comm); + exit(-1); + } + } +} + +static stz_long launcher_pid = -1; +static FILE* launcher_in = NULL; +static FILE* launcher_out = NULL; +void initialize_launcher_process (){ + if(launcher_pid < 0){ + //Create pipes + int READ = 0; + int WRITE = 1; + int lin[2]; + int lout[2]; + if(pipe(lin) < 0) exit_with_error(); + if(pipe(lout) < 0) exit_with_error(); + + //Fork + stz_long pid = (stz_long)fork(); + if(pid < 0) exit_with_error(); + + if(pid > 0){ + //Parent + launcher_pid = pid; + close(lin[READ]); + close(lout[WRITE]); + launcher_in = fdopen(lin[WRITE], "w"); + if(launcher_in == NULL) exit_with_error(); + launcher_out = fdopen(lout[READ], "r"); + if(launcher_out == NULL) exit_with_error(); + } + else{ + //Child + close(lin[WRITE]); + close(lout[READ]); + FILE* fin = fdopen(lin[READ], "r"); + if(fin == NULL) exit_with_error(); + FILE* fout = fdopen(lout[WRITE], "w"); + if(fout == NULL) exit_with_error(); + launcher_main(fin, fout); + } + } +} + +static void make_pipe_name (char* pipe_name, int pipeid) { + sprintf(pipe_name, "/tmp/stanza_exec_pipe_%ld_%ld", (long)getpid(), (long)pipeid); +} + +static int delete_process_pipe (FILE* fd, char* pipe_name, char* suffix) { + if (fd != NULL) { + int close_res = fclose(fd); + if (close_res == EOF) return -1; + char my_pipe_name[80]; + sprintf(my_pipe_name, "%s%s", pipe_name, suffix); + int rm_res = remove(my_pipe_name); + if (rm_res < 0) return -1; + } + return 0; +} + +stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error, stz_int pipeid) { + char pipe_name[80]; + make_pipe_name(pipe_name, (int)pipeid); + if (delete_process_pipe(input, pipe_name, "_in") < 0) + return -1; + if (delete_process_pipe(output, pipe_name, "_out") < 0) + return -1; + if (delete_process_pipe(error, pipe_name, "_err") < 0) + return -1; + return 0; +} + +stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, + stz_int output, stz_int error, stz_int pipeid, + stz_byte* working_dir, stz_byte** env_vars, Process* process) { + //Initialize launcher if necessary + initialize_launcher_process(); + + //Figure out unique pipe name + char pipe_name[80]; + make_pipe_name(pipe_name, (int)pipeid); + + //Compute pipe sources + int pipe_sources[NUM_STREAM_SPECS]; + for(int i=0; i= 0) + RETURN_NEG(make_pipe(pipe_name, "_in")) + if(pipe_sources[PROCESS_OUT] >= 0) + RETURN_NEG(make_pipe(pipe_name, "_out")) + if(pipe_sources[PROCESS_ERR] >= 0) + RETURN_NEG(make_pipe(pipe_name, "_err")) + + //Write in command and evaluation arguments + EvalArg earg = {STZ_STR(pipe_name), NULL, NULL, NULL, file, working_dir, env_vars, argvs}; + if(input == PROCESS_IN) earg.in_pipe = STZ_STR("_in"); + if(output == PROCESS_OUT) earg.out_pipe = STZ_STR("_out"); + if(output == PROCESS_ERR) earg.out_pipe = STZ_STR("_err"); + if(error == PROCESS_OUT) earg.err_pipe = STZ_STR("_out"); + if(error == PROCESS_ERR) earg.err_pipe = STZ_STR("_err"); + int r = fputc(LAUNCH_COMMAND, launcher_in); + if(r == EOF) return -1; + write_earg(launcher_in, &earg); + fflush(launcher_in); + + printf("input = %d, output = %d, err = %d\n", input, output, error); + + //Open pipes to child + FILE* fin = NULL; + if(pipe_sources[PROCESS_IN] >= 0){ + int fd = open_pipe(pipe_name, "_in", O_WRONLY); + RETURN_NEG(fd) + fin = fdopen(fd, "w"); + if(fin == NULL) return -1; + } + FILE* fout = NULL; + if(pipe_sources[PROCESS_OUT] >= 0){ + int fd = open_pipe(pipe_name, "_out", O_RDONLY); + RETURN_NEG(fd) + fout = fdopen(fd, "r"); + if(fout == NULL) return -1; + } + FILE* ferr = NULL; + if(pipe_sources[PROCESS_ERR] >= 0){ + int fd = open_pipe(pipe_name, "_err", O_RDONLY); + RETURN_NEG(fd) + ferr = fdopen(fd, "r"); + if(ferr == NULL) return -1; + } + + //Read back process id, and set errno if failed + stz_long pid = read_long(launcher_out); + if(pid < 0){ + errno = (int)(- pid); + return -1; + } + + //Return process structure + process->pid = pid; + process->in = fin; + process->out = fout; + process->err = ferr; + return 0; +} + +int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ + stz_int pid = process->pid; + + //Check whether launcher has been initialized + if(launcher_pid < 0){ + fprintf(stderr, "Launcher not initialized.\n"); + exit(-1); + } + + //Send command + int r = fputc(wait_for_termination? WAIT_COMMAND : STATE_COMMAND, launcher_in); + if(r == EOF) exit_with_error(); + write_long(launcher_in, pid); + fflush(launcher_in); + + //Read back process state + read_process_state(launcher_out, s); + return 0; +} +#else +#include "process-win32.c" +//============================================================ +//============== End Process Runtime ========================= +//============================================================ +#endif + +#define STACK_TYPE 6 + +stz_long stanza_entry (VMInit* init); + +// Command line arguments +// ====================== +stz_int input_argc; +stz_byte** input_argv; +stz_int input_argv_needs_free; + +// Main Driver +// =========== +static void* alloc (VMInit* init, long tag, long size){ + void* ptr = init->heap_top + 8; + *(long*)(init->heap_top) = tag; + init->heap_top += 8 + size; + return ptr; +} + +static Stack* alloc_stack (VMInit* init){ + Stack* stack = alloc(init, STACK_TYPE, sizeof(Stack)); + stz_long initial_stack_size = 8 * 1024; + StackFrame* frames = (StackFrame*)stz_malloc(initial_stack_size); + stack->size = initial_stack_size; + stack->frames = frames; + stack->stack_pointer = NULL; + stack->tail = NULL; + return stack; +} + +//Given a pointer to a struct allocated on the heap, +//add the tag bits to the pointer. +uint64_t tag_as_ref (void* p){ + return (uint64_t)p - 8 + 1; +} + +enum { + LOG_BITS_IN_BYTE = 3, + LOG_BYTES_IN_LONG = 3, + LOG_BITS_IN_LONG = LOG_BYTES_IN_LONG + LOG_BITS_IN_BYTE, + BYTES_IN_LONG = 1 << LOG_BYTES_IN_LONG, + BITS_IN_LONG = 1 << LOG_BITS_IN_LONG +}; + +#define SYSTEM_PAGE_SIZE 4096ULL +#define ROUND_UP_TO_WHOLE_PAGES(x) (((x) + (SYSTEM_PAGE_SIZE - 1)) & ~(SYSTEM_PAGE_SIZE - 1)) +#define ROUND_UP_TO_WHOLE_LONGS(x) (((x) + (sizeof(stz_long) - 1)) & ~(sizeof(stz_long) - 1)) + +static stz_long bitset_size (stz_long heap_size) { + uint64_t heap_size_in_longs = (heap_size + (BYTES_IN_LONG - 1)) >> LOG_BYTES_IN_LONG; + uint64_t bitset_size_in_longs = (heap_size_in_longs + (BITS_IN_LONG - 1)) >> LOG_BITS_IN_LONG; + return ROUND_UP_TO_WHOLE_PAGES(bitset_size_in_longs << LOG_BYTES_IN_LONG); +} + +//Use 'main' as the standard name for the C main function unless +//RENAME_STANZA_MAIN is passed as a flag. If it is, then rename 'main' +//to 'stanza_main'. +#ifdef RENAME_STANZA_MAIN + #define MAIN_FUNC stanza_main +#else + #define MAIN_FUNC main +#endif + +STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { + input_argc = (stz_int)argc; + input_argv = (stz_byte **)argv; + input_argv_needs_free = 0; + VMInit init; + + //Allocate heap + const stz_long min_heap_size = ROUND_UP_TO_WHOLE_PAGES(8 * 1024 * 1024); + const stz_long max_heap_size = ROUND_UP_TO_WHOLE_PAGES(STZ_LONG(8) * 1024 * 1024 * 1024); + init.heap_start = (stz_byte*)stz_memory_map(min_heap_size, max_heap_size); + init.heap_max_size = max_heap_size; + init.heap_size_limit = max_heap_size; + init.heap_size = min_heap_size; + + //Setup the nursery + const stz_long nursery_fraction = 8; // Must match the value in core.stanza + const stz_long nursery_size = ROUND_UP_TO_WHOLE_LONGS(min_heap_size / nursery_fraction / 2); + init.heap_old_objects_end = init.heap_start; + init.heap_top = init.heap_old_objects_end + nursery_size; + init.heap_limit = init.heap_top + nursery_size; + + //Allocate bitset for heap + const stz_long min_bitset_size = bitset_size(min_heap_size); + const stz_long max_bitset_size = bitset_size(max_heap_size); + init.heap_bitset = (stz_byte*)stz_memory_map(min_bitset_size, max_bitset_size); + init.heap_bitset_base = init.heap_bitset - ((uint64_t)init.heap_start >> 6); + memset(init.heap_bitset, 0, min_bitset_size); + + //For bitset_base computation to work: bitset must be aligned to 512-bytes boundary. + if((uint64_t)init.heap_bitset % 512 != 0){ + fprintf(stderr, "Unaligned bitset: %p.\n", init.heap_bitset); + exit(-1); + } + + //Allocate marking stack for heap + const stz_long marking_stack_size = ROUND_UP_TO_WHOLE_PAGES((1024 * 1024L) << LOG_BYTES_IN_LONG); + init.marking_stack_start = stz_memory_map(marking_stack_size, marking_stack_size); + init.marking_stack_bottom = init.marking_stack_start + marking_stack_size; + init.marking_stack_top = init.marking_stack_bottom; + + //Allocate stacks + Stack* entry_stack = alloc_stack(&init); + Stack* entry_system_stack = alloc_stack(&init); + entry_stack->tail = entry_system_stack; + init.current_stack = tag_as_ref(entry_stack); + init.system_stack = tag_as_ref(entry_system_stack); + init.stacks = entry_stack; + + //Initialize trackers to empty list. + init.trackers = NULL; + + //Call Stanza entry + stanza_entry(&init); + + //Heap and freespace are disposed by OS at process termination + return 0; +} diff --git a/runtime/driver.c b/runtime/driver.c index 88d2c87d..bdbd6a8b 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -640,6 +640,7 @@ static int open_pipe (const char* prefix, const char* suffix, int options){ } //Creating a named pipe +// TODO: free name static int make_pipe (char* prefix, char* suffix){ char* name = string_join(prefix, suffix); return mkfifo(name, S_IRUSR|S_IWUSR); @@ -685,17 +686,6 @@ static void write_optional_strings (FILE* f, stz_byte** s){ } } -static void write_earg (FILE* f, EvalArg* earg){ - write_string(f, earg->pipe); - write_string(f, earg->in_pipe); - write_string(f, earg->out_pipe); - write_string(f, earg->err_pipe); - write_string(f, earg->file); - write_string(f, earg->working_dir); - write_optional_strings(f, earg->env_vars); - write_strings(f, earg->argvs); -} - static void write_process_state (FILE* f, ProcessState* s){ write_int(f, s->state); write_int(f, s->code); @@ -842,187 +832,6 @@ static void write_error_and_exit (int fd){ exit(-1); } -static void disable_ctrl_c () { - #ifdef PLATFORM_WINDOWS - SetConsoleCtrlHandler(NULL, TRUE); - #else - signal(SIGINT, SIG_IGN); - #endif -} - -static void launcher_main (FILE* lin, FILE* lout){ - disable_ctrl_c(); - while(1){ - //Read in command - int comm = fgetc(lin); - printf("comm = %d", comm); - if(feof(lin)) exit(0); - - //Interpret launch process command - if(comm == LAUNCH_COMMAND){ - //Read in evaluation arguments - EvalArg* earg = read_earg(lin); - if(feof(lin)) exit(0); - - //Create error-code pipe - int READ = 0; - int WRITE = 1; - int exec_error[2]; - if(pipe(exec_error) < 0) exit_with_error(); - - //Fork a new child - //stz_long pid = (stz_long)fork(); - // Setup - posix_spawnattr_t attr; - posix_spawn_file_actions_t actions; - posix_spawnattr_init(&attr); - posix_spawn_file_actions_init(&actions); - // File actions - // TODO: fix all the warnings - int posix_ret; - if (posix_ret = posix_spawn_file_actions_addopen(&actions, 0, earg->in_pipe, O_RDONLY, 0)) - exit_with_error(); - if (posix_ret = posix_spawn_file_actions_addopen(&actions, 1, earg->out_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644)) - exit_with_error(); - if (posix_ret = posix_spawn_file_actions_addopen(&actions, 2, earg->err_pipe, O_WRONLY | O_CREAT | O_TRUNC, 0644)) - exit_with_error(); - if (earg->working_dir) { - if (posix_ret = posix_spawn_file_actions_addchdir_np(&actions, earg->working_dir)) - exit_with_error(); - } - // Spawn child - pid_t child_pid; - stz_long pid = (stz_long)posix_spawn(&child_pid, earg->file, &actions, &attr, earg->argvs, earg->env_vars); - //if(pid < 0) exit_with_error(); - - if(pid == 0){ - //Read from error-code pipe - //int exec_code; - //close(exec_error[WRITE]); - //int exec_r = read(exec_error[READ], &exec_code, sizeof(int)); - //close(exec_error[READ]); - int exec_r; - waitpid(pid, &exec_r, 0); - //if(exec_r == 0){ - // //Exec evaluated successfully - // //Return new process id - // write_long(lout, pid); - // fflush(lout); - //} - //else if(exec_r == sizeof(int)){ - // //Exec evaluated unsuccessfully - // //Return error code as negative long - // write_long(lout, (stz_long)-exec_code); - // fflush(lout); - //} - //else{ - // fprintf(stderr, "Unreachable code."); - // exit(-1); - //} - }else{ - close(exec_error[READ]); - fcntl(exec_error[WRITE], F_SETFD, FD_CLOEXEC); - exit_with_error(); - //Close exec pipe read, and close write end on successful exec - - //Open named pipes - //if(earg->in_pipe != NULL){ - // int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->in_pipe), O_RDONLY); - // if(fd < 0) write_error_and_exit(exec_error[WRITE]); - // dup2(fd, 0); - //} - //if(earg->out_pipe != NULL){ - // int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->out_pipe), O_WRONLY); - // if(fd < 0) write_error_and_exit(exec_error[WRITE]); - // dup2(fd, 1); - //} - //if(earg->err_pipe != NULL){ - // int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->err_pipe), O_WRONLY); - // if(fd < 0) write_error_and_exit(exec_error[WRITE]); - // dup2(fd, 2); - //} - - //Set the working directory of the child process if explicitly - //requested. - //if (earg->working_dir) { - // if (chdir(C_CSTR(earg->working_dir)) == -1) { - // write_error_and_exit(exec_error[WRITE]); - // } - //} - - //Launch child process. - //If an environment is supplied then call execvpe, otherwise call execvp. - //if(earg->env_vars == NULL) - // execvp(C_CSTR(earg->file), (char**)earg->argvs); - //else - // execvpe(C_CSTR(earg->file), (char**)earg->argvs, (char**)earg->env_vars); - - //Unsuccessful exec, write error number - //write_error_and_exit(exec_error[WRITE]); - } - // TODO: clean up spawn resources - posix_spawn_file_actions_destroy(&actions); - posix_spawnattr_destroy(&attr); - } - //Interpret state retrieval command - else if(comm == STATE_COMMAND || comm == WAIT_COMMAND){ - //Read in process id - stz_long pid = read_long(lin); - - //Retrieve state - ProcessState s; get_process_state(pid, &s, comm == WAIT_COMMAND); - write_process_state(lout, &s); - fflush(lout); - } - //Unrecognized command - else{ - fprintf(stderr, "Illegal command: %d\n", comm); - exit(-1); - } - } -} - -static stz_long launcher_pid = -1; -static FILE* launcher_in = NULL; -static FILE* launcher_out = NULL; -void initialize_launcher_process (){ - if(launcher_pid < 0){ - //Create pipes - int READ = 0; - int WRITE = 1; - int lin[2]; - int lout[2]; - if(pipe(lin) < 0) exit_with_error(); - if(pipe(lout) < 0) exit_with_error(); - - //Fork - //stz_long pid = (stz_long)fork(); - //if(pid < 0) exit_with_error(); - - //if(pid > 0){ - // //Parent - // launcher_pid = pid; - //close(lin[READ]); - //close(lout[WRITE]); - launcher_in = fdopen(lin[WRITE], "w"); - if(launcher_in == NULL) exit_with_error(); - launcher_out = fdopen(lout[READ], "r"); - if(launcher_out == NULL) exit_with_error(); - // } - // else{ - //Child - //close(lin[WRITE]); - //close(lout[READ]); - FILE* fin = fdopen(lin[READ], "r"); - if(fin == NULL) exit_with_error(); - FILE* fout = fdopen(lout[WRITE], "w"); - if(fout == NULL) exit_with_error(); - printf("launching\n"); - launcher_main(fin, fout); - // } - } -} - static void make_pipe_name (char* pipe_name, int pipeid) { sprintf(pipe_name, "/tmp/stanza_exec_pipe_%ld_%ld", (long)getpid(), (long)pipeid); } @@ -1051,12 +860,39 @@ stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error, stz_int pi return 0; } +//#ifdef PLATFORM_OS_X +//stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, +// stz_int output, stz_int error, stz_int pipeid, +// stz_byte* working_dir, stz_byte** env_vars, Process* process) { +// //Compute pipe sources: pipe_sources[i] +// int pipe_sources[NUM_STREAM_SPECS]; +// for(int i=0; i= 0) RETURN_NEG(make_pipe(pipe_name, "_err")) - //Write in command and evaluation arguments - EvalArg earg = {STZ_STR(pipe_name), NULL, NULL, NULL, file, working_dir, env_vars, argvs}; - if(input == PROCESS_IN) earg.in_pipe = STZ_STR("_in"); - if(output == PROCESS_OUT) earg.out_pipe = STZ_STR("_out"); - if(output == PROCESS_ERR) earg.out_pipe = STZ_STR("_err"); - if(error == PROCESS_OUT) earg.err_pipe = STZ_STR("_out"); - if(error == PROCESS_ERR) earg.err_pipe = STZ_STR("_err"); - printf("Writing command\n"); - int r = fputc(LAUNCH_COMMAND, launcher_in); - if(r == EOF) return -1; - write_earg(launcher_in, &earg); - fflush(launcher_in); - - //Open pipes to child + stz_byte* in_pipe = NULL; + stz_byte* out_pipe = NULL; + stz_byte* err_pipe = NULL; + + int infd = -1; + int outfd = -1; + int errfd = -1; + + //Open pipes to child process FILE* fin = NULL; if(pipe_sources[PROCESS_IN] >= 0){ - int fd = open_pipe(pipe_name, "_in", O_WRONLY); - RETURN_NEG(fd) - fin = fdopen(fd, "w"); + infd = open_pipe(pipe_name, "_in", O_RDWR); + RETURN_NEG(infd) + fin = fdopen(infd, "w"); if(fin == NULL) return -1; } FILE* fout = NULL; if(pipe_sources[PROCESS_OUT] >= 0){ - int fd = open_pipe(pipe_name, "_out", O_RDONLY); - RETURN_NEG(fd) - fout = fdopen(fd, "r"); + outfd = open_pipe(pipe_name, "_out", O_RDWR); + RETURN_NEG(outfd) + printf("fout = fdopen(%d) (%s)\n", outfd, string_join(pipe_name, "_out")); + fout = fdopen(outfd, "r"); if(fout == NULL) return -1; } FILE* ferr = NULL; if(pipe_sources[PROCESS_ERR] >= 0){ - int fd = open_pipe(pipe_name, "_err", O_RDONLY); - RETURN_NEG(fd) - ferr = fdopen(fd, "r"); + int errfd = open_pipe(pipe_name, "_err", O_RDWR); + RETURN_NEG(errfd) + printf("ferr = fdopen(%d) (%s)\n", errfd, string_join(pipe_name, "_err")); + ferr = fdopen(errfd, "r"); if(ferr == NULL) return -1; } - //Read back process id, and set errno if failed - stz_long pid = read_long(launcher_out); - if(pid < 0){ - errno = (int)(- pid); - return -1; + // Compute final file descriptors based on pipe sources + int final_outfd = outfd; + int final_errfd = errfd; + + if(input == PROCESS_IN) in_pipe = STZ_STR("_in"); + if(output == PROCESS_OUT) { + out_pipe = STZ_STR("_out"); + final_outfd = outfd; } + if(output == PROCESS_ERR) { + out_pipe = STZ_STR("_err"); + final_outfd = errfd; + } + if(error == PROCESS_OUT) { + err_pipe = STZ_STR("_out"); + final_errfd = outfd; + } + if(error == PROCESS_ERR) { + err_pipe = STZ_STR("_err"); + final_errfd = errfd; + } + + + + pid_t pid = -1; + + // OS X : use posix_spawn to initialize child process + #ifdef PLATFORM_OS_X + // Setup + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + char* child_in_pipe = in_pipe ? string_join(pipe_name, in_pipe) : NULL; + char* child_out_pipe = out_pipe ? string_join(pipe_name, out_pipe) : NULL; + char* child_err_pipe = err_pipe ? string_join(pipe_name, err_pipe) : NULL; + // File actions + // TODO: fix all the warnings + int posix_ret; + if (in_pipe) { + if ((posix_spawn_file_actions_addopen(&actions, 0, child_in_pipe, O_RDONLY, 0))) + exit_with_error(); + if ((posix_spawn_file_actions_adddup2(&actions, infd, 0))) + exit_with_error(); + } + if (out_pipe) { + if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 1, child_out_pipe, O_RDWR, 0666))) + exit_with_error(); + printf("Opened %s to fd 1\n", child_out_pipe); + printf("out: dup2(%d, 1), file = %s\n", 4, child_out_pipe); + //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 4, 1))) + //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 1, final_outfd))) + //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, final_outfd, 1))) + // exit_with_error(); + } + if (err_pipe) { + //if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 5, child_err_pipe, O_RDWR, 0666))) + // exit_with_error(); + //printf("err: dup2(%d, 2), file = %s\n", 5, child_err_pipe); + if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 2, 1))) + exit_with_error(); + printf("err: dup2(2, 1)\n"); + } + if (working_dir) { + printf("working dir: %s\n", working_dir); + if ((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, working_dir))) + exit_with_error(); + } + // Call spawn + if(posix_spawn(&pid, file, &actions, NULL, argvs, env_vars) == 0) { + // Parent process + printf("Success: child process = %d\n", pid); + //int exec_r; + //waitpid(pid, &exec_r, 0); + //printf("Child process finished\n"); + } else { + exit_with_error(); + } + + // Clean up spawn resources + posix_spawn_file_actions_destroy(&actions); + stz_free(child_in_pipe); + stz_free(child_out_pipe); + stz_free(child_err_pipe); + #endif + + // TODO: Linux: use vfork to initialize child process + #ifdef PLATFORM_LINUX + #endif + + // TODO: + //Read back process id, and set errno if failed + //stz_long pid = read_long(launcher_out); + //if(pid < 0){ + // errno = (int)(- pid); + // return -1; + //} //Return process structure process->pid = pid; process->in = fin; process->out = fout; process->err = ferr; + printf("Returning\n"); return 0; } int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ - stz_int pid = process->pid; - - //Check whether launcher has been initialized - if(launcher_pid < 0){ - fprintf(stderr, "Launcher not initialized.\n"); - exit(-1); - } - //Send command - int r = fputc(wait_for_termination? WAIT_COMMAND : STATE_COMMAND, launcher_in); - if(r == EOF) exit_with_error(); - write_long(launcher_in, pid); - fflush(launcher_in); + int status; + int ret = waitpid((pid_t)(process->pid), &status, wait_for_termination? 0 : WNOHANG); + + if(ret == 0) + *s = (ProcessState){PROCESS_RUNNING, 0}; + else if(WIFEXITED(status)) + *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; + else if(WIFSIGNALED(status)) + *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; + else if(WIFSTOPPED(status)) + *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; + else + *s = (ProcessState){PROCESS_RUNNING, 0}; - //Read back process state - read_process_state(launcher_out, s); return 0; } +// stz_int pid = process->pid; + +// //Check whether launcher has been initialized +// if(launcher_pid < 0){ +// fprintf(stderr, "Launcher not initialized.\n"); +// exit(-1); +// } + +// //Send command +// int r = fputc(wait_for_termination? WAIT_COMMAND : STATE_COMMAND, launcher_in); +// if(r == EOF) exit_with_error(); +// write_long(launcher_in, pid); +// fflush(launcher_in); + +// //Read back process state +// read_process_state(launcher_out, s); +// return 0; +//} #else #include "process-win32.c" //============================================================ From a39d1e42e29920a49e153fc7e39cf38b34282bb4 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 15 Nov 2023 14:27:55 -0800 Subject: [PATCH 04/53] sorta works --- compiler/config.stanza | 2 +- core/core.stanza | 4 - runtime/driver.c | 417 ++++++++++++++++++++++++----------------- 3 files changed, 249 insertions(+), 174 deletions(-) diff --git a/compiler/config.stanza b/compiler/config.stanza index 4934cb50..d90a018b 100644 --- a/compiler/config.stanza +++ b/compiler/config.stanza @@ -268,7 +268,7 @@ defn verify-installation () : catch (e:Exception) : ;Could not retrieve version of the executable. val msg = "Stanza install directory is set to %_, \ - but the version of the Stanza executable could not verified. %_" + but the version of the Stanza executable could not be verified. %_" throw $ Exception(msg % [STANZA-INSTALL-DIR, e]) ;Check that the version string is well-formatted. diff --git a/core/core.stanza b/core/core.stanza index 9f6a085c..53cb2696 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -10341,18 +10341,14 @@ public defn call-system-and-get-output (file:String, working-dir:String|False, env-vars:Tuple>|False) -> String : val buffer = StringBuffer() - println("Making process %_" % [file]) val proc = Process(file, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, working-dir, env-vars) - println("Made process %_" % [file]) val proc-out = output-stream(proc) let loop () : val c = get-char(proc-out) match(c:Char) : add(buffer,c) loop() - println("Waiting for %_" % [file]) wait(proc) - println("%_ done" % [file]) to-string(buffer) public defn call-system-and-get-output (file:String, args:Seqable) -> String : diff --git a/runtime/driver.c b/runtime/driver.c index bdbd6a8b..1287690c 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -851,53 +851,20 @@ static int delete_process_pipe (FILE* fd, char* pipe_name, char* suffix) { stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error, stz_int pipeid) { char pipe_name[80]; make_pipe_name(pipe_name, (int)pipeid); - if (delete_process_pipe(input, pipe_name, "_in") < 0) - return -1; - if (delete_process_pipe(output, pipe_name, "_out") < 0) - return -1; - if (delete_process_pipe(error, pipe_name, "_err") < 0) - return -1; + //if (delete_process_pipe(input, pipe_name, "_in") < 0) + // return -1; + //if (delete_process_pipe(output, pipe_name, "_out") < 0) + // return -1; + //if (delete_process_pipe(error, pipe_name, "_err") < 0) + // return -1; return 0; } -//#ifdef PLATFORM_OS_X -//stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, -// stz_int output, stz_int error, stz_int pipeid, -// stz_byte* working_dir, stz_byte** env_vars, Process* process) { -// //Compute pipe sources: pipe_sources[i] -// int pipe_sources[NUM_STREAM_SPECS]; -// for(int i=0; i= 0) - RETURN_NEG(make_pipe(pipe_name, "_in")) - if(pipe_sources[PROCESS_OUT] >= 0) - RETURN_NEG(make_pipe(pipe_name, "_out")) - if(pipe_sources[PROCESS_ERR] >= 0) - RETURN_NEG(make_pipe(pipe_name, "_err")) - - stz_byte* in_pipe = NULL; - stz_byte* out_pipe = NULL; - stz_byte* err_pipe = NULL; - - int infd = -1; - int outfd = -1; - int errfd = -1; - - //Open pipes to child process - FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] >= 0){ - infd = open_pipe(pipe_name, "_in", O_RDWR); - RETURN_NEG(infd) - fin = fdopen(infd, "w"); - if(fin == NULL) return -1; - } - FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] >= 0){ - outfd = open_pipe(pipe_name, "_out", O_RDWR); - RETURN_NEG(outfd) - printf("fout = fdopen(%d) (%s)\n", outfd, string_join(pipe_name, "_out")); - fout = fdopen(outfd, "r"); - if(fout == NULL) return -1; - } - FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] >= 0){ - int errfd = open_pipe(pipe_name, "_err", O_RDWR); - RETURN_NEG(errfd) - printf("ferr = fdopen(%d) (%s)\n", errfd, string_join(pipe_name, "_err")); - ferr = fdopen(errfd, "r"); - if(ferr == NULL) return -1; + //Generate array of pipes per-input-source + //TODO: close these at some point + int pipes[NUM_STREAM_SPECS][2]; + for(int i=0; i 0) { + // Close write end + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) + exit_with_error(); + // dup read to STDIN + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) + exit_with_error(); } - if(output == PROCESS_ERR) { - out_pipe = STZ_STR("_err"); - final_outfd = errfd; + //Setup output pipe if used + if(pipe_sources[PROCESS_OUT] > 0) { + // Close read end + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) + exit_with_error(); + // dup write to STDOUT + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) + exit_with_error(); } - if(error == PROCESS_OUT) { - err_pipe = STZ_STR("_out"); - final_errfd = outfd; + //Setup error pipe if used + if(pipe_sources[PROCESS_ERR] > 0) { + // Close read end + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) + exit_with_error(); + // dup write to STDERR + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) + exit_with_error(); } - if(error == PROCESS_ERR) { - err_pipe = STZ_STR("_err"); - final_errfd = errfd; + //Setup working directory + if(working_dir) { + if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, working_dir))) + exit_with_error(); } - - + // Spawn process pid_t pid = -1; + if(posix_spawn(&pid, file, &actions, NULL, argvs, env_vars) == 0) { + printf("Child pid: %d\n", pid); + } else { + exit_with_error(); + } - // OS X : use posix_spawn to initialize child process - #ifdef PLATFORM_OS_X - // Setup - posix_spawn_file_actions_t actions; - posix_spawn_file_actions_init(&actions); - char* child_in_pipe = in_pipe ? string_join(pipe_name, in_pipe) : NULL; - char* child_out_pipe = out_pipe ? string_join(pipe_name, out_pipe) : NULL; - char* child_err_pipe = err_pipe ? string_join(pipe_name, err_pipe) : NULL; - // File actions - // TODO: fix all the warnings - int posix_ret; - if (in_pipe) { - if ((posix_spawn_file_actions_addopen(&actions, 0, child_in_pipe, O_RDONLY, 0))) - exit_with_error(); - if ((posix_spawn_file_actions_adddup2(&actions, infd, 0))) - exit_with_error(); - } - if (out_pipe) { - if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 1, child_out_pipe, O_RDWR, 0666))) - exit_with_error(); - printf("Opened %s to fd 1\n", child_out_pipe); - printf("out: dup2(%d, 1), file = %s\n", 4, child_out_pipe); - //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 4, 1))) - //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 1, final_outfd))) - //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, final_outfd, 1))) - // exit_with_error(); - } - if (err_pipe) { - //if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 5, child_err_pipe, O_RDWR, 0666))) - // exit_with_error(); - //printf("err: dup2(%d, 2), file = %s\n", 5, child_err_pipe); - if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 2, 1))) - exit_with_error(); - printf("err: dup2(2, 1)\n"); - } - if (working_dir) { - printf("working dir: %s\n", working_dir); - if ((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, working_dir))) - exit_with_error(); - } - // Call spawn - if(posix_spawn(&pid, file, &actions, NULL, argvs, env_vars) == 0) { - // Parent process - printf("Success: child process = %d\n", pid); - //int exec_r; - //waitpid(pid, &exec_r, 0); - //printf("Child process finished\n"); - } else { - exit_with_error(); - } + // Cleanup + posix_spawn_file_actions_destroy(&actions); - // Clean up spawn resources - posix_spawn_file_actions_destroy(&actions); - stz_free(child_in_pipe); - stz_free(child_out_pipe); - stz_free(child_err_pipe); - #endif - - // TODO: Linux: use vfork to initialize child process - #ifdef PLATFORM_LINUX - #endif - - // TODO: - //Read back process id, and set errno if failed - //stz_long pid = read_long(launcher_out); - //if(pid < 0){ - // errno = (int)(- pid); - // return -1; - //} + //Parent: + //Close pipes, setup files + FILE* fin = NULL; + if(pipe_sources[PROCESS_IN] > 0) { + close(pipes[PROCESS_IN][0]); + fin = fdopen(pipes[PROCESS_IN][1], "w"); + if(fin == NULL) return -1; + } + FILE* fout = NULL; + if(pipe_sources[PROCESS_OUT] > 0) { + close(pipes[PROCESS_OUT][1]); + fout = fdopen(pipes[PROCESS_OUT][0], "r"); + if(fout == NULL) return -1; + } + FILE* ferr = NULL; + if(pipe_sources[PROCESS_ERR] > 0) { + close(pipes[PROCESS_ERR][1]); + ferr = fdopen(pipes[PROCESS_ERR][0], "r"); + if(ferr == NULL) return -1; + } - //Return process structure process->pid = pid; process->in = fin; process->out = fout; process->err = ferr; - printf("Returning\n"); return 0; } +#endif + +// +// +//#ifdef PLATFORM_LINUX +//stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, +// stz_int output, stz_int error, stz_int pipeid, +// stz_byte* working_dir, stz_byte** env_vars, Process* process) { +//} +//#endif + +// Old version +//stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, +// stz_int output, stz_int error, stz_int pipeid, +// stz_byte* working_dir, stz_byte** env_vars, Process* process) { +// //Figure out unique pipe name +// char pipe_name[80]; +// make_pipe_name(pipe_name, (int)pipeid); +// +// //Compute pipe sources +// int pipe_sources[NUM_STREAM_SPECS]; +// for(int i=0; i= 0) +// RETURN_NEG(make_pipe(pipe_name, "_in")) +// if(pipe_sources[PROCESS_OUT] >= 0) +// RETURN_NEG(make_pipe(pipe_name, "_out")) +// if(pipe_sources[PROCESS_ERR] >= 0) +// RETURN_NEG(make_pipe(pipe_name, "_err")) +// +// stz_byte* in_pipe = NULL; +// stz_byte* out_pipe = NULL; +// stz_byte* err_pipe = NULL; +// +// int infd = -1; +// int outfd = -1; +// int errfd = -1; +// +// //Open pipes to child process +// FILE* fin = NULL; +// if(pipe_sources[PROCESS_IN] >= 0){ +// infd = open_pipe(pipe_name, "_in", O_RDWR); +// RETURN_NEG(infd) +// fin = fdopen(infd, "w"); +// if(fin == NULL) return -1; +// } +// FILE* fout = NULL; +// if(pipe_sources[PROCESS_OUT] >= 0){ +// outfd = open_pipe(pipe_name, "_out", O_RDWR); +// int fd2 = open_pipe(pipe_name, "_out", O_RDONLY | O_NONBLOCK); +// printf("outfd = %d, fd2 = %d\n", outfd, fd2); +// RETURN_NEG(outfd) +// printf("fout = fdopen(%d) (%s)\n", outfd, string_join(pipe_name, "_out")); +// fout = fdopen(outfd, "r"); +// if(fout == NULL) return -1; +// } +// FILE* ferr = NULL; +// if(pipe_sources[PROCESS_ERR] >= 0){ +// int errfd = open_pipe(pipe_name, "_err", O_RDWR); +// RETURN_NEG(errfd) +// printf("ferr = fdopen(%d) (%s)\n", errfd, string_join(pipe_name, "_err")); +// ferr = fdopen(errfd, "r"); +// if(ferr == NULL) return -1; +// } +// +// // Compute final file descriptors based on pipe sources +// int final_outfd = outfd; +// int final_errfd = errfd; +// +// if(input == PROCESS_IN) in_pipe = STZ_STR("_in"); +// if(output == PROCESS_OUT) { +// out_pipe = STZ_STR("_out"); +// final_outfd = outfd; +// } +// if(output == PROCESS_ERR) { +// out_pipe = STZ_STR("_err"); +// final_outfd = errfd; +// } +// if(error == PROCESS_OUT) { +// err_pipe = STZ_STR("_out"); +// final_errfd = outfd; +// } +// if(error == PROCESS_ERR) { +// err_pipe = STZ_STR("_err"); +// final_errfd = errfd; +// } +// +// +// +// pid_t pid = -1; +// +// // OS X : use posix_spawn to initialize child process +// #ifdef PLATFORM_OS_X +// // Setup +// posix_spawn_file_actions_t actions; +// posix_spawn_file_actions_init(&actions); +// char* child_in_pipe = in_pipe ? string_join(pipe_name, in_pipe) : NULL; +// char* child_out_pipe = out_pipe ? string_join(pipe_name, out_pipe) : NULL; +// char* child_err_pipe = err_pipe ? string_join(pipe_name, err_pipe) : NULL; +// // File actions +// // TODO: fix all the warnings +// int posix_ret; +// if (in_pipe) { +// if ((posix_spawn_file_actions_addopen(&actions, 0, child_in_pipe, O_RDONLY, 0))) +// exit_with_error(); +// if ((posix_spawn_file_actions_adddup2(&actions, infd, 0))) +// exit_with_error(); +// } +// if (out_pipe) { +// if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 1, child_out_pipe, O_RDWR, 0666))) +// exit_with_error(); +// printf("Opened %s to fd 1\n", child_out_pipe); +// printf("out: dup2(%d, 1), file = %s\n", 4, child_out_pipe); +// //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 4, 1))) +// //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 1, final_outfd))) +// //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, final_outfd, 1))) +// // exit_with_error(); +// } +// if (err_pipe) { +// //if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 5, child_err_pipe, O_RDWR, 0666))) +// // exit_with_error(); +// //printf("err: dup2(%d, 2), file = %s\n", 5, child_err_pipe); +// if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 2, 1))) +// exit_with_error(); +// printf("err: dup2(2, 1)\n"); +// } +// if (working_dir) { +// printf("working dir: %s\n", working_dir); +// if ((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, working_dir))) +// exit_with_error(); +// } +// // Call spawn +// if(posix_spawn(&pid, file, &actions, NULL, argvs, env_vars) == 0) { +// // Parent process +// printf("Success: child process = %d\n", pid); +// //int exec_r; +// //waitpid(pid, &exec_r, 0); +// //printf("Child process finished\n"); +// } else { +// exit_with_error(); +// } +// +// // Clean up spawn resources +// posix_spawn_file_actions_destroy(&actions); +// stz_free(child_in_pipe); +// stz_free(child_out_pipe); +// stz_free(child_err_pipe); +// #endif +// +// // TODO: Linux: use vfork to initialize child process +// #ifdef PLATFORM_LINUX +// #endif +// +// // TODO: +// //Read back process id, and set errno if failed +// //stz_long pid = read_long(launcher_out); +// //if(pid < 0){ +// // errno = (int)(- pid); +// // return -1; +// //} +// +// //Return process structure +// process->pid = pid; +// process->in = fin; +// process->out = fout; +// process->err = ferr; +// printf("Returning\n"); +// return 0; +//} int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ int status; int ret = waitpid((pid_t)(process->pid), &status, wait_for_termination? 0 : WNOHANG); - if(ret == 0) *s = (ProcessState){PROCESS_RUNNING, 0}; else if(WIFEXITED(status)) From 3d800ba869fd275b9559fa7318876c3cbfc2ca70 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 15 Nov 2023 16:08:18 -0800 Subject: [PATCH 05/53] works on osx --- compiler/config.stanza | 5 +- compiler/main.stanza | 1 - runtime/driver-old.c | 1248 ---------------------------------------- runtime/driver.c | 341 +---------- 4 files changed, 24 insertions(+), 1571 deletions(-) delete mode 100644 runtime/driver-old.c diff --git a/compiler/config.stanza b/compiler/config.stanza index d90a018b..44a06a76 100644 --- a/compiler/config.stanza +++ b/compiler/config.stanza @@ -260,10 +260,7 @@ defn verify-installation () : val v = try : ;Retrieve the version of the executable. - println("Calling %_" % [exe-file]) - val o = call-system-and-get-output(exe-file, [exe-file, "version", "-terse"]) - println("Called %_" % [exe-file]) - o + call-system-and-get-output(exe-file, [exe-file, "version", "-terse"]) catch (e:Exception) : ;Could not retrieve version of the executable. diff --git a/compiler/main.stanza b/compiler/main.stanza index ea448d0f..b9b0eedf 100644 --- a/compiler/main.stanza +++ b/compiler/main.stanza @@ -159,7 +159,6 @@ defn build-system (verbose?:True|False) : ;First directly try calling the cc-prog driver. try : ;Call system - println("Calling %_" % [to-tuple(args)]) val return-code = call-system(to-tuple(args)) ;Return true if successful return-code == 0 diff --git a/runtime/driver-old.c b/runtime/driver-old.c deleted file mode 100644 index f2e6dcd2..00000000 --- a/runtime/driver-old.c +++ /dev/null @@ -1,1248 +0,0 @@ -#ifdef PLATFORM_WINDOWS - //This define is necessary as a workaround for accessing CreateSymbolicLink - //function. This #define is added automatically by the MSVC compiler, but - //must be added manually when using gcc. - //Must be defined before including windows.h - #define _WIN32_WINNT 0x600 - - #include -#else - #include - #include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "process.h" - -// Forward Declarations -// ==================== - -void* stz_malloc (stz_long size); -void stz_free (void* ptr); - -#ifdef PLATFORM_WINDOWS -char* get_windows_api_error() { - char* lpMsgBuf; - char* ret; - - FormatMessage( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - GetLastError(), - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (char*)&lpMsgBuf, - 0, NULL ); - - ret = strdup(lpMsgBuf); - LocalFree(lpMsgBuf); - - return ret; -} -#endif - -#ifdef PLATFORM_WINDOWS -static void exit_with_error_line_and_func (const char* file, int line) { - fprintf(stderr, "[%s:%d] %s", file, line, get_windows_api_error()); - exit(-1); -} -#endif - -#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) -static void exit_with_error_line_and_func (const char* file, int line){ - fprintf(stderr, "[%s:%d] %s\n", file, line, strerror(errno)); - exit(-1); -} -#endif - -#define exit_with_error() exit_with_error_line_and_func(__FILE__, __LINE__) - -// Stanza Defined Entities -// ======================= -typedef struct{ - stz_long returnpc; - stz_long liveness_map; - stz_long slots[]; -} StackFrame; - -typedef struct Stack{ - stz_long size; - StackFrame* frames; - StackFrame* stack_pointer; - stz_long pc; - struct Stack* tail; -} Stack; - -typedef struct{ - stz_long current_stack; - stz_long system_stack; - stz_byte* heap_top; - stz_byte* heap_limit; - stz_byte* heap_start; - stz_byte* heap_old_objects_end; - stz_byte* heap_bitset; - stz_byte* heap_bitset_base; - stz_long heap_size; - stz_long heap_size_limit; - stz_long heap_max_size; - Stack* stacks; - void* trackers; - stz_byte* marking_stack_start; - stz_byte* marking_stack_bottom; - stz_byte* marking_stack_top; -} VMInit; - -// Macro Readers -// ============= -FILE* get_stdout () {return stdout;} -FILE* get_stderr () {return stderr;} -FILE* get_stdin () {return stdin;} -stz_int get_eof () {return (stz_int)EOF;} -stz_int get_errno () {return (stz_int)errno;} - -// Time of Day -// =========== -stz_long current_time_us (void) { - struct timeval tv; - gettimeofday(&tv, NULL); - return (stz_long)tv.tv_sec * 1000 * 1000 + (stz_long)tv.tv_usec; -} - -stz_long current_time_ms (void) { - struct timeval tv; - gettimeofday(&tv, NULL); - return (stz_long)tv.tv_sec * 1000 + (stz_long)tv.tv_usec / 1000; -} - -// Random Access Files -// =================== -stz_long get_file_size (FILE* f) { - int64_t cur_pos = ftell(f); - fseek(f, 0, SEEK_END); - stz_long size = (stz_long)ftell(f); - fseek(f, cur_pos, SEEK_SET); - return size; -} - -stz_int file_seek (FILE* f, stz_long pos) { - return (stz_int)fseek(f, pos, SEEK_SET); -} - -stz_int file_skip (FILE* f, stz_long num) { - return (stz_int)fseek(f, num, SEEK_CUR); -} - -stz_int file_set_length (FILE* f, stz_long size) { - return (stz_int)ftruncate(fileno(f), size); -} - -stz_long file_read_block (FILE* f, char* data, stz_long len) { - return (stz_long)fread(data, 1, len, f); -} - -stz_long file_write_block (FILE* f, char* data, stz_long len) { - return (stz_long)fwrite(data, 1, len, f); -} - - -// Path Resolution -// =============== -#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) - stz_byte* resolve_path (const stz_byte* filename){ - //Call the Linux realpath function. - return STZ_STR(realpath(C_CSTR(filename), 0)); - } -#endif - -#if defined(PLATFORM_WINDOWS) - // Return a bitmask that represents which of the 26 letters correspond - // to valid drive letters. - stz_int windows_logical_drives_bitmask (){ - return GetLogicalDrives(); - } - - // Resolve a given file path to its fully-resolved ("final") path name. - // This function tries to return an absolute path with symbolic links - // resolved. Sometimes it returns an UNC path, which is not usable. - stz_byte* windows_final_path_name (stz_byte* path){ - // First, open the file (to get a handle to it) - HANDLE hFile = CreateFile( - /* lpFileName */ (LPCSTR)path, - /* dwDesiredAccess */ 0, - /* dwShareMode */ FILE_SHARE_READ | FILE_SHARE_WRITE, - /* lpSecurityAttributes */ NULL, - /* dwCreationDisposition */ OPEN_EXISTING, - // necessary to open directories - /* dwFlagsAndAttributes */ FILE_FLAG_BACKUP_SEMANTICS, - /* hTemplateFile */ NULL); - - // Return -1 if a handle cannot be created. - if (hFile == INVALID_HANDLE_VALUE) return NULL; - - // Then resolve it into its fully-resolved ("final") path name - LPSTR ret = stz_malloc(sizeof(CHAR) * MAX_PATH); - int numchars = GetFinalPathNameByHandle(hFile, ret, MAX_PATH, FILE_NAME_OPENED); - - // Close handle now that we no longer need it (important to do so!) - CloseHandle(hFile); - - // Return null if GetFinalPath fails. - if(numchars == 0){ - stz_free(ret); - return NULL; - } - - // Return the path. - return STZ_STR(ret); - } - - // Resolve a given file path using its "full" path name. - // This function tries to return an absolute path. Symbolic - // links are not resolved. - stz_byte* windows_full_path_name (stz_byte* filename){ - char* fileext; - char* path = (char*)stz_malloc(2048); - int numchars = GetFullPathName((LPCSTR)filename, 2048, path, &fileext); - - // Return null if GetFullPath fails. - if(numchars == 0){ - stz_free(path); - return NULL; - } - - //Return the path - return path; - } -#endif - -#ifdef PLATFORM_WINDOWS -stz_int symlink(const stz_byte* target, const stz_byte* linkpath) { - DWORD attributes, flags; - - attributes = GetFileAttributes((LPCSTR)target); - flags = (attributes & FILE_ATTRIBUTE_DIRECTORY) != 0 ? - SYMBOLIC_LINK_FLAG_DIRECTORY : 0; - - if (!CreateSymbolicLink((LPCSTR)linkpath, (LPCSTR)target, flags)) { - return -1; - } - - return 0; -} - -//This function does not follow symbolic links. If we need -//to follow symbolic links, the caller should call this -//call this function with the result of resolve-path. -stz_int get_file_type (const stz_byte* filename0) { - WIN32_FILE_ATTRIBUTE_DATA attributes; - LPCSTR filename = C_CSTR(filename0); - bool is_directory = false, - is_symlink = false; - - // First grab the file's attributes - if (!GetFileAttributesEx(filename, GetFileExInfoStandard, &attributes)) { - return -1; // Non-existent or inaccessible file - } - - // Check if it's a directory - if (attributes.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - is_directory = true; - } - - // Check for possible symlink (reparse point *may* be a symlink) - if (attributes.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - // To know for sure, find the file and check its reparse tags - WIN32_FIND_DATA find_data; - HANDLE find_handle = FindFirstFile(filename, &find_data); - - if (find_handle == INVALID_HANDLE_VALUE) { - return -1; - } - - if (// Mount point a.k.a Junction (should be treated as a symlink) - find_data.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT || - // Actual symlinks (like those created by symlink()) - find_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) { - is_symlink = true; - } - - FindClose(find_handle); - } - - // Now we can determine what kind of file it is - if (!is_directory && !is_symlink) { - return 0; // Regular file - } - else if (is_directory && !is_symlink) { - return 1; // Directory (non-symlink) - } - else if (is_symlink) { - return 2; // Symlink - } - else { - return 3; // Unknown (other) - } -} -#endif - -#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) -stz_int get_file_type (const stz_byte* filename, stz_int follow_sym_links) { - struct stat filestat; - int result; - if(follow_sym_links) result = stat(C_CSTR(filename), &filestat); - else result = lstat(C_CSTR(filename), &filestat); - - if(result == 0){ - if(S_ISREG(filestat.st_mode)) - return 0; - else if(S_ISDIR(filestat.st_mode)) - return 1; - else if(S_ISLNK(filestat.st_mode)) - return 2; - else - return 3; - } - else{ - return -1; - } -} - -#endif - -// Environment Variable Setting -// ============================ -#ifdef PLATFORM_WINDOWS - - //Retrieve all environment variables as a list. - extern char** _environ; - char** get_env_vars (){ - return _environ; - } - - stz_int setenv (const stz_byte* name, const stz_byte* value, stz_int overwrite) { - //If we don't want to overwrite previous value, then check whether it exists. - //If it does, then just return 0. - if(!overwrite){ - if(getenv(C_CSTR(name)) == 0) - return 0; - } - //(Over)write the environment variable. - char* buffer = (char*)stz_malloc(strlen(C_CSTR(name)) + strlen(C_CSTR(value)) + 10); - sprintf(buffer, "%s=%s", C_CSTR(name), C_CSTR(value)); - int r = _putenv(buffer); - stz_free(buffer); - return (stz_int)r; - } - - stz_int unsetenv (const stz_byte* name){ - char* buffer = (char*)stz_malloc(strlen(C_CSTR(name)) + 10); - sprintf(buffer, "%s=", C_CSTR(name)); - int r = _putenv(buffer); - stz_free(buffer); - return (stz_int)r; - } -#else - - //Retrieve all environment variables as a list. - extern char** environ; - char** get_env_vars (){ - return environ; - } - -#endif - -// Time Modified -// ============= - -stz_long file_time_modified (const stz_byte* filename){ - struct stat attrib; - if(stat(C_CSTR(filename), &attrib) == 0) - return (stz_long)attrib.st_mtime; - return 0; -} - -//============================================================ -//===================== String List ========================== -//============================================================ - -typedef struct { - stz_int n; - stz_int capacity; - stz_byte** strings; -} StringList; - -StringList* make_stringlist (stz_int capacity){ - StringList* list = (StringList*)malloc(sizeof(StringList)); - list->n = 0; - list->capacity = capacity; - list->strings = (stz_byte**)malloc(capacity * sizeof(stz_byte*)); - return list; -} - -static void ensure_stringlist_capacity (StringList* list, stz_int c) { - if(list->capacity < c){ - stz_int new_capacity = list->capacity; - while(new_capacity < c) new_capacity *= 2; - stz_byte** new_strings = (stz_byte**)malloc(new_capacity * sizeof(stz_byte*)); - memcpy(new_strings, list->strings, list->n * sizeof(stz_byte*)); - list->capacity = new_capacity; - free(list->strings); - list->strings = new_strings; - } -} - -void free_stringlist (StringList* list){ - for(int i=0; in; i++) - free(list->strings[i]); - free(list->strings); - free(list); -} - -void stringlist_add (StringList* list, const stz_byte* string){ - ensure_stringlist_capacity(list, list->n + 1); - char* copy = malloc(strlen(C_CSTR(string)) + 1); - strcpy(copy, C_CSTR(string)); - list->strings[list->n] = STZ_STR(copy); - list->n++; -} - -//============================================================ -//================== Directory Handling ====================== -//============================================================ - -StringList* list_dir (const stz_byte* filename){ - //Open directory - DIR* dir = opendir(C_CSTR(filename)); - if(dir == NULL) return 0; - - //Allocate memory for strings - StringList* list = make_stringlist(10); - //Loop through directory entries - while(1){ - //Read next entry - struct dirent* entry = readdir(dir); - if(entry == NULL){ - closedir(dir); - return list; - } - //Notify - stringlist_add(list, STZ_CSTR(entry->d_name)); - } - - free(list); - return 0; -} - -//============================================================ -//===================== Sleeping ============================= -//============================================================ - -stz_int sleep_us (stz_long us){ - struct timespec t1, t2; - t1.tv_sec = us / 1000000L; - t1.tv_nsec = (us % 1000000L) * 1000L; - return (stz_int)nanosleep(&t1, &t2); -} - -stz_int sleep_ms (stz_long ms){ - struct timespec t1, t2; - t1.tv_sec = ms / 1000L; - t1.tv_nsec = (ms % 1000L) * 1000000L; - return (stz_int)nanosleep(&t1, &t2); -} - -//============================================================ -//================= Stanza Memory Allocator ================== -//============================================================ - -void* stz_malloc (stz_long size){ - return malloc(size); -} - -void stz_free (void* ptr){ - free(ptr); -} - -//============================================================ -//============= Stanza Memory Mapping on POSIX =============== -//============================================================ -#if defined(PLATFORM_LINUX) | defined(PLATFORM_OS_X) - -//Set protection bits on address range p (inclusive) to p + size (exclusive). -//Fatal error if size > 0 and mprotect fails. -static void protect(void* p, stz_long size, stz_int prot) { - if (size && mprotect(p, (size_t)size, prot)) exit_with_error(); -} - -//Allocates a segment of memory that is min_size allocated, and can be -//resized up to max_size. -//This function is called from within Stanza, and min_size and max_size -//are assumed to be multiples of the system page size. -void* stz_memory_map (stz_long min_size, stz_long max_size) { - void* p = mmap(NULL, (size_t)max_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (p == MAP_FAILED) exit_with_error(); - - protect(p, min_size, PROT_READ | PROT_WRITE | PROT_EXEC); - return p; -} - -//Unmaps the region of memory. -//This function is called from within Stanza, and size is -//assumed to be a multiple of the system page size. -void stz_memory_unmap (void* p, stz_long size) { - if (p && munmap(p, (size_t)size)) exit_with_error(); -} - -//Resizes the given segment. -//old_size is assumed to be the size that is already allocated. -//new_size is the size that we desired to be allocated, and -//must be a multiple of the system page size. -void stz_memory_resize (void* p, stz_long old_size, stz_long new_size) { - stz_long min_size = old_size; - stz_long max_size = new_size; - int prot = PROT_READ | PROT_WRITE | PROT_EXEC; - - if (min_size > max_size) { - min_size = new_size; - max_size = old_size; - prot = PROT_NONE; - } - - protect((char*)p + min_size, max_size - min_size, prot); -} - -#endif - -//============================================================ -//============= Stanza Memory Mapping on Windows ============= -//============================================================ -#ifdef PLATFORM_WINDOWS - -//Allocates a segment of memory that is min_size allocated, and can be -//resized up to max_size. -//This function is called from within Stanza, and min_size and max_size -//are assumed to be multiples of the system page size. -void* stz_memory_map (stz_long min_size, stz_long max_size) { - // Reserve the max size with no access - void* p = VirtualAlloc(NULL, (SIZE_T)max_size, MEM_RESERVE, PAGE_NOACCESS); - if (p == NULL) exit_with_error(); - - // Commit the min size with RWX access. - p = VirtualAlloc(p, (SIZE_T)min_size, MEM_COMMIT, PAGE_EXECUTE_READWRITE); - if (p == NULL) exit_with_error(); - - // Return the reserved and committed pointer. - return p; -} - -//Unmaps given segment of memory. -//This function is called from within Stanza, and size is -//assumed to be a multiple of the system page size. -void stz_memory_unmap (void* p, stz_long size) { - // End doing nothing if p is null. - if (p == NULL) return; - - // Release the memory and fatal if it fails. - if (!VirtualFree(p, 0, MEM_RELEASE)) - exit_with_error(); -} - -//Resizes the given segment. -//old_size is assumed to be the size that is already allocated. -//new_size is the size that we desired to be allocated, and -//must be a multiple of the system page size. -void stz_memory_resize (void* p, stz_long old_size, stz_long new_size) { - //Case: if growing the allocated size. - if (new_size > old_size) { - // Growing the allocation: commit all memory pages from the old limit to the new limit. - if (!VirtualAlloc((char*)p + old_size, (SIZE_T)(new_size - old_size), MEM_COMMIT, PAGE_EXECUTE_READWRITE)) - exit_with_error(); - } - //Case: if shrinking the allocated size. - else if(new_size < old_size) { - // Shrinking the allocation: decommit all memory pages from the new limit to the old limit. - if (!VirtualFree((char*)p + new_size, (SIZE_T)(old_size - new_size), MEM_DECOMMIT)) - exit_with_error(); - } -} - -#endif - -//============================================================ -//================= Process Runtime ========================== -//============================================================ -#if defined(PLATFORM_OS_X) || defined(PLATFORM_LINUX) - -//------------------------------------------------------------ -//------------------- Structures ----------------------------- -//------------------------------------------------------------ - -//- working_dir: If not null, the working directory of the -// new process. -//- env_vars: If not null, the environment variables to set in the -// child process. Each string has format "MYVAR=MYVALUE" and is null-terminated. -//- argvs: The string arguments that are passed to the C main function. -typedef struct { - stz_byte* pipe; - stz_byte* in_pipe; - stz_byte* out_pipe; - stz_byte* err_pipe; - stz_byte* file; - stz_byte* working_dir; - stz_byte** env_vars; - stz_byte** argvs; -} EvalArg; - -#define RETURN_NEG(x) {int r=(x); if(r < 0) return -1;} - -//------------------------------------------------------------ -//-------------------- Utilities ----------------------------- -//------------------------------------------------------------ - -static int count_non_null (void** xs){ - int n=0; - while(xs[n] != NULL) - n++; - return n; -} - -static char* string_join (const char* a, const char* b){ - int len = strlen(a) + strlen(b); - char* buffer = (char*)stz_malloc(len + 1); - sprintf(buffer, "%s%s", a, b); - return buffer; -} - -//Opening a named pipe -static int open_pipe (const char* prefix, const char* suffix, int options){ - char* name = string_join(prefix, suffix); - int fd = open(name, options); - stz_free(name); - return fd; -} - -//Creating a named pipe -static int make_pipe (char* prefix, char* suffix){ - char* name = string_join(prefix, suffix); - return mkfifo(name, S_IRUSR|S_IWUSR); -} -#endif - -//------------------------------------------------------------ -//----------------------- Serialization ---------------------- -//------------------------------------------------------------ -#if defined(PLATFORM_LINUX) | defined(PLATFORM_OS_X) - -// ===== Serialization ===== -static void write_int (FILE* f, stz_int x){ - fwrite(&x, sizeof(stz_int), 1, f); -} -static void write_long (FILE* f, stz_long x){ - fwrite(&x, sizeof(stz_long), 1, f); -} -static void write_string (FILE* f, stz_byte* s){ - if(s == NULL) - write_int(f, -1); - else{ - size_t n = strlen(C_CSTR(s)); - write_int(f, (stz_int)n); - fwrite(s, 1, n, f); - } -} -static void write_strings (FILE* f, stz_byte** s){ - int n = count_non_null((void**)s); - write_int(f, (stz_int)n); - for(int i=0; ipipe); - write_string(f, earg->in_pipe); - write_string(f, earg->out_pipe); - write_string(f, earg->err_pipe); - write_string(f, earg->file); - write_string(f, earg->working_dir); - write_optional_strings(f, earg->env_vars); - write_strings(f, earg->argvs); -} - -static void write_process_state (FILE* f, ProcessState* s){ - write_int(f, s->state); - write_int(f, s->code); -} - -// ===== Deserialization ===== -static void bread (void* xs0, int size, int n0, FILE* f){ - char* xs = xs0; - int n = n0; - while(n > 0){ - int c = fread(xs, size, n, f); - if(c < n){ - if(ferror(f)) exit_with_error(); - if(feof(f)) return; - } - n = n - c; - xs = xs + size*c; - } -} -static stz_int read_int (FILE* f){ - stz_int n; - bread(&n, sizeof(stz_int), 1, f); - return n; -} -static stz_long read_long (FILE* f){ - stz_long n; - bread(&n, sizeof(stz_long), 1, f); - return n; -} -static stz_byte* read_string (FILE* f){ - stz_int n = read_int(f); - if(n < 0) - return NULL; - else{ - stz_byte* s = (stz_byte*)stz_malloc(n + 1); - bread(s, 1, (int)n, f); - s[n] = '\0'; - return s; - } -} -static stz_byte** read_strings (FILE* f){ - stz_int n = read_int(f); - stz_byte** xs = (stz_byte**)stz_malloc(sizeof(stz_byte*)*(n + 1)); - for(int i=0; ipipe = read_string(f); - earg->in_pipe = read_string(f); - earg->out_pipe = read_string(f); - earg->err_pipe = read_string(f); - earg->file = read_string(f); - earg->working_dir = read_string(f); - earg->env_vars = read_optional_strings(f); - earg->argvs = read_strings(f); - return earg; -} -static void read_process_state (FILE* f, ProcessState* s){ - s->state = read_int(f); - s->code = read_int(f); -} - -//===== Free ===== -static void free_strings (stz_byte** ss){ - for(int i=0; ss[i] != NULL; i++) - stz_free(ss[i]); - stz_free(ss); -} - -static void free_earg (EvalArg* arg){ - stz_free(arg->pipe); - if(arg->in_pipe != NULL) stz_free(arg->in_pipe); - if(arg->out_pipe != NULL) stz_free(arg->out_pipe); - if(arg->err_pipe != NULL) stz_free(arg->err_pipe); - stz_free(arg->file); - if(arg->working_dir != NULL) stz_free(arg->working_dir); - if(arg->env_vars != NULL) free_strings(arg->env_vars); - free_strings(arg->argvs); - stz_free(arg); -} - -//------------------------------------------------------------ -//-------------------- Process Queries ----------------------- -//------------------------------------------------------------ - -static void get_process_state (stz_long pid, ProcessState* s, int wait_for_termination){ - int status; - int ret = waitpid((pid_t)pid, &status, wait_for_termination? 0 : WNOHANG); - - if(ret == 0) - *s = (ProcessState){PROCESS_RUNNING, 0}; - else if(WIFEXITED(status)) - *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; - else if(WIFSIGNALED(status)) - *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; - else if(WIFSTOPPED(status)) - *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; - else - *s = (ProcessState){PROCESS_RUNNING, 0}; -} - -//------------------------------------------------------------ -//----------------------- Execvpe ---------------------------- -//------------------------------------------------------------ -//The 'execvpe' frontend to 'execvp' does not exist in OS-X. -//So implement a quick version of it. - -#ifdef PLATFORM_OS_X -extern char **environ; -int execvpe(const char *program, char **argv, char **envp){ - char **saved = environ; - environ = envp; - int rc = execvp(program, argv); - environ = saved; - return rc; -} -#endif - -//------------------------------------------------------------ -//---------------------- Launcher Main ----------------------- -//------------------------------------------------------------ - -#define LAUNCH_COMMAND 0 -#define STATE_COMMAND 1 -#define WAIT_COMMAND 2 - -static void write_error_and_exit (int fd){ - int code = errno; - write(fd, &code, sizeof(int)); - close(fd); - exit(-1); -} - -static void disable_ctrl_c () { - #ifdef PLATFORM_WINDOWS - SetConsoleCtrlHandler(NULL, TRUE); - #else - signal(SIGINT, SIG_IGN); - #endif -} - -static void launcher_main (FILE* lin, FILE* lout){ - disable_ctrl_c(); - while(1){ - //Read in command - int comm = fgetc(lin); - if(feof(lin)) exit(0); - - //Interpret launch process command - if(comm == LAUNCH_COMMAND){ - //Read in evaluation arguments - EvalArg* earg = read_earg(lin); - if(feof(lin)) exit(0); - - //Create error-code pipe - int READ = 0; - int WRITE = 1; - int exec_error[2]; - if(pipe(exec_error) < 0) exit_with_error(); - - //Fork a new child - stz_long pid = (stz_long)fork(); - if(pid < 0) exit_with_error(); - - if(pid > 0){ - //Read from error-code pipe - int exec_code; - close(exec_error[WRITE]); - int exec_r = read(exec_error[READ], &exec_code, sizeof(int)); - close(exec_error[READ]); - - if(exec_r == 0){ - //Exec evaluated successfully - //Return new process id - write_long(lout, pid); - fflush(lout); - } - else if(exec_r == sizeof(int)){ - //Exec evaluated unsuccessfully - //Return error code as negative long - write_long(lout, (stz_long)-exec_code); - fflush(lout); - } - else{ - fprintf(stderr, "Unreachable code."); - exit(-1); - } - }else{ - //Close exec pipe read, and close write end on successful exec - close(exec_error[READ]); - fcntl(exec_error[WRITE], F_SETFD, FD_CLOEXEC); - - //Open named pipes - if(earg->in_pipe != NULL){ - int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->in_pipe), O_RDONLY); - if(fd < 0) write_error_and_exit(exec_error[WRITE]); - dup2(fd, 0); - } - if(earg->out_pipe != NULL){ - int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->out_pipe), O_WRONLY); - if(fd < 0) write_error_and_exit(exec_error[WRITE]); - dup2(fd, 1); - } - if(earg->err_pipe != NULL){ - int fd = open_pipe(C_CSTR(earg->pipe), C_CSTR(earg->err_pipe), O_WRONLY); - if(fd < 0) write_error_and_exit(exec_error[WRITE]); - dup2(fd, 2); - } - - //Set the working directory of the child process if explicitly - //requested. - if (earg->working_dir) { - if (chdir(C_CSTR(earg->working_dir)) == -1) { - write_error_and_exit(exec_error[WRITE]); - } - } - - //Launch child process. - //If an environment is supplied then call execvpe, otherwise call execvp. - if(earg->env_vars == NULL) - execvp(C_CSTR(earg->file), (char**)earg->argvs); - else - execvpe(C_CSTR(earg->file), (char**)earg->argvs, (char**)earg->env_vars); - - //Unsuccessful exec, write error number - write_error_and_exit(exec_error[WRITE]); - } - } - //Interpret state retrieval command - else if(comm == STATE_COMMAND || comm == WAIT_COMMAND){ - //Read in process id - stz_long pid = read_long(lin); - - //Retrieve state - ProcessState s; get_process_state(pid, &s, comm == WAIT_COMMAND); - write_process_state(lout, &s); - fflush(lout); - } - //Unrecognized command - else{ - fprintf(stderr, "Illegal command: %d\n", comm); - exit(-1); - } - } -} - -static stz_long launcher_pid = -1; -static FILE* launcher_in = NULL; -static FILE* launcher_out = NULL; -void initialize_launcher_process (){ - if(launcher_pid < 0){ - //Create pipes - int READ = 0; - int WRITE = 1; - int lin[2]; - int lout[2]; - if(pipe(lin) < 0) exit_with_error(); - if(pipe(lout) < 0) exit_with_error(); - - //Fork - stz_long pid = (stz_long)fork(); - if(pid < 0) exit_with_error(); - - if(pid > 0){ - //Parent - launcher_pid = pid; - close(lin[READ]); - close(lout[WRITE]); - launcher_in = fdopen(lin[WRITE], "w"); - if(launcher_in == NULL) exit_with_error(); - launcher_out = fdopen(lout[READ], "r"); - if(launcher_out == NULL) exit_with_error(); - } - else{ - //Child - close(lin[WRITE]); - close(lout[READ]); - FILE* fin = fdopen(lin[READ], "r"); - if(fin == NULL) exit_with_error(); - FILE* fout = fdopen(lout[WRITE], "w"); - if(fout == NULL) exit_with_error(); - launcher_main(fin, fout); - } - } -} - -static void make_pipe_name (char* pipe_name, int pipeid) { - sprintf(pipe_name, "/tmp/stanza_exec_pipe_%ld_%ld", (long)getpid(), (long)pipeid); -} - -static int delete_process_pipe (FILE* fd, char* pipe_name, char* suffix) { - if (fd != NULL) { - int close_res = fclose(fd); - if (close_res == EOF) return -1; - char my_pipe_name[80]; - sprintf(my_pipe_name, "%s%s", pipe_name, suffix); - int rm_res = remove(my_pipe_name); - if (rm_res < 0) return -1; - } - return 0; -} - -stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error, stz_int pipeid) { - char pipe_name[80]; - make_pipe_name(pipe_name, (int)pipeid); - if (delete_process_pipe(input, pipe_name, "_in") < 0) - return -1; - if (delete_process_pipe(output, pipe_name, "_out") < 0) - return -1; - if (delete_process_pipe(error, pipe_name, "_err") < 0) - return -1; - return 0; -} - -stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int pipeid, - stz_byte* working_dir, stz_byte** env_vars, Process* process) { - //Initialize launcher if necessary - initialize_launcher_process(); - - //Figure out unique pipe name - char pipe_name[80]; - make_pipe_name(pipe_name, (int)pipeid); - - //Compute pipe sources - int pipe_sources[NUM_STREAM_SPECS]; - for(int i=0; i= 0) - RETURN_NEG(make_pipe(pipe_name, "_in")) - if(pipe_sources[PROCESS_OUT] >= 0) - RETURN_NEG(make_pipe(pipe_name, "_out")) - if(pipe_sources[PROCESS_ERR] >= 0) - RETURN_NEG(make_pipe(pipe_name, "_err")) - - //Write in command and evaluation arguments - EvalArg earg = {STZ_STR(pipe_name), NULL, NULL, NULL, file, working_dir, env_vars, argvs}; - if(input == PROCESS_IN) earg.in_pipe = STZ_STR("_in"); - if(output == PROCESS_OUT) earg.out_pipe = STZ_STR("_out"); - if(output == PROCESS_ERR) earg.out_pipe = STZ_STR("_err"); - if(error == PROCESS_OUT) earg.err_pipe = STZ_STR("_out"); - if(error == PROCESS_ERR) earg.err_pipe = STZ_STR("_err"); - int r = fputc(LAUNCH_COMMAND, launcher_in); - if(r == EOF) return -1; - write_earg(launcher_in, &earg); - fflush(launcher_in); - - printf("input = %d, output = %d, err = %d\n", input, output, error); - - //Open pipes to child - FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] >= 0){ - int fd = open_pipe(pipe_name, "_in", O_WRONLY); - RETURN_NEG(fd) - fin = fdopen(fd, "w"); - if(fin == NULL) return -1; - } - FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] >= 0){ - int fd = open_pipe(pipe_name, "_out", O_RDONLY); - RETURN_NEG(fd) - fout = fdopen(fd, "r"); - if(fout == NULL) return -1; - } - FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] >= 0){ - int fd = open_pipe(pipe_name, "_err", O_RDONLY); - RETURN_NEG(fd) - ferr = fdopen(fd, "r"); - if(ferr == NULL) return -1; - } - - //Read back process id, and set errno if failed - stz_long pid = read_long(launcher_out); - if(pid < 0){ - errno = (int)(- pid); - return -1; - } - - //Return process structure - process->pid = pid; - process->in = fin; - process->out = fout; - process->err = ferr; - return 0; -} - -int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ - stz_int pid = process->pid; - - //Check whether launcher has been initialized - if(launcher_pid < 0){ - fprintf(stderr, "Launcher not initialized.\n"); - exit(-1); - } - - //Send command - int r = fputc(wait_for_termination? WAIT_COMMAND : STATE_COMMAND, launcher_in); - if(r == EOF) exit_with_error(); - write_long(launcher_in, pid); - fflush(launcher_in); - - //Read back process state - read_process_state(launcher_out, s); - return 0; -} -#else -#include "process-win32.c" -//============================================================ -//============== End Process Runtime ========================= -//============================================================ -#endif - -#define STACK_TYPE 6 - -stz_long stanza_entry (VMInit* init); - -// Command line arguments -// ====================== -stz_int input_argc; -stz_byte** input_argv; -stz_int input_argv_needs_free; - -// Main Driver -// =========== -static void* alloc (VMInit* init, long tag, long size){ - void* ptr = init->heap_top + 8; - *(long*)(init->heap_top) = tag; - init->heap_top += 8 + size; - return ptr; -} - -static Stack* alloc_stack (VMInit* init){ - Stack* stack = alloc(init, STACK_TYPE, sizeof(Stack)); - stz_long initial_stack_size = 8 * 1024; - StackFrame* frames = (StackFrame*)stz_malloc(initial_stack_size); - stack->size = initial_stack_size; - stack->frames = frames; - stack->stack_pointer = NULL; - stack->tail = NULL; - return stack; -} - -//Given a pointer to a struct allocated on the heap, -//add the tag bits to the pointer. -uint64_t tag_as_ref (void* p){ - return (uint64_t)p - 8 + 1; -} - -enum { - LOG_BITS_IN_BYTE = 3, - LOG_BYTES_IN_LONG = 3, - LOG_BITS_IN_LONG = LOG_BYTES_IN_LONG + LOG_BITS_IN_BYTE, - BYTES_IN_LONG = 1 << LOG_BYTES_IN_LONG, - BITS_IN_LONG = 1 << LOG_BITS_IN_LONG -}; - -#define SYSTEM_PAGE_SIZE 4096ULL -#define ROUND_UP_TO_WHOLE_PAGES(x) (((x) + (SYSTEM_PAGE_SIZE - 1)) & ~(SYSTEM_PAGE_SIZE - 1)) -#define ROUND_UP_TO_WHOLE_LONGS(x) (((x) + (sizeof(stz_long) - 1)) & ~(sizeof(stz_long) - 1)) - -static stz_long bitset_size (stz_long heap_size) { - uint64_t heap_size_in_longs = (heap_size + (BYTES_IN_LONG - 1)) >> LOG_BYTES_IN_LONG; - uint64_t bitset_size_in_longs = (heap_size_in_longs + (BITS_IN_LONG - 1)) >> LOG_BITS_IN_LONG; - return ROUND_UP_TO_WHOLE_PAGES(bitset_size_in_longs << LOG_BYTES_IN_LONG); -} - -//Use 'main' as the standard name for the C main function unless -//RENAME_STANZA_MAIN is passed as a flag. If it is, then rename 'main' -//to 'stanza_main'. -#ifdef RENAME_STANZA_MAIN - #define MAIN_FUNC stanza_main -#else - #define MAIN_FUNC main -#endif - -STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { - input_argc = (stz_int)argc; - input_argv = (stz_byte **)argv; - input_argv_needs_free = 0; - VMInit init; - - //Allocate heap - const stz_long min_heap_size = ROUND_UP_TO_WHOLE_PAGES(8 * 1024 * 1024); - const stz_long max_heap_size = ROUND_UP_TO_WHOLE_PAGES(STZ_LONG(8) * 1024 * 1024 * 1024); - init.heap_start = (stz_byte*)stz_memory_map(min_heap_size, max_heap_size); - init.heap_max_size = max_heap_size; - init.heap_size_limit = max_heap_size; - init.heap_size = min_heap_size; - - //Setup the nursery - const stz_long nursery_fraction = 8; // Must match the value in core.stanza - const stz_long nursery_size = ROUND_UP_TO_WHOLE_LONGS(min_heap_size / nursery_fraction / 2); - init.heap_old_objects_end = init.heap_start; - init.heap_top = init.heap_old_objects_end + nursery_size; - init.heap_limit = init.heap_top + nursery_size; - - //Allocate bitset for heap - const stz_long min_bitset_size = bitset_size(min_heap_size); - const stz_long max_bitset_size = bitset_size(max_heap_size); - init.heap_bitset = (stz_byte*)stz_memory_map(min_bitset_size, max_bitset_size); - init.heap_bitset_base = init.heap_bitset - ((uint64_t)init.heap_start >> 6); - memset(init.heap_bitset, 0, min_bitset_size); - - //For bitset_base computation to work: bitset must be aligned to 512-bytes boundary. - if((uint64_t)init.heap_bitset % 512 != 0){ - fprintf(stderr, "Unaligned bitset: %p.\n", init.heap_bitset); - exit(-1); - } - - //Allocate marking stack for heap - const stz_long marking_stack_size = ROUND_UP_TO_WHOLE_PAGES((1024 * 1024L) << LOG_BYTES_IN_LONG); - init.marking_stack_start = stz_memory_map(marking_stack_size, marking_stack_size); - init.marking_stack_bottom = init.marking_stack_start + marking_stack_size; - init.marking_stack_top = init.marking_stack_bottom; - - //Allocate stacks - Stack* entry_stack = alloc_stack(&init); - Stack* entry_system_stack = alloc_stack(&init); - entry_stack->tail = entry_system_stack; - init.current_stack = tag_as_ref(entry_stack); - init.system_stack = tag_as_ref(entry_system_stack); - init.stacks = entry_stack; - - //Initialize trackers to empty list. - init.trackers = NULL; - - //Call Stanza entry - stanza_entry(&init); - - //Heap and freespace are disposed by OS at process termination - return 0; -} diff --git a/runtime/driver.c b/runtime/driver.c index 1287690c..b4091b91 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -592,31 +592,11 @@ void stz_memory_resize (void* p, stz_long old_size, stz_long new_size) { #if defined(PLATFORM_OS_X) || defined(PLATFORM_LINUX) //------------------------------------------------------------ -//------------------- Structures ----------------------------- +//-------------------- Utilities ----------------------------- //------------------------------------------------------------ -//- working_dir: If not null, the working directory of the -// new process. -//- env_vars: If not null, the environment variables to set in the -// child process. Each string has format "MYVAR=MYVALUE" and is null-terminated. -//- argvs: The string arguments that are passed to the C main function. -typedef struct { - stz_byte* pipe; - stz_byte* in_pipe; - stz_byte* out_pipe; - stz_byte* err_pipe; - stz_byte* file; - stz_byte* working_dir; - stz_byte** env_vars; - stz_byte** argvs; -} EvalArg; - #define RETURN_NEG(x) {int r=(x); if(r < 0) return -1;} -//------------------------------------------------------------ -//-------------------- Utilities ----------------------------- -//------------------------------------------------------------ - static int count_non_null (void** xs){ int n=0; while(xs[n] != NULL) @@ -624,27 +604,8 @@ static int count_non_null (void** xs){ return n; } -static char* string_join (const char* a, const char* b){ - int len = strlen(a) + strlen(b); - char* buffer = (char*)stz_malloc(len + 1); - sprintf(buffer, "%s%s", a, b); - return buffer; -} -//Opening a named pipe -static int open_pipe (const char* prefix, const char* suffix, int options){ - char* name = string_join(prefix, suffix); - int fd = open(name, options); - stz_free(name); - return fd; -} -//Creating a named pipe -// TODO: free name -static int make_pipe (char* prefix, char* suffix){ - char* name = string_join(prefix, suffix); - return mkfifo(name, S_IRUSR|S_IWUSR); -} #endif //------------------------------------------------------------ @@ -744,18 +705,6 @@ static stz_byte** read_optional_strings (FILE* f){ } } -static EvalArg* read_earg (FILE* f){ - EvalArg* earg = (EvalArg*)stz_malloc(sizeof(EvalArg)); - earg->pipe = read_string(f); - earg->in_pipe = read_string(f); - earg->out_pipe = read_string(f); - earg->err_pipe = read_string(f); - earg->file = read_string(f); - earg->working_dir = read_string(f); - earg->env_vars = read_optional_strings(f); - earg->argvs = read_strings(f); - return earg; -} static void read_process_state (FILE* f, ProcessState* s){ s->state = read_int(f); s->code = read_int(f); @@ -768,55 +717,6 @@ static void free_strings (stz_byte** ss){ stz_free(ss); } -static void free_earg (EvalArg* arg){ - stz_free(arg->pipe); - if(arg->in_pipe != NULL) stz_free(arg->in_pipe); - if(arg->out_pipe != NULL) stz_free(arg->out_pipe); - if(arg->err_pipe != NULL) stz_free(arg->err_pipe); - stz_free(arg->file); - if(arg->working_dir != NULL) stz_free(arg->working_dir); - if(arg->env_vars != NULL) free_strings(arg->env_vars); - free_strings(arg->argvs); - stz_free(arg); -} - -//------------------------------------------------------------ -//-------------------- Process Queries ----------------------- -//------------------------------------------------------------ - -static void get_process_state (stz_long pid, ProcessState* s, int wait_for_termination){ - int status; - int ret = waitpid((pid_t)pid, &status, wait_for_termination? 0 : WNOHANG); - - if(ret == 0) - *s = (ProcessState){PROCESS_RUNNING, 0}; - else if(WIFEXITED(status)) - *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; - else if(WIFSIGNALED(status)) - *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; - else if(WIFSTOPPED(status)) - *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; - else - *s = (ProcessState){PROCESS_RUNNING, 0}; -} - -//------------------------------------------------------------ -//----------------------- Execvpe ---------------------------- -//------------------------------------------------------------ -//The 'execvpe' frontend to 'execvp' does not exist in OS-X. -//So implement a quick version of it. - -#ifdef PLATFORM_OS_X -extern char **environ; -int execvpe(const char *program, char **argv, char **envp){ - char **saved = environ; - environ = envp; - int rc = execvp(program, argv); - environ = saved; - return rc; -} -#endif - //------------------------------------------------------------ //---------------------- Launcher Main ----------------------- //------------------------------------------------------------ @@ -832,31 +732,21 @@ static void write_error_and_exit (int fd){ exit(-1); } -static void make_pipe_name (char* pipe_name, int pipeid) { - sprintf(pipe_name, "/tmp/stanza_exec_pipe_%ld_%ld", (long)getpid(), (long)pipeid); -} - -static int delete_process_pipe (FILE* fd, char* pipe_name, char* suffix) { +static int delete_process_pipe (FILE* fd) { if (fd != NULL) { int close_res = fclose(fd); if (close_res == EOF) return -1; - char my_pipe_name[80]; - sprintf(my_pipe_name, "%s%s", pipe_name, suffix); - int rm_res = remove(my_pipe_name); - if (rm_res < 0) return -1; } return 0; } stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error, stz_int pipeid) { - char pipe_name[80]; - make_pipe_name(pipe_name, (int)pipeid); - //if (delete_process_pipe(input, pipe_name, "_in") < 0) - // return -1; - //if (delete_process_pipe(output, pipe_name, "_out") < 0) - // return -1; - //if (delete_process_pipe(error, pipe_name, "_err") < 0) - // return -1; + if (delete_process_pipe(input) < 0) + return -1; + if (delete_process_pipe(output) < 0) + return -1; + if (delete_process_pipe(error) < 0) + return -1; return 0; } @@ -884,44 +774,46 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, posix_spawn_file_actions_init(&actions); int posix_ret; - // TODO: close all pipes that get dup2'ed //Setup input pipe if used if(pipe_sources[PROCESS_IN] > 0) { - // Close write end if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) exit_with_error(); - // dup read to STDIN if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) exit_with_error(); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0]))) + exit_with_error(); } //Setup output pipe if used if(pipe_sources[PROCESS_OUT] > 0) { - // Close read end if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) exit_with_error(); - // dup write to STDOUT if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) exit_with_error(); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1]))) + exit_with_error(); } //Setup error pipe if used if(pipe_sources[PROCESS_ERR] > 0) { - // Close read end if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) exit_with_error(); - // dup write to STDERR if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) exit_with_error(); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1]))) + exit_with_error(); } //Setup working directory if(working_dir) { - if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, working_dir))) + // TODO: is this cast OK? + if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, (const char *) working_dir))) exit_with_error(); } // Spawn process pid_t pid = -1; - if(posix_spawn(&pid, file, &actions, NULL, argvs, env_vars) == 0) { - printf("Child pid: %d\n", pid); + // TODO: are these casts OK? + if(posix_spawn(&pid, (const char *) file, &actions, NULL, (char * const *) argvs, (char * const *) env_vars) == 0) { + // TODO: setup SIGCHILD handler + //printf("Child pid: %d\n", pid); } else { exit_with_error(); } @@ -958,178 +850,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } #endif -// -// -//#ifdef PLATFORM_LINUX -//stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, -// stz_int output, stz_int error, stz_int pipeid, -// stz_byte* working_dir, stz_byte** env_vars, Process* process) { -//} -//#endif - -// Old version -//stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, -// stz_int output, stz_int error, stz_int pipeid, -// stz_byte* working_dir, stz_byte** env_vars, Process* process) { -// //Figure out unique pipe name -// char pipe_name[80]; -// make_pipe_name(pipe_name, (int)pipeid); -// -// //Compute pipe sources -// int pipe_sources[NUM_STREAM_SPECS]; -// for(int i=0; i= 0) -// RETURN_NEG(make_pipe(pipe_name, "_in")) -// if(pipe_sources[PROCESS_OUT] >= 0) -// RETURN_NEG(make_pipe(pipe_name, "_out")) -// if(pipe_sources[PROCESS_ERR] >= 0) -// RETURN_NEG(make_pipe(pipe_name, "_err")) -// -// stz_byte* in_pipe = NULL; -// stz_byte* out_pipe = NULL; -// stz_byte* err_pipe = NULL; -// -// int infd = -1; -// int outfd = -1; -// int errfd = -1; -// -// //Open pipes to child process -// FILE* fin = NULL; -// if(pipe_sources[PROCESS_IN] >= 0){ -// infd = open_pipe(pipe_name, "_in", O_RDWR); -// RETURN_NEG(infd) -// fin = fdopen(infd, "w"); -// if(fin == NULL) return -1; -// } -// FILE* fout = NULL; -// if(pipe_sources[PROCESS_OUT] >= 0){ -// outfd = open_pipe(pipe_name, "_out", O_RDWR); -// int fd2 = open_pipe(pipe_name, "_out", O_RDONLY | O_NONBLOCK); -// printf("outfd = %d, fd2 = %d\n", outfd, fd2); -// RETURN_NEG(outfd) -// printf("fout = fdopen(%d) (%s)\n", outfd, string_join(pipe_name, "_out")); -// fout = fdopen(outfd, "r"); -// if(fout == NULL) return -1; -// } -// FILE* ferr = NULL; -// if(pipe_sources[PROCESS_ERR] >= 0){ -// int errfd = open_pipe(pipe_name, "_err", O_RDWR); -// RETURN_NEG(errfd) -// printf("ferr = fdopen(%d) (%s)\n", errfd, string_join(pipe_name, "_err")); -// ferr = fdopen(errfd, "r"); -// if(ferr == NULL) return -1; -// } -// -// // Compute final file descriptors based on pipe sources -// int final_outfd = outfd; -// int final_errfd = errfd; -// -// if(input == PROCESS_IN) in_pipe = STZ_STR("_in"); -// if(output == PROCESS_OUT) { -// out_pipe = STZ_STR("_out"); -// final_outfd = outfd; -// } -// if(output == PROCESS_ERR) { -// out_pipe = STZ_STR("_err"); -// final_outfd = errfd; -// } -// if(error == PROCESS_OUT) { -// err_pipe = STZ_STR("_out"); -// final_errfd = outfd; -// } -// if(error == PROCESS_ERR) { -// err_pipe = STZ_STR("_err"); -// final_errfd = errfd; -// } -// -// -// -// pid_t pid = -1; -// -// // OS X : use posix_spawn to initialize child process -// #ifdef PLATFORM_OS_X -// // Setup -// posix_spawn_file_actions_t actions; -// posix_spawn_file_actions_init(&actions); -// char* child_in_pipe = in_pipe ? string_join(pipe_name, in_pipe) : NULL; -// char* child_out_pipe = out_pipe ? string_join(pipe_name, out_pipe) : NULL; -// char* child_err_pipe = err_pipe ? string_join(pipe_name, err_pipe) : NULL; -// // File actions -// // TODO: fix all the warnings -// int posix_ret; -// if (in_pipe) { -// if ((posix_spawn_file_actions_addopen(&actions, 0, child_in_pipe, O_RDONLY, 0))) -// exit_with_error(); -// if ((posix_spawn_file_actions_adddup2(&actions, infd, 0))) -// exit_with_error(); -// } -// if (out_pipe) { -// if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 1, child_out_pipe, O_RDWR, 0666))) -// exit_with_error(); -// printf("Opened %s to fd 1\n", child_out_pipe); -// printf("out: dup2(%d, 1), file = %s\n", 4, child_out_pipe); -// //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 4, 1))) -// //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 1, final_outfd))) -// //if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, final_outfd, 1))) -// // exit_with_error(); -// } -// if (err_pipe) { -// //if ((posix_ret = posix_spawn_file_actions_addopen(&actions, 5, child_err_pipe, O_RDWR, 0666))) -// // exit_with_error(); -// //printf("err: dup2(%d, 2), file = %s\n", 5, child_err_pipe); -// if ((posix_ret = posix_spawn_file_actions_adddup2(&actions, 2, 1))) -// exit_with_error(); -// printf("err: dup2(2, 1)\n"); -// } -// if (working_dir) { -// printf("working dir: %s\n", working_dir); -// if ((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, working_dir))) -// exit_with_error(); -// } -// // Call spawn -// if(posix_spawn(&pid, file, &actions, NULL, argvs, env_vars) == 0) { -// // Parent process -// printf("Success: child process = %d\n", pid); -// //int exec_r; -// //waitpid(pid, &exec_r, 0); -// //printf("Child process finished\n"); -// } else { -// exit_with_error(); -// } -// -// // Clean up spawn resources -// posix_spawn_file_actions_destroy(&actions); -// stz_free(child_in_pipe); -// stz_free(child_out_pipe); -// stz_free(child_err_pipe); -// #endif -// -// // TODO: Linux: use vfork to initialize child process -// #ifdef PLATFORM_LINUX -// #endif -// -// // TODO: -// //Read back process id, and set errno if failed -// //stz_long pid = read_long(launcher_out); -// //if(pid < 0){ -// // errno = (int)(- pid); -// // return -1; -// //} -// -// //Return process structure -// process->pid = pid; -// process->in = fin; -// process->out = fout; -// process->err = ferr; -// printf("Returning\n"); -// return 0; -//} + int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ @@ -1148,24 +869,8 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ return 0; } -// stz_int pid = process->pid; - -// //Check whether launcher has been initialized -// if(launcher_pid < 0){ -// fprintf(stderr, "Launcher not initialized.\n"); -// exit(-1); -// } - -// //Send command -// int r = fputc(wait_for_termination? WAIT_COMMAND : STATE_COMMAND, launcher_in); -// if(r == EOF) exit_with_error(); -// write_long(launcher_in, pid); -// fflush(launcher_in); - -// //Read back process state -// read_process_state(launcher_out, s); -// return 0; -//} + + #else #include "process-win32.c" //============================================================ From e9cbc3faa29f2c21fddc487d6dabc74e008d1e12 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 15 Nov 2023 16:11:56 -0800 Subject: [PATCH 06/53] cleanup --- compiler/config.stanza | 1 - core/core.stanza | 8 -------- 2 files changed, 9 deletions(-) diff --git a/compiler/config.stanza b/compiler/config.stanza index 44a06a76..0a1610b4 100644 --- a/compiler/config.stanza +++ b/compiler/config.stanza @@ -261,7 +261,6 @@ defn verify-installation () : try : ;Retrieve the version of the executable. call-system-and-get-output(exe-file, [exe-file, "version", "-terse"]) - catch (e:Exception) : ;Could not retrieve version of the executable. val msg = "Stanza install directory is set to %_, \ diff --git a/core/core.stanza b/core/core.stanza index 53cb2696..90c907d2 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -87,7 +87,6 @@ protected extern stz_memory_resize: (ptr, long, long) -> int #else: protected extern launch_process: (ptr, ptr>, int, int, int, int, ptr, ptr>, ptr) -> int protected extern delete_process_pipes: (ptr, ptr, ptr, int) -> int - ;protected extern initialize_launcher_process: () -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int @@ -10236,13 +10235,6 @@ public lostanza defn error-stream (p:ref) -> ref : p.error-stream = new FileInputStream{p.error, 0} return p.error-stream as ref -; Initialization -; ============== -;public lostanza defn initialize-process-launcher () -> ref : -; #if-not-defined(PLATFORM-WINDOWS): -; call-c clib/initialize_launcher_process() -; return false - ; State API ; ========= lostanza deftype StateStruct : From 27798cdbfa47fa90f9b85ef8c6f08673628e5e35 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 15 Nov 2023 17:23:12 -0800 Subject: [PATCH 07/53] starting sig handler --- runtime/driver.c | 147 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/runtime/driver.c b/runtime/driver.c index b4091b91..876b2f1c 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -605,6 +605,54 @@ static int count_non_null (void** xs){ } +//------------------------------------------------------------ +//------------------ Child Processes ------------------------- +//------------------------------------------------------------ + +// This is the max that the parent can _ever_ allocate +// this approach is dumb +// but simple +const int MAX_PROCESSES = 64; +typedef struct { + pid_t pid; + int* pipe_arr; + FILE* fin; + FILE* fout; + FILE* ferr; + int exit_status; // init to -1 +} ChildProcess; + +ChildProcess* child_processes[MAX_PROCESSES]; +int num_processes = 0; + +static void register_proc (ChildProcess* c) { + child_processes[num_processes] = c; + num_processes++; +} + +static ChildProcess* get_child (pid_t pid) { + for(int i = 0; i < num_processes; i++) + if(child_processes[i]->pid == pid) return child_processes[i]; + return NULL; +} + +static void kill_child (ChildProcess* c, int exit_code) { + c->exit_status = exit_code; + // Close files + fclose(c->fin); + fclose(c->fout); + fclose(c->ferr); + // Close pipes + for(int i = 0; ipipe_arr) + (i * sizeof(int)))[0]); + close(((c->pipe_arr) + (i * sizeof(int)))[1]); + } + // Destroy refs + c->pipe_arr = NULL; + c->fin = NULL; + c->fout = NULL; + c->ferr = NULL; +} #endif @@ -750,6 +798,104 @@ stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error, stz_int pi return 0; } +#ifdef PLATFORM_LINUX +stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, + stz_int output, stz_int error, stz_int pipeid, + stz_byte* working_dir, stz_byte** env_vars, Process* process) { + //Compute pipe sources: + int pipe_sources[NUM_STREAM_SPECS]; + for(int i=0; i 0) { + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) + exit_with_error(); + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) + exit_with_error(); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0]))) + exit_with_error(); + } + //Setup output pipe if used + if(pipe_sources[PROCESS_OUT] > 0) { + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) + exit_with_error(); + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) + exit_with_error(); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1]))) + exit_with_error(); + } + //Setup error pipe if used + if(pipe_sources[PROCESS_ERR] > 0) { + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) + exit_with_error(); + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) + exit_with_error(); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1]))) + exit_with_error(); + } + //Setup working directory + if(working_dir) { + // TODO: is this cast OK? + if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, (const char *) working_dir))) + exit_with_error(); + } + + // Spawn process + pid_t pid = -1; + // TODO: are these casts OK? + if(posix_spawn(&pid, (const char *) file, &actions, NULL, (char * const *) argvs, (char * const *) env_vars) == 0) { + // TODO: setup SIGCHILD handler + //printf("Child pid: %d\n", pid); + } else { + exit_with_error(); + } + + // Cleanup + posix_spawn_file_actions_destroy(&actions); + + //Parent: + //Close pipes, setup files + FILE* fin = NULL; + if(pipe_sources[PROCESS_IN] > 0) { + close(pipes[PROCESS_IN][0]); + fin = fdopen(pipes[PROCESS_IN][1], "w"); + if(fin == NULL) return -1; + } + FILE* fout = NULL; + if(pipe_sources[PROCESS_OUT] > 0) { + close(pipes[PROCESS_OUT][1]); + fout = fdopen(pipes[PROCESS_OUT][0], "r"); + if(fout == NULL) return -1; + } + FILE* ferr = NULL; + if(pipe_sources[PROCESS_ERR] > 0) { + close(pipes[PROCESS_ERR][1]); + ferr = fdopen(pipes[PROCESS_ERR][0], "r"); + if(ferr == NULL) return -1; + } + + process->pid = pid; + process->in = fin; + process->out = fout; + process->err = ferr; + return 0; +} +#endif + + + #ifdef PLATFORM_OS_X stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int output, stz_int error, stz_int pipeid, @@ -852,6 +998,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, + int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ int status; From bd7760320af8edb13b393fd05c117cc1b2d34ecd Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 09:18:32 -0800 Subject: [PATCH 08/53] no more pipeids --- core/core.stanza | 15 +++++---------- runtime/driver.c | 8 +++++--- runtime/process-win32.c | 1 - runtime/process.h | 3 --- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 90c907d2..3bb87992 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -9901,7 +9901,6 @@ public val CURRENT-PLATFORM:Platform = public lostanza deftype Process <: Unique : var pid: long var handle: ptr - var pipeid: int var input: ptr var output: ptr var error: ptr @@ -10000,7 +9999,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, env-var-mode:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) - val proc = new Process{0, null, -1, null, null, null, false, false, false, false} + val proc = new Process{0, null, null, null, null, false, false, false, false} ; Create a command line from the given filename and args (discarding the first argument). ; This is necessary because Windows' process API expects a command line (not a list of arguments). @@ -10070,8 +10069,6 @@ defn process-env-var-mode (env-vars:Tuple>|False, return copy-buffer-to-stable-memory(buffer) #else: - val PIPE-ID-GENERATOR = Random() - public lostanza defn Process (filename:ref, args0:ref>, input:ref, @@ -10083,8 +10080,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) - val pipe-id = next-int(PIPE-ID-GENERATOR).value - val proc = new Process{0, null, pipe-id, null, null, null, false, false, false, false} + val proc = new Process{0, null, null, null, null, false, false, false, false} val input_v = value(input).value val output_v = value(output).value val error_v = value(error).value @@ -10106,7 +10102,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ;Launch the process val launch_succ = call-c clib/launch_process(addr!(filename.chars), argvs, - input_v, output_v, error_v, proc.pipeid, working-dir-chars, env-var-string, addr!([proc])) + input_v, output_v, error_v, working-dir-chars, env-var-string, addr!([proc])) ;Free the memory created using malloc. free-linux-env-var-string(env-var-string) @@ -10253,11 +10249,10 @@ lostanza defn retrieve-state (p:ref, wait-for-termination?:refpid = (stz_long)proc_info.dwProcessId; - process->pipeid = -1; // -1 signals we didn't create named pipes for this process process->handle = (void*)proc_info.hProcess; process->in = file_from_handle(stdin_write, FT_WRITE); process->out = file_from_handle(stdout_read, FT_READ); diff --git a/runtime/process.h b/runtime/process.h index c6abfcbc..41429337 100644 --- a/runtime/process.h +++ b/runtime/process.h @@ -7,15 +7,12 @@ //communicating with it. //- pid: The id of the process. //- handle: The Windows handle to the process. Not used by other platforms. -//- pipeid: A unique integer used to generate the names of the named pipes for communication. -// Will be set to -1 for Windows to indicate that no named pipes are created. //- in: The standard input stream of the Process. //- out: The standard output stream of the Process. //- err: The standard error stream of the Process. typedef struct { stz_long pid; void* handle; - stz_int pipeid; FILE* in; FILE* out; FILE* err; From 3a7ce8deca1cc701a647f18b14621261110d684f Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 09:47:47 -0800 Subject: [PATCH 09/53] fix casts, most of linux impl --- runtime/driver.c | 139 +++++++++++++++++++++-------------------------- 1 file changed, 63 insertions(+), 76 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 5d7c200b..5a058307 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -818,80 +818,69 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } // TODO: don't spawn + stz_long pid = (stz_long)vfork(); + if(pid < 0) exit_with_error(); + if(pid > 0) { // Parent + FILE* fin = NULL; + if(pipe_sources[PROCESS_IN] > 0) { + close(pipes[PROCESS_IN][0]); + fin = fdopen(pipes[PROCESS_IN][1], "w"); + if(fin == NULL) return -1; + } + FILE* fout = NULL; + if(pipe_sources[PROCESS_OUT] > 0) { + close(pipes[PROCESS_OUT][1]); + fout = fdopen(pipes[PROCESS_OUT][0], "r"); + if(fout == NULL) return -1; + } + FILE* ferr = NULL; + if(pipe_sources[PROCESS_ERR] > 0) { + close(pipes[PROCESS_ERR][1]); + ferr = fdopen(pipes[PROCESS_ERR][0], "r"); + if(ferr == NULL) return -1; + } - int posix_ret; - //Setup input pipe if used - if(pipe_sources[PROCESS_IN] > 0) { - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) - exit_with_error(); - if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) - exit_with_error(); - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0]))) - exit_with_error(); - } - //Setup output pipe if used - if(pipe_sources[PROCESS_OUT] > 0) { - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) - exit_with_error(); - if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) - exit_with_error(); - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1]))) - exit_with_error(); - } - //Setup error pipe if used - if(pipe_sources[PROCESS_ERR] > 0) { - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) - exit_with_error(); - if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) - exit_with_error(); - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1]))) - exit_with_error(); - } - //Setup working directory - if(working_dir) { - // TODO: is this cast OK? - if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, (const char *) working_dir))) - exit_with_error(); - } - - // Spawn process - pid_t pid = -1; - // TODO: are these casts OK? - if(posix_spawn(&pid, (const char *) file, &actions, NULL, (char * const *) argvs, (char * const *) env_vars) == 0) { - // TODO: setup SIGCHILD handler - //printf("Child pid: %d\n", pid); - } else { - exit_with_error(); - } - - // Cleanup - posix_spawn_file_actions_destroy(&actions); - - //Parent: - //Close pipes, setup files - FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] > 0) { - close(pipes[PROCESS_IN][0]); - fin = fdopen(pipes[PROCESS_IN][1], "w"); - if(fin == NULL) return -1; - } - FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] > 0) { - close(pipes[PROCESS_OUT][1]); - fout = fdopen(pipes[PROCESS_OUT][0], "r"); - if(fout == NULL) return -1; - } - FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] > 0) { - close(pipes[PROCESS_ERR][1]); - ferr = fdopen(pipes[PROCESS_ERR][0], "r"); - if(ferr == NULL) return -1; + process->pid = pid; + process->in = fin; + process->out = fout; + process->err = ferr; + // TODO: register with SIGCHILD handler + } else { // Child + //Setup input pipe if used + if(pipe_sources[PROCESS_IN] > 0) { + if(close(pipes[PROCESS_IN][1]) < 0) + exit_with_error(); + if(dup2(pipes[PROCESS_IN][0], STDIN_FILENO) < 0) + exit_with_error(); + if(close(pipes[PROCESS_IN][0]) < 0) + exit_with_error(); + } + //Setup output pipe if used + if(pipe_sources[PROCESS_OUT] > 0) { + if(close(pipes[PROCESS_OUT][0]) < 0) + exit_with_error(); + if(dup2(pipes[PROCESS_OUT][1], STDOUT_FILENO) < 0) + exit_with_error(); + if(close(pipes[PROCESS_OUT][1]) < 0) + exit_with_error(); + } + //Setup error pipe if used + if(pipe_sources[PROCESS_ERR] > 0) { + if(close(pipes[PROCESS_ERR][0]) < 0) + exit_with_error(); + if(dup2(pipes[PROCESS_ERR][1], STDERR_FILENO) < 0) + exit_with_error(); + if(close(pipes[PROCESS_ERR][1]) < 0) + exit_with_error(); + } + //Setup working directory + if(working_dir) { + // TODO: is this cast OK? + if(chdir(C_CSTR(working_dir)) < 0) + exit_with_error(); + } + // TODO: exec } - - process->pid = pid; - process->in = fin; - process->out = fout; - process->err = ferr; return 0; } #endif @@ -951,15 +940,13 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } //Setup working directory if(working_dir) { - // TODO: is this cast OK? - if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, (const char *) working_dir))) + if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, C_CSTR(working_dir)))) exit_with_error(); } // Spawn process pid_t pid = -1; - // TODO: are these casts OK? - if(posix_spawn(&pid, (const char *) file, &actions, NULL, (char * const *) argvs, (char * const *) env_vars) == 0) { + if(posix_spawn(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char **)env_vars) == 0) { // TODO: setup SIGCHILD handler //printf("Child pid: %d\n", pid); } else { From 5e45a6cc576258ba65443abae1c001576b8ebb6d Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 10:23:12 -0800 Subject: [PATCH 10/53] update runtime for signature changes, add linux implementation --- core/core.stanza | 4 ++-- runtime/driver.c | 43 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 3bb87992..f8eb67ca 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -85,8 +85,8 @@ protected extern stz_memory_resize: (ptr, long, long) -> int protected extern launch_process: (ptr, int, int, int, ptr, ptr, ptr) -> int protected extern close_process_handle: (ptr) -> int #else: - protected extern launch_process: (ptr, ptr>, int, int, int, int, ptr, ptr>, ptr) -> int - protected extern delete_process_pipes: (ptr, ptr, ptr, int) -> int + protected extern launch_process: (ptr, ptr>, int, int, int, ptr, ptr>, ptr) -> int + protected extern delete_process_pipes: (ptr, ptr, ptr) -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int diff --git a/runtime/driver.c b/runtime/driver.c index 5a058307..2058a94d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -798,7 +798,21 @@ stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error) { return 0; } +// TODO: delete this +// Only included for testing linux implementation on OSX +#ifdef PLATFORM_OS_X +extern char **environ; +int execvpe(const char *program, char **argv, char **envp){ + char **saved = environ; + environ = envp; + int rc = execvp(program, argv); + environ = saved; + return rc; +} +#endif + #ifdef PLATFORM_LINUX +//if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int output, stz_int error, stz_byte* working_dir, stz_byte** env_vars, Process* process) { @@ -811,16 +825,18 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, pipe_sources[error] = 2; //Generate array of pipes per-input-source - //TODO: close these at some point int pipes[NUM_STREAM_SPECS][2]; for(int i=0; i 0) { // Parent + + // Parent: open files, register with signal handler + if(pid > 0) { FILE* fin = NULL; if(pipe_sources[PROCESS_IN] > 0) { close(pipes[PROCESS_IN][0]); @@ -845,7 +861,9 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; // TODO: register with SIGCHILD handler - } else { // Child + } + // Child: setup pipes, exec + else { //Setup input pipe if used if(pipe_sources[PROCESS_IN] > 0) { if(close(pipes[PROCESS_IN][1]) < 0) @@ -875,11 +893,19 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } //Setup working directory if(working_dir) { - // TODO: is this cast OK? if(chdir(C_CSTR(working_dir)) < 0) exit_with_error(); } - // TODO: exec + + //Launch child process. + //If an environment is supplied then call execvpe, otherwise call execvp. + int cstatus; + if(env_vars == NULL) + cstatus = execvp(C_CSTR(file), (char**)argvs); + else + cstatus = execvpe(C_CSTR(file), (char**)argvs, (char**)env_vars); + if(cstatus < 0) + exit_with_error(); } return 0; } @@ -900,7 +926,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, pipe_sources[error] = 2; //Generate array of pipes per-input-source - //TODO: close these at some point int pipes[NUM_STREAM_SPECS][2]; for(int i=0; i Date: Thu, 16 Nov 2023 12:40:56 -0800 Subject: [PATCH 11/53] some cleanup code, not yet handler not yet hooked in --- runtime/driver.c | 124 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 107 insertions(+), 17 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 2058a94d..876c9ed6 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -613,6 +613,8 @@ static int count_non_null (void** xs){ // this approach is dumb // but simple const int MAX_PROCESSES = 64; + +// Process metadata struct typedef struct { pid_t pid; int* pipe_arr; @@ -622,27 +624,14 @@ typedef struct { int exit_status; // init to -1 } ChildProcess; -ChildProcess* child_processes[MAX_PROCESSES]; -int num_processes = 0; - -static void register_proc (ChildProcess* c) { - child_processes[num_processes] = c; - num_processes++; -} - -static ChildProcess* get_child (pid_t pid) { - for(int i = 0; i < num_processes; i++) - if(child_processes[i]->pid == pid) return child_processes[i]; - return NULL; -} - -static void kill_child (ChildProcess* c, int exit_code) { - c->exit_status = exit_code; +// Free everything allocated by ChildProcess +static void cleanup_child (ChildProcess* c, int status) { + c->exit_status = status; // Close files fclose(c->fin); fclose(c->fout); fclose(c->ferr); - // Close pipes + // Close pipes (TODO: is the indexing OK?) for(int i = 0; ipipe_arr) + (i * sizeof(int)))[0]); close(((c->pipe_arr) + (i * sizeof(int)))[1]); @@ -654,6 +643,105 @@ static void kill_child (ChildProcess* c, int exit_code) { c->ferr = NULL; } +// Dead process exit code struct +typedef struct { + pid_t pid; + int status; +} DeadProcess; + +// Linked lists of alive/dead processes +typedef struct ProcessNode { + ChildProcess* proc; + struct ProcessNode* next; +} ProcessNode; + +typedef struct DeadNode { + DeadProcess* proc; + struct DeadNode* next; +} DeadNode; + +ProcessNode* proc_head = NULL; +DeadNode* dead_head = NULL; + +// Record process metadata +static void register_proc (ChildProcess* c) { + ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); + if(new_node == NULL) + exit_with_error(); + new_node->proc = c; + new_node->next = proc_head; + proc_head = new_node; +} + +// Record a dead process +static void register_dead_proc (DeadProcess* c) { + DeadNode* new_node = (DeadNode*)malloc(sizeof(DeadNode)); + if(new_node == NULL) + exit_with_error(); + new_node->proc = c; + new_node->next = dead_head; + dead_head = new_node; +} + +static void kill_child (pid_t pid, int status) { + ProcessNode* curr = proc_head; + ProcessNode* prev = NULL; + // Find matching Node + while(curr != NULL && curr->proc->pid != pid) { + prev = curr; + curr = curr->next; + } + if(curr != NULL) { + // Remove node from list of alive processes + if(prev == NULL) { + proc_head = curr->next; + } else { + prev->next = curr->next; + } + // Cleanup resources, generate a DeadProcess node to store exit status + cleanup_child(curr->proc, status); + DeadProcess* dead_proc = (DeadProcess*)malloc(sizeof(DeadProcess)); + dead_proc->pid = pid; + dead_proc->status = status; + register_dead_proc(dead_proc); + } +} + +void sigchild_handler(int signo) { + int status; + pid_t pid; + + // Cleanup zombie child processes + while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { + kill_child(pid, status); + } +} + +static void free_processes () { + ProcessNode* curr = proc_head; + ProcessNode* tmp = NULL; + // Free all nodes and their contents + while(curr != NULL) { + tmp = curr; + curr = curr->next; + free(tmp->proc); + free(tmp); + } +} + +static void free_dead_processes () { + DeadNode* curr = dead_head; + DeadNode* tmp = NULL; + // Free all nodes and their contents + while(curr != NULL) { + tmp = curr; + curr = curr->next; + free(tmp->proc); + free(tmp); + } +} + + #endif //------------------------------------------------------------ @@ -1158,6 +1246,8 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Call Stanza entry stanza_entry(&init); + free_processes(); + free_dead_processes(); //Heap and freespace are disposed by OS at process termination return 0; } From da16ffca78c1b5749cb2dd2509fa05c3a919631d Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 13:58:08 -0800 Subject: [PATCH 12/53] actually use the cleanup sighandler --- runtime/driver.c | 87 ++++++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 876c9ed6..50f966c5 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -609,32 +609,25 @@ static int count_non_null (void** xs){ //------------------ Child Processes ------------------------- //------------------------------------------------------------ -// This is the max that the parent can _ever_ allocate -// this approach is dumb -// but simple -const int MAX_PROCESSES = 64; - // Process metadata struct typedef struct { pid_t pid; - int* pipe_arr; + int (*pipe_arr)[NUM_STREAM_SPECS][2]; FILE* fin; FILE* fout; FILE* ferr; - int exit_status; // init to -1 } ChildProcess; // Free everything allocated by ChildProcess -static void cleanup_child (ChildProcess* c, int status) { - c->exit_status = status; +static void cleanup_proc (ChildProcess* c, int status) { // Close files fclose(c->fin); fclose(c->fout); fclose(c->ferr); - // Close pipes (TODO: is the indexing OK?) + // Close pipes for(int i = 0; ipipe_arr) + (i * sizeof(int)))[0]); - close(((c->pipe_arr) + (i * sizeof(int)))[1]); + close(*(c->pipe_arr)[i][0]); + close(*(c->pipe_arr)[i][1]); } // Destroy refs c->pipe_arr = NULL; @@ -664,11 +657,18 @@ ProcessNode* proc_head = NULL; DeadNode* dead_head = NULL; // Record process metadata -static void register_proc (ChildProcess* c) { +static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { + // Init ChildProcess struct + ChildProcess* child = (ChildProcess*)malloc(sizeof(ChildProcess)); + if(child == NULL) exit_with_error(); + child->pid = pid; + child->pipe_arr = pipes; + child->fin = fin; + child->ferr = ferr; + // Store child in ProcessNode ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); - if(new_node == NULL) - exit_with_error(); - new_node->proc = c; + if(new_node == NULL) exit_with_error(); + new_node->proc = child; new_node->next = proc_head; proc_head = new_node; } @@ -676,14 +676,14 @@ static void register_proc (ChildProcess* c) { // Record a dead process static void register_dead_proc (DeadProcess* c) { DeadNode* new_node = (DeadNode*)malloc(sizeof(DeadNode)); - if(new_node == NULL) - exit_with_error(); + if(new_node == NULL) exit_with_error(); new_node->proc = c; new_node->next = dead_head; dead_head = new_node; } -static void kill_child (pid_t pid, int status) { +// Cleanup all resources for process pid +static void cleanup_child (pid_t pid, int status) { ProcessNode* curr = proc_head; ProcessNode* prev = NULL; // Find matching Node @@ -699,7 +699,7 @@ static void kill_child (pid_t pid, int status) { prev->next = curr->next; } // Cleanup resources, generate a DeadProcess node to store exit status - cleanup_child(curr->proc, status); + cleanup_proc(curr->proc, status); DeadProcess* dead_proc = (DeadProcess*)malloc(sizeof(DeadProcess)); dead_proc->pid = pid; dead_proc->status = status; @@ -707,20 +707,21 @@ static void kill_child (pid_t pid, int status) { } } -void sigchild_handler(int signo) { +void sigchld_handler(int sig) { int status; pid_t pid; // Cleanup zombie child processes while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { - kill_child(pid, status); + if(WIFEXITED(status) || WIFSIGNALED(status)) + cleanup_child(pid, status); } } +// Free all ChildProcess nodes and their contents static void free_processes () { ProcessNode* curr = proc_head; ProcessNode* tmp = NULL; - // Free all nodes and their contents while(curr != NULL) { tmp = curr; curr = curr->next; @@ -729,10 +730,10 @@ static void free_processes () { } } +// Free all DeadProcess nodes and their contents static void free_dead_processes () { DeadNode* curr = dead_head; DeadNode* tmp = NULL; - // Free all nodes and their contents while(curr != NULL) { tmp = curr; curr = curr->next; @@ -886,21 +887,20 @@ stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error) { return 0; } -// TODO: delete this -// Only included for testing linux implementation on OSX -#ifdef PLATFORM_OS_X -extern char **environ; -int execvpe(const char *program, char **argv, char **envp){ - char **saved = environ; - environ = envp; - int rc = execvp(program, argv); - environ = saved; - return rc; -} -#endif +// Useful for testing linux implementation on OSX +//#ifdef PLATFORM_OS_X +//extern char **environ; +//int execvpe(const char *program, char **argv, char **envp){ +// char **saved = environ; +// environ = envp; +// int rc = execvp(program, argv); +// environ = saved; +// return rc; +//} +//#endif #ifdef PLATFORM_LINUX -//if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) +//#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int output, stz_int error, stz_byte* working_dir, stz_byte** env_vars, Process* process) { @@ -948,7 +948,8 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->in = fin; process->out = fout; process->err = ferr; - // TODO: register with SIGCHILD handler + + register_proc(pid, &pipes, fin, fout, ferr); } // Child: setup pipes, exec else { @@ -1062,7 +1063,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Spawn process pid_t pid = -1; if(posix_spawn(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars) == 0) { - // TODO: setup SIGCHILD handler //printf("Child pid: %d\n", pid); } else { exit_with_error(); @@ -1096,6 +1096,8 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->in = fin; process->out = fout; process->err = ferr; + + register_proc(pid, &pipes, fin, fout, ferr); return 0; } #endif @@ -1243,6 +1245,13 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Initialize trackers to empty list. init.trackers = NULL; + //Setup SIGCHLD handler + struct sigaction sa; + sa.sa_handler = sigchld_handler; + sa.sa_flags = SA_RESTART|SA_NOCLDSTOP; + sigaction(SIGCHLD, &sa, NULL); + + //Call Stanza entry stanza_entry(&init); From 0369fa09493fdbbb0648f72c5f6331af9b4d1486 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 14:00:58 -0800 Subject: [PATCH 13/53] trimmed --- runtime/driver.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 50f966c5..dc70d11d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -730,7 +730,7 @@ static void free_processes () { } } -// Free all DeadProcess nodes and their contents +// Free all DeadProcess nodes and their contents static void free_dead_processes () { DeadNode* curr = dead_head; DeadNode* tmp = NULL; @@ -781,7 +781,7 @@ static void write_optional_strings (FILE* f, stz_byte** s){ }else{ write_int(f, 1); write_strings(f, s); - } + } } static void write_process_state (FILE* f, ProcessState* s){ @@ -925,7 +925,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Parent: open files, register with signal handler if(pid > 0) { - FILE* fin = NULL; + FILE* fin = NULL; if(pipe_sources[PROCESS_IN] > 0) { close(pipes[PROCESS_IN][0]); fin = fdopen(pipes[PROCESS_IN][1], "w"); @@ -950,7 +950,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->err = ferr; register_proc(pid, &pipes, fin, fout, ferr); - } + } // Child: setup pipes, exec else { //Setup input pipe if used From ec52a80d69ed97ac8f8ef14eade150f2cef4ba6c Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 14:02:11 -0800 Subject: [PATCH 14/53] oops --- scripts/finish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/finish.sh b/scripts/finish.sh index 597e28f0..8ee98466 100755 --- a/scripts/finish.sh +++ b/scripts/finish.sh @@ -1,4 +1,4 @@ -# scripts/make-asmjit.sh os-x +scripts/make-asmjit.sh os-x gcc -std=gnu99 -c core/sha256.c -O3 -o build/sha256.o -I include gcc -std=gnu99 -c compiler/cvm.c -O3 -o build/cvm.o -I include gcc -c runtime/linenoise-ng/linenoise.cpp -O3 -o build/linenoise.o -I include -I runtime/linenoise-ng From f4cd0d9de2cc1ab6d1de8a5bb96d25e774fed260 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 15:55:59 -0800 Subject: [PATCH 15/53] windows compilation issue --- runtime/driver.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/driver.c b/runtime/driver.c index dc70d11d..0b425c33 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -9,8 +9,8 @@ #else #include #include + #include #endif -#include #include #include #include From 5b7d035705b5e2b02dbb0cb4f2300b2a3b1315bc Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 16:34:39 -0800 Subject: [PATCH 16/53] start revamp of retrieve_process_state --- core/core.stanza | 7 ----- runtime/driver.c | 75 ++++++++++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 304035d1..0bd87611 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -86,7 +86,6 @@ protected extern stz_memory_resize: (ptr, long, long) -> int protected extern close_process_handle: (ptr) -> int #else: protected extern launch_process: (ptr, ptr>, int, int, int, ptr, ptr>, ptr) -> int - protected extern delete_process_pipes: (ptr, ptr, ptr) -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int @@ -10266,12 +10265,6 @@ lostanza defn retrieve-state (p:ref, wait-for-termination?:reffin); + // Close files. TODO: should I check for EOF to throw an error? + fclose(c->fin); fclose(c->fout); fclose(c->ferr); // Close pipes @@ -682,6 +682,20 @@ static void register_dead_proc (DeadProcess* c) { dead_head = new_node; } +// Retrieve state of a dead process +static int get_state (pid_t pid, int* status) { + DeadNode* curr = dead_head; + while(curr != NULL && curr->proc->pid != pid) { + curr = curr->next; + } + if(curr != NULL) { + *status = curr->proc->status; + return 0; + } else { // pid not found + return -1; + } +} + // Cleanup all resources for process pid static void cleanup_child (pid_t pid, int status) { ProcessNode* curr = proc_head; @@ -869,24 +883,6 @@ static void write_error_and_exit (int fd){ exit(-1); } -static int delete_process_pipe (FILE* fd) { - if (fd != NULL) { - int close_res = fclose(fd); - if (close_res == EOF) return -1; - } - return 0; -} - -stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error) { - if (delete_process_pipe(input) < 0) - return -1; - if (delete_process_pipe(output) < 0) - return -1; - if (delete_process_pipe(error) < 0) - return -1; - return 0; -} - // Useful for testing linux implementation on OSX //#ifdef PLATFORM_OS_X //extern char **environ; @@ -1108,17 +1104,34 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ int status; - int ret = waitpid((pid_t)(process->pid), &status, wait_for_termination? 0 : WNOHANG); - if(ret == 0) - *s = (ProcessState){PROCESS_RUNNING, 0}; - else if(WIFEXITED(status)) - *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; - else if(WIFSIGNALED(status)) - *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; - else if(WIFSTOPPED(status)) - *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; - else - *s = (ProcessState){PROCESS_RUNNING, 0}; + + sigset_t sigchld_mask; + sigemptyset(&sigchld_mask); + sigaddset(&sigchld_mask, SIGCHLD); + + // block SIGCHLD during check + if(sigprocmask(SIG_BLOCK, &sigchld_mask, NULL)) + exit_with_error(); + + if(get_state(process->pid, &status) == 0) { + if(WIFEXITED(status)) + *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; + else if(WIFSIGNALED(status)) + *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; + else if(WIFSTOPPED(status)) + *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; + else exit_with_error(); + } else { + *s = (ProcessState){PROCESS_RUNNING, 0}; //exit_with_error(); + } + if(wait_for_termination) { + sleep(1); + // TODO: check again + } + + // TODO: maybe use the last argument and reset to initial state? + if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) + exit_with_error(); return 0; } From 28e1524ebf08c6588c34a63ccbfd9591b97c9e82 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 16:48:06 -0800 Subject: [PATCH 17/53] store STOPPED status --- runtime/driver.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/runtime/driver.c b/runtime/driver.c index ee901d62..9c046c7d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -616,6 +616,7 @@ typedef struct { FILE* fin; FILE* fout; FILE* ferr; + int status; } ChildProcess; // Free everything allocated by ChildProcess @@ -665,6 +666,7 @@ static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* f child->pipe_arr = pipes; child->fin = fin; child->ferr = ferr; + child->status = -1; // Store child in ProcessNode ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); if(new_node == NULL) exit_with_error(); @@ -683,6 +685,9 @@ static void register_dead_proc (DeadProcess* c) { } // Retrieve state of a dead process +// TODO: this will not detect STOPPED since it will still be in the +// list of live processes +// maybe make a separate list of process statuses? static int get_state (pid_t pid, int* status) { DeadNode* curr = dead_head; while(curr != NULL && curr->proc->pid != pid) { @@ -721,6 +726,21 @@ static void cleanup_child (pid_t pid, int status) { } } + +// Cleanup all resources for process pid +static void update_child (pid_t pid, int status) { + ProcessNode* curr = proc_head; + // Find matching Node + while(curr != NULL && curr->proc->pid != pid) { + curr = curr->next; + } + if(curr != NULL) { + curr->proc->status = status; + } +} + + + void sigchld_handler(int sig) { int status; pid_t pid; @@ -729,6 +749,7 @@ void sigchld_handler(int sig) { while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { if(WIFEXITED(status) || WIFSIGNALED(status)) cleanup_child(pid, status); + else update_child(pid, status); } } @@ -1261,7 +1282,7 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Setup SIGCHLD handler struct sigaction sa; sa.sa_handler = sigchld_handler; - sa.sa_flags = SA_RESTART|SA_NOCLDSTOP; + sa.sa_flags = SA_RESTART; sigaction(SIGCHLD, &sa, NULL); From 30d7cace84e5bd2dc6865f9c27bab456eeb1988a Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 16 Nov 2023 16:58:35 -0800 Subject: [PATCH 18/53] more updates, need to address possible race condition --- runtime/driver.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 9c046c7d..10dccfbc 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -1145,15 +1145,21 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ } else { *s = (ProcessState){PROCESS_RUNNING, 0}; //exit_with_error(); } - if(wait_for_termination) { - sleep(1); - // TODO: check again - } // TODO: maybe use the last argument and reset to initial state? + // TODO: use pselect to unblock and wait for SIGCHLD if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) exit_with_error(); + // TODO: possible race condition if SIGCHLD hits here + + if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { + // sleep should be interrupted by SIGCHLD + sleep(100); + // TODO: check again + retrieve_process_state(process, s, wait_for_termination); + } + return 0; } From eadddad5ed4aa3393e92afa23d41c619bf9bcf60 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Fri, 17 Nov 2023 09:33:36 -0800 Subject: [PATCH 19/53] store all statuses --- runtime/driver.c | 100 ++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 10dccfbc..8740bb54 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -606,11 +606,11 @@ static int count_non_null (void** xs){ //------------------------------------------------------------ -//------------------ Child Processes ------------------------- +//---------- Managing process resources and state ------------ //------------------------------------------------------------ // Process metadata struct -typedef struct { +typedef struct ChildProcess { pid_t pid; int (*pipe_arr)[NUM_STREAM_SPECS][2]; FILE* fin; @@ -619,6 +619,13 @@ typedef struct { int status; } ChildProcess; +// Linked lists of process metadata +typedef struct ProcessNode { + ChildProcess* proc; + struct ProcessNode* next; +} ProcessNode; + + // Free everything allocated by ChildProcess static void cleanup_proc (ChildProcess* c, int status) { // Close files. TODO: should I check for EOF to throw an error? @@ -637,25 +644,30 @@ static void cleanup_proc (ChildProcess* c, int status) { c->ferr = NULL; } -// Dead process exit code struct -typedef struct { +// Process status struct +typedef struct ProcessStatus { pid_t pid; int status; -} DeadProcess; +} ProcessStatus; -// Linked lists of alive/dead processes -typedef struct ProcessNode { - ChildProcess* proc; - struct ProcessNode* next; -} ProcessNode; - -typedef struct DeadNode { - DeadProcess* proc; - struct DeadNode* next; -} DeadNode; +typedef struct StatusNode { + ProcessStatus* proc; + struct StatusNode* next; +} StatusNode; +// Linked list of live processes ProcessNode* proc_head = NULL; -DeadNode* dead_head = NULL; +// Linked list of all process statuses +StatusNode* status_head = NULL; + +// Record a process' status +static void register_proc_status (ProcessStatus* c) { + StatusNode* new_node = (StatusNode*)malloc(sizeof(StatusNode)); + if(new_node == NULL) exit_with_error(); + new_node->proc = c; + new_node->next = status_head; + status_head = new_node; +} // Record process metadata static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { @@ -673,23 +685,19 @@ static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* f new_node->proc = child; new_node->next = proc_head; proc_head = new_node; -} -// Record a dead process -static void register_dead_proc (DeadProcess* c) { - DeadNode* new_node = (DeadNode*)malloc(sizeof(DeadNode)); - if(new_node == NULL) exit_with_error(); - new_node->proc = c; - new_node->next = dead_head; - dead_head = new_node; + // Init and register ProcessStatus struct + ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); + pstatus->pid = pid; + // TODO: is this OK? Maybe I don't even need to initialize + pstatus->status = -1; + register_proc_status(pstatus); } -// Retrieve state of a dead process -// TODO: this will not detect STOPPED since it will still be in the -// list of live processes -// maybe make a separate list of process statuses? + +// Retrieve process state static int get_state (pid_t pid, int* status) { - DeadNode* curr = dead_head; + StatusNode* curr = status_head; while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; } @@ -717,19 +725,14 @@ static void cleanup_child (pid_t pid, int status) { } else { prev->next = curr->next; } - // Cleanup resources, generate a DeadProcess node to store exit status + // Cleanup resources cleanup_proc(curr->proc, status); - DeadProcess* dead_proc = (DeadProcess*)malloc(sizeof(DeadProcess)); - dead_proc->pid = pid; - dead_proc->status = status; - register_dead_proc(dead_proc); } } - -// Cleanup all resources for process pid -static void update_child (pid_t pid, int status) { - ProcessNode* curr = proc_head; +// Update a process' status code +static void update_status (pid_t pid, int status) { + StatusNode* curr = status_head; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; @@ -739,21 +742,21 @@ static void update_child (pid_t pid, int status) { } } - - +// SIGCHLD handler: cleanup dead processes, record status change void sigchld_handler(int sig) { int status; pid_t pid; - // Cleanup zombie child processes while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { + // Cleanup dead process if(WIFEXITED(status) || WIFSIGNALED(status)) cleanup_child(pid, status); - else update_child(pid, status); + // Update process status + update_status(pid, status); } } -// Free all ChildProcess nodes and their contents +// Free all ProcessNodes and their contents static void free_processes () { ProcessNode* curr = proc_head; ProcessNode* tmp = NULL; @@ -765,10 +768,10 @@ static void free_processes () { } } -// Free all DeadProcess nodes and their contents -static void free_dead_processes () { - DeadNode* curr = dead_head; - DeadNode* tmp = NULL; +// Free all StatusNodes and their contents +static void free_status_nodes() { + StatusNode* curr = status_head; + StatusNode* tmp = NULL; while(curr != NULL) { tmp = curr; curr = curr->next; @@ -777,7 +780,6 @@ static void free_dead_processes () { } } - #endif //------------------------------------------------------------ @@ -1296,7 +1298,7 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { stanza_entry(&init); free_processes(); - free_dead_processes(); + free_status_nodes(); //Heap and freespace are disposed by OS at process termination return 0; } From 997fc9578f7c8f720e0c93deece1ebc03e8d9e2c Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Fri, 17 Nov 2023 13:10:51 -0800 Subject: [PATCH 20/53] fixed up seems to work --- runtime/driver.c | 63 ++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 8740bb54..c23f066c 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -75,6 +75,11 @@ static void exit_with_error_line_and_func (const char* file, int line){ #define exit_with_error() exit_with_error_line_and_func(__FILE__, __LINE__) +static void throw_error (const char * msg) { + fprintf(stderr, "%s\n", msg); + exit(-1); +} + // Stanza Defined Entities // ======================= typedef struct{ @@ -689,7 +694,7 @@ static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* f // Init and register ProcessStatus struct ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); pstatus->pid = pid; - // TODO: is this OK? Maybe I don't even need to initialize + // TODO: is status=-1 OK? pstatus->status = -1; register_proc_status(pstatus); } @@ -746,7 +751,6 @@ static void update_status (pid_t pid, int status) { void sigchld_handler(int sig) { int status; pid_t pid; - while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { // Cleanup dead process if(WIFEXITED(status) || WIFSIGNALED(status)) @@ -754,6 +758,8 @@ void sigchld_handler(int sig) { // Update process status update_status(pid, status); } + // it's OK if there are no more children + if(pid < 0 && errno != ECHILD) exit_with_error(); } // Free all ProcessNodes and their contents @@ -941,7 +947,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_long pid = (stz_long)vfork(); if(pid < 0) exit_with_error(); - // Parent: open files, register with signal handler if(pid > 0) { FILE* fin = NULL; @@ -1078,15 +1083,14 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, exit_with_error(); } - - // Spawn process pid_t pid = -1; - if(posix_spawn(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars) == 0) { - //printf("Child pid: %d\n", pid); + int spawn_ret; + if((spawn_ret = posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars)) == 0) { + // success } else { - exit_with_error(); + errno = spawn_ret; // might not want to do this manually? + return -1; } - // Cleanup posix_spawn_file_actions_destroy(&actions); @@ -1128,9 +1132,11 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ int status; - sigset_t sigchld_mask; + sigset_t sigchld_mask, empty_mask; sigemptyset(&sigchld_mask); sigaddset(&sigchld_mask, SIGCHLD); + // useful later + sigemptyset(&empty_mask); // block SIGCHLD during check if(sigprocmask(SIG_BLOCK, &sigchld_mask, NULL)) @@ -1143,25 +1149,36 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; else if(WIFSTOPPED(status)) *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; - else exit_with_error(); + else // exit_with_error(); + *s = (ProcessState){PROCESS_RUNNING, 0}; } else { - *s = (ProcessState){PROCESS_RUNNING, 0}; //exit_with_error(); + char* err; + sprintf(err, "Process %lld not found", process->pid); + throw_error(err); } - // TODO: maybe use the last argument and reset to initial state? - // TODO: use pselect to unblock and wait for SIGCHLD - if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) - exit_with_error(); - - // TODO: possible race condition if SIGCHLD hits here - if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { - // sleep should be interrupted by SIGCHLD - sleep(100); - // TODO: check again - retrieve_process_state(process, s, wait_for_termination); + // TODO: maybe parameterize? This is kinda silly + struct timespec to = {100, 0}; + if(pselect(0, NULL, NULL, NULL, &to, &empty_mask) < 0) { + if(errno == EINTR) {// Signal interrupt; try again + // reset mask to original state + if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) + exit_with_error(); + // try again + retrieve_process_state(process, s, wait_for_termination); + } else { + exit_with_error(); + } + } + else { // timeout, try again + retrieve_process_state(process, s, wait_for_termination); + } } + // reset mask to original state + if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) exit_with_error(); + return 0; } From 8d7d4c3364b81c5fcde55f1be9dd5cbf0fc318e9 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Fri, 17 Nov 2023 14:15:53 -0800 Subject: [PATCH 21/53] unbreak windows? --- runtime/driver.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index c23f066c..d3bfddfb 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -1304,18 +1304,22 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Initialize trackers to empty list. init.trackers = NULL; - //Setup SIGCHLD handler - struct sigaction sa; - sa.sa_handler = sigchld_handler; - sa.sa_flags = SA_RESTART; - sigaction(SIGCHLD, &sa, NULL); + #if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) + //Setup SIGCHLD handler + struct sigaction sa; + sa.sa_handler = sigchld_handler; + sa.sa_flags = SA_RESTART; + sigaction(SIGCHLD, &sa, NULL); + #endif //Call Stanza entry stanza_entry(&init); - free_processes(); - free_status_nodes(); + #if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) + free_processes(); + free_status_nodes(); + #endif //Heap and freespace are disposed by OS at process termination return 0; } From 936f6a2d933655da2362f4c1fb774a19132a9151 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 27 Nov 2023 09:41:09 -0800 Subject: [PATCH 22/53] small fixes --- runtime/driver.c | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index d3bfddfb..3ec3cba2 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -642,7 +642,7 @@ static void cleanup_proc (ChildProcess* c, int status) { close(*(c->pipe_arr)[i][0]); close(*(c->pipe_arr)[i][1]); } - // Destroy refs + // Destroy refs (might not need to do this?) c->pipe_arr = NULL; c->fin = NULL; c->fout = NULL; @@ -950,19 +950,19 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Parent: open files, register with signal handler if(pid > 0) { FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] > 0) { + if(pipe_sources[PROCESS_IN] >= 0) { close(pipes[PROCESS_IN][0]); fin = fdopen(pipes[PROCESS_IN][1], "w"); if(fin == NULL) return -1; } FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] > 0) { + if(pipe_sources[PROCESS_OUT] >= 0) { close(pipes[PROCESS_OUT][1]); fout = fdopen(pipes[PROCESS_OUT][0], "r"); if(fout == NULL) return -1; } FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] > 0) { + if(pipe_sources[PROCESS_ERR] >= 0) { close(pipes[PROCESS_ERR][1]); ferr = fdopen(pipes[PROCESS_ERR][0], "r"); if(ferr == NULL) return -1; @@ -978,7 +978,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Child: setup pipes, exec else { //Setup input pipe if used - if(pipe_sources[PROCESS_IN] > 0) { + if(pipe_sources[PROCESS_IN] >= 0) { if(close(pipes[PROCESS_IN][1]) < 0) exit_with_error(); if(dup2(pipes[PROCESS_IN][0], STDIN_FILENO) < 0) @@ -987,7 +987,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, exit_with_error(); } //Setup output pipe if used - if(pipe_sources[PROCESS_OUT] > 0) { + if(pipe_sources[PROCESS_OUT] >= 0) { if(close(pipes[PROCESS_OUT][0]) < 0) exit_with_error(); if(dup2(pipes[PROCESS_OUT][1], STDOUT_FILENO) < 0) @@ -996,7 +996,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, exit_with_error(); } //Setup error pipe if used - if(pipe_sources[PROCESS_ERR] > 0) { + if(pipe_sources[PROCESS_ERR] >= 0) { if(close(pipes[PROCESS_ERR][0]) < 0) exit_with_error(); if(dup2(pipes[PROCESS_ERR][1], STDERR_FILENO) < 0) @@ -1050,7 +1050,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int posix_ret; //Setup input pipe if used - if(pipe_sources[PROCESS_IN] > 0) { + if(pipe_sources[PROCESS_IN] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) exit_with_error(); if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) @@ -1059,7 +1059,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, exit_with_error(); } //Setup output pipe if used - if(pipe_sources[PROCESS_OUT] > 0) { + if(pipe_sources[PROCESS_OUT] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) exit_with_error(); if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) @@ -1068,7 +1068,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, exit_with_error(); } //Setup error pipe if used - if(pipe_sources[PROCESS_ERR] > 0) { + if(pipe_sources[PROCESS_ERR] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) exit_with_error(); if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) @@ -1097,19 +1097,19 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Parent: //Close pipes, setup files FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] > 0) { + if(pipe_sources[PROCESS_IN] >= 0) { close(pipes[PROCESS_IN][0]); fin = fdopen(pipes[PROCESS_IN][1], "w"); if(fin == NULL) return -1; } FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] > 0) { + if(pipe_sources[PROCESS_OUT] >= 0) { close(pipes[PROCESS_OUT][1]); fout = fdopen(pipes[PROCESS_OUT][0], "r"); if(fout == NULL) return -1; } FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] > 0) { + if(pipe_sources[PROCESS_ERR] >= 0) { close(pipes[PROCESS_ERR][1]); ferr = fdopen(pipes[PROCESS_ERR][0], "r"); if(ferr == NULL) return -1; @@ -1132,14 +1132,14 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ int status; - sigset_t sigchld_mask, empty_mask; + sigset_t sigchld_mask, old_mask; sigemptyset(&sigchld_mask); sigaddset(&sigchld_mask, SIGCHLD); // useful later - sigemptyset(&empty_mask); + // sigemptyset(&empty_mask); // block SIGCHLD during check - if(sigprocmask(SIG_BLOCK, &sigchld_mask, NULL)) + if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) exit_with_error(); if(get_state(process->pid, &status) == 0) { @@ -1158,9 +1158,8 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ } if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { - // TODO: maybe parameterize? This is kinda silly - struct timespec to = {100, 0}; - if(pselect(0, NULL, NULL, NULL, &to, &empty_mask) < 0) { + // no file descriptors or timeout; just wait indefinitely for an interrupt + if(pselect(0, NULL, NULL, NULL, NULL, &old_mask) < 0) { if(errno == EINTR) {// Signal interrupt; try again // reset mask to original state if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) @@ -1177,7 +1176,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ } // reset mask to original state - if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) exit_with_error(); + if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); return 0; } From fdd525ce77414c803790d952ab7cd5b551ad0067 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 27 Nov 2023 10:38:04 -0800 Subject: [PATCH 23/53] errors --- runtime/driver.c | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 3ec3cba2..089d57a5 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -666,19 +666,20 @@ ProcessNode* proc_head = NULL; StatusNode* status_head = NULL; // Record a process' status -static void register_proc_status (ProcessStatus* c) { +static int register_proc_status (ProcessStatus* c) { StatusNode* new_node = (StatusNode*)malloc(sizeof(StatusNode)); - if(new_node == NULL) exit_with_error(); + if(new_node == NULL) return -1; new_node->proc = c; new_node->next = status_head; status_head = new_node; + return 0; } // Record process metadata -static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { +static int register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { // Init ChildProcess struct ChildProcess* child = (ChildProcess*)malloc(sizeof(ChildProcess)); - if(child == NULL) exit_with_error(); + if(child == NULL) return -1; child->pid = pid; child->pipe_arr = pipes; child->fin = fin; @@ -686,7 +687,7 @@ static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* f child->status = -1; // Store child in ProcessNode ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); - if(new_node == NULL) exit_with_error(); + if(new_node == NULL) return -1; new_node->proc = child; new_node->next = proc_head; proc_head = new_node; @@ -696,7 +697,8 @@ static void register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* f pstatus->pid = pid; // TODO: is status=-1 OK? pstatus->status = -1; - register_proc_status(pstatus); + RETURN_NEG(register_proc_status(pstatus)) + return 0; } @@ -946,7 +948,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Fork child process stz_long pid = (stz_long)vfork(); - if(pid < 0) exit_with_error(); + if(pid < 0) return -1; // Parent: open files, register with signal handler if(pid > 0) { FILE* fin = NULL; @@ -973,7 +975,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - register_proc(pid, &pipes, fin, fout, ferr); + RETURN_NEG(register_proc(pid, &pipes, fin, fout, ferr)) } // Child: setup pipes, exec else { @@ -1012,13 +1014,11 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Launch child process. //If an environment is supplied then call execvpe, otherwise call execvp. - int cstatus; if(env_vars == NULL) - cstatus = execvp(C_CSTR(file), (char**)argvs); + execvp(C_CSTR(file), (char**)argvs); else - cstatus = execvpe(C_CSTR(file), (char**)argvs, (char**)env_vars); - if(cstatus < 0) - exit_with_error(); + execvpe(C_CSTR(file), (char**)argvs, (char**)env_vars); + exit_with_error(); } return 0; } @@ -1052,35 +1052,35 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Setup input pipe if used if(pipe_sources[PROCESS_IN] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) - exit_with_error(); + return -1; if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) - exit_with_error(); + return -1; if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0]))) - exit_with_error(); + return -1; } //Setup output pipe if used if(pipe_sources[PROCESS_OUT] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) - exit_with_error(); + return -1; if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) - exit_with_error(); + return -1; if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1]))) - exit_with_error(); + return -1; } //Setup error pipe if used if(pipe_sources[PROCESS_ERR] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) - exit_with_error(); + return -1; if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) - exit_with_error(); + return -1; if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1]))) - exit_with_error(); + return -1; } //Setup working directory if(working_dir) { if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, C_CSTR(working_dir)))) - exit_with_error(); + return -1; } pid_t pid = -1; @@ -1120,7 +1120,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - register_proc(pid, &pipes, fin, fout, ferr); + RETURN_NEG(register_proc(pid, &pipes, fin, fout, ferr)) return 0; } #endif From 87c3cd2eebef02ce398b37903a416e685becdf6a Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 27 Nov 2023 14:13:11 -0800 Subject: [PATCH 24/53] briging back unique ids --- core/core.stanza | 11 +++++++---- runtime/driver.c | 40 ++++++++++++++++++++++------------------ runtime/process-win32.c | 1 + runtime/process.h | 1 + 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 0bd87611..8ddaae35 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -85,7 +85,7 @@ protected extern stz_memory_resize: (ptr, long, long) -> int protected extern launch_process: (ptr, int, int, int, ptr, ptr, ptr) -> int protected extern close_process_handle: (ptr) -> int #else: - protected extern launch_process: (ptr, ptr>, int, int, int, ptr, ptr>, ptr) -> int + protected extern launch_process: (ptr, ptr>, int, int, int, int, ptr, ptr>, ptr) -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int @@ -9918,6 +9918,7 @@ public val CURRENT-PLATFORM:Platform = public lostanza deftype Process <: Unique : var pid: long var handle: ptr + var pipeid: int var input: ptr var output: ptr var error: ptr @@ -10016,7 +10017,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, env-var-mode:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) - val proc = new Process{0, null, null, null, null, false, false, false, false} + val proc = new Process{0, null, -1, null, null, null, false, false, false, false} ; Create a command line from the given filename and args (discarding the first argument). ; This is necessary because Windows' process API expects a command line (not a list of arguments). @@ -10086,6 +10087,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, return copy-buffer-to-stable-memory(buffer) #else: + val PIPE-ID-GENERATOR = Random() public lostanza defn Process (filename:ref, args0:ref>, input:ref, @@ -10097,7 +10099,8 @@ defn process-env-var-mode (env-vars:Tuple>|False, ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) - val proc = new Process{0, null, null, null, null, false, false, false, false} + val pipe-id = next-int(PIPE-ID-GENERATOR).value + val proc = new Process{0, null, pipe-id, null, null, null, false, false, false, false} val input_v = value(input).value val output_v = value(output).value val error_v = value(error).value @@ -10119,7 +10122,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ;Launch the process val launch_succ = call-c clib/launch_process(addr!(filename.chars), argvs, - input_v, output_v, error_v, working-dir-chars, env-var-string, addr!([proc])) + input_v, output_v, error_v, pipe-id, working-dir-chars, env-var-string, addr!([proc])) ;Free the memory created using malloc. free-linux-env-var-string(env-var-string) diff --git a/runtime/driver.c b/runtime/driver.c index 089d57a5..8304e90a 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -621,7 +621,7 @@ typedef struct ChildProcess { FILE* fin; FILE* fout; FILE* ferr; - int status; + int* status; } ChildProcess; // Linked lists of process metadata @@ -632,7 +632,7 @@ typedef struct ProcessNode { // Free everything allocated by ChildProcess -static void cleanup_proc (ChildProcess* c, int status) { +static void cleanup_proc (ChildProcess* c) { // Close files. TODO: should I check for EOF to throw an error? fclose(c->fin); fclose(c->fout); @@ -652,6 +652,7 @@ static void cleanup_proc (ChildProcess* c, int status) { // Process status struct typedef struct ProcessStatus { pid_t pid; + stz_int pipeid; int status; } ProcessStatus; @@ -677,6 +678,15 @@ static int register_proc_status (ProcessStatus* c) { // Record process metadata static int register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { + + // Init and register ProcessStatus struct + ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); + pstatus->pid = pid; + pstatus->pipeid = -1; + // TODO: is status=-1 OK? + pstatus->status = -1; + RETURN_NEG(register_proc_status(pstatus)) + // Init ChildProcess struct ChildProcess* child = (ChildProcess*)malloc(sizeof(ChildProcess)); if(child == NULL) return -1; @@ -684,7 +694,7 @@ static int register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fi child->pipe_arr = pipes; child->fin = fin; child->ferr = ferr; - child->status = -1; + child->status = &(pstatus->status); // Store child in ProcessNode ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); if(new_node == NULL) return -1; @@ -692,18 +702,12 @@ static int register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fi new_node->next = proc_head; proc_head = new_node; - // Init and register ProcessStatus struct - ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); - pstatus->pid = pid; - // TODO: is status=-1 OK? - pstatus->status = -1; - RETURN_NEG(register_proc_status(pstatus)) return 0; } // Retrieve process state -static int get_state (pid_t pid, int* status) { +static int get_status (pid_t pid, int* status) { StatusNode* curr = status_head; while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; @@ -733,19 +737,19 @@ static void cleanup_child (pid_t pid, int status) { prev->next = curr->next; } // Cleanup resources - cleanup_proc(curr->proc, status); + cleanup_proc(curr->proc); } } // Update a process' status code static void update_status (pid_t pid, int status) { - StatusNode* curr = status_head; + ProcessNode* curr = proc_head; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; } if(curr != NULL) { - curr->proc->status = status; + *(curr->proc->status) = status; } } @@ -754,11 +758,11 @@ void sigchld_handler(int sig) { int status; pid_t pid; while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { + // Update process status + update_status(pid, status); // Cleanup dead process if(WIFEXITED(status) || WIFSIGNALED(status)) cleanup_child(pid, status); - // Update process status - update_status(pid, status); } // it's OK if there are no more children if(pid < 0 && errno != ECHILD) exit_with_error(); @@ -929,7 +933,7 @@ static void write_error_and_exit (int fd){ #ifdef PLATFORM_LINUX //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, + stz_int output, stz_int error, stz_int pipeid, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -1028,7 +1032,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, #ifdef PLATFORM_OS_X stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, + stz_int output, stz_int error, stz_int pipeid, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -1142,7 +1146,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) exit_with_error(); - if(get_state(process->pid, &status) == 0) { + if(get_status(process->pid, &status) == 0) { if(WIFEXITED(status)) *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; else if(WIFSIGNALED(status)) diff --git a/runtime/process-win32.c b/runtime/process-win32.c index a618225d..8e1b5a4c 100644 --- a/runtime/process-win32.c +++ b/runtime/process-win32.c @@ -273,6 +273,7 @@ stz_int launch_process(stz_byte* command_line, if (success) { // Populate process with the relevant info process->pid = (stz_long)proc_info.dwProcessId; + process->pipeid = -1; // no pipe ids on Windows process->handle = (void*)proc_info.hProcess; process->in = file_from_handle(stdin_write, FT_WRITE); process->out = file_from_handle(stdout_read, FT_READ); diff --git a/runtime/process.h b/runtime/process.h index 41429337..9121fa6c 100644 --- a/runtime/process.h +++ b/runtime/process.h @@ -13,6 +13,7 @@ typedef struct { stz_long pid; void* handle; + stz_int pipeid; FILE* in; FILE* out; FILE* err; From 3422d5d8458ffbdce8c71741d5260fcd1f41e53b Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 27 Nov 2023 14:18:20 -0800 Subject: [PATCH 25/53] actually index status by unique id --- runtime/driver.c | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 8304e90a..17915870 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -651,7 +651,6 @@ static void cleanup_proc (ChildProcess* c) { // Process status struct typedef struct ProcessStatus { - pid_t pid; stz_int pipeid; int status; } ProcessStatus; @@ -677,12 +676,11 @@ static int register_proc_status (ProcessStatus* c) { } // Record process metadata -static int register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { +static int register_proc (pid_t pid, stz_int pipeid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { // Init and register ProcessStatus struct ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); - pstatus->pid = pid; - pstatus->pipeid = -1; + pstatus->pipeid = pipeid; // TODO: is status=-1 OK? pstatus->status = -1; RETURN_NEG(register_proc_status(pstatus)) @@ -707,9 +705,9 @@ static int register_proc (pid_t pid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fi // Retrieve process state -static int get_status (pid_t pid, int* status) { +static int get_status (stz_int pipeid, int* status) { StatusNode* curr = status_head; - while(curr != NULL && curr->proc->pid != pid) { + while(curr != NULL && curr->proc->pipeid != pipeid) { curr = curr->next; } if(curr != NULL) { @@ -907,17 +905,6 @@ static void free_strings (stz_byte** ss){ //---------------------- Launcher Main ----------------------- //------------------------------------------------------------ -#define LAUNCH_COMMAND 0 -#define STATE_COMMAND 1 -#define WAIT_COMMAND 2 - -static void write_error_and_exit (int fd){ - int code = errno; - write(fd, &code, sizeof(int)); - close(fd); - exit(-1); -} - // Useful for testing linux implementation on OSX //#ifdef PLATFORM_OS_X //extern char **environ; @@ -975,11 +962,12 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } process->pid = pid; + process->pipeid = pipeid; process->in = fin; process->out = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, &pipes, fin, fout, ferr)) + RETURN_NEG(register_proc(pid, pipeid, &pipes, fin, fout, ferr)) } // Child: setup pipes, exec else { @@ -1120,11 +1108,12 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } process->pid = pid; + process->pipeid = pipeid; process->in = fin; process->out = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, &pipes, fin, fout, ferr)) + RETURN_NEG(register_proc(pid, pipeid, &pipes, fin, fout, ferr)) return 0; } #endif @@ -1139,21 +1128,19 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ sigset_t sigchld_mask, old_mask; sigemptyset(&sigchld_mask); sigaddset(&sigchld_mask, SIGCHLD); - // useful later - // sigemptyset(&empty_mask); // block SIGCHLD during check if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) exit_with_error(); - if(get_status(process->pid, &status) == 0) { + if(get_status(process->pipeid, &status) == 0) { if(WIFEXITED(status)) *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; else if(WIFSIGNALED(status)) *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; else if(WIFSTOPPED(status)) *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; - else // exit_with_error(); + else *s = (ProcessState){PROCESS_RUNNING, 0}; } else { char* err; From 632d47b28f553c6e64c970f3779040b597af098a Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 27 Nov 2023 14:31:48 -0800 Subject: [PATCH 26/53] actually unique ids --- core/core.stanza | 9 ++++----- runtime/driver.c | 24 ++++++++++++------------ runtime/process.h | 2 +- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 8ddaae35..7a36e589 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -9918,7 +9918,7 @@ public val CURRENT-PLATFORM:Platform = public lostanza deftype Process <: Unique : var pid: long var handle: ptr - var pipeid: int + var stz-proc-id: int var input: ptr var output: ptr var error: ptr @@ -10087,7 +10087,6 @@ defn process-env-var-mode (env-vars:Tuple>|False, return copy-buffer-to-stable-memory(buffer) #else: - val PIPE-ID-GENERATOR = Random() public lostanza defn Process (filename:ref, args0:ref>, input:ref, @@ -10099,8 +10098,8 @@ defn process-env-var-mode (env-vars:Tuple>|False, ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) - val pipe-id = next-int(PIPE-ID-GENERATOR).value - val proc = new Process{0, null, pipe-id, null, null, null, false, false, false, false} + val stz-proc-id = genid().value + val proc = new Process{0, null, stz-proc-id, null, null, null, false, false, false, false} val input_v = value(input).value val output_v = value(output).value val error_v = value(error).value @@ -10122,7 +10121,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ;Launch the process val launch_succ = call-c clib/launch_process(addr!(filename.chars), argvs, - input_v, output_v, error_v, pipe-id, working-dir-chars, env-var-string, addr!([proc])) + input_v, output_v, error_v, stz-proc-id, working-dir-chars, env-var-string, addr!([proc])) ;Free the memory created using malloc. free-linux-env-var-string(env-var-string) diff --git a/runtime/driver.c b/runtime/driver.c index 17915870..a8749f2d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -651,7 +651,7 @@ static void cleanup_proc (ChildProcess* c) { // Process status struct typedef struct ProcessStatus { - stz_int pipeid; + stz_int stz_proc_id; int status; } ProcessStatus; @@ -676,11 +676,11 @@ static int register_proc_status (ProcessStatus* c) { } // Record process metadata -static int register_proc (pid_t pid, stz_int pipeid, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { +static int register_proc (pid_t pid, stz_int stz_proc_id, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { // Init and register ProcessStatus struct ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); - pstatus->pipeid = pipeid; + pstatus->stz_proc_id = stz_proc_id; // TODO: is status=-1 OK? pstatus->status = -1; RETURN_NEG(register_proc_status(pstatus)) @@ -705,9 +705,9 @@ static int register_proc (pid_t pid, stz_int pipeid, int (*pipes)[NUM_STREAM_SPE // Retrieve process state -static int get_status (stz_int pipeid, int* status) { +static int get_status (stz_int stz_proc_id, int* status) { StatusNode* curr = status_head; - while(curr != NULL && curr->proc->pipeid != pipeid) { + while(curr != NULL && curr->proc->stz_proc_id != stz_proc_id) { curr = curr->next; } if(curr != NULL) { @@ -920,7 +920,7 @@ static void free_strings (stz_byte** ss){ #ifdef PLATFORM_LINUX //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int pipeid, + stz_int output, stz_int error, stz_int stz_proc_id, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -962,12 +962,12 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } process->pid = pid; - process->pipeid = pipeid; + process->stz_proc_id = stz_proc_id; process->in = fin; process->out = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, pipeid, &pipes, fin, fout, ferr)) + RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr)) } // Child: setup pipes, exec else { @@ -1020,7 +1020,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, #ifdef PLATFORM_OS_X stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int pipeid, + stz_int output, stz_int error, stz_int stz_proc_id, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -1108,12 +1108,12 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } process->pid = pid; - process->pipeid = pipeid; + process->stz_proc_id = stz_proc_id; process->in = fin; process->out = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, pipeid, &pipes, fin, fout, ferr)) + RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr)) return 0; } #endif @@ -1133,7 +1133,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) exit_with_error(); - if(get_status(process->pipeid, &status) == 0) { + if(get_status(process->stz_proc_id, &status) == 0) { if(WIFEXITED(status)) *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; else if(WIFSIGNALED(status)) diff --git a/runtime/process.h b/runtime/process.h index 9121fa6c..2a7f3589 100644 --- a/runtime/process.h +++ b/runtime/process.h @@ -13,7 +13,7 @@ typedef struct { stz_long pid; void* handle; - stz_int pipeid; + stz_int stz_proc_id; FILE* in; FILE* out; FILE* err; From 6bab95dc2a91825cf3e4c2b42c96d9fec1645ab0 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 27 Nov 2023 15:05:33 -0800 Subject: [PATCH 27/53] no recursion --- core/core.stanza | 58 ++++++++++++++++++++++++------------------------ runtime/driver.c | 56 ++++++++++++++++++++++++---------------------- 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 7a36e589..ae1521f3 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -2435,7 +2435,7 @@ public lostanza defn collect-garbage (allocation-size:long, vms:ptr) -> ;Step 1. Define the desired size of the nursery. val nursery-size = compute-nursery-size(allocation-size, heap) - + ;Determine whether we should attempt a partial GC at all. ;Skip the partial GC if the desired nursery size is less than the ;amount of space available. @@ -3520,7 +3520,7 @@ defstruct DyCtxtFrame : ;Create a new non-finally dynamic context frame. defn DyCtxtFrame (in:False|(() -> ?), out:False|(() -> ?)) -> DyCtxtFrame : - DyCtxtFrame(false, in, out) + DyCtxtFrame(false, in, out) ;Create a new finally dynamic context frame. defn DyCtxtFinallyFrame (out:False|(() -> ?)) -> DyCtxtFrame : @@ -3626,7 +3626,7 @@ defn empty? (s:DyCtxtState) -> True|False : empty?(frames(s)) ;Execute and push a new frame onto the state. -;The frame is pushed *after* it executes. +;The frame is pushed *after* it executes. defn push-frame (s:DyCtxtState, f:DyCtxtFrame) -> False : match(in(f)) : (f:False) : false @@ -3685,7 +3685,7 @@ defn wind-in (winder:DyCtxtWinder, c:RawCoroutine) -> False : exec-frame(cos, key(entry), value(entry)) defn* exec-frame (cos:List, co-id:Int, frame:DyCtxtFrame) : val co = head(cos) - if id(co) == co-id : + if id(co) == co-id : push-frame(dy-ctxt-state(co), frame) pop-next-frame(cos) else : @@ -3865,7 +3865,7 @@ lostanza defmethod resume (c:ref, x:ref) -> ref : ;Adjust state attach-coroutine(c) - + ;Begin execution val result = call-prim yield(current-coroutine.stack, x) @@ -3979,7 +3979,7 @@ lostanza defn free-coroutine (c:ref) -> int : ;Fix up the .parent, .status, and .crsp links from ;attaching the given coroutine to 'current-coroutine'. ;The current coroutine will be set to the last parent in the chain. -lostanza defn attach-coroutine (c:ref) -> int : +lostanza defn attach-coroutine (c:ref) -> int : labels : begin : goto loop(c) @@ -5174,7 +5174,7 @@ lostanza defmethod get-int (i:ref) -> ref : if n == 4 : val int-ptr = addr!(READ-BUFFER.chars) as ptr return new Int{[int-ptr]} - return false + return false ;Optimized implementation for FileInputStream. lostanza defmethod get-long (i:ref) -> ref : @@ -5196,7 +5196,7 @@ lostanza defmethod get-float (i:ref) -> ref : if n == 4 : val float-ptr = addr!(READ-BUFFER.chars) as ptr return new Float{[float-ptr]} - return false + return false ;Generic implementation for all input streams. defmethod get-double (i:InputStream) -> False|Double : @@ -6084,7 +6084,7 @@ defmethod print (o:OutputStream, e:CreateDirException) : ;Recursively create all directories that lead up to filepath. ;Behaves like 'mkdir -p'. ;Throws an exception if the filepath (or a path leading up to filepath) -;is not a directory. +;is not a directory. public defn create-dir-recursive (filepath:String, permissions:Int) -> False : val path = parse-path(filepath) @@ -6174,7 +6174,7 @@ public lostanza defn get-env-vars () -> ref>> : ;Helper: Add x to vector v if x is not empty. defn add? (v:Vector, x:Maybe) -> False : - add(v,value!(x)) when not empty?(x) + add(v,value!(x)) when not empty?(x) ;Helper: Parse an env-var string with foramt: "VAR=VALUE". ;Returns None() if it is not in that format. @@ -8381,7 +8381,7 @@ public defn map (f: Int -> ?R, r:Range) -> List : ;An efficient implementation for the most commonly-used ;type of Range: a starting integer, an exclusive ending integer, -;and a step of +1. +;and a step of +1. public deftype SimpleRange <: Range ;A SimpleRange always has a step of +1, and has @@ -8577,7 +8577,7 @@ defmethod to-seq (s:Seq) : s ;must not be empty. public defmulti next (s:Seq) -> T -;Test whether the sequence is empty. +;Test whether the sequence is empty. public defmulti empty? (s:Seq) -> True|False ;Release any resources required by the seq. Default @@ -8615,16 +8615,16 @@ public defn PeekSeq (xs-seq:Seq) -> PeekSeq : ;not be empty. This function guarantees that the buffer ;is full. defn fill () : (item = next(xs-seq)) when item is Sentinel - + new PeekSeq : - + defmethod next (this) : ;Retrieve the next item in the buffer, and then ;empty the buffer. val x = peek(this) item = sentinel x - + defmethod peek (this) : ;Ensure that the buffer is full. If the underlying ;Seq is empty, this call will properly fail. @@ -8632,7 +8632,7 @@ public defn PeekSeq (xs-seq:Seq) -> PeekSeq : ;If the fill() succeeds, then we're guaranteed that ;the buffer is full. item as T - + defmethod empty? (this) : ;The PeekSeq is empty if both the buffer is empty ;and the underlying sequence is empty. @@ -8982,7 +8982,7 @@ public defn first! (f: (T,S) -> Maybe, xs:Seqable, ys:Seqable< ;Helper function that creates a sequence from a generating ;function. The generating function returns Sentinel when it -;is exhausted. +;is exhausted. ;- f: The callback that is called repeatedly to retrieve ; each item in the sequence. When the sequence is empty, f must ; return and continue returning Sentinel. @@ -9323,7 +9323,7 @@ public defn cat-all (input-xss:Seqable>) -> Seq : ;Hold the currently active sequence. Initialized ;to false in the beginning. var xs:Seq|False = false - + ;Advance 'xs' to first non empty? seq. Returns true if any is available. ;Guarantees that 'xs' is a non empty seq in this case. defn* ready? () -> True|False : @@ -9335,7 +9335,7 @@ public defn cat-all (input-xss:Seqable>) -> Seq : ready?() else : true - + new Seq : defmethod next (this) : ready?() @@ -10098,7 +10098,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) - val stz-proc-id = genid().value + val stz-proc-id = genid().value val proc = new Process{0, null, stz-proc-id, null, null, null, false, false, false, false} val input_v = value(input).value val output_v = value(output).value @@ -10137,7 +10137,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, match(env-vars) : (env-vars:ref) : return null - (env-vars:ref>>) : + (env-vars:ref>>) : val n = env-vars.length val scratch = ByteBuffer() val env-array:ptr> = call-c clib/malloc((n + 1) * sizeof(ptr)) @@ -10219,7 +10219,7 @@ public defn Process (filename:String, working-dir:String|False, env-vars:Tuple>|False) -> Process : Process(filename, args, input, output, error, working-dir, env-vars, AddEnvVars) - + ;Default working-dir is false. Default env-vars is false. public defn Process (filename:String, args:Seqable, input:StreamSpecifier, output:StreamSpecifier, error:StreamSpecifier) -> Process : @@ -10315,7 +10315,7 @@ public defn call-system (filename:String, env-vars:Tuple>|False) -> Int : match(wait(Process(filename, args, working-dir, env-vars))) : (s:ProcessDone) : value(s) - (s) : throw(ProcessAbortedError(s)) + (s) : throw(ProcessAbortedError(s)) public defn call-system (file:String, args:Seqable) -> Int : call-system(file, args, false, false) @@ -10323,12 +10323,12 @@ public defn call-system (file:String, args:Seqable) -> Int : public defn call-system (args:Tuple, working-dir:String|False, env-vars:Tuple>|False) -> Int : - if empty?(args) : + if empty?(args) : fatal("Arguments must have length 1. Program name is expected to be first argument.") call-system(args[0], args, working-dir, env-vars) public defn call-system (args:Tuple) -> Int : - if empty?(args) : + if empty?(args) : fatal("Arguments must have minimum length 1. Program name is expected to be first argument.") call-system(args[0], args, false, false) @@ -10357,16 +10357,16 @@ public defn call-system-and-get-output (file:String, args:Seqable) -> St public defn call-system-and-get-output (args:Tuple, working-dir:String|False, env-vars:Tuple>|False) -> String : - if empty?(args) : - fatal("Arguments must have minimum length 1. Program name is expected to be first argument.") + if empty?(args) : + fatal("Arguments must have minimum length 1. Program name is expected to be first argument.") call-system-and-get-output(args[0], args, working-dir, env-vars) public defn call-system-and-get-output (args:Tuple) -> String : - if empty?(args) : - fatal("Arguments must have minimum length 1. Program name is expected to be first argument.") + if empty?(args) : + fatal("Arguments must have minimum length 1. Program name is expected to be first argument.") call-system-and-get-output(args[0], args, false, false) ;============================================================ diff --git a/runtime/driver.c b/runtime/driver.c index a8749f2d..1e97f5d5 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -1125,6 +1125,8 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ int status; + bool state_unknown = true; + sigset_t sigchld_mask, old_mask; sigemptyset(&sigchld_mask); sigaddset(&sigchld_mask, SIGCHLD); @@ -1133,36 +1135,38 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) exit_with_error(); - if(get_status(process->stz_proc_id, &status) == 0) { - if(WIFEXITED(status)) - *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; - else if(WIFSIGNALED(status)) - *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; - else if(WIFSTOPPED(status)) - *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; - else - *s = (ProcessState){PROCESS_RUNNING, 0}; - } else { - char* err; - sprintf(err, "Process %lld not found", process->pid); - throw_error(err); - } + while(state_unknown) { + if(get_status(process->stz_proc_id, &status) == 0) { + if(WIFEXITED(status)) + *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; + else if(WIFSIGNALED(status)) + *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; + else if(WIFSTOPPED(status)) + *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; + else + *s = (ProcessState){PROCESS_RUNNING, 0}; + state_unknown = false; + } else { + char* err; + sprintf(err, "Process %lld not found", process->pid); + throw_error(err); + } - if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { - // no file descriptors or timeout; just wait indefinitely for an interrupt - if(pselect(0, NULL, NULL, NULL, NULL, &old_mask) < 0) { - if(errno == EINTR) {// Signal interrupt; try again - // reset mask to original state - if(sigprocmask(SIG_UNBLOCK, &sigchld_mask, NULL)) + if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { + // no file descriptors or timeout; just wait indefinitely for an interrupt + if(pselect(0, NULL, NULL, NULL, NULL, &old_mask) < 0) { + if(errno == EINTR) { + // interrupt: try again + state_unknown = true; + } else { + // non-interrupt error: impossible exit_with_error(); - // try again - retrieve_process_state(process, s, wait_for_termination); - } else { - exit_with_error(); + } } } - else { // timeout, try again - retrieve_process_state(process, s, wait_for_termination); + if(!wait_for_termination) { // TODO: should this be an else? + // do not loop if we don't need the process to terminate + state_unknown = false; } } From 66bdd910b60c2af7e2fe17c72ff7d93d7ece4c27 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 27 Nov 2023 15:53:27 -0800 Subject: [PATCH 28/53] touch ups --- runtime/driver.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 1e97f5d5..03e755a1 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -633,20 +633,19 @@ typedef struct ProcessNode { // Free everything allocated by ChildProcess static void cleanup_proc (ChildProcess* c) { - // Close files. TODO: should I check for EOF to throw an error? - fclose(c->fin); - fclose(c->fout); - fclose(c->ferr); + // Close files + if(c->fin != NULL) + if(fclose(c->fin) == EOF) exit_with_error(); + if(c->fout != NULL) + if(fclose(c->fout) == EOF) exit_with_error(); + if(c->ferr != NULL) + if(fclose(c->ferr) == EOF) exit_with_error(); // Close pipes for(int i = 0; ipipe_arr)[i][0]); close(*(c->pipe_arr)[i][1]); } - // Destroy refs (might not need to do this?) - c->pipe_arr = NULL; - c->fin = NULL; - c->fout = NULL; - c->ferr = NULL; + free(c); } // Process status struct @@ -756,7 +755,7 @@ void sigchld_handler(int sig) { int status; pid_t pid; while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { - // Update process status + // Update process status update_status(pid, status); // Cleanup dead process if(WIFEXITED(status) || WIFSIGNALED(status)) @@ -939,7 +938,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Fork child process stz_long pid = (stz_long)vfork(); - if(pid < 0) return -1; + if(pid < 0) return -1; // Parent: open files, register with signal handler if(pid > 0) { FILE* fin = NULL; @@ -1164,7 +1163,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ } } } - if(!wait_for_termination) { // TODO: should this be an else? + else if(!wait_for_termination) { // do not loop if we don't need the process to terminate state_unknown = false; } From fa6888331e5b7e6dcc718f53d168e2e50d1b949c Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 28 Nov 2023 12:32:51 -0800 Subject: [PATCH 29/53] small change to eliminate linux compilation warning --- runtime/driver.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 03e755a1..ca331051 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -712,7 +712,7 @@ static int get_status (stz_int stz_proc_id, int* status) { if(curr != NULL) { *status = curr->proc->status; return 0; - } else { // pid not found + } else { // stz_proc_id not found return -1; } } @@ -1146,9 +1146,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ *s = (ProcessState){PROCESS_RUNNING, 0}; state_unknown = false; } else { - char* err; - sprintf(err, "Process %lld not found", process->pid); - throw_error(err); + return -1; // process not found } if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { From 3d25d6b045ccd135ba967cf6473131388da783dd Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 29 Nov 2023 10:19:21 -0800 Subject: [PATCH 30/53] send errors to parent --- runtime/driver.c | 106 ++++++++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 37 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index ca331051..96c2853c 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -690,6 +690,7 @@ static int register_proc (pid_t pid, stz_int stz_proc_id, int (*pipes)[NUM_STREA child->pid = pid; child->pipe_arr = pipes; child->fin = fin; + child->fout = fout; child->ferr = ferr; child->status = &(pstatus->status); // Store child in ProcessNode @@ -904,6 +905,13 @@ static void free_strings (stz_byte** ss){ //---------------------- Launcher Main ----------------------- //------------------------------------------------------------ +static void write_error_and_exit (int fd){ + int code = errno; + write(fd, &code, sizeof(int)); + close(fd); + exit(-1); +} + // Useful for testing linux implementation on OSX //#ifdef PLATFORM_OS_X //extern char **environ; @@ -935,72 +943,97 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, if(pipe(pipes[i])) return -1; } + //Create error-code pipe + int READ = 0; + int WRITE = 1; + int exec_error[2]; + if(pipe(exec_error) < 0) exit_with_error(); + // Fork child process stz_long pid = (stz_long)vfork(); if(pid < 0) return -1; - // Parent: open files, register with signal handler + // Parent: if exec succeeded, open files, register with signal handler if(pid > 0) { - FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] >= 0) { - close(pipes[PROCESS_IN][0]); - fin = fdopen(pipes[PROCESS_IN][1], "w"); - if(fin == NULL) return -1; - } - FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] >= 0) { - close(pipes[PROCESS_OUT][1]); - fout = fdopen(pipes[PROCESS_OUT][0], "r"); - if(fout == NULL) return -1; - } - FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] >= 0) { - close(pipes[PROCESS_ERR][1]); - ferr = fdopen(pipes[PROCESS_ERR][0], "r"); - if(ferr == NULL) return -1; - } - process->pid = pid; - process->stz_proc_id = stz_proc_id; - process->in = fin; - process->out = fout; - process->err = ferr; + //Read from error-code pipe + int exec_code; + close(exec_error[WRITE]); + int exec_r = read(exec_error[READ], &exec_code, sizeof(int)); + close(exec_error[READ]); + + if(exec_r == 0) { + FILE* fin = NULL; + if(pipe_sources[PROCESS_IN] >= 0) { + close(pipes[PROCESS_IN][0]); + fin = fdopen(pipes[PROCESS_IN][1], "w"); + if(fin == NULL) return -1; + } + FILE* fout = NULL; + if(pipe_sources[PROCESS_OUT] >= 0) { + close(pipes[PROCESS_OUT][1]); + fout = fdopen(pipes[PROCESS_OUT][0], "r"); + if(fout == NULL) return -1; + } + FILE* ferr = NULL; + if(pipe_sources[PROCESS_ERR] >= 0) { + close(pipes[PROCESS_ERR][1]); + ferr = fdopen(pipes[PROCESS_ERR][0], "r"); + if(ferr == NULL) return -1; + } - RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr)) + process->pid = pid; + process->stz_proc_id = stz_proc_id; + process->in = fin; + process->out = fout; + process->err = ferr; + + RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr)) + } else if(exec_r == sizeof(int)) { + errno = exec_code; + return -1; + } else { + fprintf(stderr, "Unreachable code."); + exit(-1); + } } // Child: setup pipes, exec else { + //Close exec pipe read, and close write end on successful exec + close(exec_error[READ]); + fcntl(exec_error[WRITE], F_SETFD, FD_CLOEXEC); + //Setup input pipe if used if(pipe_sources[PROCESS_IN] >= 0) { if(close(pipes[PROCESS_IN][1]) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); if(dup2(pipes[PROCESS_IN][0], STDIN_FILENO) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); if(close(pipes[PROCESS_IN][0]) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); } //Setup output pipe if used if(pipe_sources[PROCESS_OUT] >= 0) { if(close(pipes[PROCESS_OUT][0]) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); if(dup2(pipes[PROCESS_OUT][1], STDOUT_FILENO) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); if(close(pipes[PROCESS_OUT][1]) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); } //Setup error pipe if used if(pipe_sources[PROCESS_ERR] >= 0) { if(close(pipes[PROCESS_ERR][0]) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); if(dup2(pipes[PROCESS_ERR][1], STDERR_FILENO) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); if(close(pipes[PROCESS_ERR][1]) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); } //Setup working directory if(working_dir) { if(chdir(C_CSTR(working_dir)) < 0) - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); } //Launch child process. @@ -1009,14 +1042,13 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, execvp(C_CSTR(file), (char**)argvs); else execvpe(C_CSTR(file), (char**)argvs, (char**)env_vars); - exit_with_error(); + write_error_and_exit(exec_error[WRITE]); } return 0; } #endif - #ifdef PLATFORM_OS_X stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int output, stz_int error, stz_int stz_proc_id, From 97e297c46b260dff878493a5f0f48b13ade84822 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 29 Nov 2023 14:41:45 -0800 Subject: [PATCH 31/53] add option to manually cleanup files --- core/core.stanza | 11 +++++++++-- runtime/driver.c | 51 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index ae1521f3..19ef7eea 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -85,7 +85,8 @@ protected extern stz_memory_resize: (ptr, long, long) -> int protected extern launch_process: (ptr, int, int, int, ptr, ptr, ptr) -> int protected extern close_process_handle: (ptr) -> int #else: - protected extern launch_process: (ptr, ptr>, int, int, int, int, ptr, ptr>, ptr) -> int + protected extern launch_process: (ptr, ptr>, int, int, int, int, int, ptr, ptr>, ptr) -> int + protected extern delete_process_pipes: (ptr, ptr, ptr) -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int @@ -10121,7 +10122,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ;Launch the process val launch_succ = call-c clib/launch_process(addr!(filename.chars), argvs, - input_v, output_v, error_v, stz-proc-id, working-dir-chars, env-var-string, addr!([proc])) + input_v, output_v, error_v, stz-proc-id, 0, working-dir-chars, env-var-string, addr!([proc])) ;Free the memory created using malloc. free-linux-env-var-string(env-var-string) @@ -10267,6 +10268,12 @@ lostanza defn retrieve-state (p:ref, wait-for-termination?:reffin != NULL) - if(fclose(c->fin) == EOF) exit_with_error(); - if(c->fout != NULL) - if(fclose(c->fout) == EOF) exit_with_error(); - if(c->ferr != NULL) - if(fclose(c->ferr) == EOF) exit_with_error(); + if(c->auto_cleanup) { + if(c->fin != NULL) + if(fclose(c->fin) == EOF) exit_with_error(); + if(c->fout != NULL) + if(fclose(c->fout) == EOF) exit_with_error(); + if(c->ferr != NULL) + if(fclose(c->ferr) == EOF) exit_with_error(); + } // Close pipes for(int i = 0; ipipe_arr)[i][0]); @@ -675,7 +678,14 @@ static int register_proc_status (ProcessStatus* c) { } // Record process metadata -static int register_proc (pid_t pid, stz_int stz_proc_id, int (*pipes)[NUM_STREAM_SPECS][2], FILE* fin, FILE* fout, FILE* ferr) { +static int register_proc ( + pid_t pid, + stz_int stz_proc_id, + int (*pipes)[NUM_STREAM_SPECS][2], + FILE* fin, + FILE* fout, + FILE* ferr, + bool auto_cleanup) { // Init and register ProcessStatus struct ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); @@ -693,6 +703,7 @@ static int register_proc (pid_t pid, stz_int stz_proc_id, int (*pipes)[NUM_STREA child->fout = fout; child->ferr = ferr; child->status = &(pstatus->status); + child->auto_cleanup = auto_cleanup; // Store child in ProcessNode ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); if(new_node == NULL) return -1; @@ -912,6 +923,24 @@ static void write_error_and_exit (int fd){ exit(-1); } +static int delete_process_pipe (FILE* fd) { + if (fd != NULL) { + int close_res = fclose(fd); + if (close_res == EOF) return -1; + } + return 0; +} + +stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error) { + if (delete_process_pipe(input) < 0) + return -1; + if (delete_process_pipe(output) < 0) + return -1; + if (delete_process_pipe(error) < 0) + return -1; + return 0; +} + // Useful for testing linux implementation on OSX //#ifdef PLATFORM_OS_X //extern char **environ; @@ -927,7 +956,7 @@ static void write_error_and_exit (int fd){ #ifdef PLATFORM_LINUX //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int stz_proc_id, + stz_int output, stz_int error, stz_int stz_proc_id, stz_int cleanup_files, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -988,7 +1017,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr)) + RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, cleanup_files > 0)) } else if(exec_r == sizeof(int)) { errno = exec_code; return -1; @@ -1051,7 +1080,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, #ifdef PLATFORM_OS_X stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int stz_proc_id, + stz_int output, stz_int error, stz_int stz_proc_id, stz_int cleanup_files, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -1144,7 +1173,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr)) + RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, cleanup_files > 0)) return 0; } #endif From d3782943e853e5a6a63fcb8dfe1dcf83b483c45b Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 29 Nov 2023 15:47:45 -0800 Subject: [PATCH 32/53] process api changes --- core/core.stanza | 58 ++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 19ef7eea..3c604b9e 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -10008,14 +10008,15 @@ defn process-env-var-mode (env-vars:Tuple>|False, ; Constructor ; =========== #if-defined(PLATFORM-WINDOWS): - public lostanza defn Process (filename:ref, - args0:ref>, - input:ref, - output:ref, - error:ref, - working-dir:ref, - env-vars:ref>|False> - env-var-mode:ref) -> ref : + lostanza defn Process (filename:ref, + args0:ref>, + input:ref, + output:ref, + error:ref, + working-dir:ref, + env-vars:ref>|False> + env-var-mode:ref, + placeholder-ignore:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val proc = new Process{0, null, -1, null, null, null, false, false, false, false} @@ -10088,14 +10089,15 @@ defn process-env-var-mode (env-vars:Tuple>|False, return copy-buffer-to-stable-memory(buffer) #else: - public lostanza defn Process (filename:ref, - args0:ref>, - input:ref, - output:ref, - error:ref, - working-dir:ref - env-vars:ref>|False> - env-var-mode:ref) -> ref : + lostanza defn Process (filename:ref, + args0:ref>, + input:ref, + output:ref, + error:ref, + working-dir:ref + env-vars:ref>|False> + env-var-mode:ref, + cleanup-files:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) @@ -10218,20 +10220,24 @@ public defn Process (filename:String, output:StreamSpecifier, error:StreamSpecifier, working-dir:String|False, - env-vars:Tuple>|False) -> Process : - Process(filename, args, input, output, error, working-dir, env-vars, AddEnvVars) + env-vars:Tuple>|False + -- cleanup-files:True|False = true) -> Process : + Process(filename, args, input, output, error, working-dir, env-vars, AddEnvVars, cleanup-files) ;Default working-dir is false. Default env-vars is false. public defn Process (filename:String, args:Seqable, - input:StreamSpecifier, output:StreamSpecifier, error:StreamSpecifier) -> Process : - Process(filename, args, input, output, error, false, false) + input:StreamSpecifier, output:StreamSpecifier, error:StreamSpecifier, + -- cleanup-files:True|False = true) -> Process : + Process(filename, args, input, output, error, false, false, cleanup-files = cleanup-files) public defn Process (filename:String, args:Seqable, - working-dir:String|False, env:Tuple>|False) -> Process : - Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, working-dir, env) + working-dir:String|False, env:Tuple>|False, + -- cleanup-files:True|False = true) -> Process : + Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, working-dir, env, cleanup-files = cleanup-files) -public defn Process (filename:String, args:Seqable) : - Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, false, false) +public defn Process (filename:String, args:Seqable, + -- cleanup-files:True|False = true) : + Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, false, false, cleanup-files = cleanup-files) ; Stream API ; ========== @@ -10320,7 +10326,7 @@ public defn call-system (filename:String, args:Seqable, working-dir:String|False, env-vars:Tuple>|False) -> Int : - match(wait(Process(filename, args, working-dir, env-vars))) : + match(wait(Process(filename, args, working-dir, env-vars, cleanup-files = false))) : (s:ProcessDone) : value(s) (s) : throw(ProcessAbortedError(s)) @@ -10348,7 +10354,7 @@ public defn call-system-and-get-output (file:String, working-dir:String|False, env-vars:Tuple>|False) -> String : val buffer = StringBuffer() - val proc = Process(file, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, working-dir, env-vars) + val proc = Process(file, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, working-dir, env-vars, cleanup-files = false) val proc-out = output-stream(proc) let loop () : val c = get-char(proc-out) From 43bdabbc0e19bb504900248a68404c5433b127f0 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 5 Mar 2024 11:09:59 -0800 Subject: [PATCH 33/53] hopefully now it builds on windows --- runtime/process-win32.c | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime/process-win32.c b/runtime/process-win32.c index 8e1b5a4c..a618225d 100644 --- a/runtime/process-win32.c +++ b/runtime/process-win32.c @@ -273,7 +273,6 @@ stz_int launch_process(stz_byte* command_line, if (success) { // Populate process with the relevant info process->pid = (stz_long)proc_info.dwProcessId; - process->pipeid = -1; // no pipe ids on Windows process->handle = (void*)proc_info.hProcess; process->in = file_from_handle(stdin_write, FT_WRITE); process->out = file_from_handle(stdout_read, FT_READ); From 0eb91839afbf74505bce54405473983954e77fa1 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 5 Mar 2024 13:45:19 -0800 Subject: [PATCH 34/53] do not close pipes, this is handled by fclose --- compiler/package-manager-system.stanza | 2 +- runtime/driver.c | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/compiler/package-manager-system.stanza b/compiler/package-manager-system.stanza index b7717410..fc1814a2 100644 --- a/compiler/package-manager-system.stanza +++ b/compiler/package-manager-system.stanza @@ -31,7 +31,7 @@ public defn package-manager-system () -> System : match(redirect-output:String) : val out = FileOutputStream(redirect-output) try : - val proc = Process(prog, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, workdir, env-vars) + val proc = Process(prog, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, workdir, env-vars, cleanup-files = false) val proc-out = output-stream(proc) let loop () : val c = get-char(proc-out) diff --git a/runtime/driver.c b/runtime/driver.c index 9acf35a8..28e1fe03 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -617,7 +617,6 @@ static int count_non_null (void** xs){ // Process metadata struct typedef struct ChildProcess { pid_t pid; - int (*pipe_arr)[NUM_STREAM_SPECS][2]; FILE* fin; FILE* fout; FILE* ferr; @@ -643,11 +642,6 @@ static void cleanup_proc (ChildProcess* c) { if(c->ferr != NULL) if(fclose(c->ferr) == EOF) exit_with_error(); } - // Close pipes - for(int i = 0; ipipe_arr)[i][0]); - close(*(c->pipe_arr)[i][1]); - } free(c); } @@ -698,7 +692,6 @@ static int register_proc ( ChildProcess* child = (ChildProcess*)malloc(sizeof(ChildProcess)); if(child == NULL) return -1; child->pid = pid; - child->pipe_arr = pipes; child->fin = fin; child->fout = fout; child->ferr = ferr; From 5fb988865da0296106bd6e33de02a9b8cbf950ab Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 7 Mar 2024 11:31:57 -0800 Subject: [PATCH 35/53] working commit --- compiler/auto-doc.stanza | 2 +- core/core.stanza | 21 +++- runtime/driver.c | 206 +++++++++++++++++++++++---------------- runtime/process.h | 2 + 4 files changed, 139 insertions(+), 92 deletions(-) diff --git a/compiler/auto-doc.stanza b/compiler/auto-doc.stanza index ddebecf9..0777196f 100644 --- a/compiler/auto-doc.stanza +++ b/compiler/auto-doc.stanza @@ -121,7 +121,7 @@ defn slurp-from (s:InputStream) -> String : to-string(buffer) defn call-plugin (name:String, arguments:Seqable) -> String : - val proc = Process(name, cat([name], arguments), PROCESS-IN, PROCESS-OUT, PROCESS-ERR) + val proc = Process(name, cat([name], arguments), PROCESS-IN, PROCESS-OUT, PROCESS-ERR, cleanup-files = false) val response = trim $ slurp-from(output-stream(proc)) val state = wait(proc) match(state:ProcessDone) : diff --git a/core/core.stanza b/core/core.stanza index 3c604b9e..e0d0386c 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -9923,6 +9923,7 @@ public lostanza deftype Process <: Unique : var input: ptr var output: ptr var error: ptr + var status: int var input-stream: ref var output-stream: ref var error-stream: ref @@ -10019,7 +10020,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, placeholder-ignore:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) - val proc = new Process{0, null, -1, null, null, null, false, false, false, false} + val proc = new Process{0, null, -1, null, null, null, -1, false, false, false, false} ; Create a command line from the given filename and args (discarding the first argument). ; This is necessary because Windows' process API expects a command line (not a list of arguments). @@ -10102,7 +10103,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) val stz-proc-id = genid().value - val proc = new Process{0, null, stz-proc-id, null, null, null, false, false, false, false} + val proc = new Process{0, null, stz-proc-id, null, null, null, -1, false, false, false, false} val input_v = value(input).value val output_v = value(output).value val error_v = value(error).value @@ -10257,6 +10258,8 @@ public lostanza defn error-stream (p:ref) -> ref : p.error-stream = new FileInputStream{p.error, 0} return p.error-stream as ref +public lostanza defn pid (p:ref) -> ref : return (new Int{p.pid as int}) + ; State API ; ========= lostanza deftype StateStruct : @@ -10305,6 +10308,9 @@ public lostanza defn state (p:ref) -> ref : ;ProcessRunning. (s:ref) : return s + ; TODO: should we wait on stopped processes? + (s:ref) : + return s ;If the process has finished, then save its terminating ;state before returning it. (s:ref) : @@ -10317,9 +10323,12 @@ public lostanza defn state (p:ref) -> ref : public defn* wait (p:Process) -> ProcessState : val s = retrieve-state(p, true) - match(s:ProcessRunning) : wait(p) + ; TODO: should we wait on stopped processes? + match(s:ProcessRunning|ProcessStopped) : wait(p) + ;match(s:ProcessRunning) : wait(p) else : s + ; System Call API ; =============== public defn call-system (filename:String, @@ -10327,8 +10336,10 @@ public defn call-system (filename:String, working-dir:String|False, env-vars:Tuple>|False) -> Int : match(wait(Process(filename, args, working-dir, env-vars, cleanup-files = false))) : - (s:ProcessDone) : value(s) - (s) : throw(ProcessAbortedError(s)) + (s:ProcessDone) : + value(s) + (s) : + throw(ProcessAbortedError(s)) public defn call-system (file:String, args:Seqable) -> Int : call-system(file, args, false, false) diff --git a/runtime/driver.c b/runtime/driver.c index 28e1fe03..fda6e25c 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -617,6 +617,7 @@ static int count_non_null (void** xs){ // Process metadata struct typedef struct ChildProcess { pid_t pid; + stz_int stz_proc_id; FILE* fin; FILE* fout; FILE* ferr; @@ -648,28 +649,11 @@ static void cleanup_proc (ChildProcess* c) { // Process status struct typedef struct ProcessStatus { stz_int stz_proc_id; - int status; + int* status; } ProcessStatus; -typedef struct StatusNode { - ProcessStatus* proc; - struct StatusNode* next; -} StatusNode; - // Linked list of live processes ProcessNode* proc_head = NULL; -// Linked list of all process statuses -StatusNode* status_head = NULL; - -// Record a process' status -static int register_proc_status (ProcessStatus* c) { - StatusNode* new_node = (StatusNode*)malloc(sizeof(StatusNode)); - if(new_node == NULL) return -1; - new_node->proc = c; - new_node->next = status_head; - status_head = new_node; - return 0; -} // Record process metadata static int register_proc ( @@ -679,23 +663,19 @@ static int register_proc ( FILE* fin, FILE* fout, FILE* ferr, + int* status, bool auto_cleanup) { - // Init and register ProcessStatus struct - ProcessStatus* pstatus = (ProcessStatus*)malloc(sizeof(ProcessStatus)); - pstatus->stz_proc_id = stz_proc_id; - // TODO: is status=-1 OK? - pstatus->status = -1; - RETURN_NEG(register_proc_status(pstatus)) - // Init ChildProcess struct ChildProcess* child = (ChildProcess*)malloc(sizeof(ChildProcess)); if(child == NULL) return -1; child->pid = pid; + child->stz_proc_id = stz_proc_id; child->fin = fin; child->fout = fout; child->ferr = ferr; - child->status = &(pstatus->status); + child->status = status; + *(child->status) = -1; // TODO: is this OK? child->auto_cleanup = auto_cleanup; // Store child in ProcessNode ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); @@ -709,22 +689,16 @@ static int register_proc ( // Retrieve process state -static int get_status (stz_int stz_proc_id, int* status) { - StatusNode* curr = status_head; - while(curr != NULL && curr->proc->stz_proc_id != stz_proc_id) { - curr = curr->next; - } - if(curr != NULL) { - *status = curr->proc->status; - return 0; - } else { // stz_proc_id not found - return -1; - } +// TODO: just get rid of this +static int get_status (Process* process, int* status) { + *status = process->status; + return 0; } // Cleanup all resources for process pid -static void cleanup_child (pid_t pid, int status) { - ProcessNode* curr = proc_head; +static void cleanup_child (pid_t pid) { + + ProcessNode* curr = proc_head; //find_child(pid); ProcessNode* prev = NULL; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { @@ -746,53 +720,47 @@ static void cleanup_child (pid_t pid, int status) { // Update a process' status code static void update_status (pid_t pid, int status) { ProcessNode* curr = proc_head; + // Find matching Node while(curr != NULL && curr->proc->pid != pid) { + //prev = curr; curr = curr->next; } if(curr != NULL) { *(curr->proc->status) = status; - } + } } // SIGCHLD handler: cleanup dead processes, record status change void sigchld_handler(int sig) { int status; pid_t pid; - while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { - // Update process status + + //while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { + pid = waitpid(-1, &status, WNOHANG); + if(pid){ + // Update status update_status(pid, status); // Cleanup dead process - if(WIFEXITED(status) || WIFSIGNALED(status)) - cleanup_child(pid, status); + //if(WIFEXITED(status) || WIFSIGNALED(status)) + // cleanup_child(pid); } // it's OK if there are no more children if(pid < 0 && errno != ECHILD) exit_with_error(); -} -// Free all ProcessNodes and their contents -static void free_processes () { - ProcessNode* curr = proc_head; - ProcessNode* tmp = NULL; - while(curr != NULL) { - tmp = curr; - curr = curr->next; - free(tmp->proc); - free(tmp); - } } -// Free all StatusNodes and their contents -static void free_status_nodes() { - StatusNode* curr = status_head; - StatusNode* tmp = NULL; - while(curr != NULL) { - tmp = curr; - curr = curr->next; - free(tmp->proc); - free(tmp); - } -} +// Free all ProcessNodes and their contents +//static void free_processes () { +// ProcessNode* curr = proc_head; +// ProcessNode* tmp = NULL; +// while(curr != NULL) { +// tmp = curr; +// curr = curr->next; +// free(tmp->proc); +// free(tmp); +// } +//} #endif @@ -946,6 +914,24 @@ stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error) { //} //#endif +// block/unblock utilities +static sigset_t block () { + sigset_t sigchld_mask, old_mask; + sigemptyset(&sigchld_mask); + sigaddset(&sigchld_mask, SIGCHLD); + + if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) + exit_with_error(); + + return old_mask; +} + +static void unblock (sigset_t* old_mask) { + if(sigprocmask(SIG_SETMASK, old_mask, NULL)) exit_with_error(); +} + + + #ifdef PLATFORM_LINUX //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, @@ -977,6 +963,14 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, if(pid < 0) return -1; // Parent: if exec succeeded, open files, register with signal handler if(pid > 0) { + + // Block SIGCHLD until setup is finished + sigset_t old_mask = block(); //sigchld_mask, old_mask; + //sigemptyset(&sigchld_mask); + //sigaddset(&sigchld_mask, SIGCHLD); + + //if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) + // exit_with_error(); //Read from error-code pipe int exec_code; @@ -1010,7 +1004,13 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, cleanup_files > 0)) + int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0) + + unblock(&old_mask); + // Unblock SIGCHLD + //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); + if(r < 0) return -1; + return 0; } else if(exec_r == sizeof(int)) { errno = exec_code; return -1; @@ -1075,6 +1075,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int output, stz_int error, stz_int stz_proc_id, stz_int cleanup_files, stz_byte* working_dir, stz_byte** env_vars, Process* process) { + sigset_t old_mask = block(); //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; for(int i=0; iout = fout; process->err = ferr; - RETURN_NEG(register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, cleanup_files > 0)) + int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0); + + // Unblock SIGCHLD + unblock(&old_mask); + //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); + if(r < 0) return -1; return 0; } #endif - int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ + // block SIGCHLD + sigset_t old_mask = block(); + int status; bool state_unknown = true; - - sigset_t sigchld_mask, old_mask; - sigemptyset(&sigchld_mask); - sigaddset(&sigchld_mask, SIGCHLD); + //sigset_t sigchld_mask, old_mask; + //sigemptyset(&sigchld_mask); + //sigaddset(&sigchld_mask, SIGCHLD); // block SIGCHLD during check - if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) - exit_with_error(); + //if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) + // exit_with_error(); + + // check if SIGCHLD was blocked previously + bool sigchld_blocked = sigismember(&old_mask, SIGCHLD); while(state_unknown) { - if(get_status(process->stz_proc_id, &status) == 0) { + + if(get_status(process, &status) == 0) { if(WIFEXITED(status)) *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; else if(WIFSIGNALED(status)) @@ -1198,12 +1220,17 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; else *s = (ProcessState){PROCESS_RUNNING, 0}; + //printf("STATE OF %lld: signaled? %d, exited? %d, stopped? %d\n", process->pid, WIFSIGNALED(status), WIFEXITED(status), WIFSTOPPED(status)); state_unknown = false; } else { return -1; // process not found } - if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { + // Make sure SIGCHLD can get through + if(sigchld_blocked) { + sigdelset(&old_mask, SIGCHLD); + } + // no file descriptors or timeout; just wait indefinitely for an interrupt if(pselect(0, NULL, NULL, NULL, NULL, &old_mask) < 0) { if(errno == EINTR) { @@ -1214,16 +1241,20 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ exit_with_error(); } } + } else if(!wait_for_termination) { // do not loop if we don't need the process to terminate state_unknown = false; } } - - // reset mask to original state - if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); - + // Re-add SIGCHLD if it was previously blocked + if(sigchld_blocked){ + sigaddset(&old_mask, SIGCHLD); + } + // Reset to old_mask + unblock(&old_mask); + //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); return 0; } @@ -1351,9 +1382,13 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { #if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) //Setup SIGCHLD handler + sigset_t sigchld_mask; + sigemptyset(&sigchld_mask); + sigaddset(&sigchld_mask, SIGCHLD); struct sigaction sa; sa.sa_handler = sigchld_handler; - sa.sa_flags = SA_RESTART; + sa.sa_mask = sigchld_mask; + sa.sa_flags = SA_RESTART | SA_NODEFER; sigaction(SIGCHLD, &sa, NULL); #endif @@ -1361,10 +1396,9 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Call Stanza entry stanza_entry(&init); - #if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) - free_processes(); - free_status_nodes(); - #endif + //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) + // free_processes(); + //#endif //Heap and freespace are disposed by OS at process termination return 0; } diff --git a/runtime/process.h b/runtime/process.h index 2a7f3589..34459eec 100644 --- a/runtime/process.h +++ b/runtime/process.h @@ -10,10 +10,12 @@ //- in: The standard input stream of the Process. //- out: The standard output stream of the Process. //- err: The standard error stream of the Process. +//- status: A pointer to the status code of the Process. Not used on Windows. typedef struct { stz_long pid; void* handle; stz_int stz_proc_id; + int status; FILE* in; FILE* out; FILE* err; From f64a2f07911d4561dd057154a037e4cc2d271b88 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 7 Mar 2024 16:09:28 -0800 Subject: [PATCH 36/53] working commit for fredrik --- runtime/driver.c | 92 +++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 56 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index fda6e25c..a5e71e4b 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -655,6 +655,9 @@ typedef struct ProcessStatus { // Linked list of live processes ProcessNode* proc_head = NULL; +// flag to update process statuses +sig_atomic_t proc_dirty = 0; + // Record process metadata static int register_proc ( pid_t pid, @@ -727,40 +730,30 @@ static void update_status (pid_t pid, int status) { curr = curr->next; } if(curr != NULL) { + //printf(" found node for %d, updating (%d)\n", pid, getpid()); *(curr->proc->status) = status; } } -// SIGCHLD handler: cleanup dead processes, record status change -void sigchld_handler(int sig) { +// If needed, wait on any un-waited processes and update their statuses +// Precondition: SIGCHLD is blocked +int update_all_statuses () { int status; pid_t pid; - - //while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { - pid = waitpid(-1, &status, WNOHANG); - if(pid){ - // Update status + while ((pid = waitpid(-1, &status, WNOHANG | WUNTRACED | WCONTINUED)) > 0) { update_status(pid, status); - // Cleanup dead process //if(WIFEXITED(status) || WIFSIGNALED(status)) // cleanup_child(pid); } - // it's OK if there are no more children if(pid < 0 && errno != ECHILD) exit_with_error(); - + return 0; } -// Free all ProcessNodes and their contents -//static void free_processes () { -// ProcessNode* curr = proc_head; -// ProcessNode* tmp = NULL; -// while(curr != NULL) { -// tmp = curr; -// curr = curr->next; -// free(tmp->proc); -// free(tmp); -// } -//} +// SIGCHLD handler: cleanup dead processes, record status change +void sigchld_handler(int sig) { + //proc_dirty = 1; + update_all_statuses(); // Safe because sigaction blocks SIGCHLD +} #endif @@ -926,7 +919,7 @@ static sigset_t block () { return old_mask; } -static void unblock (sigset_t* old_mask) { +static void restore (sigset_t* old_mask) { if(sigprocmask(SIG_SETMASK, old_mask, NULL)) exit_with_error(); } @@ -1006,7 +999,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0) - unblock(&old_mask); + restore(&old_mask); // Unblock SIGCHLD //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); if(r < 0) return -1; @@ -1075,6 +1068,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int output, stz_int error, stz_int stz_proc_id, stz_int cleanup_files, stz_byte* working_dir, stz_byte** env_vars, Process* process) { + //block sigchld sigset_t old_mask = block(); //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -1129,19 +1123,11 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, return -1; } - // Block SIGCHLD until setup is complete - //sigset_t sigchld_mask, old_mask; - //sigemptyset(&sigchld_mask); - //sigaddset(&sigchld_mask, SIGCHLD); - - //if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) - // exit_with_error(); - //sigset_t old_mask = block(); - // Spawn child process pid_t pid = -1; int spawn_ret; sleep(1); + // printf("LAUNCHING %s (%d)\n", file, getpid()); if((spawn_ret = posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars)) == 0) { // success } else { @@ -1181,7 +1167,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0); // Unblock SIGCHLD - unblock(&old_mask); + restore(&old_mask); //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); if(r < 0) return -1; return 0; @@ -1192,25 +1178,19 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ + // printf("Getting state of %lld (%d)\n", process->pid, getpid()); // block SIGCHLD sigset_t old_mask = block(); int status; bool state_unknown = true; - //sigset_t sigchld_mask, old_mask; - //sigemptyset(&sigchld_mask); - //sigaddset(&sigchld_mask, SIGCHLD); - - // block SIGCHLD during check - //if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) - // exit_with_error(); - - // check if SIGCHLD was blocked previously - bool sigchld_blocked = sigismember(&old_mask, SIGCHLD); while(state_unknown) { + // update_all_statuses(); // Safe because SIGCHLD is blocked above + + // printf(" checking status of %lld (%d)\n", process->pid, getpid()); if(get_status(process, &status) == 0) { if(WIFEXITED(status)) *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; @@ -1220,21 +1200,27 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; else *s = (ProcessState){PROCESS_RUNNING, 0}; - //printf("STATE OF %lld: signaled? %d, exited? %d, stopped? %d\n", process->pid, WIFSIGNALED(status), WIFEXITED(status), WIFSTOPPED(status)); + // printf(" STATE OF %lld: signaled? %d, exited? %d, stopped? %d (%d)\n", process->pid, WIFSIGNALED(status), WIFEXITED(status), WIFSTOPPED(status), getpid()); state_unknown = false; } else { return -1; // process not found } if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { - // Make sure SIGCHLD can get through - if(sigchld_blocked) { - sigdelset(&old_mask, SIGCHLD); - } + // Allow SIGCHLD during sigsuspend + sigset_t allow_sigchld; + // This is overkill, but ensures SIGCHLD is allowed + sigfillset(&allow_sigchld); + // SIGCHLD is unmasked + sigdelset(&allow_sigchld, SIGCHLD); // no file descriptors or timeout; just wait indefinitely for an interrupt - if(pselect(0, NULL, NULL, NULL, NULL, &old_mask) < 0) { + //if(pselect(0, NULL, NULL, NULL, NULL, &old_mask) < 0) { + if(sigsuspend(&allow_sigchld) < 0) { if(errno == EINTR) { + // printf(" interruped (%d)\n", getpid()); + //update_all_statuses() // interrupt: try again + //proc_dirty = 1; state_unknown = true; } else { // non-interrupt error: impossible @@ -1248,13 +1234,8 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ state_unknown = false; } } - // Re-add SIGCHLD if it was previously blocked - if(sigchld_blocked){ - sigaddset(&old_mask, SIGCHLD); - } // Reset to old_mask - unblock(&old_mask); - //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); + restore(&old_mask); return 0; } @@ -1388,7 +1369,6 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { struct sigaction sa; sa.sa_handler = sigchld_handler; sa.sa_mask = sigchld_mask; - sa.sa_flags = SA_RESTART | SA_NODEFER; sigaction(SIGCHLD, &sa, NULL); #endif From a08df8b04e57830d5d441c197b71cf3c47e01b8d Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 2 Apr 2024 08:48:18 -0700 Subject: [PATCH 37/53] spot fixes for linux testing --- runtime/driver.c | 21 +++++++++++---------- scripts/finish.sh | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index a5e71e4b..2cdbbd17 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -628,7 +628,7 @@ typedef struct ChildProcess { // Linked lists of process metadata typedef struct ProcessNode { ChildProcess* proc; - struct ProcessNode* next; + volatile struct ProcessNode* next; } ProcessNode; @@ -653,7 +653,7 @@ typedef struct ProcessStatus { } ProcessStatus; // Linked list of live processes -ProcessNode* proc_head = NULL; +volatile ProcessNode* proc_head = NULL; // flag to update process statuses sig_atomic_t proc_dirty = 0; @@ -681,7 +681,7 @@ static int register_proc ( *(child->status) = -1; // TODO: is this OK? child->auto_cleanup = auto_cleanup; // Store child in ProcessNode - ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); + volatile ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); if(new_node == NULL) return -1; new_node->proc = child; new_node->next = proc_head; @@ -701,8 +701,8 @@ static int get_status (Process* process, int* status) { // Cleanup all resources for process pid static void cleanup_child (pid_t pid) { - ProcessNode* curr = proc_head; //find_child(pid); - ProcessNode* prev = NULL; + volatile ProcessNode* curr = proc_head; //find_child(pid); + volatile ProcessNode* prev = NULL; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { prev = curr; @@ -722,7 +722,7 @@ static void cleanup_child (pid_t pid) { // Update a process' status code static void update_status (pid_t pid, int status) { - ProcessNode* curr = proc_head; + volatile ProcessNode* curr = proc_head; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { @@ -997,7 +997,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0) + int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0); restore(&old_mask); // Unblock SIGCHLD @@ -1366,9 +1366,10 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { sigset_t sigchld_mask; sigemptyset(&sigchld_mask); sigaddset(&sigchld_mask, SIGCHLD); - struct sigaction sa; - sa.sa_handler = sigchld_handler; - sa.sa_mask = sigchld_mask; + struct sigaction sa = { + .sa_handler = sigchld_handler, + .sa_mask = sigchld_mask, + }; sigaction(SIGCHLD, &sa, NULL); #endif diff --git a/scripts/finish.sh b/scripts/finish.sh index 8ee98466..597e28f0 100755 --- a/scripts/finish.sh +++ b/scripts/finish.sh @@ -1,4 +1,4 @@ -scripts/make-asmjit.sh os-x +# scripts/make-asmjit.sh os-x gcc -std=gnu99 -c core/sha256.c -O3 -o build/sha256.o -I include gcc -std=gnu99 -c compiler/cvm.c -O3 -o build/cvm.o -I include gcc -c runtime/linenoise-ng/linenoise.cpp -O3 -o build/linenoise.o -I include -I runtime/linenoise-ng From 12b73b7dc4de9c5b82c2552f510dfd0ab07bc5bd Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 2 Apr 2024 15:04:45 -0700 Subject: [PATCH 38/53] volatile volatile --- runtime/driver.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 2cdbbd17..766b905c 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -628,7 +628,7 @@ typedef struct ChildProcess { // Linked lists of process metadata typedef struct ProcessNode { ChildProcess* proc; - volatile struct ProcessNode* next; + volatile struct ProcessNode * volatile next; } ProcessNode; @@ -653,7 +653,7 @@ typedef struct ProcessStatus { } ProcessStatus; // Linked list of live processes -volatile ProcessNode* proc_head = NULL; +volatile ProcessNode * volatile proc_head = NULL; // flag to update process statuses sig_atomic_t proc_dirty = 0; @@ -681,7 +681,7 @@ static int register_proc ( *(child->status) = -1; // TODO: is this OK? child->auto_cleanup = auto_cleanup; // Store child in ProcessNode - volatile ProcessNode* new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); + volatile ProcessNode * volatile new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); if(new_node == NULL) return -1; new_node->proc = child; new_node->next = proc_head; @@ -701,8 +701,8 @@ static int get_status (Process* process, int* status) { // Cleanup all resources for process pid static void cleanup_child (pid_t pid) { - volatile ProcessNode* curr = proc_head; //find_child(pid); - volatile ProcessNode* prev = NULL; + volatile ProcessNode * volatile curr = proc_head; //find_child(pid); + volatile ProcessNode * volatile prev = NULL; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { prev = curr; @@ -722,7 +722,7 @@ static void cleanup_child (pid_t pid) { // Update a process' status code static void update_status (pid_t pid, int status) { - volatile ProcessNode* curr = proc_head; + volatile ProcessNode * volatile curr = proc_head; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { @@ -732,7 +732,9 @@ static void update_status (pid_t pid, int status) { if(curr != NULL) { //printf(" found node for %d, updating (%d)\n", pid, getpid()); *(curr->proc->status) = status; - } + } else { + //printf(" proc_head = NULL for %d (%d)\n", pid, getpid()); + } } // If needed, wait on any un-waited processes and update their statuses @@ -1127,7 +1129,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, pid_t pid = -1; int spawn_ret; sleep(1); - // printf("LAUNCHING %s (%d)\n", file, getpid()); + //printf("LAUNCHING %s (%d)\n", file, getpid()); if((spawn_ret = posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars)) == 0) { // success } else { @@ -1178,7 +1180,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ - // printf("Getting state of %lld (%d)\n", process->pid, getpid()); + //printf("Getting state of %lld (%d)\n", process->pid, getpid()); // block SIGCHLD sigset_t old_mask = block(); From 2d884c63d2d34ec8a13aaae551d1a7d6e191b0a5 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Mon, 8 Apr 2024 14:29:33 -0700 Subject: [PATCH 39/53] defer to existing handler when pid is not found --- runtime/driver.c | 79 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 766b905c..1c81b925 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -654,6 +654,7 @@ typedef struct ProcessStatus { // Linked list of live processes volatile ProcessNode * volatile proc_head = NULL; +struct sigaction oldact; // flag to update process statuses sig_atomic_t proc_dirty = 0; @@ -720,41 +721,62 @@ static void cleanup_child (pid_t pid) { } } +// is process p registered to this handler? +static bool pid_is_registered (pid_t pid) { + volatile ProcessNode * volatile curr = proc_head; + // Find matching Node + while(curr != NULL && curr->proc->pid != pid) { + curr = curr->next; + } + return (curr != NULL); +} + // Update a process' status code +// Precondition: SIGCHLD is blocked static void update_status (pid_t pid, int status) { volatile ProcessNode * volatile curr = proc_head; - // Find matching Node while(curr != NULL && curr->proc->pid != pid) { - //prev = curr; curr = curr->next; } if(curr != NULL) { - //printf(" found node for %d, updating (%d)\n", pid, getpid()); *(curr->proc->status) = status; } else { - //printf(" proc_head = NULL for %d (%d)\n", pid, getpid()); } } -// If needed, wait on any un-waited processes and update their statuses // Precondition: SIGCHLD is blocked -int update_all_statuses () { +int wait_and_update (pid_t pid) { int status; - pid_t pid; - while ((pid = waitpid(-1, &status, WNOHANG | WUNTRACED | WCONTINUED)) > 0) { + if(waitpid(pid, &status, WNOHANG | WUNTRACED | WCONTINUED) > 0) { update_status(pid, status); - //if(WIFEXITED(status) || WIFSIGNALED(status)) - // cleanup_child(pid); + // TODO: cleanup if finished? } - if(pid < 0 && errno != ECHILD) exit_with_error(); return 0; } -// SIGCHLD handler: cleanup dead processes, record status change -void sigchld_handler(int sig) { - //proc_dirty = 1; - update_all_statuses(); // Safe because sigaction blocks SIGCHLD +void sigchld_action(int sig, siginfo_t* info, void* ctx) { + if(pid_is_registered(info->si_pid)) { + wait_and_update(info->si_pid); + } else { + oldact.sa_sigaction(sig, info, ctx); + } +} + +// Register SIGCHLD handler +void register_handler () { + #if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) + //Setup SIGCHLD handler + sigset_t sigchld_mask; + sigemptyset(&sigchld_mask); + sigaddset(&sigchld_mask, SIGCHLD); + struct sigaction sa = { + .sa_sigaction = sigchld_action, + .sa_mask = sigchld_mask, + .sa_flags = SA_RESTART | SA_SIGINFO + }; + sigaction(SIGCHLD, &sa, &oldact); + #endif } #endif @@ -1128,8 +1150,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Spawn child process pid_t pid = -1; int spawn_ret; - sleep(1); - //printf("LAUNCHING %s (%d)\n", file, getpid()); if((spawn_ret = posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars)) == 0) { // success } else { @@ -1179,8 +1199,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ - - //printf("Getting state of %lld (%d)\n", process->pid, getpid()); // block SIGCHLD sigset_t old_mask = block(); @@ -1190,9 +1208,6 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ while(state_unknown) { - // update_all_statuses(); // Safe because SIGCHLD is blocked above - - // printf(" checking status of %lld (%d)\n", process->pid, getpid()); if(get_status(process, &status) == 0) { if(WIFEXITED(status)) *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; @@ -1202,7 +1217,6 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; else *s = (ProcessState){PROCESS_RUNNING, 0}; - // printf(" STATE OF %lld: signaled? %d, exited? %d, stopped? %d (%d)\n", process->pid, WIFSIGNALED(status), WIFEXITED(status), WIFSTOPPED(status), getpid()); state_unknown = false; } else { return -1; // process not found @@ -1216,13 +1230,8 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ sigdelset(&allow_sigchld, SIGCHLD); // no file descriptors or timeout; just wait indefinitely for an interrupt - //if(pselect(0, NULL, NULL, NULL, NULL, &old_mask) < 0) { if(sigsuspend(&allow_sigchld) < 0) { if(errno == EINTR) { - // printf(" interruped (%d)\n", getpid()); - //update_all_statuses() - // interrupt: try again - //proc_dirty = 1; state_unknown = true; } else { // non-interrupt error: impossible @@ -1363,22 +1372,12 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Initialize trackers to empty list. init.trackers = NULL; - #if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) - //Setup SIGCHLD handler - sigset_t sigchld_mask; - sigemptyset(&sigchld_mask); - sigaddset(&sigchld_mask, SIGCHLD); - struct sigaction sa = { - .sa_handler = sigchld_handler, - .sa_mask = sigchld_mask, - }; - sigaction(SIGCHLD, &sa, NULL); - #endif - + register_handler(); //Call Stanza entry stanza_entry(&init); + // TODO: cleanup //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) // free_processes(); //#endif From 8375b0806ee3194b8915fe4ee73a3dfe70453922 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 9 Apr 2024 13:35:19 -0700 Subject: [PATCH 40/53] some cleanup and improvements: no longer reliant on sa_pid in siginfo struct --- runtime/driver.c | 64 ++++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 1c81b925..74671a4d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -653,6 +653,7 @@ typedef struct ProcessStatus { } ProcessStatus; // Linked list of live processes +// TODO: volatility annotations volatile ProcessNode * volatile proc_head = NULL; struct sigaction oldact; @@ -682,7 +683,7 @@ static int register_proc ( *(child->status) = -1; // TODO: is this OK? child->auto_cleanup = auto_cleanup; // Store child in ProcessNode - volatile ProcessNode * volatile new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); + volatile ProcessNode * new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); if(new_node == NULL) return -1; new_node->proc = child; new_node->next = proc_head; @@ -702,8 +703,8 @@ static int get_status (Process* process, int* status) { // Cleanup all resources for process pid static void cleanup_child (pid_t pid) { - volatile ProcessNode * volatile curr = proc_head; //find_child(pid); - volatile ProcessNode * volatile prev = NULL; + volatile ProcessNode * curr = proc_head; + volatile ProcessNode * prev = NULL; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { prev = curr; @@ -723,7 +724,7 @@ static void cleanup_child (pid_t pid) { // is process p registered to this handler? static bool pid_is_registered (pid_t pid) { - volatile ProcessNode * volatile curr = proc_head; + volatile ProcessNode * curr = proc_head; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; @@ -731,35 +732,52 @@ static bool pid_is_registered (pid_t pid) { return (curr != NULL); } +static bool dead_status (int st) { + return WIFSIGNALED(st) || WIFEXITED(st); +} + // Update a process' status code // Precondition: SIGCHLD is blocked static void update_status (pid_t pid, int status) { - volatile ProcessNode * volatile curr = proc_head; + volatile ProcessNode * curr = proc_head; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; } if(curr != NULL) { *(curr->proc->status) = status; - } else { - } + } } // Precondition: SIGCHLD is blocked -int wait_and_update (pid_t pid) { +static int wait_and_update (pid_t pid) { int status; if(waitpid(pid, &status, WNOHANG | WUNTRACED | WCONTINUED) > 0) { update_status(pid, status); - // TODO: cleanup if finished? + // remove metadata if dead + if(dead_status(status)) cleanup_child(pid); } return 0; } -void sigchld_action(int sig, siginfo_t* info, void* ctx) { - if(pid_is_registered(info->si_pid)) { - wait_and_update(info->si_pid); - } else { - oldact.sa_sigaction(sig, info, ctx); +// waitpid on all live process +static void waitpid_all () { + volatile ProcessNode * curr = proc_head; + // Find matching Node + while(curr != NULL) { + wait_and_update(curr->proc->pid); + curr = curr->next; + } +} + +void sigchld_handler(int sig) { + waitpid_all(); + // TODO: check if I should run sigaction or sighandler + if(!(oldact.sa_flags & SA_SIGINFO)) { + if(oldact.sa_handler != SIG_IGN && oldact.sa_handler != SIG_DFL) { + // TODO: why does this work without dereferencing pointer? + oldact.sa_handler(sig); + } } } @@ -771,9 +789,9 @@ void register_handler () { sigemptyset(&sigchld_mask); sigaddset(&sigchld_mask, SIGCHLD); struct sigaction sa = { - .sa_sigaction = sigchld_action, + .sa_handler = sigchld_handler, .sa_mask = sigchld_mask, - .sa_flags = SA_RESTART | SA_SIGINFO + .sa_flags = SA_RESTART }; sigaction(SIGCHLD, &sa, &oldact); #endif @@ -982,12 +1000,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, if(pid > 0) { // Block SIGCHLD until setup is finished - sigset_t old_mask = block(); //sigchld_mask, old_mask; - //sigemptyset(&sigchld_mask); - //sigaddset(&sigchld_mask, SIGCHLD); - - //if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) - // exit_with_error(); + sigset_t old_mask = block(); //Read from error-code pipe int exec_code; @@ -1025,7 +1038,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, restore(&old_mask); // Unblock SIGCHLD - //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); if(r < 0) return -1; return 0; } else if(exec_r == sizeof(int)) { @@ -1190,7 +1202,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Unblock SIGCHLD restore(&old_mask); - //if(sigprocmask(SIG_SETMASK, &old_mask, NULL)) exit_with_error(); if(r < 0) return -1; return 0; } @@ -1201,7 +1212,6 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ // block SIGCHLD sigset_t old_mask = block(); - int status; bool state_unknown = true; @@ -1221,7 +1231,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ } else { return -1; // process not found } - if(wait_for_termination && !(WIFSIGNALED(status) || WIFEXITED(status))) { + if(wait_for_termination && !dead_status(status)) { // Allow SIGCHLD during sigsuspend sigset_t allow_sigchld; // This is overkill, but ensures SIGCHLD is allowed @@ -1377,7 +1387,7 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Call Stanza entry stanza_entry(&init); - // TODO: cleanup + // TODO: cleanup? //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) // free_processes(); //#endif From ee6f2a5609cf8e4e6408509ad2bc35f496c1342b Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 9 Apr 2024 14:16:45 -0700 Subject: [PATCH 41/53] polish --- runtime/driver.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 74671a4d..493789e8 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -722,25 +722,13 @@ static void cleanup_child (pid_t pid) { } } -// is process p registered to this handler? -static bool pid_is_registered (pid_t pid) { - volatile ProcessNode * curr = proc_head; - // Find matching Node - while(curr != NULL && curr->proc->pid != pid) { - curr = curr->next; - } - return (curr != NULL); -} - static bool dead_status (int st) { return WIFSIGNALED(st) || WIFEXITED(st); } // Update a process' status code -// Precondition: SIGCHLD is blocked static void update_status (pid_t pid, int status) { volatile ProcessNode * curr = proc_head; - // Find matching Node while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; } @@ -749,7 +737,6 @@ static void update_status (pid_t pid, int status) { } } -// Precondition: SIGCHLD is blocked static int wait_and_update (pid_t pid) { int status; if(waitpid(pid, &status, WNOHANG | WUNTRACED | WCONTINUED) > 0) { @@ -760,23 +747,30 @@ static int wait_and_update (pid_t pid) { return 0; } -// waitpid on all live process -static void waitpid_all () { +static void waitpid_all_registered () { volatile ProcessNode * curr = proc_head; - // Find matching Node while(curr != NULL) { wait_and_update(curr->proc->pid); curr = curr->next; } } +static void waitpid_global () { + int status; + pid_t pid; + while((pid = waitpid(-1, &status, WNOHANG | WUNTRACED | WCONTINUED)) > 0) { + update_status(pid, status); + if(dead_status(status)) cleanup_child(pid); + } +} + void sigchld_handler(int sig) { - waitpid_all(); - // TODO: check if I should run sigaction or sighandler if(!(oldact.sa_flags & SA_SIGINFO)) { - if(oldact.sa_handler != SIG_IGN && oldact.sa_handler != SIG_DFL) { - // TODO: why does this work without dereferencing pointer? - oldact.sa_handler(sig); + if(oldact.sa_handler != SIG_DFL) { + waitpid_all_registered(); // wait for all registered children + oldact.sa_handler(sig); // defer to prior handler for unregistered pids + } else { // no existing handler: waitpid for any child + waitpid_global(); } } } @@ -949,7 +943,7 @@ stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error) { //} //#endif -// block/unblock utilities +// block SIGCHLD static sigset_t block () { sigset_t sigchld_mask, old_mask; sigemptyset(&sigchld_mask); From 42df55a470ec213ada59e079344a058604c4641d Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 9 Apr 2024 15:41:33 -0700 Subject: [PATCH 42/53] restore signal mask on early returns --- runtime/driver.c | 83 +++++++++++++++++++----------------------------- 1 file changed, 33 insertions(+), 50 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 493789e8..948b8a01 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -653,7 +653,6 @@ typedef struct ProcessStatus { } ProcessStatus; // Linked list of live processes -// TODO: volatility annotations volatile ProcessNode * volatile proc_head = NULL; struct sigaction oldact; @@ -680,7 +679,7 @@ static int register_proc ( child->fout = fout; child->ferr = ferr; child->status = status; - *(child->status) = -1; // TODO: is this OK? + *(child->status) = -1; child->auto_cleanup = auto_cleanup; // Store child in ProcessNode volatile ProcessNode * new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); @@ -692,14 +691,6 @@ static int register_proc ( return 0; } - -// Retrieve process state -// TODO: just get rid of this -static int get_status (Process* process, int* status) { - *status = process->status; - return 0; -} - // Cleanup all resources for process pid static void cleanup_child (pid_t pid) { @@ -959,6 +950,10 @@ static void restore (sigset_t* old_mask) { if(sigprocmask(SIG_SETMASK, old_mask, NULL)) exit_with_error(); } +static int restore_and_err (sigset_t* old_mask) { + restore(old_mask); + return -1; +} #ifdef PLATFORM_LINUX @@ -1111,7 +1106,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Generate array of pipes per-input-source int pipes[NUM_STREAM_SPECS][2]; for(int i=0; i= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) - return -1; + restore_and_err(&old_mask); if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) - return -1; + restore_and_err(&old_mask); if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0]))) - return -1; + restore_and_err(&old_mask); } //Setup output pipe if used if(pipe_sources[PROCESS_OUT] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) - return -1; + restore_and_err(&old_mask); if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) - return -1; + restore_and_err(&old_mask); if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1]))) - return -1; + restore_and_err(&old_mask); } //Setup error pipe if used if(pipe_sources[PROCESS_ERR] >= 0) { if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) - return -1; + restore_and_err(&old_mask); if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) - return -1; + restore_and_err(&old_mask); if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1]))) - return -1; + restore_and_err(&old_mask); } //Setup working directory if(working_dir) { if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, C_CSTR(working_dir)))) - return -1; + restore_and_err(&old_mask); } // Spawn child process @@ -1160,7 +1155,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // success } else { errno = spawn_ret; // might not want to do this manually? - return -1; + restore_and_err(&old_mask); } // Cleanup posix_spawn_file_actions_destroy(&actions); @@ -1172,18 +1167,19 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, close(pipes[PROCESS_IN][0]); fin = fdopen(pipes[PROCESS_IN][1], "w"); if(fin == NULL) return -1; + restore_and_err(&old_mask); } FILE* fout = NULL; if(pipe_sources[PROCESS_OUT] >= 0) { close(pipes[PROCESS_OUT][1]); fout = fdopen(pipes[PROCESS_OUT][0], "r"); - if(fout == NULL) return -1; + if(fout == NULL) restore_and_err(&old_mask); } FILE* ferr = NULL; if(pipe_sources[PROCESS_ERR] >= 0) { close(pipes[PROCESS_ERR][1]); ferr = fdopen(pipes[PROCESS_ERR][0], "r"); - if(ferr == NULL) return -1; + if(ferr == NULL) restore_and_err(&old_mask); } process->pid = pid; @@ -1211,45 +1207,36 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ bool state_unknown = true; while(state_unknown) { - - if(get_status(process, &status) == 0) { - if(WIFEXITED(status)) - *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; - else if(WIFSIGNALED(status)) - *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; - else if(WIFSTOPPED(status)) - *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; - else - *s = (ProcessState){PROCESS_RUNNING, 0}; - state_unknown = false; - } else { - return -1; // process not found - } + status = process->status; + if(WIFEXITED(status)) + *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; + else if(WIFSIGNALED(status)) + *s = (ProcessState){PROCESS_TERMINATED, WTERMSIG(status)}; + else if(WIFSTOPPED(status)) + *s = (ProcessState){PROCESS_STOPPED, WSTOPSIG(status)}; + else + *s = (ProcessState){PROCESS_RUNNING, 0}; + state_unknown = false; if(wait_for_termination && !dead_status(status)) { - // Allow SIGCHLD during sigsuspend + // We allow SICHLD during sigsuspend sigset_t allow_sigchld; - // This is overkill, but ensures SIGCHLD is allowed sigfillset(&allow_sigchld); - // SIGCHLD is unmasked sigdelset(&allow_sigchld, SIGCHLD); - // no file descriptors or timeout; just wait indefinitely for an interrupt if(sigsuspend(&allow_sigchld) < 0) { if(errno == EINTR) { state_unknown = true; } else { - // non-interrupt error: impossible - exit_with_error(); + exit_with_error(); // non-interrupt error: impossible } } } + // do not loop if we don't need the process to terminate else if(!wait_for_termination) { - // do not loop if we don't need the process to terminate state_unknown = false; } } - // Reset to old_mask restore(&old_mask); return 0; } @@ -1381,10 +1368,6 @@ STANZA_API_FUNC int MAIN_FUNC (int argc, char* argv[]) { //Call Stanza entry stanza_entry(&init); - // TODO: cleanup? - //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) - // free_processes(); - //#endif //Heap and freespace are disposed by OS at process termination return 0; } From def5f95abdc0b07f9f258fdb0e4c171980348e99 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Tue, 9 Apr 2024 15:56:06 -0700 Subject: [PATCH 43/53] remove unnecessary msg pipe on linux --- runtime/driver.c | 120 +++++++++++++++++------------------------------ 1 file changed, 43 insertions(+), 77 deletions(-) diff --git a/runtime/driver.c b/runtime/driver.c index 948b8a01..dfaffd5d 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -725,7 +725,7 @@ static void update_status (pid_t pid, int status) { } if(curr != NULL) { *(curr->proc->status) = status; - } + } } static int wait_and_update (pid_t pid) { @@ -942,7 +942,7 @@ static sigset_t block () { if(sigprocmask(SIG_BLOCK, &sigchld_mask, &old_mask)) exit_with_error(); - + return old_mask; } @@ -975,105 +975,71 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, if(pipe(pipes[i])) return -1; } - //Create error-code pipe - int READ = 0; - int WRITE = 1; - int exec_error[2]; - if(pipe(exec_error) < 0) exit_with_error(); - // Fork child process stz_long pid = (stz_long)vfork(); if(pid < 0) return -1; // Parent: if exec succeeded, open files, register with signal handler if(pid > 0) { - + // Block SIGCHLD until setup is finished sigset_t old_mask = block(); - //Read from error-code pipe - int exec_code; - close(exec_error[WRITE]); - int exec_r = read(exec_error[READ], &exec_code, sizeof(int)); - close(exec_error[READ]); - - if(exec_r == 0) { - FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] >= 0) { - close(pipes[PROCESS_IN][0]); - fin = fdopen(pipes[PROCESS_IN][1], "w"); - if(fin == NULL) return -1; - } - FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] >= 0) { - close(pipes[PROCESS_OUT][1]); - fout = fdopen(pipes[PROCESS_OUT][0], "r"); - if(fout == NULL) return -1; - } - FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] >= 0) { - close(pipes[PROCESS_ERR][1]); - ferr = fdopen(pipes[PROCESS_ERR][0], "r"); - if(ferr == NULL) return -1; - } + FILE* fin = NULL; + if(pipe_sources[PROCESS_IN] >= 0) { + close(pipes[PROCESS_IN][0]); + fin = fdopen(pipes[PROCESS_IN][1], "w"); + if(fin == NULL) return -1; + } + FILE* fout = NULL; + if(pipe_sources[PROCESS_OUT] >= 0) { + close(pipes[PROCESS_OUT][1]); + fout = fdopen(pipes[PROCESS_OUT][0], "r"); + if(fout == NULL) return -1; + } + FILE* ferr = NULL; + if(pipe_sources[PROCESS_ERR] >= 0) { + close(pipes[PROCESS_ERR][1]); + ferr = fdopen(pipes[PROCESS_ERR][0], "r"); + if(ferr == NULL) return -1; + } - process->pid = pid; - process->stz_proc_id = stz_proc_id; - process->in = fin; - process->out = fout; - process->err = ferr; + process->pid = pid; + process->stz_proc_id = stz_proc_id; + process->in = fin; + process->out = fout; + process->err = ferr; - int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0); + int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0); - restore(&old_mask); - // Unblock SIGCHLD - if(r < 0) return -1; - return 0; - } else if(exec_r == sizeof(int)) { - errno = exec_code; - return -1; - } else { - fprintf(stderr, "Unreachable code."); - exit(-1); - } + // Unblock SIGCHLD + restore(&old_mask); + if(r < 0) return -1; + return 0; } // Child: setup pipes, exec else { - //Close exec pipe read, and close write end on successful exec - close(exec_error[READ]); - fcntl(exec_error[WRITE], F_SETFD, FD_CLOEXEC); - //Setup input pipe if used if(pipe_sources[PROCESS_IN] >= 0) { - if(close(pipes[PROCESS_IN][1]) < 0) - write_error_and_exit(exec_error[WRITE]); - if(dup2(pipes[PROCESS_IN][0], STDIN_FILENO) < 0) - write_error_and_exit(exec_error[WRITE]); - if(close(pipes[PROCESS_IN][0]) < 0) - write_error_and_exit(exec_error[WRITE]); + if(close(pipes[PROCESS_IN][1]) < 0) exit(-1); + if(dup2(pipes[PROCESS_IN][0], STDIN_FILENO) < 0) exit(-1); + if(close(pipes[PROCESS_IN][0]) < 0) exit(-1); } //Setup output pipe if used if(pipe_sources[PROCESS_OUT] >= 0) { - if(close(pipes[PROCESS_OUT][0]) < 0) - write_error_and_exit(exec_error[WRITE]); - if(dup2(pipes[PROCESS_OUT][1], STDOUT_FILENO) < 0) - write_error_and_exit(exec_error[WRITE]); - if(close(pipes[PROCESS_OUT][1]) < 0) - write_error_and_exit(exec_error[WRITE]); + if(close(pipes[PROCESS_OUT][0]) < 0) exit(-1); + if(dup2(pipes[PROCESS_OUT][1], STDOUT_FILENO) < 0) exit(-1); + if(close(pipes[PROCESS_OUT][1]) < 0) exit(-1); } //Setup error pipe if used if(pipe_sources[PROCESS_ERR] >= 0) { - if(close(pipes[PROCESS_ERR][0]) < 0) - write_error_and_exit(exec_error[WRITE]); - if(dup2(pipes[PROCESS_ERR][1], STDERR_FILENO) < 0) - write_error_and_exit(exec_error[WRITE]); - if(close(pipes[PROCESS_ERR][1]) < 0) - write_error_and_exit(exec_error[WRITE]); + if(close(pipes[PROCESS_ERR][0]) < 0) exit(-1); + if(dup2(pipes[PROCESS_ERR][1], STDERR_FILENO) < 0) exit(-1); + if(close(pipes[PROCESS_ERR][1]) < 0) exit(-1); } //Setup working directory if(working_dir) { - if(chdir(C_CSTR(working_dir)) < 0) - write_error_and_exit(exec_error[WRITE]); + if(chdir(C_CSTR(working_dir)) < 0) exit(-1); } //Launch child process. @@ -1082,7 +1048,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, execvp(C_CSTR(file), (char**)argvs); else execvpe(C_CSTR(file), (char**)argvs, (char**)env_vars); - write_error_and_exit(exec_error[WRITE]); + exit(-1); } return 0; } @@ -1222,7 +1188,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ sigset_t allow_sigchld; sigfillset(&allow_sigchld); sigdelset(&allow_sigchld, SIGCHLD); - + if(sigsuspend(&allow_sigchld) < 0) { if(errno == EINTR) { state_unknown = true; From 52793d2a6a78cbe25deea9a0cb8647d982862714 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 10 Apr 2024 14:19:36 -0700 Subject: [PATCH 44/53] fix windows build --- core/core.stanza | 3 +-- runtime/driver.c | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index e0d0386c..50bb7cf1 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -10016,8 +10016,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, error:ref, working-dir:ref, env-vars:ref>|False> - env-var-mode:ref, - placeholder-ignore:ref) -> ref : + env-var-mode:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val proc = new Process{0, null, -1, null, null, null, -1, false, false, false, false} diff --git a/runtime/driver.c b/runtime/driver.c index dfaffd5d..f3f9b7c8 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -766,6 +766,8 @@ void sigchld_handler(int sig) { } } +#endif + // Register SIGCHLD handler void register_handler () { #if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) @@ -782,8 +784,6 @@ void register_handler () { #endif } -#endif - //------------------------------------------------------------ //----------------------- Serialization ---------------------- //------------------------------------------------------------ From 0d76032557805ee8c836f5a1743b0b39c58d8109 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 10 Apr 2024 14:32:50 -0700 Subject: [PATCH 45/53] fix windows yet again --- core/core.stanza | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index 9ec01a3d..e0df6800 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -10018,8 +10018,9 @@ defn process-env-var-mode (env-vars:Tuple>|False, output:ref, error:ref, working-dir:ref, - env-vars:ref>|False> - env-var-mode:ref) -> ref : + env-vars:ref>|False>, + env-var-mode:ref, + placeholder-ignore:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val proc = new Process{0, null, -1, null, null, null, -1, false, false, false, false} From 9501f45c75fdc1c3cd6da5e6a1d2c3a1940231ea Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Wed, 10 Apr 2024 17:48:30 -0700 Subject: [PATCH 46/53] yet another commit to get CI to work --- scripts/finish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/finish.sh b/scripts/finish.sh index 597e28f0..8ee98466 100755 --- a/scripts/finish.sh +++ b/scripts/finish.sh @@ -1,4 +1,4 @@ -# scripts/make-asmjit.sh os-x +scripts/make-asmjit.sh os-x gcc -std=gnu99 -c core/sha256.c -O3 -o build/sha256.o -I include gcc -std=gnu99 -c compiler/cvm.c -O3 -o build/cvm.o -I include gcc -c runtime/linenoise-ng/linenoise.cpp -O3 -o build/linenoise.o -I include -I runtime/linenoise-ng From 9371bfb9e043b2d613be4958c3522b860b04d2eb Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Thu, 11 Apr 2024 13:41:20 -0700 Subject: [PATCH 47/53] fix two bugs: incorrect response to posix_spawn error and need to malloc status code in C --- compiler/auto-doc.stanza | 2 +- compiler/package-manager-system.stanza | 2 +- core/core.stanza | 50 +++---- runtime/driver.c | 199 +++++++++++-------------- runtime/process.h | 7 +- 5 files changed, 116 insertions(+), 144 deletions(-) diff --git a/compiler/auto-doc.stanza b/compiler/auto-doc.stanza index 0777196f..ddebecf9 100644 --- a/compiler/auto-doc.stanza +++ b/compiler/auto-doc.stanza @@ -121,7 +121,7 @@ defn slurp-from (s:InputStream) -> String : to-string(buffer) defn call-plugin (name:String, arguments:Seqable) -> String : - val proc = Process(name, cat([name], arguments), PROCESS-IN, PROCESS-OUT, PROCESS-ERR, cleanup-files = false) + val proc = Process(name, cat([name], arguments), PROCESS-IN, PROCESS-OUT, PROCESS-ERR) val response = trim $ slurp-from(output-stream(proc)) val state = wait(proc) match(state:ProcessDone) : diff --git a/compiler/package-manager-system.stanza b/compiler/package-manager-system.stanza index fc1814a2..b7717410 100644 --- a/compiler/package-manager-system.stanza +++ b/compiler/package-manager-system.stanza @@ -31,7 +31,7 @@ public defn package-manager-system () -> System : match(redirect-output:String) : val out = FileOutputStream(redirect-output) try : - val proc = Process(prog, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, workdir, env-vars, cleanup-files = false) + val proc = Process(prog, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, workdir, env-vars) val proc-out = output-stream(proc) let loop () : val c = get-char(proc-out) diff --git a/core/core.stanza b/core/core.stanza index e0df6800..6dabc2af 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -85,8 +85,7 @@ protected extern stz_memory_resize: (ptr, long, long) -> int protected extern launch_process: (ptr, int, int, int, ptr, ptr, ptr) -> int protected extern close_process_handle: (ptr) -> int #else: - protected extern launch_process: (ptr, ptr>, int, int, int, int, int, ptr, ptr>, ptr) -> int - protected extern delete_process_pipes: (ptr, ptr, ptr) -> int + protected extern launch_process: (ptr, ptr>, int, int, int, int, ptr, ptr>, ptr) -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int @@ -9926,7 +9925,7 @@ public lostanza deftype Process <: Unique : var input: ptr var output: ptr var error: ptr - var status: int + var status: ptr var input-stream: ref var output-stream: ref var error-stream: ref @@ -10019,11 +10018,10 @@ defn process-env-var-mode (env-vars:Tuple>|False, error:ref, working-dir:ref, env-vars:ref>|False>, - env-var-mode:ref, - placeholder-ignore:ref) -> ref : + env-var-mode:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) - val proc = new Process{0, null, -1, null, null, null, -1, false, false, false, false} + val proc = new Process{0, null, -1, null, null, null, null, false, false, false, false} ; Create a command line from the given filename and args (discarding the first argument). ; This is necessary because Windows' process API expects a command line (not a list of arguments). @@ -10100,13 +10098,12 @@ defn process-env-var-mode (env-vars:Tuple>|False, error:ref, working-dir:ref env-vars:ref>|False> - env-var-mode:ref, - cleanup-files:ref) -> ref : + env-var-mode:ref) -> ref : ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) val stz-proc-id = genid().value - val proc = new Process{0, null, stz-proc-id, null, null, null, -1, false, false, false, false} + val proc = new Process{0, null, stz-proc-id, null, null, null, null, false, false, false, false} val input_v = value(input).value val output_v = value(output).value val error_v = value(error).value @@ -10128,7 +10125,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ;Launch the process val launch_succ = call-c clib/launch_process(addr!(filename.chars), argvs, - input_v, output_v, error_v, stz-proc-id, 0, working-dir-chars, env-var-string, addr!([proc])) + input_v, output_v, error_v, stz-proc-id, working-dir-chars, env-var-string, addr!([proc])) ;Free the memory created using malloc. free-linux-env-var-string(env-var-string) @@ -10224,24 +10221,20 @@ public defn Process (filename:String, output:StreamSpecifier, error:StreamSpecifier, working-dir:String|False, - env-vars:Tuple>|False - -- cleanup-files:True|False = true) -> Process : - Process(filename, args, input, output, error, working-dir, env-vars, AddEnvVars, cleanup-files) + env-vars:Tuple>|False) -> Process : + Process(filename, args, input, output, error, working-dir, env-vars, AddEnvVars) ;Default working-dir is false. Default env-vars is false. public defn Process (filename:String, args:Seqable, - input:StreamSpecifier, output:StreamSpecifier, error:StreamSpecifier, - -- cleanup-files:True|False = true) -> Process : - Process(filename, args, input, output, error, false, false, cleanup-files = cleanup-files) + input:StreamSpecifier, output:StreamSpecifier, error:StreamSpecifier) -> Process : + Process(filename, args, input, output, error, false, false) public defn Process (filename:String, args:Seqable, - working-dir:String|False, env:Tuple>|False, - -- cleanup-files:True|False = true) -> Process : - Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, working-dir, env, cleanup-files = cleanup-files) + working-dir:String|False, env:Tuple>|False) -> Process : + Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, working-dir, env) -public defn Process (filename:String, args:Seqable, - -- cleanup-files:True|False = true) : - Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, false, false, cleanup-files = cleanup-files) +public defn Process (filename:String, args:Seqable) : + Process(filename, args, STANDARD-IN, STANDARD-OUT, STANDARD-ERR, false, false) ; Stream API ; ========== @@ -10280,12 +10273,6 @@ lostanza defn retrieve-state (p:ref, wait-for-termination?:ref) -> ref : ;ProcessRunning. (s:ref) : return s - ; TODO: should we wait on stopped processes? (s:ref) : return s ;If the process has finished, then save its terminating @@ -10326,9 +10312,7 @@ public lostanza defn state (p:ref) -> ref : public defn* wait (p:Process) -> ProcessState : val s = retrieve-state(p, true) - ; TODO: should we wait on stopped processes? match(s:ProcessRunning|ProcessStopped) : wait(p) - ;match(s:ProcessRunning) : wait(p) else : s @@ -10338,7 +10322,7 @@ public defn call-system (filename:String, args:Seqable, working-dir:String|False, env-vars:Tuple>|False) -> Int : - match(wait(Process(filename, args, working-dir, env-vars, cleanup-files = false))) : + match(wait(Process(filename, args, working-dir, env-vars))) : (s:ProcessDone) : value(s) (s) : @@ -10368,7 +10352,7 @@ public defn call-system-and-get-output (file:String, working-dir:String|False, env-vars:Tuple>|False) -> String : val buffer = StringBuffer() - val proc = Process(file, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, working-dir, env-vars, cleanup-files = false) + val proc = Process(file, args, STANDARD-IN, PROCESS-OUT, PROCESS-OUT, working-dir, env-vars) val proc-out = output-stream(proc) let loop () : val c = get-char(proc-out) diff --git a/runtime/driver.c b/runtime/driver.c index f3f9b7c8..2ac1a317 100644 --- a/runtime/driver.c +++ b/runtime/driver.c @@ -621,8 +621,7 @@ typedef struct ChildProcess { FILE* fin; FILE* fout; FILE* ferr; - int* status; - bool auto_cleanup; + stz_int* status; } ChildProcess; // Linked lists of process metadata @@ -631,35 +630,12 @@ typedef struct ProcessNode { volatile struct ProcessNode * volatile next; } ProcessNode; - -// Free everything allocated by ChildProcess -static void cleanup_proc (ChildProcess* c) { - // Close files - if(c->auto_cleanup) { - if(c->fin != NULL) - if(fclose(c->fin) == EOF) exit_with_error(); - if(c->fout != NULL) - if(fclose(c->fout) == EOF) exit_with_error(); - if(c->ferr != NULL) - if(fclose(c->ferr) == EOF) exit_with_error(); - } - free(c); -} - -// Process status struct -typedef struct ProcessStatus { - stz_int stz_proc_id; - int* status; -} ProcessStatus; - // Linked list of live processes volatile ProcessNode * volatile proc_head = NULL; struct sigaction oldact; -// flag to update process statuses -sig_atomic_t proc_dirty = 0; - // Record process metadata +// Precondition: SIGCHLD is blocked static int register_proc ( pid_t pid, stz_int stz_proc_id, @@ -667,9 +643,12 @@ static int register_proc ( FILE* fin, FILE* fout, FILE* ferr, - int* status, - bool auto_cleanup) { + ProcessStatus** status) { + // Init and save ProcessStatus struct + ProcessStatus* st = (ProcessStatus*)malloc(sizeof(ProcessStatus)); + st->val = -1; + *status = st; // Init ChildProcess struct ChildProcess* child = (ChildProcess*)malloc(sizeof(ChildProcess)); if(child == NULL) return -1; @@ -678,11 +657,9 @@ static int register_proc ( child->fin = fin; child->fout = fout; child->ferr = ferr; - child->status = status; - *(child->status) = -1; - child->auto_cleanup = auto_cleanup; + child->status = &(st->val); // Store child in ProcessNode - volatile ProcessNode * new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); + volatile ProcessNode * volatile new_node = (ProcessNode*)malloc(sizeof(ProcessNode)); if(new_node == NULL) return -1; new_node->proc = child; new_node->next = proc_head; @@ -691,35 +668,36 @@ static int register_proc ( return 0; } +static bool dead_status (int st) { + return WIFSIGNALED(st) || WIFEXITED(st); +} + // Cleanup all resources for process pid +// Precondition: SIGCHLD is blocked static void cleanup_child (pid_t pid) { - volatile ProcessNode * curr = proc_head; - volatile ProcessNode * prev = NULL; + volatile ProcessNode * volatile curr = proc_head; + volatile ProcessNode * volatile prev = NULL; // Find matching Node while(curr != NULL && curr->proc->pid != pid) { prev = curr; curr = curr->next; } + // Remove node from list of processes, free memory if(curr != NULL) { - // Remove node from list of alive processes if(prev == NULL) { proc_head = curr->next; } else { prev->next = curr->next; } - // Cleanup resources - cleanup_proc(curr->proc); + free((void*) curr); } } -static bool dead_status (int st) { - return WIFSIGNALED(st) || WIFEXITED(st); -} - // Update a process' status code +// Precondition: SIGCHLD is blocked static void update_status (pid_t pid, int status) { - volatile ProcessNode * curr = proc_head; + volatile ProcessNode * volatile curr = proc_head; while(curr != NULL && curr->proc->pid != pid) { curr = curr->next; } @@ -728,6 +706,7 @@ static void update_status (pid_t pid, int status) { } } +// Precondition: SIGCHLD is blocked static int wait_and_update (pid_t pid) { int status; if(waitpid(pid, &status, WNOHANG | WUNTRACED | WCONTINUED) > 0) { @@ -738,14 +717,16 @@ static int wait_and_update (pid_t pid) { return 0; } +// Precondition: SIGCHLD is blocked static void waitpid_all_registered () { - volatile ProcessNode * curr = proc_head; + volatile ProcessNode * volatile curr = proc_head; while(curr != NULL) { wait_and_update(curr->proc->pid); curr = curr->next; } } +// Precondition: SIGCHLD is blocked static void waitpid_global () { int status; pid_t pid; @@ -755,6 +736,7 @@ static void waitpid_global () { } } +// Precondition: SIGCHLD is blocked void sigchld_handler(int sig) { if(!(oldact.sa_flags & SA_SIGINFO)) { if(oldact.sa_handler != SIG_DFL) { @@ -897,31 +879,6 @@ static void free_strings (stz_byte** ss){ //---------------------- Launcher Main ----------------------- //------------------------------------------------------------ -static void write_error_and_exit (int fd){ - int code = errno; - write(fd, &code, sizeof(int)); - close(fd); - exit(-1); -} - -static int delete_process_pipe (FILE* fd) { - if (fd != NULL) { - int close_res = fclose(fd); - if (close_res == EOF) return -1; - } - return 0; -} - -stz_int delete_process_pipes (FILE* input, FILE* output, FILE* error) { - if (delete_process_pipe(input) < 0) - return -1; - if (delete_process_pipe(output) < 0) - return -1; - if (delete_process_pipe(error) < 0) - return -1; - return 0; -} - // Useful for testing linux implementation on OSX //#ifdef PLATFORM_OS_X //extern char **environ; @@ -950,16 +907,10 @@ static void restore (sigset_t* old_mask) { if(sigprocmask(SIG_SETMASK, old_mask, NULL)) exit_with_error(); } -static int restore_and_err (sigset_t* old_mask) { - restore(old_mask); - return -1; -} - - #ifdef PLATFORM_LINUX //#if defined(PLATFORM_LINUX) || defined(PLATFORM_OS_X) stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int stz_proc_id, stz_int cleanup_files, + stz_int output, stz_int error, stz_int stz_proc_id, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources: int pipe_sources[NUM_STREAM_SPECS]; @@ -1010,7 +961,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0); + int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status)); // Unblock SIGCHLD restore(&old_mask); @@ -1057,7 +1008,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, #ifdef PLATFORM_OS_X stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int stz_proc_id, stz_int cleanup_files, + stz_int output, stz_int error, stz_int stz_proc_id, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //block sigchld sigset_t old_mask = block(); @@ -1072,7 +1023,10 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Generate array of pipes per-input-source int pipes[NUM_STREAM_SPECS][2]; for(int i=0; i= 0) { - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) - restore_and_err(&old_mask); - if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) - restore_and_err(&old_mask); - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0]))) - restore_and_err(&old_mask); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1]))) { + restore(&old_mask); + return -1; + } + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO))) { + restore(&old_mask); + return -1; + } + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0]))) { + restore(&old_mask); + return -1; + } } //Setup output pipe if used if(pipe_sources[PROCESS_OUT] >= 0) { - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) - restore_and_err(&old_mask); - if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) - restore_and_err(&old_mask); - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1]))) - restore_and_err(&old_mask); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0]))) { + restore(&old_mask); + return -1; + } + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO))) { + restore(&old_mask); + return -1; + } + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1]))) { + restore(&old_mask); + return -1; + } } //Setup error pipe if used if(pipe_sources[PROCESS_ERR] >= 0) { - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) - restore_and_err(&old_mask); - if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) - restore_and_err(&old_mask); - if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1]))) - restore_and_err(&old_mask); + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0]))) { + restore(&old_mask); + return -1; + } + if((posix_ret = posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO))) { + restore(&old_mask); + return -1; + } + if((posix_ret = posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1]))) { + restore(&old_mask); + return -1; + } } //Setup working directory if(working_dir) { - if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, C_CSTR(working_dir)))) - restore_and_err(&old_mask); + if((posix_ret = posix_spawn_file_actions_addchdir_np(&actions, C_CSTR(working_dir)))) { + restore(&old_mask); + return -1; + } } // Spawn child process @@ -1120,8 +1094,9 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, if((spawn_ret = posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars)) == 0) { // success } else { - errno = spawn_ret; // might not want to do this manually? - restore_and_err(&old_mask); + errno = spawn_ret; + restore(&old_mask); + return -1; } // Cleanup posix_spawn_file_actions_destroy(&actions); @@ -1132,20 +1107,28 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, if(pipe_sources[PROCESS_IN] >= 0) { close(pipes[PROCESS_IN][0]); fin = fdopen(pipes[PROCESS_IN][1], "w"); - if(fin == NULL) return -1; - restore_and_err(&old_mask); + if(fin == NULL) { + restore(&old_mask); + return -1; + } } FILE* fout = NULL; if(pipe_sources[PROCESS_OUT] >= 0) { close(pipes[PROCESS_OUT][1]); fout = fdopen(pipes[PROCESS_OUT][0], "r"); - if(fout == NULL) restore_and_err(&old_mask); + if(fout == NULL) { + restore(&old_mask); + return -1; + } } FILE* ferr = NULL; if(pipe_sources[PROCESS_ERR] >= 0) { close(pipes[PROCESS_ERR][1]); ferr = fdopen(pipes[PROCESS_ERR][0], "r"); - if(ferr == NULL) restore_and_err(&old_mask); + if(ferr == NULL) { + restore(&old_mask); + return -1; + } } process->pid = pid; @@ -1154,7 +1137,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, process->out = fout; process->err = ferr; - int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status), cleanup_files > 0); + int r = register_proc(pid, stz_proc_id, &pipes, fin, fout, ferr, &(process->status)); // Unblock SIGCHLD restore(&old_mask); @@ -1165,7 +1148,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, -int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ +stz_int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_termination){ // block SIGCHLD sigset_t old_mask = block(); int status; @@ -1173,7 +1156,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ bool state_unknown = true; while(state_unknown) { - status = process->status; + status = process->status->val; if(WIFEXITED(status)) *s = (ProcessState){PROCESS_DONE, WEXITSTATUS(status)}; else if(WIFSIGNALED(status)) @@ -1184,7 +1167,7 @@ int retrieve_process_state (Process* process, ProcessState* s, stz_int wait_for_ *s = (ProcessState){PROCESS_RUNNING, 0}; state_unknown = false; if(wait_for_termination && !dead_status(status)) { - // We allow SICHLD during sigsuspend + // We allow SIGCHLD during sigsuspend sigset_t allow_sigchld; sigfillset(&allow_sigchld); sigdelset(&allow_sigchld, SIGCHLD); diff --git a/runtime/process.h b/runtime/process.h index 34459eec..02028276 100644 --- a/runtime/process.h +++ b/runtime/process.h @@ -3,6 +3,11 @@ #include #include +// Process status struct +typedef struct ProcessStatus { + stz_int val; +} ProcessStatus; + //Represents the Process, and the channels for //communicating with it. //- pid: The id of the process. @@ -15,10 +20,10 @@ typedef struct { stz_long pid; void* handle; stz_int stz_proc_id; - int status; FILE* in; FILE* out; FILE* err; + ProcessStatus* status; } Process; //Represents the state of the process. From ba41e0f08aa1f056a1e6d8a47b0cc83c630afd24 Mon Sep 17 00:00:00 2001 From: Tristan Knoth Date: Sun, 14 Apr 2024 15:48:27 -0700 Subject: [PATCH 48/53] forgot to commit the RTLD_LOCAL selection for OSX --- core/dynamic-library.stanza | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/dynamic-library.stanza b/core/dynamic-library.stanza index d32d4156..af2ac795 100644 --- a/core/dynamic-library.stanza +++ b/core/dynamic-library.stanza @@ -64,7 +64,8 @@ public lostanza defn name (dl:ref) -> ref : public lostanza defn dynamic-library-open (name: ref) -> ref: #if-defined(PLATFORM-OS-X) : val RTLD_NOW = 0x2 - val dlflags = RTLD_NOW + val RTLD_LOCAL = 0x4 + val dlflags = RTLD_NOW | RTLD_LOCAL #if-defined(PLATFORM-LINUX) : val RTLD_NOW = 0x2 val RTLD_DEEPBIND = 0x8 From 0fc589543d93f5ca4da9d1e886d0eb86d95f17a5 Mon Sep 17 00:00:00 2001 From: Patrick Li Date: Mon, 15 Apr 2024 23:34:21 -0400 Subject: [PATCH 49/53] Remove stz-proc-id field. --- core/core.stanza | 8 +++----- runtime/process-posix.c | 14 ++++---------- runtime/process.h | 2 -- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/core/core.stanza b/core/core.stanza index d0606ad2..904f3d27 100644 --- a/core/core.stanza +++ b/core/core.stanza @@ -85,7 +85,7 @@ protected extern stz_memory_resize: (ptr, long, long) -> int protected extern launch_process: (ptr, int, int, int, ptr, ptr, ptr) -> int protected extern close_process_handle: (ptr) -> int #else: - protected extern launch_process: (ptr, ptr>, int, int, int, int, ptr, ptr>, ptr) -> int + protected extern launch_process: (ptr, ptr>, int, int, int, ptr, ptr>, ptr) -> int protected extern retrieve_process_state: (ptr, ptr, int) -> int @@ -9922,7 +9922,6 @@ public val CURRENT-PLATFORM:Platform = public lostanza deftype Process <: Unique : var pid: long var handle: ptr - var stz-proc-id: int var input: ptr var output: ptr var error: ptr @@ -10103,8 +10102,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ensure-valid-env-var-names!(env-vars) ensure-valid-stream-specifiers(input, output, error) val args = to-tuple(args0) - val stz-proc-id = genid().value - val proc = new Process{0, null, stz-proc-id, null, null, null, null, false, false, false, false} + val proc = new Process{0, null, null, null, null, null, false, false, false, false} val input_v = value(input).value val output_v = value(output).value val error_v = value(error).value @@ -10126,7 +10124,7 @@ defn process-env-var-mode (env-vars:Tuple>|False, ;Launch the process val launch_succ = call-c clib/launch_process(addr!(filename.chars), argvs, - input_v, output_v, error_v, stz-proc-id, working-dir-chars, env-var-string, addr!([proc])) + input_v, output_v, error_v, working-dir-chars, env-var-string, addr!([proc])) ;Free the memory created using malloc. free-linux-env-var-string(env-var-string) diff --git a/runtime/process-posix.c b/runtime/process-posix.c index 063ef507..2822dc23 100644 --- a/runtime/process-posix.c +++ b/runtime/process-posix.c @@ -10,14 +10,12 @@ // Holds all metadata for a spawned child process. // - pid: The process id of the child process itself. -// - stz_proc_id: A unique id for identifying this spawned child. // - fin/fout/ferr: The stdin/stdout/stderr streams for communicating with the child. // NULL if child uses stdin/stdout/stderr directly. // - status: Pointer to location to write status code when child is terminated. // Used by signal handler. typedef struct ChildProcess { pid_t pid; - stz_int stz_proc_id; FILE* fin; FILE* fout; FILE* ferr; @@ -89,7 +87,6 @@ static void remove_child_process (pid_t pid) { // is written to the provided pointer. static void register_child_process ( pid_t pid, - stz_int stz_proc_id, FILE* fin, FILE* fout, FILE* ferr, @@ -103,7 +100,6 @@ static void register_child_process ( // Initialize ChildProcess struct. ChildProcess* child = (ChildProcess*)malloc(sizeof(ChildProcess)); child->pid = pid; - child->stz_proc_id = stz_proc_id; child->fin = fin; child->fout = fout; child->ferr = ferr; @@ -315,7 +311,7 @@ stz_int retrieve_process_state (Process* process, //} stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int stz_proc_id, + stz_int output, stz_int error, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //block sigchld sigset_t old_signal_mask = block_sigchild(); @@ -438,14 +434,13 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Store the process details in the process structure. process->pid = pid; - process->stz_proc_id = stz_proc_id; process->in = fin; process->out = fout; process->err = ferr; //Register the child process so that it is automatically //reaped by the autoreap handler. - register_child_process(pid, stz_proc_id, fin, fout, ferr, &(process->status)); + register_child_process(pid, fin, fout, ferr, &(process->status)); //Process spawn was successful. goto return_success; @@ -473,7 +468,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, #ifdef PLATFORM_LINUX stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, stz_int stz_proc_id, + stz_int output, stz_int error, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //Compute pipe sources. Examples of entries: @@ -531,14 +526,13 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Store the process details in the process structure. process->pid = pid; - process->stz_proc_id = stz_proc_id; process->in = fin; process->out = fout; process->err = ferr; //Register the child process so that it is automatically //repeated by the autoreap handler. - register_child_process(pid, stz_proc_id, fin, fout, ferr, &(process->status)); + register_child_process(pid, fin, fout, ferr, &(process->status)); //Perform cleanup and return -1 to indicate error. return_error: { diff --git a/runtime/process.h b/runtime/process.h index 56f0005d..19f39a95 100644 --- a/runtime/process.h +++ b/runtime/process.h @@ -14,7 +14,6 @@ typedef struct ProcessStatus { //communicating with it. //- pid: The id of the process. //- handle: The Windows handle to the process. Not used by other platforms. -//- stz_proc_id: A unique integer for identifying this process. //- in: The standard input stream of the Process. //- out: The standard output stream of the Process. //- err: The standard error stream of the Process. @@ -22,7 +21,6 @@ typedef struct ProcessStatus { typedef struct { stz_long pid; void* handle; - stz_int stz_proc_id; FILE* in; FILE* out; FILE* err; From 9433e138391fb4eb63d67c126e73c23cc389dafe Mon Sep 17 00:00:00 2001 From: Patrick Li Date: Tue, 16 Apr 2024 00:33:36 -0400 Subject: [PATCH 50/53] Fix bug during refactor. --- runtime/process-posix.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/runtime/process-posix.c b/runtime/process-posix.c index 2822dc23..3354db35 100644 --- a/runtime/process-posix.c +++ b/runtime/process-posix.c @@ -152,8 +152,12 @@ static void update_child_status (pid_t pid) { static void update_all_child_statuses () { volatile ChildProcessList * volatile curr = child_processes; while(curr != NULL) { + //NOTE: curr->next is retrieved preemptively because + //update_child_status may call free() on curr, and make + //curr->next inaccessible. + ChildProcessList* next = curr->next; update_child_status(curr->proc->pid); - curr = curr->next; + curr = next; } } @@ -174,11 +178,16 @@ void autoreaping_sigchld_handler(int sig){ //Update the statuses of all registered child processes. update_all_child_statuses(); - //Test whether the previous signal handler was - //created via the "sigaction" system. Forward the signal - //if it was. - if(!(old_sigchild_action.sa_flags & SA_SIGINFO)) + //Test whether we are able to forward the signal + //to the previous signal handler. It must: + //1. Not be using the sigaction system. (SA_SIGINFO) + //2. Not be the default signal handler. (SIG_DFL) + //3. Not be explicitly ignored. (SIG_IGN) + if(!(old_sigchild_action.sa_flags & SA_SIGINFO) && + old_sigchild_action.sa_handler != SIG_DFL && + old_sigchild_action.sa_handler != SIG_IGN){ old_sigchild_action.sa_handler(sig); + } } // This installs the autoreaping signal handler. @@ -406,11 +415,8 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Spawn the child process. pid_t pid = -1; - int spawn_ret = posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars); - if(spawn_ret != 0){ - errno = spawn_ret; + if(posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars)) goto return_error; - } //Set up the pipes in the parent process. FILE* fin = NULL; From 3b67627029fb983a2e256ed9238d798cef0cc508 Mon Sep 17 00:00:00 2001 From: Patrick Li Date: Tue, 16 Apr 2024 03:20:06 -0400 Subject: [PATCH 51/53] Fix pipe redirection bugs in OS-X launch_process. --- runtime/process-posix.c | 123 +++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 71 deletions(-) diff --git a/runtime/process-posix.c b/runtime/process-posix.c index 3354db35..1f311e79 100644 --- a/runtime/process-posix.c +++ b/runtime/process-posix.c @@ -319,26 +319,24 @@ stz_int retrieve_process_state (Process* process, // return rc; //} -stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, - stz_int output, stz_int error, +stz_int launch_process(stz_byte* file, stz_byte** argvs, + stz_int input, stz_int output, stz_int error, stz_byte* working_dir, stz_byte** env_vars, Process* process) { //block sigchld sigset_t old_signal_mask = block_sigchild(); - //Compute pipe sources. Examples of entries: - // pipe_sources[PROCESS_IN] = 0, indicates that - // the input pipe to the process is served by POSIX file descriptor 0 (stdin). - // pipe_sources[PROCESS_IN] = -1, indicates that - // no input pipe to the process should be created. - // pipe_sources[STANDARD_ERR] = 1, indicates that - // the process standard error stream should be served by POSIX file descriptor 1 (stdout), - // which means child writes to its error stream should automatically go to stdout. - int pipe_sources[NUM_STREAM_SPECS]; + //Compute which pipes to create for the process. + //has_pipes[PROCESS_IN] = 1, indicates that a process input pipe + //needs to be created. + int has_pipes[NUM_STREAM_SPECS]; for(int i=0; i= 0) - if(pipe(pipes[process_io[i]])) - goto return_error; - - //Redirect stdout if necessary. - { - int i = STANDARD_OUT; - if(pipe_sources[i] >= 0 && pipe_sources[i] != STDOUT_FILENO){ - if(posix_spawn_file_actions_adddup2(&actions, STDOUT_FILENO, pipe_sources[i])) - goto return_error; - } - } - //Redirect stderr if necessary. - { - int i = STANDARD_ERR; - if(pipe_sources[i] >= 0 && pipe_sources[i] != STDERR_FILENO){ - if(posix_spawn_file_actions_adddup2(&actions, STDERR_FILENO, pipe_sources[i])) - goto return_error; - } - } + for(int i=0; i= 0) { - if(posix_spawn_file_actions_addclose(&actions, pipes[i][1])) - goto return_error; - if(posix_spawn_file_actions_adddup2(&actions, pipes[i][0], pipe_sources[i])) - goto return_error; - if(posix_spawn_file_actions_addclose(&actions, pipes[i][0])) - goto return_error; - } + //Connect process input pipe if necessary. + if(has_pipes[PROCESS_IN]){ + if(posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][1])) + goto return_error; + if(posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_IN][0], STDIN_FILENO)) + goto return_error; + if(posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_IN][0])) + goto return_error; } - - //Redirect stdout/stderr to process output pipe if necessary. - { - int i = PROCESS_OUT; - if(pipe_sources[i] >= 0) { - if(posix_spawn_file_actions_addclose(&actions, pipes[i][0])) - goto return_error; - if(posix_spawn_file_actions_adddup2(&actions, pipes[i][1], pipe_sources[i])) - goto return_error; - if(posix_spawn_file_actions_addclose(&actions, pipes[i][1])) + //Connect process output pipe if necessary. + if(has_pipes[PROCESS_OUT]){ + if(posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][0])) + goto return_error; + if(output == PROCESS_OUT) + if(posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDOUT_FILENO)) goto return_error; - } + if(error == PROCESS_OUT) + if(posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_OUT][1], STDERR_FILENO)) + goto return_error; + if(posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_OUT][1])) + goto return_error; } - { - int i = PROCESS_ERR; - if(pipe_sources[i] >= 0) { - if(posix_spawn_file_actions_addclose(&actions, pipes[i][0])) - goto return_error; - if(posix_spawn_file_actions_adddup2(&actions, pipes[i][1], pipe_sources[i])) + //Connect process error pipe if necessary. + if(has_pipes[PROCESS_ERR]){ + if(posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][0])) + goto return_error; + if(error == PROCESS_ERR) + if(posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDERR_FILENO)) goto return_error; - if(posix_spawn_file_actions_addclose(&actions, pipes[i][1])) + if(output == PROCESS_ERR) + if(posix_spawn_file_actions_adddup2(&actions, pipes[PROCESS_ERR][1], STDOUT_FILENO)) goto return_error; - } + if(posix_spawn_file_actions_addclose(&actions, pipes[PROCESS_ERR][1])) + goto return_error; } //Setup working directory @@ -415,24 +393,27 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //Spawn the child process. pid_t pid = -1; - if(posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars)) + int spawn_ret = posix_spawnp(&pid, C_CSTR(file), &actions, NULL, (char**)argvs, (char**)env_vars); + if(spawn_ret){ + errno = spawn_ret; goto return_error; + } //Set up the pipes in the parent process. FILE* fin = NULL; - if(pipe_sources[PROCESS_IN] >= 0) { + if(has_pipes[PROCESS_IN]) { close(pipes[PROCESS_IN][0]); fin = fdopen(pipes[PROCESS_IN][1], "w"); if(fin == NULL) goto return_error; } FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] >= 0) { + if(has_pipes[PROCESS_OUT]) { close(pipes[PROCESS_OUT][1]); fout = fdopen(pipes[PROCESS_OUT][0], "r"); if(fout == NULL) goto return_error; } FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] >= 0) { + if(has_pipes[PROCESS_ERR]) { close(pipes[PROCESS_ERR][1]); ferr = fdopen(pipes[PROCESS_ERR][0], "r"); if(ferr == NULL) goto return_error; From f13e925e04b24cb34005aefe2cb3fa19869744c5 Mon Sep 17 00:00:00 2001 From: Patrick Li Date: Tue, 16 Apr 2024 03:28:12 -0400 Subject: [PATCH 52/53] Fix pipe redirection bugs on Linux. --- runtime/process-posix.c | 102 +++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 59 deletions(-) diff --git a/runtime/process-posix.c b/runtime/process-posix.c index 1f311e79..b805ee91 100644 --- a/runtime/process-posix.c +++ b/runtime/process-posix.c @@ -155,7 +155,7 @@ static void update_all_child_statuses () { //NOTE: curr->next is retrieved preemptively because //update_child_status may call free() on curr, and make //curr->next inaccessible. - ChildProcessList* next = curr->next; + volatile ChildProcessList* next = curr->next; update_child_status(curr->proc->pid); curr = next; } @@ -458,28 +458,24 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, stz_int output, stz_int error, stz_byte* working_dir, stz_byte** env_vars, Process* process) { - //Compute pipe sources. Examples of entries: - // pipe_sources[PROCESS_IN] = 0, indicates that - // the input pipe to the process is served by POSIX file descriptor 0 (stdin). - // pipe_sources[PROCESS_IN] = -1, indicates that - // no input pipe to the process should be created. - // pipe_sources[STANDARD_ERR] = 1, indicates that - // the process standard error stream should be served by POSIX file descriptor 1 (stdout), - // which means child writes to its error stream should automatically go to stdout. - int pipe_sources[NUM_STREAM_SPECS]; + //Compute which pipes to create for the process. + //has_pipes[PROCESS_IN] = 1, indicates that a process input pipe + //needs to be created. + int has_pipes[NUM_STREAM_SPECS]; for(int i=0; i= 0) - if(pipe(pipes[process_io[i]])) - return -1; + for(int i=0; i= 0) { + if(has_pipes[PROCESS_IN]) { close(pipes[PROCESS_IN][0]); fin = fdopen(pipes[PROCESS_IN][1], "w"); if(fin == NULL) goto return_error; } FILE* fout = NULL; - if(pipe_sources[PROCESS_OUT] >= 0) { + if(has_pipes[PROCESS_OUT]) { close(pipes[PROCESS_OUT][1]); fout = fdopen(pipes[PROCESS_OUT][0], "r"); if(fout == NULL) goto return_error; } FILE* ferr = NULL; - if(pipe_sources[PROCESS_ERR] >= 0) { + if(has_pipes[PROCESS_ERR]) { close(pipes[PROCESS_ERR][1]); ferr = fdopen(pipes[PROCESS_ERR][0], "r"); if(ferr == NULL) goto return_error; @@ -521,6 +517,9 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, //repeated by the autoreap handler. register_child_process(pid, fin, fout, ferr, &(process->status)); + //Child process successfully launched and registered. + goto return_success; + //Perform cleanup and return -1 to indicate error. return_error: { restore_signal_mask(&old_signal_mask); @@ -528,7 +527,7 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, } //Perform cleanup and return 0 to indicate success. - return_success:{ + return_success: { restore_signal_mask(&old_signal_mask); return 0; } @@ -536,44 +535,29 @@ stz_int launch_process(stz_byte* file, stz_byte** argvs, stz_int input, // Child: setup pipes, exec else { - //Redirect stdout if necessary - { - int i = STANDARD_OUT; - if(pipe_sources[i] >= 0 && pipe_sources[i] != STDOUT_FILENO) - if(dup2(STDOUT_FILENO, pipe_sources[i]) < 0) exit(-1); - } - //Redirect stderr if necessary - { - int i = STANDARD_ERR; - if(pipe_sources[i] >= 0 && pipe_sources[i] != STDERR_FILENO) - if(dup2(STDERR_FILENO, pipe_sources[i]) < 0) exit(-1); - } - //Setup input pipe if used - { - int i = PROCESS_IN; - if(pipe_sources[i] >= 0) { - if(close(pipes[i][1]) < 0) exit(-1); - if(dup2(pipes[i][0], pipe_sources[i]) < 0) exit(-1); - if(close(pipes[i][0]) < 0) exit(-1); - } + //Connect process input pipe if necessary. + if(has_pipes[PROCESS_IN]){ + if(close(pipes[PROCESS_IN][1]) < 0) exit(-1); + if(dup2(pipes[PROCESS_IN][0], STDIN_FILENO) < 0) exit(-1); + if(close(pipes[PROCESS_IN][0]) < 0) exit(-1); } - //Setup output pipe if used - { - int i = PROCESS_OUT; - if(pipe_sources[i] >= 0) { - if(close(pipes[i][0]) < 0) exit(-1); - if(dup2(pipes[i][1], pipe_sources[i]) < 0) exit(-1); - if(close(pipes[i][1]) < 0) exit(-1); - } + //Connect process output pipe if necessary. + if(has_pipes[PROCESS_OUT]){ + if(close(pipes[PROCESS_OUT][0]) < 0) exit(-1); + if(output == PROCESS_OUT) + if(dup2(pipes[PROCESS_OUT][1], STDOUT_FILENO) < 0) exit(-1); + if(error == PROCESS_OUT) + if(dup2(pipes[PROCESS_OUT][1], STDERR_FILENO) < 0) exit(-1); + if(close(pipes[PROCESS_OUT][1]) < 0) exit(-1); } - //Setup error pipe if used - { - int i = PROCESS_ERR; - if(pipe_sources[i] >= 0) { - if(close(pipes[i][0]) < 0) exit(-1); - if(dup2(pipes[i][1], pipe_sources[i]) < 0) exit(-1); - if(close(pipes[i][1]) < 0) exit(-1); - } + //Connect process error pipe if necessary. + if(has_pipes[PROCESS_ERR]){ + if(close(pipes[PROCESS_ERR][0]) < 0) exit(-1); + if(output == PROCESS_ERR) + if(dup2(pipes[PROCESS_ERR][1], STDOUT_FILENO) < 0) exit(-1); + if(error == PROCESS_ERR) + if(dup2(pipes[PROCESS_ERR][1], STDERR_FILENO) < 0) exit(-1); + if(close(pipes[PROCESS_ERR][1]) < 0) exit(-1); } //Setup working directory From 01e1bade26269c9699fe163b2f4125cb3eadbab6 Mon Sep 17 00:00:00 2001 From: Patrick Li Date: Tue, 16 Apr 2024 04:01:25 -0400 Subject: [PATCH 53/53] Increment version. --- ci/build-stanza-version.txt | 2 +- compiler/params.stanza | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build-stanza-version.txt b/ci/build-stanza-version.txt index 9b5e2382..abd6926f 100644 --- a/ci/build-stanza-version.txt +++ b/ci/build-stanza-version.txt @@ -7,4 +7,4 @@ # like 1.23.45 # # Use version 0.17.56 to compile 0.18.0 -0.18.45 +0.18.59 diff --git a/compiler/params.stanza b/compiler/params.stanza index 288201d7..d294da17 100644 --- a/compiler/params.stanza +++ b/compiler/params.stanza @@ -18,7 +18,7 @@ public defn compiler-flags () : to-tuple(COMPILE-FLAGS) ;========= Stanza Configuration ======== -public val STANZA-VERSION = [0 18 59] +public val STANZA-VERSION = [0 18 60] public var STANZA-INSTALL-DIR:String = "" public var OUTPUT-PLATFORM:Symbol = `platform public var STANZA-PKG-DIRS:List = List()