From 8ee20a0fbd44b14ace8e6f2d5ac26ef8f1722679 Mon Sep 17 00:00:00 2001 From: Morgan Rodgers Date: Fri, 1 Nov 2019 14:46:43 -0400 Subject: [PATCH 1/5] Conditionally add IJulia 1.x kernel only if the user has it installed for themselves --- template/script.sh.erb | 57 +++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/template/script.sh.erb b/template/script.sh.erb index 90c9870..985178d 100755 --- a/template/script.sh.erb +++ b/template/script.sh.erb @@ -1,6 +1,7 @@ #!/usr/bin/env bash <%- + require 'pathname' cuda = (context.node_type == "gpu") ? context.cuda_version : "" wrapper = session.staged_root.join("launch_wrapper.sh") wrapper_log = session.staged_root.join("launch_wrapper.log") @@ -97,24 +98,44 @@ env: { MODULES: "xalt/latest julia/0.6.4 #{cuda}" } - }, - julia100: { - display_name: "Julia 1.0.0 [julia/1.0.0 #{cuda}]", - language: "julia", - argv: [ - wrapper, - "julia", - "-i", - "--startup-file=yes", - "--color=yes", - "/usr/local/julia/1.0.0/site/packages/IJulia/fRegO/src/kernel.jl", - "{connection_file}" - ], - env: { - MODULES: "xalt/latest julia/1.0.0 #{cuda}" - } - }, - } + } + }.tap do |hsh| + # Due to problems using a mix of user and site modules the site module for IJulia 1.x will be removed + # This detect if a user has IJulia installed for themselves and if so will provide a kernel + manifest = Pathname.new("~/.julia/environments/v1.1/Manifest.toml").expand_path + ijulia_installation_root = Pathname.new("~/.julia/packages/IJulia").expand_path + + if manifest.exist? and ijulia_installation_root.exist? + if matches = manifest.read.match(/\[\[IJulia\]\].+?version\s*=\s*"(?\d+\.\d+\.\d+)"\n/m) + active_ijulia_version = matches[:version] + + ijulia_installation_root.children.select do |installation| + begin + installation.join('Project.toml').read.match(/version\s*=\s*"(?\d+\.\d+\.\d+)"/)[:version] == active_ijulia_version + rescue StandardError => e + return false + end + end.each do |installation| + hsh["julia#{active_ijulia_version.gsub('.', '').to_sym}"] = { + display_name: "Julia #{active_ijulia_version}" + ((cuda.empty?) ? "" : " [#{cuda}]"), + language: "julia", + argv: [ + wrapper, + "julia", + "-i", + "--startup-file=yes", + "--color=yes", + installation.join("src/kernel.jl").to_s, + "{connection_file}" + ], + env: { + MODULES: "xalt/latest #{cuda}" + } + } + end + end + end + end -%> echo "Starting main script..." From 62982d844ed5bbfc822158c8d674e514d133a47c Mon Sep 17 00:00:00 2001 From: Morgan Rodgers Date: Fri, 1 Nov 2019 16:32:25 -0400 Subject: [PATCH 2/5] Add static kernel for Julia 1.0.0 back Users who have installed the kernel for themselves will be able to use that, and users who have not installed it for themselves will get the static kernel instead --- template/script.sh.erb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/template/script.sh.erb b/template/script.sh.erb index 985178d..fecf52d 100755 --- a/template/script.sh.erb +++ b/template/script.sh.erb @@ -97,7 +97,24 @@ ], env: { MODULES: "xalt/latest julia/0.6.4 #{cuda}" + }, + julia100: { + display_name: "Julia 1.0.0 [julia/1.0.0 #{cuda}]", + language: "julia", + argv: [ + wrapper, + "julia", + "-i", + "--startup-file=yes", + "--color=yes", + "/usr/local/julia/1.0.0/site/packages/IJulia/fRegO/src/kernel.jl", + "{connection_file}" + ], + env: { + MODULES: "xalt/latest julia/1.0.0 #{cuda}" } + }, + } } }.tap do |hsh| # Due to problems using a mix of user and site modules the site module for IJulia 1.x will be removed @@ -116,8 +133,8 @@ return false end end.each do |installation| - hsh["julia#{active_ijulia_version.gsub('.', '').to_sym}"] = { - display_name: "Julia #{active_ijulia_version}" + ((cuda.empty?) ? "" : " [#{cuda}]"), + hsh[:julia100] = { + display_name: "User Defined Julia #{active_ijulia_version}" + ((cuda.empty?) ? "" : " [#{cuda}]"), language: "julia", argv: [ wrapper, From 12545a6c77ea0ff8ce8587eb7ffbdf73837ea8de Mon Sep 17 00:00:00 2001 From: Morgan Rodgers Date: Mon, 4 Nov 2019 16:19:37 -0500 Subject: [PATCH 3/5] Update IJulia kernel generation - Fix bug where Julia module wasn't loaded - Get Julia versions from an Owens login node via SSH --- template/script.sh.erb | 128 +++++++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 36 deletions(-) diff --git a/template/script.sh.erb b/template/script.sh.erb index fecf52d..5bb1cfe 100755 --- a/template/script.sh.erb +++ b/template/script.sh.erb @@ -2,6 +2,81 @@ <%- require 'pathname' + + # @returns [[major_minor_version, "lmod_load_string"], ["0.6", "julia/0.6.4"]] + def get_lmod_julia_versions + `ssh -o BatchMode=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no owens.osc.edu 'bash --login -c "module spider julia"' 2>&1`.scan( + /julia\/(?:(?\d+)\.(?\d+)\.(?\d+))/ + ).uniq.map do |mtch| + [ + "#{mtch[0]}.#{mtch[1]}", + "#{mtch.join('.')}" + ] + end + end + + # @returns [[major_minor_version, kernel_version], ["1.0", "1.20.0"]] + def get_active_user_ijulia_kernel_versions + user_julia_envs = Pathname.new("~/.julia/environments/").expand_path + + if ! user_julia_envs.exist? + [] + else + user_julia_envs.children.select do |child| + child.directory? && child.to_s.match?(/v\d+\.\d+/) && child.join('Manifest.toml').exist? + end.map do |child| + [ + child.basename.to_s.gsub('v', ''), + child.join('Manifest.toml').read.match(/\[\[IJulia\]\].+?version\s*=\s*"(?\d+\.\d+\.\d+)"\n/m) + ] + end.select do |pair| + pair[-1] + end.map do |pair| + pair[-1] = pair[-1][:version] + pair + end + end + end + + # @returns [[kernel_version, path_to_kernel], ["1.19.0", #]] + def get_installed_user_ijulia_kernels + ijulia_installation_root = Pathname.new("~/.julia/packages/IJulia").expand_path + + if ! ijulia_installation_root.exist? + [] + else + ijulia_installation_root.children.select do |child| + child.join('Project.toml').exist? + end.map do |child| + [ + child.join('Project.toml').read.match(/version\s*=\s*"(?\d+\.\d+\.\d+)"/)[:version], + child.join('src/kernel.jl') + ] + end + end + end + + def kernel_hash( + cuda:, installation:, lmod_version:, wrapper: + ) + { + display_name: "User Defined Julia #{lmod_version}" + ((cuda.empty?) ? "" : " [#{cuda}]"), + language: "julia", + argv: [ + wrapper, + "julia", + "-i", + "--startup-file=yes", + "--color=yes", + installation.join("src/kernel.jl").to_s, + "{connection_file}" + ], + env: { + MODULES: "xalt/latest julia/#{lmod_version} #{cuda}" + } + } + end + cuda = (context.node_type == "gpu") ? context.cuda_version : "" wrapper = session.staged_root.join("launch_wrapper.sh") wrapper_log = session.staged_root.join("launch_wrapper.log") @@ -98,6 +173,7 @@ env: { MODULES: "xalt/latest julia/0.6.4 #{cuda}" }, + }, julia100: { display_name: "Julia 1.0.0 [julia/1.0.0 #{cuda}]", language: "julia", @@ -114,43 +190,23 @@ MODULES: "xalt/latest julia/1.0.0 #{cuda}" } }, - } - } }.tap do |hsh| - # Due to problems using a mix of user and site modules the site module for IJulia 1.x will be removed - # This detect if a user has IJulia installed for themselves and if so will provide a kernel - manifest = Pathname.new("~/.julia/environments/v1.1/Manifest.toml").expand_path - ijulia_installation_root = Pathname.new("~/.julia/packages/IJulia").expand_path - - if manifest.exist? and ijulia_installation_root.exist? - if matches = manifest.read.match(/\[\[IJulia\]\].+?version\s*=\s*"(?\d+\.\d+\.\d+)"\n/m) - active_ijulia_version = matches[:version] - - ijulia_installation_root.children.select do |installation| - begin - installation.join('Project.toml').read.match(/version\s*=\s*"(?\d+\.\d+\.\d+)"/)[:version] == active_ijulia_version - rescue StandardError => e - return false - end - end.each do |installation| - hsh[:julia100] = { - display_name: "User Defined Julia #{active_ijulia_version}" + ((cuda.empty?) ? "" : " [#{cuda}]"), - language: "julia", - argv: [ - wrapper, - "julia", - "-i", - "--startup-file=yes", - "--color=yes", - installation.join("src/kernel.jl").to_s, - "{connection_file}" - ], - env: { - MODULES: "xalt/latest #{cuda}" - } - } - end - end + # Any existing Julia kernelspec may be clobbered if the user has their own version of that kernel installed + lmod_julia_versions = Hash[get_lmod_julia_versions] + installed_user_ijulia_kernels = Hash[get_installed_user_ijulia_kernels] + + get_active_user_ijulia_kernel_versions.each do |maj_min_ver, kernel_version| + lmod_version = lmod_julia_versions[maj_min_ver] + + next unless lmod_version + next unless installed_user_ijulia_kernels[kernel_version] + + hsh["julia#{lmod_version.gsub('.', '')}".to_sym] = kernel_hash( + cuda: cuda, + installation: installed_user_ijulia_kernels[kernel_version], + lmod_version: lmod_version, + wrapper: wrapper + ) end end -%> From 4df62e5f093cac50cc7f233da749398fbb70344b Mon Sep 17 00:00:00 2001 From: Morgan Rodgers Date: Tue, 5 Nov 2019 09:50:48 -0500 Subject: [PATCH 4/5] Fix bad kernel path --- template/script.sh.erb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/template/script.sh.erb b/template/script.sh.erb index 5bb1cfe..daa5e62 100755 --- a/template/script.sh.erb +++ b/template/script.sh.erb @@ -17,7 +17,7 @@ # @returns [[major_minor_version, kernel_version], ["1.0", "1.20.0"]] def get_active_user_ijulia_kernel_versions - user_julia_envs = Pathname.new("~/.julia/environments/").expand_path + user_julia_envs = Pathname.new("~zyou/.julia/environments/").expand_path if ! user_julia_envs.exist? [] @@ -40,7 +40,7 @@ # @returns [[kernel_version, path_to_kernel], ["1.19.0", #]] def get_installed_user_ijulia_kernels - ijulia_installation_root = Pathname.new("~/.julia/packages/IJulia").expand_path + ijulia_installation_root = Pathname.new("~zyou/.julia/packages/IJulia").expand_path if ! ijulia_installation_root.exist? [] @@ -50,7 +50,7 @@ end.map do |child| [ child.join('Project.toml').read.match(/version\s*=\s*"(?\d+\.\d+\.\d+)"/)[:version], - child.join('src/kernel.jl') + child.join('src/kernel.jl').to_s ] end end @@ -68,7 +68,7 @@ "-i", "--startup-file=yes", "--color=yes", - installation.join("src/kernel.jl").to_s, + installation, "{connection_file}" ], env: { From e2f416c6e2587897d5d6825efddd89f7372031b0 Mon Sep 17 00:00:00 2001 From: Morgan Rodgers Date: Tue, 5 Nov 2019 10:03:45 -0500 Subject: [PATCH 5/5] Remove ZQ's user name from the paths --- template/script.sh.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template/script.sh.erb b/template/script.sh.erb index daa5e62..8c2d398 100755 --- a/template/script.sh.erb +++ b/template/script.sh.erb @@ -17,7 +17,7 @@ # @returns [[major_minor_version, kernel_version], ["1.0", "1.20.0"]] def get_active_user_ijulia_kernel_versions - user_julia_envs = Pathname.new("~zyou/.julia/environments/").expand_path + user_julia_envs = Pathname.new("~/.julia/environments/").expand_path if ! user_julia_envs.exist? [] @@ -40,7 +40,7 @@ # @returns [[kernel_version, path_to_kernel], ["1.19.0", #]] def get_installed_user_ijulia_kernels - ijulia_installation_root = Pathname.new("~zyou/.julia/packages/IJulia").expand_path + ijulia_installation_root = Pathname.new("~/.julia/packages/IJulia").expand_path if ! ijulia_installation_root.exist? []