From a967d34a159acb49972ce2ea137d7c4d2447a4db Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sat, 25 Feb 2023 19:34:35 +1300 Subject: [PATCH 1/8] Add support for custom evaluators using compiled langauges --- app/controllers/evaluators_controller.rb | 2 +- app/models/evaluator.rb | 1 + app/views/evaluators/_form.html.erb | 4 ++ app/views/evaluators/index.html.erb | 2 + app/views/evaluators/show.html.erb | 4 +- app/views/problems/_admin.html.erb | 2 +- app/workers/judge_submission_worker.rb | 50 +++++++++++++------ ...225054132_add_language_id_to_evaluators.rb | 5 ++ db/schema.rb | 3 +- 9 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 db/migrate/20230225054132_add_language_id_to_evaluators.rb diff --git a/app/controllers/evaluators_controller.rb b/app/controllers/evaluators_controller.rb index 43b87745..1c61e1fa 100644 --- a/app/controllers/evaluators_controller.rb +++ b/app/controllers/evaluators_controller.rb @@ -2,7 +2,7 @@ class EvaluatorsController < ApplicationController def permitted_params @_permitted_params ||= begin - permitted_attributes = [:name, :description, :source] + permitted_attributes = [:name, :description, :source, :language_id] permitted_attributes << :owner_id if policy(@evaluator || Evaluator).transfer? params.require(:evaluator).permit(*permitted_attributes) end diff --git a/app/models/evaluator.rb b/app/models/evaluator.rb index 1e4d32a3..1011a1e2 100644 --- a/app/models/evaluator.rb +++ b/app/models/evaluator.rb @@ -3,6 +3,7 @@ class Evaluator < ActiveRecord::Base has_many :problems belongs_to :owner, :class_name => :User + belongs_to :language validates :name, :presence => true diff --git a/app/views/evaluators/_form.html.erb b/app/views/evaluators/_form.html.erb index 35f86b4d..33f4d741 100644 --- a/app/views/evaluators/_form.html.erb +++ b/app/views/evaluators/_form.html.erb @@ -23,6 +23,10 @@ <%= f.label :source %>
<%= f.text_area :source %> +
+ <%= f.label :language_id %>
+ <%= f.select :language_id, grouped_options_for_select(Language.grouped_submission_options, @evaluator.language_id), :include_blank => true %> +
<%= f.label :owner_id %>
<% if policy(@evaluator).transfer? %> diff --git a/app/views/evaluators/index.html.erb b/app/views/evaluators/index.html.erb index 6d5de645..13d0928e 100644 --- a/app/views/evaluators/index.html.erb +++ b/app/views/evaluators/index.html.erb @@ -7,6 +7,7 @@ Name Description User + Langauge <% if policy(Evaluator).update? %> @@ -22,6 +23,7 @@ <%= evaluator.name %> <%= evaluator.description %> <%= evaluator.owner_id %> + <%= evaluator.language&.name %> <%= link_to 'Show', evaluator %> <% if policy(Evaluator).update? %> <%= link_to 'Edit', edit_evaluator_path(evaluator) if policy(evaluator).update? %> diff --git a/app/views/evaluators/show.html.erb b/app/views/evaluators/show.html.erb index 9de7a7c9..d91657de 100644 --- a/app/views/evaluators/show.html.erb +++ b/app/views/evaluators/show.html.erb @@ -8,8 +8,10 @@

<% if policy(@evaluator).inspect? %>

+ Language: + <%= @evaluator.language&.name %>
Source: - 

<%= @evaluator.source %>
+ <%= predisplay @evaluator.source, language: @evaluator.language&.lexer %>

<% end %> diff --git a/app/views/problems/_admin.html.erb b/app/views/problems/_admin.html.erb index f8128d3f..c91dec9b 100644 --- a/app/views/problems/_admin.html.erb +++ b/app/views/problems/_admin.html.erb @@ -54,7 +54,7 @@ <% if @problem.evaluator %> <%= link_to @problem.evaluator.name, @problem.evaluator %> <% if policy(@problem.evaluator).inspect? # privilege required to see evaluator source %> - <%= predisplay @problem.evaluator.source, language: :sh %> + <%= predisplay @problem.evaluator.source, language: @problem.evaluator.language&.lexer %> <% end %> <% else %> Default evaluator diff --git a/app/workers/judge_submission_worker.rb b/app/workers/judge_submission_worker.rb index 3939ca9e..febf86e8 100644 --- a/app/workers/judge_submission_worker.rb +++ b/app/workers/judge_submission_worker.rb @@ -52,7 +52,7 @@ def perform(submission_id) raise end - EvalFileName = "eval.sh" + EvalFileName = "eval" OutputBaseLimit = 1024 * 1024 * 2 attr_accessor :submission, :exe_filename @@ -68,14 +68,35 @@ def judge result = {} setup_judging do if submission.language.compiled - result['compile'] = compile!(exe_filename) # possible caching - return result.merge!(grade_compile_error(result['compile'])) if result['compile']['stat'] != 0 #error + result['compile'] = compile!(submission.source, submission.language, exe_filename) # possible caching + return result.merge!(grade_compile_error(result['compile'])) if result['compile']['stat'] != 0 # error else File.open(File.expand_path(exe_filename, tmpdir),"w") { |f| f.write(submission.source) } end run_command = submission.language.run_command(exe_filename) + if problem.evaluator.nil? + eval_command = nil + else + if problem.evaluator.language.nil? + eval_command = "./#{EvalFileName}" + else + eval_command = problem.evaluator.language.run_command(EvalFileName) + end + + if problem.evaluator.language&.compiled + evaluator_compilation = compile!(problem.evaluator.source, problem.evaluator.language, EvalFileName) # possible caching + raise evaluator_compilation['log'] if evaluator_compilation['stat'] != 0 # error + else + File.open(File.expand_path(EvalFileName, tmpdir),"w") do |file| + file.chmod(0700) + file.write(problem.evaluator.source.gsub(/\r\n?/, "\n")) + end + end + end + + result['test_cases'] = {} result['test_sets'] = {} @@ -85,7 +106,7 @@ def judge prereqs = problem.test_cases.where(:id => problem.prerequisite_sets.joins(:test_case_relations).select(:test_case_relations => :test_case_id)) prereqs.each do |test_case| - result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, resource_limits) unless result['test_cases'].has_key?(test_case.id) + result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, eval_command, resource_limits) unless result['test_cases'].has_key?(test_case.id) end problem.prerequisite_sets.each do |test_set| @@ -100,7 +121,7 @@ def judge # test cases (problem.test_cases - prereqs).each do |test_case| - result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, resource_limits) unless result['test_cases'].has_key?(test_case.id) + result['test_cases'][test_case.id] = judge_test_case(test_case, run_command, eval_command, resource_limits) unless result['test_cases'].has_key?(test_case.id) end # test sets @@ -148,18 +169,18 @@ def setup_judging end end - def compile! output - result = submission.language.compile(box, submission.source, output, :mem => 393216, :wall_time => 60) + def compile!(source, language, output) + result = language.compile(box, source, output, :mem => 393216, :wall_time => 60) FileUtils.copy(box.expand_path(output), File.expand_path(output, tmpdir)) if result['stat'] == 0 return result ensure box.clean! end - def judge_test_case(test_case, run_command, resource_limits) + def judge_test_case(test_case, run_command, eval_command, resource_limits) FileUtils.copy(File.expand_path(exe_filename, tmpdir), box.expand_path(exe_filename)) result = run_test_case(test_case, run_command, resource_limits) - result['evaluator'] = evaluate_output(test_case, result['output'], result['output_size'], problem.evaluator) + result['evaluator'] = evaluate_output(test_case, result['output'], result['output_size'], eval_command) result['log'] = truncate_output(result['log']) # log only a small portion result['output'] = truncate_output(result['output'].slice(0,100)) # store only a small portion result @@ -190,21 +211,18 @@ def run_test_case(test_case, run_command, resource_limits = {}) box.clean! end - def evaluate_output(test_case, output, output_size, evaluator) + def evaluate_output(test_case, output, output_size, eval_command) stream_limit = OutputBaseLimit + test_case.output.bytesize*2 if output_size > stream_limit return {'evaluation' => 0, 'log' => "Output exceeded the streamsize limit of #{stream_limit}.", 'meta' => {'status' => 'OK'}} end expected = conditioned_output(test_case.output) actual = conditioned_output(output) - if evaluator.nil? + if eval_command.nil? {'evaluation' => (actual == expected ? 1 : 0), 'meta' => {'status' => 'OK'}} else r = {} - box.fopen(EvalFileName,"w") do |file| - file.chmod(0700) - file.write(problem.evaluator.source.gsub(/\r\n?/, "\n")) - end + FileUtils.copy(File.expand_path(EvalFileName, tmpdir), box.expand_path(EvalFileName)) resource_limits = { :mem => 524288, :time => time_limit*3+15, :wall_time => time_limit*3+30 } box.fopen("actual","w") { |f| f.write(actual) } # DEPRECATED box.fopen("input","w") { |f| f.write(test_case.input) } # DEPRECATED @@ -213,7 +231,7 @@ def evaluate_output(test_case, output, output_size, evaluator) eval_output = nil str_to_pipe(test_case.input, expected) do |input_stream, output_stream| run_opts = resource_limits.reverse_merge(:processes => true, 3 => input_stream, 4 => output_stream, :stdin_data => actual, :output_limit => OutputBaseLimit + test_case.output.bytesize*4, :clean_utf8 => true, :inherit_fds => true) - (stdout,), (r['log'],r['log_size']), (r['box'],), r['meta'], status = box.capture5("./#{EvalFileName} #{deprecated_args}", run_opts ) + (stdout,), (r['log'],r['log_size']), (r['box'],), r['meta'], status = box.capture5("#{eval_command} #{deprecated_args}", run_opts ) r['log'] = truncate_output(r['log']) return r.merge('stat' => 2, 'box' => 'Output was not a valid UTF-8 encoding\n'+r['box']) if !output.force_encoding("UTF-8").valid_encoding? eval_output = stdout.strip.split(nil,2) diff --git a/db/migrate/20230225054132_add_language_id_to_evaluators.rb b/db/migrate/20230225054132_add_language_id_to_evaluators.rb new file mode 100644 index 00000000..43721961 --- /dev/null +++ b/db/migrate/20230225054132_add_language_id_to_evaluators.rb @@ -0,0 +1,5 @@ +class AddLanguageIdToEvaluators < ActiveRecord::Migration + def change + add_column :evaluators, :language_id, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index a096daf6..0c980263 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20200418113601) do +ActiveRecord::Schema.define(version: 20230225054132) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -93,6 +93,7 @@ t.integer "owner_id", null: false t.datetime "created_at" t.datetime "updated_at" + t.integer "language_id" end create_table "file_attachments", force: :cascade do |t| From 8d84fcd1a7e25100eed3c07585798f83ed4a20a4 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Mon, 29 Jan 2024 17:28:42 +1300 Subject: [PATCH 2/8] fixup! Add support for custom evaluators using compiled langauges --- app/workers/judge_submission_worker.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/app/workers/judge_submission_worker.rb b/app/workers/judge_submission_worker.rb index febf86e8..e93a7228 100644 --- a/app/workers/judge_submission_worker.rb +++ b/app/workers/judge_submission_worker.rb @@ -96,7 +96,6 @@ def judge end end - result['test_cases'] = {} result['test_sets'] = {} From 34aa09182a42d20ee448040f98de25e75ec1e8a6 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sat, 27 Apr 2024 23:39:28 +1200 Subject: [PATCH 3/8] Show evaluator compilation log to staff on error --- app/assets/stylesheets/submission.css | 1 + app/models/submission/judge_data.rb | 8 ++++++++ app/views/submissions/show.html.erb | 23 +++++++++++++++++------ app/workers/judge_submission_worker.rb | 2 +- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/app/assets/stylesheets/submission.css b/app/assets/stylesheets/submission.css index 0e90889b..8adfcacb 100644 --- a/app/assets/stylesheets/submission.css +++ b/app/assets/stylesheets/submission.css @@ -19,6 +19,7 @@ table.results { border-spacing: 0; border-collapse: collapse; width: 100%; + margin: 15px 0px; } .results th { diff --git a/app/models/submission/judge_data.rb b/app/models/submission/judge_data.rb index 23a577db..ed8f3c87 100644 --- a/app/models/submission/judge_data.rb +++ b/app/models/submission/judge_data.rb @@ -285,10 +285,18 @@ def compiled? data.has_key?('compile') end + def evaluator_compiled? + data.has_key?('evaluator_compile') + end + def compilation @compilation ||= Compilation.new(data['compile']) end + def evaluator_compilation + @evaluator_compilation ||= Compilation.new(data['evaluator_compile']) + end + def prerequisite_sets test_sets.slice(@presets) end diff --git a/app/views/submissions/show.html.erb b/app/views/submissions/show.html.erb index 13954682..abee414f 100644 --- a/app/views/submissions/show.html.erb +++ b/app/views/submissions/show.html.erb @@ -47,8 +47,8 @@

<% @judge_data = @submission.judge_data %> - - <% if @judge_data.compiled? %> +<% if @judge_data.compiled? %> +
@@ -59,9 +59,22 @@ - <% end %> +
Compilation: <%= @judge_data.compilation.command %><%= @judge_data.compilation.log %>
+<% end %> +<% if @judge_data.evaluator_compiled? && @submission.problem.evaluator && policy(@submission.problem.evaluator).inspect? %> + + + + + + + + + + +
Evaluator Compilation: <%= @judge_data.evaluator_compilation.command %><%= @judge_data.evaluator_compilation.judgement %> 
<%= @judge_data.evaluator_compilation.log %>
-  +<% end %> <% if !@judge_data.compiled? || @judge_data.compilation.status == :success %> @@ -152,8 +165,6 @@
-
-

Source: <%= predisplay(@submission.source || "", language: @submission.language.lexer) %> diff --git a/app/workers/judge_submission_worker.rb b/app/workers/judge_submission_worker.rb index e93a7228..238e5973 100644 --- a/app/workers/judge_submission_worker.rb +++ b/app/workers/judge_submission_worker.rb @@ -87,7 +87,7 @@ def judge if problem.evaluator.language&.compiled evaluator_compilation = compile!(problem.evaluator.source, problem.evaluator.language, EvalFileName) # possible caching - raise evaluator_compilation['log'] if evaluator_compilation['stat'] != 0 # error + return result.merge!('evaluator_compile' => evaluator_compilation, 'status' => 2) if evaluator_compilation['stat'] != 0 # error else File.open(File.expand_path(EvalFileName, tmpdir),"w") do |file| file.chmod(0700) From bdde01843306820bfb2e577e0e7d1d088121b4e4 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sat, 27 Apr 2024 23:51:21 +1200 Subject: [PATCH 4/8] fixup! Show evaluator compilation log to staff on error --- app/views/submissions/show.html.erb | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app/views/submissions/show.html.erb b/app/views/submissions/show.html.erb index abee414f..8ebfaae9 100644 --- a/app/views/submissions/show.html.erb +++ b/app/views/submissions/show.html.erb @@ -62,18 +62,18 @@ <% end %> <% if @judge_data.evaluator_compiled? && @submission.problem.evaluator && policy(@submission.problem.evaluator).inspect? %> - - - - - - - - - - - -
Evaluator Compilation: <%= @judge_data.evaluator_compilation.command %><%= @judge_data.evaluator_compilation.judgement %> 
<%= @judge_data.evaluator_compilation.log %>
+ + + + + + + + + + + +
Evaluator Compilation: <%= @judge_data.evaluator_compilation.command %><%= @judge_data.evaluator_compilation.judgement %> 
<%= @judge_data.evaluator_compilation.log %>
<% end %> <% if !@judge_data.compiled? || @judge_data.compilation.status == :success %> From 406b25f4ad7ac7a42ced6987ab7e21797b83ff58 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sat, 27 Apr 2024 23:54:18 +1200 Subject: [PATCH 5/8] Pin isolate to 1.10.1 This is the last version that supports cgroups v1; isolate 2.0 supports only cgroups v2. --- .github/workflows/ci.yml | 2 +- script/install/config.bash | 2 +- script/install/isolate.bash | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fd34ad0..22562e79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: SCHEDULE_BACKUPS: 0 ISOLATE_ROOT: / ISOLATE_CGROUPS: false - ISOLATE_BRANCH: master + ISOLATE_BRANCH: v1.10.1 - name: Back up db/schema.rb # it will be overwritten when install.bash runs migrate.bash; back up the original so we can check if it's up to date run: cp db/schema.rb db/schema.rb.git diff --git a/script/install/config.bash b/script/install/config.bash index 62d830dd..250231f4 100644 --- a/script/install/config.bash +++ b/script/install/config.bash @@ -203,7 +203,7 @@ declare -p ISOLATE_CGROUPS &> /dev/null || while [ -z "$ISOLATE_CGROUPS" ] ; do else ISOLATE_CGROUPS=false; fi done -declare -p ISOLATE_BRANCH &> /dev/null || ISOLATE_BRANCH=master # no prompt +declare -p ISOLATE_BRANCH &> /dev/null || ISOLATE_BRANCH=v1.10.1 # no prompt shopt -u nocasematch; diff --git a/script/install/isolate.bash b/script/install/isolate.bash index d8d3dc8c..a6ecc154 100644 --- a/script/install/isolate.bash +++ b/script/install/isolate.bash @@ -9,7 +9,7 @@ cd $srclocation if [ -d "isolate" ]; then cd isolate done=true - git pull --force | grep -q -v 'Already up-to-date.' && done=false + # git pull --force | grep -q -v 'Already up-to-date.' && done=false if $done; then exit fi From 9e7aa3ef2f49f83eb98d2db86dc33debf9894d35 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sun, 28 Apr 2024 20:33:50 +1200 Subject: [PATCH 6/8] Show error message and hide test cases on judging failure --- app/models/submission/judge_data.rb | 2 +- app/views/submissions/show.html.erb | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/models/submission/judge_data.rb b/app/models/submission/judge_data.rb index ed8f3c87..78b843f4 100644 --- a/app/models/submission/judge_data.rb +++ b/app/models/submission/judge_data.rb @@ -274,7 +274,7 @@ def initialize(log, test_sets, test_cases, prerequisite_sets = []) end def errored? - data.has_key?('error') + data.has_key?('error') || status == :error end def completed? diff --git a/app/views/submissions/show.html.erb b/app/views/submissions/show.html.erb index 8ebfaae9..e2719429 100644 --- a/app/views/submissions/show.html.erb +++ b/app/views/submissions/show.html.erb @@ -5,7 +5,12 @@ <%= stylesheet_link_tag "submission" %> -<% if @submission.score == nil %> +<% @judge_data = @submission.judge_data %> +<% if @judge_data.errored? %> +

+ An unexpected error has occurred during judging. Please retry the submission, or contact us at <%= mail_to "nzic@nzoi.org.nz" %> if the error persists. +

+<% elsif @submission.score == nil %>

This submission has not finished judging. Refresh this page in a minute or two to see the submission's score.

@@ -46,7 +51,6 @@ <% end %>

-<% @judge_data = @submission.judge_data %> <% if @judge_data.compiled? %>
@@ -76,7 +80,7 @@
<% end %> - <% if !@judge_data.compiled? || @judge_data.compilation.status == :success %> + <% if !@judge_data.errored? && (!@judge_data.compiled? || @judge_data.compilation.status == :success) %> From fb5bdb6fce907c4f8676ac53e99934d4dac264b5 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sun, 28 Apr 2024 21:31:07 +1200 Subject: [PATCH 7/8] Set status on judging error --- app/workers/judge_submission_worker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/workers/judge_submission_worker.rb b/app/workers/judge_submission_worker.rb index 238e5973..8d862905 100644 --- a/app/workers/judge_submission_worker.rb +++ b/app/workers/judge_submission_worker.rb @@ -46,7 +46,7 @@ def perform(submission_id) rescue StandardError => e unless self.submission.nil? submission.reload - submission.judge_log = {'error' => {'message' => e.message, 'backtrace' => e.backtrace}}.to_json + submission.judge_log = {'error' => {'message' => e.message, 'backtrace' => e.backtrace}, 'status' => 2}.to_json submission.save end raise From 6b95ed6e677be6c901b32ce4f33b26f62a981d01 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sun, 28 Apr 2024 21:41:31 +1200 Subject: [PATCH 8/8] Show evaluator errors in submission test messages --- app/models/submission.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/submission.rb b/app/models/submission.rb index d1b6dcda..ed0bf29f 100644 --- a/app/models/submission.rb +++ b/app/models/submission.rb @@ -149,7 +149,7 @@ def update_test_messages judge_data = self.judge_data return if judge_data.status == :pending # incomplete judge_log - return if judge_data.errored? # judge errored - very bad + return if judge_data.data.has_key?('error') # judge errored - very bad errors = [] warnings = []
Time