From 5894a2b8e06d7830048ef9cd2610c8d29f7fa495 Mon Sep 17 00:00:00 2001 From: Jonathan Khoo Date: Sat, 27 Jan 2024 18:02:25 +1300 Subject: [PATCH] Add tests for interactive problems --- spec/factories/problems.rb | 167 ++++++++++++++++++++++++++++++--- spec/factories/submissions.rb | 122 ++++++++++++++++++++++++ spec/models/submission_spec.rb | 56 +++++++++++ 3 files changed, 331 insertions(+), 14 deletions(-) diff --git a/spec/factories/problems.rb b/spec/factories/problems.rb index d50edc1e..13a10226 100644 --- a/spec/factories/problems.rb +++ b/spec/factories/problems.rb @@ -4,34 +4,173 @@ factory :problem do sequence(:name) {|n| "Problem #{n}" } statement { "Do nothing" } - sequence(:input) {|n| "#{n}.in" } - sequence(:output) {|n| "#{n}.out"} + input { nil } + output { nil } memory_limit { 1 } time_limit { 1 } owner_id { 0 } - factory :adding_problem do + + test_sets { test_cases.map{FactoryBot.create(:test_set)} } + + after(:create) do |problem| + problem.test_cases.zip(problem.test_sets).each do |test_case, test_set| + FactoryBot.create(:test_case_relation, :test_case => test_case, :test_set => test_set) + end + end + + factory :adding_problem_stdio do sequence(:name) {|n| "Adding problem #{n}" } statement { "Read two integers from input and output the sum." } - input { "add.in" } - output { "add.out" } memory_limit { 30 } time_limit { 1 } test_cases { [FactoryBot.create(:test_case, :input => "5 9", :output => "14"), FactoryBot.create(:test_case, :input => "100 -50", :output => "50"), FactoryBot.create(:test_case, :input => "1235 942", :output => "2177"), FactoryBot.create(:test_case, :input => "-4000 123", :output => "-3877")] } - test_sets { (0...4).map{FactoryBot.create(:test_set)} } - after(:create) do |problem| - (0...4).each do |i| - FactoryBot.create(:test_case_relation, :test_case => problem.test_cases[i], :test_set => problem.test_sets[i]) - end + factory :adding_problem do + input { "add.in" } + output { "add.out" } end + end - factory :adding_problem_stdio do - input { nil } - output { nil } - end + factory :binary_search_problem do + name { "Binary search problem" } + statement { "Find the target number within Q guesses. After each guess you are told whether the target is lower, higher, or correct." } + memory_limit { 16 } + time_limit { 0.1 } + test_cases { [FactoryBot.create(:test_case, :input => "100 100 98"), + FactoryBot.create(:test_case, :input => "100000 17 37")] } + + evaluator { FactoryBot.create(:evaluator, :language => LanguageGroup.find_by_identifier("c++").current_language, + :interactive_processes => 1, :source => <<~'sourcecode' ) } + #include + #include + #include + + void grade(int score, const char* message = NULL) { + fprintf(stdout, "%d\n", score); + if (message) + fprintf(stderr, "%s\n", message); + exit(0); + } + + int main() { + { + // Keep alive on broken pipes + struct sigaction sa; + sa.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sa, NULL); + } + + FILE* user_in = fdopen(5, "r"); + FILE* user_out = fdopen(6, "w"); + + int N, Q, K; + scanf("%d %d %d", &N, &Q, &K); + fclose(stdin); + fprintf(user_out, "%d %d\n", N, Q); + fflush(user_out); + + int guess; + for (int i = 0; i < Q; ++i) { + if (fscanf(user_in, "%d", &guess) != 1) { + grade(0, "Could not read guess"); + } + if (guess == K) { + fprintf(user_out, "0\n"); + fflush(user_out); + break; + } else if (guess < K) { + fprintf(user_out, "1\n"); + fflush(user_out); + } else { + fprintf(user_out, "-1\n"); + fflush(user_out); + } + if (i == Q - 1) { + grade(0, "Too many guesses"); + } + } + + if (fscanf(user_in, "%d", &guess) != EOF) + grade(0, "Wrong output format, trailing garbage"); + + grade(1); + } + sourcecode + end + + factory :integer_encoding_problem do + name { "Integer encoding problem" } + statement { "Send the input number between two processes using only alphabetic characters." } + memory_limit { 16 } + time_limit { 0.5 } + test_cases { [FactoryBot.create(:test_case, :input => "0"), + FactoryBot.create(:test_case, :input => "42"), + FactoryBot.create(:test_case, :input => "9999")] } + + evaluator { FactoryBot.create(:evaluator, :interactive_processes => 2, :source => <<~'sourcecode' ) } + #!/usr/bin/env python3 + import os + import sys + import functools + import traceback + import time + + print = functools.partial(print, flush=True) # Always flush + + user1_in = os.fdopen(5, 'r') + user1_out = os.fdopen(6, 'w') + user2_in = os.fdopen(7, 'r') + user2_out = os.fdopen(8, 'w') + + def grade(score, admin_message=None, user_message=None): + if not user2_out.closed: + try: + print(-1, file=user2_out) + except: + pass + print(score) + if user_message is not None: + print(user_message) + if admin_message is not None: + print(admin_message, file=sys.stderr) + sys.exit(0) + + N = int(input()) + + try: + print(1, file=user1_out) + print(N, file=user1_out) + user1_out.close() + encoded_string = user1_in.read(100000).strip() + except (BrokenPipeError, ValueError): + grade(0, traceback.format_exc()) + + if user1_in.read(100000).strip(): + grade(0, "Wrong output format, trailing garbage") + user1_in.close() + + if not encoded_string.isalpha(): + grade(0, "Invalid encoded string: " + encoded_string) + + try: + print(2, file=user2_out) + print(encoded_string, file=user2_out) + user2_out.close() + decoded_integer = int(user2_in.readline(100000)) + except (BrokenPipeError, ValueError): + grade(0, traceback.format_exc()) + + if user2_in.read(100000).strip(): + grade(0, "Wrong output format, trailing garbage") + user2_in.close() + + if decoded_integer != N: + grade(0, "Wrong answer") + grade(1) + sourcecode end end end diff --git a/spec/factories/submissions.rb b/spec/factories/submissions.rb index f658bbc6..33c9c869 100644 --- a/spec/factories/submissions.rb +++ b/spec/factories/submissions.rb @@ -70,6 +70,128 @@ fprintf(out, "%u\\n",(int)c); return 0; } +sourcecode + end + factory :binary_search_submission do + language { LanguageGroup.find_by_identifier("c++").current_language } + source { < + using namespace std; + int main() { + int lo=0, hi, attempts, result; + cin >> hi >> attempts; + while (hi - lo > 1) { + int mid = (lo + hi) / 2; + cout << mid << endl; + cin >> result; + if ( result == 0 ) + return 0; + else if ( result < 0 ) + hi = mid; + else + lo = mid + 1; + } + cout << lo << endl; + } +sourcecode + end + factory :binary_search_submission_incorrect do + language { LanguageGroup.find_by_identifier("c++").current_language } + source { < + using namespace std; + int main() { + int hi, attempts, result; + cin >> hi >> attempts; + for (int i = 0; i < hi; i++) { + cout << i << endl; + cin >> result; + if (result == 0) + break; + } + } +sourcecode + end + factory :binary_search_submission_wall_tle do + language { LanguageGroup.find_by_identifier("c++").current_language } + source { < + using namespace std; + int main() { + int hi, attempts, result; + cin >> hi >> attempts; + for (int i = 0; i < hi; i++) { + //cout << i << endl; + cin >> result; + } + } +sourcecode + end + factory :integer_encoding_submission do + language { LanguageGroup.find_by_identifier("c++").current_language } + source { < + #include + #include + using namespace std; + int main() { + int mode, N = 0; + std::string encoded_string; + cin >> mode; + if (mode == 1) { + cin >> N; + while (N) { + encoded_string += 'a' + (N & 1); + N >>= 1; + } + encoded_string += 'a'; + reverse(encoded_string.begin(), encoded_string.end()); + cout << encoded_string << endl; + } + if (mode == 2) { + cin >> encoded_string; + for (char c : encoded_string) { + N <<= 1; + N += c > 'a'; + } + cout << N << endl; + } + } +sourcecode + end + factory :integer_encoding_submission_mle do + language { LanguageGroup.find_by_identifier("c++").current_language } + source { < + #include + #include + using namespace std; + int main() { + std::array arr; // x2 = 20 MiB; should MLE + arr.fill(-1); + + int mode, N = 0; + std::string encoded_string; + cin >> mode; + if (mode == 1) { + cin >> N; + while (N) { + encoded_string += 'a' + (N & 1); + N >>= 1; + } + encoded_string += 'a'; + reverse(encoded_string.begin(), encoded_string.end()); + cout << encoded_string << endl; + } + if (mode == 2) { + cin >> encoded_string; + for (char c : encoded_string) { + N <<= 1; + N += c > 'a'; + } + cout << N << endl; + } + } sourcecode end end diff --git a/spec/models/submission_spec.rb b/spec/models/submission_spec.rb index dd66ff1d..ab8c0bff 100644 --- a/spec/models/submission_spec.rb +++ b/spec/models/submission_spec.rb @@ -40,4 +40,60 @@ expect(@unsigned_submission.evaluation).to eq(0.75) end end + + context 'on "binary search" problem' do + before(:all) do + @user = FactoryBot.create(:user) + @problem = FactoryBot.create(:binary_search_problem) + end + after(:all) do + [@user, @problem].reverse_each { |object| object.destroy } + end + it 'judges submission' do + submission = FactoryBot.create(:binary_search_submission, :problem => @problem, :user => @user) + expect(submission.score).to be_nil + submission.judge + submission.reload + expect(submission.evaluation).to eq(1) + end + it 'judges incorrect submission' do + submission = FactoryBot.create(:binary_search_submission_incorrect, :problem => @problem, :user => @user) + expect(submission.evaluation).to be_nil + submission.judge + submission.reload + expect(submission.evaluation).to eq(0.5) + end + it 'judges wall timeout submission' do + submission = FactoryBot.create(:binary_search_submission_wall_tle, :problem => @problem, :user => @user) + expect(submission.evaluation).to be_nil + submission.judge + submission.reload + expect(submission.evaluation).to eq(0) + end + end + + context 'on "integer encoding" problem' do + before(:all) do + @user = FactoryBot.create(:user) + @problem = FactoryBot.create(:integer_encoding_problem) + end + after(:all) do + [@user, @problem].reverse_each { |object| object.destroy } + end + it 'judges submission' do + submission = FactoryBot.create(:integer_encoding_submission, :problem => @problem, :user => @user) + expect(submission.score).to be_nil + submission.judge + submission.reload + expect(submission.evaluation).to eq(1) + end + it 'judges memory limit exceeded submission' do + submission = FactoryBot.create(:integer_encoding_submission_mle, :problem => @problem, :user => @user) + expect(submission.score).to be_nil + submission.judge + submission.reload + expect(submission.judge_data.test_cases.first[1].status).to eq(:memory) + expect(submission.evaluation).to eq(0) + end + end end