From 98ad9a243ad1bb26b051808fbbcb14a35d779e4f Mon Sep 17 00:00:00 2001 From: Edoardo Morassutto Date: Wed, 6 Nov 2024 12:21:32 +0100 Subject: [PATCH 01/10] Fix submissions_status RPC when UNION ALL doesn't keep the order (#1201) UNION ALL is not guaranteed to keep the order of the results. In fact with (at least) Postgres 13 the order is not well defined and can change between executions. This patch adds an extra column to the query, tagging each statistics with its key, so that we can reconstruct the correct order of the results. --- cms/server/admin/server.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cms/server/admin/server.py b/cms/server/admin/server.py index 796ab84698..525df4d151 100644 --- a/cms/server/admin/server.py +++ b/cms/server/admin/server.py @@ -27,7 +27,7 @@ import logging -from sqlalchemy import func, not_ +from sqlalchemy import func, not_, literal_column from cms import config, ServiceCoord, get_service_shards from cms.db import SessionGen, Dataset, Submission, SubmissionResult, Task @@ -176,13 +176,17 @@ def submissions_status(contest_id): .filter(Task.contest_id == contest_id) queries['total'] = total_query - stats = {} + # Add a "key" column for keeping track of each stats, in case they + # get shuffled. + for key, query in queries.items(): + key_column = literal_column(f"'{key}'").label("key") + queries[key] = query.add_columns(key_column) + keys = list(queries.keys()) results = queries[keys[0]].union_all( *(queries[key] for key in keys[1:])).all() - for i, k in enumerate(keys): - stats[k] = results[i][0] + stats = {key: value for value, key in results} stats['compiling'] += 2 * stats['total'] - sum(stats.values()) return stats From fd9354542adfcdb1859ab0e6a528e131ba94240e Mon Sep 17 00:00:00 2001 From: Luca Versari Date: Sun, 17 Nov 2024 00:20:03 +0100 Subject: [PATCH 02/10] Fail in RunFunctionalTests if tests throw exceptions. (#1273) --- cmstestsuite/RunFunctionalTests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmstestsuite/RunFunctionalTests.py b/cmstestsuite/RunFunctionalTests.py index 38dcfe6da0..260628e31c 100755 --- a/cmstestsuite/RunFunctionalTests.py +++ b/cmstestsuite/RunFunctionalTests.py @@ -225,6 +225,8 @@ def main(): runner = TestRunner(test_list, contest_id=args.contest, workers=4) failures = [] + testing_general_failure = False + try: # Submit and wait for all tests to complete. runner.submit_tests() @@ -238,6 +240,8 @@ def main(): with open("./log/cms/last.log", "rt", encoding="utf-8") as f: print(f.read()) print("\n\n===== END OF LOG DUMP =====\n\n") + logging.error("Failure while running tests", exc_info=True) + testing_general_failure = True finally: # And good night! runner.shutdown() @@ -245,6 +249,9 @@ def main(): combine_coverage() + if testing_general_failure: + return 1 + logger.info("Executed: %s", tests) logger.info("Failed: %s", len(failures)) if not failures: From c786bfbc7442cbffbb551502c8d8f3d7086d853f Mon Sep 17 00:00:00 2001 From: William Di Luigi Date: Sun, 17 Nov 2024 00:37:47 +0100 Subject: [PATCH 03/10] Fix CI not reporting error when unit tests fail but functional tests pass. (#1274) --- _cms-test-internal.sh | 28 ++++++++++++++++++++++++++++ docker-compose.test.yml | 13 +------------ 2 files changed, 29 insertions(+), 12 deletions(-) create mode 100755 _cms-test-internal.sh diff --git a/_cms-test-internal.sh b/_cms-test-internal.sh new file mode 100755 index 0000000000..a2e6556291 --- /dev/null +++ b/_cms-test-internal.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +sudo chown cmsuser:cmsuser ./codecov + +dropdb --host=testdb --username=postgres cmsdbfortesting +createdb --host=testdb --username=postgres cmsdbfortesting +cmsInitDB + +pytest --cov . --cov-report xml:codecov/unittests.xml +UNIT=$? + +dropdb --host=testdb --username=postgres cmsdbfortesting +createdb --host=testdb --username=postgres cmsdbfortesting +cmsInitDB + +cmsRunFunctionalTests -v --coverage codecov/functionaltests.xml +FUNC=$? + +# This check is needed because otherwise failing unit tests aren't reported in +# the CI as long as the functional tests are passing. Ideally we should get rid +# of `cmsRunFunctionalTests` and make those tests work with pytest so they can +# be auto-discovered and run in a single command. +if [ $UNIT -ne 0 ] || [ $FUNC -ne 0 ] +then + exit 1 +else + exit 0 +fi diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 64a35faf71..27157afe60 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -18,15 +18,4 @@ services: - "./codecov:/home/cmsuser/cms/codecov" privileged: true cgroup: host - command: > - wait-for-it testdb:5432 -- sh -c " - dropdb --host=testdb --username=postgres cmsdbfortesting ; - createdb --host=testdb --username=postgres cmsdbfortesting ; - cmsInitDB ; - sudo chown cmsuser:cmsuser ./codecov ; - pytest --cov . --cov-report xml:codecov/unittests.xml ; - dropdb --host=testdb --username=postgres cmsdbfortesting ; - createdb --host=testdb --username=postgres cmsdbfortesting ; - cmsInitDB ; - cmsRunFunctionalTests -v --coverage codecov/functionaltests.xml ; - " + command: wait-for-it testdb:5432 -- ./_cms-test-internal.sh From eb51ac1b654e0bfc709f37fc1d58177c71e72ac0 Mon Sep 17 00:00:00 2001 From: Luca Versari Date: Sun, 17 Nov 2024 00:38:51 +0100 Subject: [PATCH 04/10] Remove python2. (#1272) It is not supported on the latest Ubuntu LTS, and has been unsupported for almost 5 years. Co-authored-by: William Di Luigi --- Dockerfile | 1 - cms/conf.py | 2 +- cms/grading/languages/python2_cpython.py | 80 ------------------- cmsranking/Config.py | 2 +- cmstestsuite/Tests.py | 5 +- .../tasks/batch_fileio_managed/code/checker | 7 +- .../communication_fifoio_stubbed/code/manager | 15 ++-- .../code/manager | 23 +++--- .../code/manager | 23 +++--- .../tasks/communication_stdio/code/manager | 15 ++-- .../communication_stdio_stubbed/code/manager | 15 ++-- .../tasks/outputonly_comparator/code/checker | 3 +- .../tasks/twosteps_comparator/code/checker | 3 +- codecov/.gitkeep | 0 docs/Configuring a contest.rst | 2 +- docs/Installation.rst | 6 +- setup.py | 1 - 17 files changed, 56 insertions(+), 147 deletions(-) delete mode 100644 cms/grading/languages/python2_cpython.py delete mode 100644 codecov/.gitkeep diff --git a/Dockerfile b/Dockerfile index 699e279344..63f767f995 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,6 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ openjdk-8-jdk-headless \ php7.4-cli \ postgresql-client \ - python2 \ python3-pip \ python3.8 \ python3.8-dev \ diff --git a/cms/conf.py b/cms/conf.py index 9ccc2b8e74..87ca680ec1 100644 --- a/cms/conf.py +++ b/cms/conf.py @@ -167,7 +167,7 @@ def __init__(self): # the prefix (or real_prefix to accommodate virtualenvs). bin_path = os.path.join(os.getcwd(), sys.argv[0]) bin_name = os.path.basename(bin_path) - bin_is_python = bin_name in ["ipython", "python", "python2", "python3"] + bin_is_python = bin_name in ["ipython", "python", "python3"] bin_in_installed_path = bin_path.startswith(sys.prefix) or ( hasattr(sys, 'real_prefix') and bin_path.startswith(sys.real_prefix)) diff --git a/cms/grading/languages/python2_cpython.py b/cms/grading/languages/python2_cpython.py deleted file mode 100644 index bc55df1f26..0000000000 --- a/cms/grading/languages/python2_cpython.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 - -# Contest Management System - http://cms-dev.github.io/ -# Copyright © 2016-2018 Stefano Maggiolo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Python programming language, version 2, definition.""" - -import os - -from cms.grading import CompiledLanguage - - -__all__ = ["Python2CPython"] - - -class Python2CPython(CompiledLanguage): - """This defines the Python programming language, version 2 (more - precisely, the subversion of Python 2 available on the system, - usually 2.7) using the default interpeter in the system. - - """ - - MAIN_FILENAME = "__main__.pyc" - - @property - def name(self): - """See Language.name.""" - return "Python 2 / CPython" - - @property - def source_extensions(self): - """See Language.source_extensions.""" - return [".py"] - - @property - def executable_extension(self): - """See Language.executable_extension.""" - return ".zip" - - def get_compilation_commands(self, - source_filenames, executable_filename, - for_evaluation=True): - """See Language.get_compilation_commands.""" - - commands = [] - files_to_package = [] - commands.append(["/usr/bin/python2", "-m", "compileall", "."]) - for idx, source_filename in enumerate(source_filenames): - basename = os.path.splitext(os.path.basename(source_filename))[0] - pyc_filename = "%s.pyc" % basename - # The file with the entry point must be in first position. - if idx == 0: - commands.append(["/bin/mv", pyc_filename, self.MAIN_FILENAME]) - files_to_package.append(self.MAIN_FILENAME) - else: - files_to_package.append(pyc_filename) - - commands.append(["/usr/bin/zip", executable_filename] - + files_to_package) - - return commands - - def get_evaluation_commands( - self, executable_filename, main=None, args=None): - """See Language.get_evaluation_commands.""" - args = args if args is not None else [] - return [["/usr/bin/python2", executable_filename] + args] diff --git a/cmsranking/Config.py b/cmsranking/Config.py index e7d624f328..da0511aa33 100644 --- a/cmsranking/Config.py +++ b/cmsranking/Config.py @@ -61,7 +61,7 @@ def __init__(self): # TODO: move to cmscommon as it is used both here and in cms/conf.py bin_path = os.path.join(os.getcwd(), sys.argv[0]) bin_name = os.path.basename(bin_path) - bin_is_python = bin_name in ["ipython", "python", "python2", "python3"] + bin_is_python = bin_name in ["ipython", "python", "python3"] bin_in_installed_path = bin_path.startswith(sys.prefix) or ( hasattr(sys, 'real_prefix') and bin_path.startswith(sys.real_prefix)) diff --git a/cmstestsuite/Tests.py b/cmstestsuite/Tests.py index 7cf8416ac9..2cf440d4af 100644 --- a/cmstestsuite/Tests.py +++ b/cmstestsuite/Tests.py @@ -48,20 +48,19 @@ LANG_JAVA = "Java / JDK" LANG_PASCAL = "Pascal / fpc" LANG_PHP = "PHP" -LANG_PYTHON = "Python 2 / CPython" LANG_PYTHON3 = "Python 3 / CPython" LANG_RUST = "Rust" LANG_C_SHARP = "C# / Mono" ALL_LANGUAGES = ( LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_C, LANG_HS, LANG_JAVA, LANG_PASCAL, - LANG_PHP, LANG_PYTHON, LANG_PYTHON3, LANG_RUST, LANG_C_SHARP + LANG_PHP, LANG_PYTHON3, LANG_RUST, LANG_C_SHARP ) NON_INTERPRETED_LANGUAGES = ( LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_PASCAL ) COMPILED_LANGUAGES = ( LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_PASCAL, LANG_JAVA, - LANG_PYTHON, LANG_PYTHON3, LANG_HS, LANG_RUST, LANG_C_SHARP + LANG_PYTHON3, LANG_HS, LANG_RUST, LANG_C_SHARP ) ALL_TESTS = [ diff --git a/cmstestsuite/tasks/batch_fileio_managed/code/checker b/cmstestsuite/tasks/batch_fileio_managed/code/checker index a508bed8be..d8fcdb6c56 100644 --- a/cmstestsuite/tasks/batch_fileio_managed/code/checker +++ b/cmstestsuite/tasks/batch_fileio_managed/code/checker @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys @@ -17,11 +16,11 @@ try: f = io.open(output_file, 'rb') line = f.readline().strip() more = f.readline() - if line == '%s %d' % (solution_word, number) and more == '': + if line == b'%s %d' % (solution_word, number) and more == b'': print("1.0") print("Correcto", file=sys.stderr) else: - assert more == '' + assert more == b'' print("0.0") print("Plain wrong", file=sys.stderr) except: diff --git a/cmstestsuite/tasks/communication_fifoio_stubbed/code/manager b/cmstestsuite/tasks/communication_fifoio_stubbed/code/manager index 935e8d85be..18b19bdcf8 100644 --- a/cmstestsuite/tasks/communication_fifoio_stubbed/code/manager +++ b/cmstestsuite/tasks/communication_fifoio_stubbed/code/manager @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys @@ -17,28 +16,28 @@ correct = True # Speak the to program a few times, check what it does, and output the last # line. -for i in range(10,20) + [0]: +for i in list(range(10, 20)) + [0]: x = i + input_value # Write a question to the candidate executable. - fifo_to_user.write("%d\n" % x) + fifo_to_user.write(b"%d\n" % x) # Read their response. l = fifo_from_user.readline() # EOF? - if l == '': + if l == b'': correct = False break # These are the only things we expect from our stub. - if l.strip() != 'correct %d' % x: + if l.strip() != b'correct %d' % x: correct = False break correct = correct and (int(l.split()[1]) == x) else: # Tell stub to exit. - fifo_to_user.write("0\n") + fifo_to_user.write(b"0\n") # This file exists just for convenience. -io.open("output.txt", "wb").write(l + "\n") +io.open("output.txt", "wb").write(l + b"\n") # This is the final score. if correct: diff --git a/cmstestsuite/tasks/communication_many_fifoio_stubbed/code/manager b/cmstestsuite/tasks/communication_many_fifoio_stubbed/code/manager index 6de8de3262..469d70edfe 100644 --- a/cmstestsuite/tasks/communication_many_fifoio_stubbed/code/manager +++ b/cmstestsuite/tasks/communication_many_fifoio_stubbed/code/manager @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys @@ -20,44 +19,44 @@ correct = True # Speak the to program a few times, check what it does, and output the last # line. -for i in range(10, 20) + [0]: +for i in list(range(10, 20)) + [0]: x = i + input_value # Write a question to the candidate executable. - fifo_to_user1.write("%d\n" % x) + fifo_to_user1.write(b"%d\n" % x) # Read their response. l = fifo_from_user1.readline() # EOF? - if l == '': + if l == b'': correct = False break # These are the only things we expect from our stub. - if l.strip() != 'correct %d' % x: + if l.strip() != b'correct %d' % x: correct = False break correct = correct and (int(l.split()[1]) == x) # Write a question to the candidate executable. - fifo_to_user2.write("%d\n" % x) + fifo_to_user2.write(b"%d\n" % x) # Read their response. l = fifo_from_user2.readline() # EOF? - if l == '': + if l == b'': correct = False break # These are the only things we expect from our stub. - if l.strip() != 'correct %d' % (-x): + if l.strip() != b'correct %d' % (-x): correct = False break correct = correct and (int(l.split()[1]) == -x) else: # Tell stub to exit. - fifo_to_user1.write("0\n") - fifo_to_user2.write("0\n") + fifo_to_user1.write(b"0\n") + fifo_to_user2.write(b"0\n") # This file exists just for convenience. -io.open("output.txt", "wb").write(l + "\n") +io.open("output.txt", "wb").write(l + b"\n") # This is the final score. if correct: diff --git a/cmstestsuite/tasks/communication_many_stdio_stubbed/code/manager b/cmstestsuite/tasks/communication_many_stdio_stubbed/code/manager index b2f9f66e43..c2652e3577 100644 --- a/cmstestsuite/tasks/communication_many_stdio_stubbed/code/manager +++ b/cmstestsuite/tasks/communication_many_stdio_stubbed/code/manager @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys @@ -22,44 +21,44 @@ correct = True # Speak the to program a few times, check what it does, and output the last # line. -for i in range(10, 20) + [0]: +for i in list(range(10, 20)) + [0]: x = i + input_value # Write a question to the candidate executable. - fifo_to_user1.write("%d\n" % x) + fifo_to_user1.write(b"%d\n" % x) # Read their response. l = fifo_from_user1.readline() # EOF? - if l == '': + if l == b'': correct = False break # These are the only things we expect from our stub. - if l.strip() != 'correct %d' % x: + if l.strip() != b'correct %d' % x: correct = False break correct = correct and (int(l.split()[1]) == x) # Write a question to the candidate executable. - fifo_to_user2.write("%d\n" % x) + fifo_to_user2.write(b"%d\n" % x) # Read their response. l = fifo_from_user2.readline() # EOF? - if l == '': + if l == b'': correct = False break # These are the only things we expect from our stub. - if l.strip() != 'correct %d' % (-x): + if l.strip() != b'correct %d' % (-x): correct = False break correct = correct and (int(l.split()[1]) == -x) else: # Tell stub to exit. - fifo_to_user1.write("0\n") - fifo_to_user2.write("0\n") + fifo_to_user1.write(b"0\n") + fifo_to_user2.write(b"0\n") # This file exists just for convenience. -io.open("output.txt", "wb").write(l + "\n") +io.open("output.txt", "wb").write(l + b"\n") # This is the final score. if correct: diff --git a/cmstestsuite/tasks/communication_stdio/code/manager b/cmstestsuite/tasks/communication_stdio/code/manager index 27e68c024d..cca1157f28 100644 --- a/cmstestsuite/tasks/communication_stdio/code/manager +++ b/cmstestsuite/tasks/communication_stdio/code/manager @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys @@ -19,28 +18,28 @@ correct = True # Speak the to program a few times, check what it does, and output the last # line. -for i in range(10,20) + [0]: +for i in list(range(10, 20)) + [0]: x = i + input_value # Write a question to the candidate executable. - fifo_to_user.write("%d\n" % x) + fifo_to_user.write(b"%d\n" % x) # Read their response. l = fifo_from_user.readline() # EOF? - if l == '': + if l == b'': correct = False break # These are the only things we expect from our stub. - if l.strip() != 'correct %d' % x: + if l.strip() != b'correct %d' % x: correct = False break correct = correct and (int(l.split()[1]) == x) else: # Tell stub to exit. - fifo_to_user.write("0\n") + fifo_to_user.write(b"0\n") # This file exists just for convenience. -io.open("output.txt", "wb").write(l + "\n") +io.open("output.txt", "wb").write(l + b"\n") # This is the final score. if correct: diff --git a/cmstestsuite/tasks/communication_stdio_stubbed/code/manager b/cmstestsuite/tasks/communication_stdio_stubbed/code/manager index 27e68c024d..cca1157f28 100644 --- a/cmstestsuite/tasks/communication_stdio_stubbed/code/manager +++ b/cmstestsuite/tasks/communication_stdio_stubbed/code/manager @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys @@ -19,28 +18,28 @@ correct = True # Speak the to program a few times, check what it does, and output the last # line. -for i in range(10,20) + [0]: +for i in list(range(10, 20)) + [0]: x = i + input_value # Write a question to the candidate executable. - fifo_to_user.write("%d\n" % x) + fifo_to_user.write(b"%d\n" % x) # Read their response. l = fifo_from_user.readline() # EOF? - if l == '': + if l == b'': correct = False break # These are the only things we expect from our stub. - if l.strip() != 'correct %d' % x: + if l.strip() != b'correct %d' % x: correct = False break correct = correct and (int(l.split()[1]) == x) else: # Tell stub to exit. - fifo_to_user.write("0\n") + fifo_to_user.write(b"0\n") # This file exists just for convenience. -io.open("output.txt", "wb").write(l + "\n") +io.open("output.txt", "wb").write(l + b"\n") # This is the final score. if correct: diff --git a/cmstestsuite/tasks/outputonly_comparator/code/checker b/cmstestsuite/tasks/outputonly_comparator/code/checker index 4380aa75ae..c1e89a5eae 100644 --- a/cmstestsuite/tasks/outputonly_comparator/code/checker +++ b/cmstestsuite/tasks/outputonly_comparator/code/checker @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys diff --git a/cmstestsuite/tasks/twosteps_comparator/code/checker b/cmstestsuite/tasks/twosteps_comparator/code/checker index 4380aa75ae..c1e89a5eae 100644 --- a/cmstestsuite/tasks/twosteps_comparator/code/checker +++ b/cmstestsuite/tasks/twosteps_comparator/code/checker @@ -1,6 +1,5 @@ -#!/usr/bin/python2 -sS +#!/usr/bin/python3 -sS -from __future__ import print_function import io import sys diff --git a/codecov/.gitkeep b/codecov/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/Configuring a contest.rst b/docs/Configuring a contest.rst index 820b922afd..ca8b1509ad 100644 --- a/docs/Configuring a contest.rst +++ b/docs/Configuring a contest.rst @@ -225,7 +225,7 @@ Language details * Pascal support is provided by ``fpc``, and submissions are optimized with ``-O2``. -* Python submissions are executed using the system Python interpreter (you need to have ``/usr/bin/python2`` or ``/usr/bin/python3``, respectively). +* Python submissions are executed using the system Python interpreter (you need to have ``/usr/bin/python3``). * PHP submissions are interpreted by ``/usr/bin/php``. diff --git a/docs/Installation.rst b/docs/Installation.rst index 9ad179c2db..7693d5e3c5 100644 --- a/docs/Installation.rst +++ b/docs/Installation.rst @@ -34,7 +34,7 @@ Then you require the compilation and execution environments for the languages yo * `Free Pascal `_ (for Pascal, with executable ``fpc``); -* `Python `_ >= 2.7 (for Python, with executable ``python2`` or ``python3``; in addition you will need ``zip``); +* `Python `_ >= 3.8 (for Python, with executable ``python3``; in addition you will need ``zip``); * `PHP `_ >= 5 (for PHP, with executable ``php``); @@ -63,7 +63,7 @@ On Ubuntu 20.04, one will need to run the following script to satisfy all depend libffi-dev python3-pip # Optional - sudo apt-get install nginx-full python2.7 php7.4-cli php7.4-fpm \ + sudo apt-get install nginx-full python3.8 php7.4-cli php7.4-fpm \ phppgadmin texlive-latex-base a2ps haskell-platform rustc mono-mcs The above commands provide a very essential Pascal environment. Consider installing the following packages for additional units: `fp-units-base`, `fp-units-fcl`, `fp-units-misc`, `fp-units-math` and `fp-units-rtl`. @@ -86,7 +86,7 @@ On Arch Linux, unofficial AUR packages can be found: `cms Date: Sun, 17 Nov 2024 01:03:26 +0100 Subject: [PATCH 05/10] [italy_yaml] Load all contest parameters from contest.yaml (#1276) --- cmscontrib/loaders/italy_yaml.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/cmscontrib/loaders/italy_yaml.py b/cmscontrib/loaders/italy_yaml.py index f307522f9e..c292bd44d4 100644 --- a/cmscontrib/loaders/italy_yaml.py +++ b/cmscontrib/loaders/italy_yaml.py @@ -125,9 +125,11 @@ def load(src, dst, src_name, dst_name=None, conv=lambda i: i): def parse_datetime(val): + if isinstance(val, datetime): + return val.astimezone(timezone.utc) if isinstance(val, (int, float)): return datetime.fromtimestamp(val, timezone.utc) - return datetime.fromisoformat(val) + raise ValueError("Invalid datetime format.") def make_timedelta(t): @@ -173,11 +175,26 @@ def get_contest(self): args = {} + # Contest information load(conf, args, ["name", "nome_breve"]) load(conf, args, ["description", "nome"]) + load(conf, args, "allowed_localizations") + load(conf, args, "languages") + load(conf, args, "submissions_download_allowed") + load(conf, args, "allow_questions") + load(conf, args, "allow_user_tests") + load(conf, args, "score_precision") logger.info("Loading parameters for contest %s.", args["name"]) + # Logging in + load(conf, args, "block_hidden_participations") + load(conf, args, "allow_password_authentication") + load(conf, args, "allow_registration") + load(conf, args, "ip_restriction") + load(conf, args, "ip_autologin") + + # Token parameters # Use the new token settings format if detected. if "token_mode" in conf: load(conf, args, "token_mode") @@ -219,16 +236,23 @@ def get_contest(self): if args["token_gen_interval"].total_seconds() == 0: args["token_gen_interval"] = timedelta(minutes=1) + # Times load(conf, args, ["start", "inizio"], conv=parse_datetime) load(conf, args, ["stop", "fine"], conv=parse_datetime) - load(conf, args, ["per_user_time"], conv=make_timedelta) load(conf, args, ["timezone"]) + load(conf, args, ["per_user_time"], conv=make_timedelta) + # Limits load(conf, args, "max_submission_number") load(conf, args, "max_user_test_number") load(conf, args, "min_submission_interval", conv=make_timedelta) load(conf, args, "min_user_test_interval", conv=make_timedelta) + # Analysis mode + load(conf, args, "analysis_enabled") + load(conf, args, "analysis_start", conv=parse_datetime) + load(conf, args, "analysis_stop", conv=parse_datetime) + tasks = load(conf, None, ["tasks", "problemi"]) participations = load(conf, None, ["users", "utenti"]) participations = [] if participations is None else participations From 2125d5ede3f8beebd3449fda5c102a166ae05d11 Mon Sep 17 00:00:00 2001 From: Luca Versari Date: Sun, 17 Nov 2024 01:23:52 +0100 Subject: [PATCH 06/10] Compatibility with Python 3.12. (#1271) Inspired by https://github.com/ioi-2024/cms/commit/0f044fd92c3d0b79e1588df015d6b0fdca930dee Also update CI and Dockerfile to Ubuntu 24.04. --- Dockerfile | 15 ++--- cms/io/web_service.py | 7 +++ cms/server/admin/handlers/base.py | 7 +++ .../admin/handlers/contestannouncement.py | 7 +++ cms/server/admin/handlers/contestquestion.py | 7 +++ cms/server/admin/handlers/contestuser.py | 7 +++ cms/server/admin/handlers/dataset.py | 7 +++ cms/server/admin/handlers/task.py | 7 +++ cms/server/contest/handlers/base.py | 7 +++ cms/server/contest/handlers/communication.py | 7 +++ cms/server/contest/handlers/contest.py | 7 +++ cms/server/contest/handlers/main.py | 7 +++ cms/server/contest/handlers/task.py | 7 +++ cms/server/contest/handlers/tasksubmission.py | 7 +++ cms/server/contest/handlers/taskusertest.py | 7 +++ cms/server/util.py | 7 +++ cmscontrib/loaders/polygon.py | 10 ++-- cmstestsuite/code/oom-heap.c | 22 ++++---- .../unit_tests/grading/ParameterTypesTest.py | 7 +++ cmstestsuite/unit_tests/locale/locale_test.py | 55 +++++++++---------- docs/Installation.rst | 12 ++-- requirements.txt | 22 ++++---- 22 files changed, 179 insertions(+), 69 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63f767f995..2c71a46068 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM ubuntu:20.04 +FROM ubuntu:24.04 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ build-essential \ @@ -7,7 +7,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ cppreference-doc-en-html \ fp-compiler \ git \ - haskell-platform \ + ghc \ libcap-dev \ libcups2-dev \ libffi-dev \ @@ -15,12 +15,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ libyaml-dev \ mono-mcs \ openjdk-8-jdk-headless \ - php7.4-cli \ + php-cli \ postgresql-client \ python3-pip \ - python3.8 \ - python3.8-dev \ + python3.12 \ + python3.12-dev \ rustc \ + shared-mime-info \ sudo \ wait-for-it \ zip @@ -38,8 +39,8 @@ COPY --chown=cmsuser:cmsuser requirements.txt dev-requirements.txt /home/cmsuser WORKDIR /home/cmsuser/cms -RUN sudo pip3 install -r requirements.txt -RUN sudo pip3 install -r dev-requirements.txt +RUN sudo pip3 install --break-system-packages -r requirements.txt +RUN sudo pip3 install --break-system-packages -r dev-requirements.txt COPY --chown=cmsuser:cmsuser . /home/cmsuser/cms diff --git a/cms/io/web_service.py b/cms/io/web_service.py index 41d2e2829c..21e1580514 100644 --- a/cms/io/web_service.py +++ b/cms/io/web_service.py @@ -21,6 +21,13 @@ import logging +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.wsgi as tornado_wsgi except ImportError: diff --git a/cms/server/admin/handlers/base.py b/cms/server/admin/handlers/base.py index e38be1fbbf..43df71b4b8 100644 --- a/cms/server/admin/handlers/base.py +++ b/cms/server/admin/handlers/base.py @@ -34,6 +34,13 @@ from datetime import datetime, timedelta from functools import wraps +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/admin/handlers/contestannouncement.py b/cms/server/admin/handlers/contestannouncement.py index 464ff0cced..20f87fe3d5 100644 --- a/cms/server/admin/handlers/contestannouncement.py +++ b/cms/server/admin/handlers/contestannouncement.py @@ -26,6 +26,13 @@ """ +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/admin/handlers/contestquestion.py b/cms/server/admin/handlers/contestquestion.py index 625248f376..57ad1f13b7 100644 --- a/cms/server/admin/handlers/contestquestion.py +++ b/cms/server/admin/handlers/contestquestion.py @@ -27,6 +27,13 @@ import logging +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/admin/handlers/contestuser.py b/cms/server/admin/handlers/contestuser.py index 1706ce8f50..9ec869039b 100644 --- a/cms/server/admin/handlers/contestuser.py +++ b/cms/server/admin/handlers/contestuser.py @@ -31,6 +31,13 @@ import logging +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/admin/handlers/dataset.py b/cms/server/admin/handlers/dataset.py index d01716a90c..5adb03d093 100644 --- a/cms/server/admin/handlers/dataset.py +++ b/cms/server/admin/handlers/dataset.py @@ -32,6 +32,13 @@ import re import zipfile +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/admin/handlers/task.py b/cms/server/admin/handlers/task.py index ff5d5e8e48..c9df69ff08 100644 --- a/cms/server/admin/handlers/task.py +++ b/cms/server/admin/handlers/task.py @@ -29,6 +29,13 @@ import logging import traceback +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/contest/handlers/base.py b/cms/server/contest/handlers/base.py index 00e24c273f..faeb031f58 100644 --- a/cms/server/contest/handlers/base.py +++ b/cms/server/contest/handlers/base.py @@ -32,6 +32,13 @@ import logging import traceback +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/contest/handlers/communication.py b/cms/server/contest/handlers/communication.py index 8bac98e539..572b13be64 100644 --- a/cms/server/contest/handlers/communication.py +++ b/cms/server/contest/handlers/communication.py @@ -29,6 +29,13 @@ import logging +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/contest/handlers/contest.py b/cms/server/contest/handlers/contest.py index 4fe6936ec4..724d5886b8 100644 --- a/cms/server/contest/handlers/contest.py +++ b/cms/server/contest/handlers/contest.py @@ -32,6 +32,13 @@ import ipaddress import logging +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/contest/handlers/main.py b/cms/server/contest/handlers/main.py index 6d2fb2a51d..3ea2129d01 100644 --- a/cms/server/contest/handlers/main.py +++ b/cms/server/contest/handlers/main.py @@ -33,6 +33,13 @@ import logging import re +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/contest/handlers/task.py b/cms/server/contest/handlers/task.py index 3f6f163a28..cfdfe1b69a 100644 --- a/cms/server/contest/handlers/task.py +++ b/cms/server/contest/handlers/task.py @@ -30,6 +30,13 @@ import logging +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/contest/handlers/tasksubmission.py b/cms/server/contest/handlers/tasksubmission.py index 9199c0bcd7..e233291a76 100644 --- a/cms/server/contest/handlers/tasksubmission.py +++ b/cms/server/contest/handlers/tasksubmission.py @@ -32,6 +32,13 @@ import logging import re +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/contest/handlers/taskusertest.py b/cms/server/contest/handlers/taskusertest.py index c78c87c1c8..07da1f9a59 100644 --- a/cms/server/contest/handlers/taskusertest.py +++ b/cms/server/contest/handlers/taskusertest.py @@ -31,6 +31,13 @@ import logging import re +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: import tornado4.web as tornado_web except ImportError: diff --git a/cms/server/util.py b/cms/server/util.py index 953d5eb1c3..7b787595cb 100644 --- a/cms/server/util.py +++ b/cms/server/util.py @@ -30,6 +30,13 @@ from functools import wraps from urllib.parse import quote, urlencode +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: from tornado4.web import RequestHandler except ImportError: diff --git a/cmscontrib/loaders/polygon.py b/cmscontrib/loaders/polygon.py index fb4a4b4ade..200dd38c8d 100644 --- a/cmscontrib/loaders/polygon.py +++ b/cmscontrib/loaders/polygon.py @@ -20,7 +20,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import imp import logging import os import subprocess @@ -145,10 +144,11 @@ def get_task(self, get_statement=True): task_cms_conf = None if os.path.exists(task_cms_conf_path): logger.info("Found additional CMS options for task %s.", name) - with open(task_cms_conf_path, 'rb') as f: - task_cms_conf = imp.load_module('cms_conf', f, - task_cms_conf_path, - ('.py', 'r', imp.PY_SOURCE)) + import importlib.util + spec = importlib.util.spec_from_file_location( + 'cms_conf', task_cms_conf_path) + task_cms_conf = importlib.util.module_from_spec(spec) + spec.loader.exec_module(task_cms_conf) if task_cms_conf is not None and hasattr(task_cms_conf, "general"): args.update(task_cms_conf.general) diff --git a/cmstestsuite/code/oom-heap.c b/cmstestsuite/code/oom-heap.c index 3300d17175..ce21c8aa46 100644 --- a/cmstestsuite/code/oom-heap.c +++ b/cmstestsuite/code/oom-heap.c @@ -1,17 +1,17 @@ #include #include -int *big; +volatile int *big; int main() { - int i; - big = malloc(128 * 1024 * 1024); - // If we don't do this cycle, the compiler is smart enough not to - // map the array into resident memory. - for (i = 0; i < 128 * 1024 * 1024 / sizeof(int); i++) { - big[i] = 0; - } - scanf("%d", &big[10000]); - printf("correct %d\n", big[10000]); - return 0; + int i; + big = malloc(128 * 1024 * 1024); + // If we don't do this cycle, the compiler is smart enough not to + // map the array into resident memory. + for (i = 0; i < 128 * 1024 * 1024 / sizeof(int); i++) { + big[i] = 0; + } + scanf("%d", &big[10000]); + printf("correct %d\n", big[10000]); + return 0; } diff --git a/cmstestsuite/unit_tests/grading/ParameterTypesTest.py b/cmstestsuite/unit_tests/grading/ParameterTypesTest.py index 21e20ec38a..c7c56af41b 100755 --- a/cmstestsuite/unit_tests/grading/ParameterTypesTest.py +++ b/cmstestsuite/unit_tests/grading/ParameterTypesTest.py @@ -20,6 +20,13 @@ import unittest +import collections +try: + collections.MutableMapping +except: + # Monkey-patch: Tornado 4.5.3 does not work on Python 3.11 by default + collections.MutableMapping = collections.abc.MutableMapping + try: from tornado4.web import MissingArgumentError except ImportError: diff --git a/cmstestsuite/unit_tests/locale/locale_test.py b/cmstestsuite/unit_tests/locale/locale_test.py index 170a488473..e81d0fe458 100755 --- a/cmstestsuite/unit_tests/locale/locale_test.py +++ b/cmstestsuite/unit_tests/locale/locale_test.py @@ -103,20 +103,20 @@ def test_utc(self): self.assertEqual( ENGLISH.format_datetime(datetime(2018, 1, 1, 12, 34, 56), timezone=UTC), - "Jan 1, 2018, 12:34:56 PM") + "Jan 1, 2018, 12:34:56\N{Narrow No-Break Space}PM") def test_other_timezone_winter(self): # Other timezone, in winter (no DST). self.assertEqual( ENGLISH.format_datetime(datetime(2018, 1, 1, 12, 34, 56), timezone=ROME), - "Jan 1, 2018, 1:34:56 PM") + "Jan 1, 2018, 1:34:56\N{Narrow No-Break Space}PM") def test_other_timezone_summer(self): self.assertEqual( ENGLISH.format_datetime(datetime(2018, 7, 1, 12, 34, 56), timezone=ROME), - "Jul 1, 2018, 2:34:56 PM") + "Jul 1, 2018, 2:34:56\N{Narrow No-Break Space}PM") # As above, localized (use a language with a 24h clock and with a # different day/month/year order). @@ -146,19 +146,19 @@ def test_utc(self): # UTC, in English self.assertEqual(ENGLISH.format_time(datetime(2018, 1, 1, 12, 34, 56), timezone=UTC), - "12:34:56 PM") + "12:34:56\N{Narrow No-Break Space}PM") def test_other_timezone_winter(self): # Other timezone, in winter (no DST). self.assertEqual(ENGLISH.format_time(datetime(2018, 1, 1, 12, 34, 56), timezone=ROME), - "1:34:56 PM") + "1:34:56\N{Narrow No-Break Space}PM") def test_other_timezone_in_summer(self): # Other timezone, in summer (DST). self.assertEqual(ENGLISH.format_time(datetime(2018, 7, 1, 12, 34, 56), timezone=ROME), - "2:34:56 PM") + "2:34:56\N{Narrow No-Break Space}PM") # As above, localized (use Danish as they use periods rather # than colons and have a 24h clock). @@ -187,17 +187,17 @@ def test_utc(self): ENGLISH.format_datetime_smart(datetime(2018, 1, 1, 12, 34, 56), datetime(2018, 1, 1, 23, 30), timezone=UTC), - "12:34:56 PM") + "12:34:56\N{Narrow No-Break Space}PM") self.assertEqual( ENGLISH.format_datetime_smart(datetime(2018, 1, 1, 12, 34, 56), datetime(2018, 1, 2, 0, 30), timezone=UTC), - "Jan 1, 2018, 12:34:56 PM") + "Jan 1, 2018, 12:34:56\N{Narrow No-Break Space}PM") self.assertEqual( ENGLISH.format_datetime_smart(datetime(2018, 1, 1, 12, 34, 56), datetime(2017, 12, 31, 23, 30), timezone=UTC), - "Jan 1, 2018, 12:34:56 PM") + "Jan 1, 2018, 12:34:56\N{Narrow No-Break Space}PM") def test_other_timezone_winter(self): # Other timezone, in winter (no DST). @@ -205,17 +205,17 @@ def test_other_timezone_winter(self): ENGLISH.format_datetime_smart(datetime(2018, 1, 1, 12, 34, 56), datetime(2018, 1, 1, 22, 30), timezone=ROME), - "1:34:56 PM") + "1:34:56\N{Narrow No-Break Space}PM") self.assertEqual( ENGLISH.format_datetime_smart(datetime(2018, 1, 1, 12, 34, 56), datetime(2018, 1, 1, 23, 30), timezone=ROME), - "Jan 1, 2018, 1:34:56 PM") + "Jan 1, 2018, 1:34:56\N{Narrow No-Break Space}PM") self.assertEqual( ENGLISH.format_datetime_smart(datetime(2018, 1, 1, 12, 34, 56), datetime(2017, 12, 31, 22, 30), timezone=ROME), - "Jan 1, 2018, 1:34:56 PM") + "Jan 1, 2018, 1:34:56\N{Narrow No-Break Space}PM") def test_other_timezone_summer(self): # Other timezone, in summer (DST). @@ -223,17 +223,17 @@ def test_other_timezone_summer(self): ENGLISH.format_datetime_smart(datetime(2018, 7, 1, 12, 34, 56), datetime(2018, 7, 1, 21, 30), timezone=ROME), - "2:34:56 PM") + "2:34:56\N{Narrow No-Break Space}PM") self.assertEqual( ENGLISH.format_datetime_smart(datetime(2018, 7, 1, 12, 34, 56), datetime(2018, 7, 1, 22, 30), timezone=ROME), - "Jul 1, 2018, 2:34:56 PM") + "Jul 1, 2018, 2:34:56\N{Narrow No-Break Space}PM") self.assertEqual( ENGLISH.format_datetime_smart(datetime(2018, 7, 1, 12, 34, 56), datetime(2018, 6, 30, 21, 30), timezone=ROME), - "Jul 1, 2018, 2:34:56 PM") + "Jul 1, 2018, 2:34:56\N{Narrow No-Break Space}PM") # As above, localized. @@ -508,23 +508,23 @@ def test_large(self): def test_localized_zero(self): self.assertEqual(FRENCH.format_size(0), - "0 octet") + "0\N{No-Break Space}octet") def test_localized_small_values(self): self.assertEqual(FRENCH.format_size(1), - "1 octet") + "1\N{No-Break Space}octet") self.assertEqual(FRENCH.format_size(2), - "2 octets") + "2\N{No-Break Space}octets") def test_localized_cutoff_kib(self): self.assertEqual(FRENCH.format_size(999), - "999 octets") + "999\N{No-Break Space}octets") self.assertEqual(FRENCH.format_size(1000), - "1\N{NO-BREAK SPACE}000 octets") + "1\N{Narrow No-Break Space}000\N{No-Break Space}octets") self.assertEqual(FRENCH.format_size(1001), - "1\N{NO-BREAK SPACE}001 octets") + "1\N{Narrow No-Break Space}001\N{No-Break Space}octets") self.assertEqual(FRENCH.format_size(1023), - "1\N{NO-BREAK SPACE}023 octets") + "1\N{Narrow No-Break Space}023\N{No-Break Space}octets") self.assertEqual(FRENCH.format_size(1024), "1,00 Kio") self.assertEqual(FRENCH.format_size(1025), @@ -534,17 +534,16 @@ def test_localized_cutoff_mib(self): self.assertEqual(FRENCH.format_size(999 * 1024), "999 Kio") self.assertEqual(FRENCH.format_size(1000 * 1024), - "1\N{NO-BREAK SPACE}000 Kio") + "1\N{Narrow No-Break Space}000 Kio") self.assertEqual(FRENCH.format_size(1001 * 1024), - "1\N{NO-BREAK SPACE}001 Kio") + "1\N{Narrow No-Break Space}001 Kio") self.assertEqual(FRENCH.format_size(1023 * 1024), - "1\N{NO-BREAK SPACE}023 Kio") + "1\N{Narrow No-Break Space}023 Kio") self.assertEqual(FRENCH.format_size(1024 * 1024), "1,00 Mio") self.assertEqual(FRENCH.format_size(1025 * 1024), "1,00 Mio") - def test_localized_large(self): self.assertEqual(FRENCH.format_size(2_345_000), "2,24 Mio") @@ -553,7 +552,7 @@ def test_localized_large(self): self.assertEqual(FRENCH.format_size(456_789_000_000_000), "415 Tio") self.assertEqual(FRENCH.format_size(5_678_912_300_000_000), - "5\N{NO-BREAK SPACE}165 Tio") + "5\N{Narrow No-Break Space}165 Tio") class TestFormatDecimal(unittest.TestCase): @@ -594,7 +593,7 @@ class TestTranslateMimetype(unittest.TestCase): @unittest.skipIf(not os.path.isfile( "/usr/share/locale/it/LC_MESSAGES/shared-mime-info.mo"), - reason="need Italian shared-mime-info translation") + reason="need Italian shared-mime-info translation") def test_translate_mimetype(self): self.assertEqual(ENGLISH.translate_mimetype("PDF document"), "PDF document") diff --git a/docs/Installation.rst b/docs/Installation.rst index 7693d5e3c5..2f5936d2e1 100644 --- a/docs/Installation.rst +++ b/docs/Installation.rst @@ -49,22 +49,22 @@ All dependencies can be installed automatically on most Linux distributions. Ubuntu ------ -On Ubuntu 20.04, one will need to run the following script to satisfy all dependencies: +On Ubuntu 24.04, one will need to run the following script to satisfy all dependencies: .. sourcecode:: bash # Feel free to change OpenJDK packages with your preferred JDK. sudo apt-get install build-essential openjdk-11-jdk-headless fp-compiler \ - postgresql postgresql-client python3.8 cppreference-doc-en-html \ + postgresql postgresql-client python3.12 cppreference-doc-en-html \ cgroup-lite libcap-dev zip # Only if you are going to use pip/venv to install python dependencies - sudo apt-get install python3.8-dev libpq-dev libcups2-dev libyaml-dev \ + sudo apt-get install python3.12-dev libpq-dev libcups2-dev libyaml-dev \ libffi-dev python3-pip # Optional - sudo apt-get install nginx-full python3.8 php7.4-cli php7.4-fpm \ - phppgadmin texlive-latex-base a2ps haskell-platform rustc mono-mcs + sudo apt-get install nginx-full php-cli texlive-latex-base \ + a2ps ghc rustc mono-mcs The above commands provide a very essential Pascal environment. Consider installing the following packages for additional units: `fp-units-base`, `fp-units-fcl`, `fp-units-misc`, `fp-units-math` and `fp-units-rtl`. @@ -86,7 +86,7 @@ On Arch Linux, unofficial AUR packages can be found: `cms =4.5,<4.6 # http://www.tornadoweb.org/en/stable/releases.html -psycopg2>=2.8,<2.9 # http://initd.org/psycopg/articles/tag/release/ +tornado==4.5.3 # http://www.tornadoweb.org/en/stable/releases.html +psycopg2==2.9.7 # http://initd.org/psycopg/articles/tag/release/ sqlalchemy>=1.3,<1.4 # http://docs.sqlalchemy.org/en/latest/changelog/index.html netifaces>=0.10,<0.11 # https://bitbucket.org/al45tair/netifaces/src/ -pycryptodomex>=3.6,<3.7 # https://github.com/Legrandin/pycryptodome/blob/master/Changelog.rst +pycryptodomex==3.19.0 # https://github.com/Legrandin/pycryptodome/blob/master/Changelog.rst psutil>=5.5,<5.6 # https://github.com/giampaolo/psutil/blob/master/HISTORY.rst -requests>=2.22,<2.23 # https://pypi.python.org/pypi/requests -gevent==20.12.0 # http://www.gevent.org/changelog.html -# Limit greenlet version for binary compatibility with gevent 20.12 wheels -greenlet==1.0.0 -werkzeug>=0.16,<0.17 # https://github.com/pallets/werkzeug/blob/master/CHANGES +requests==2.32.3 # https://pypi.python.org/pypi/requests +gevent==23.9.0.post1 # http://www.gevent.org/changelog.html +greenlet>=3.0rc1 +werkzeug<1.0 # https://github.com/pallets/werkzeug/blob/master/CHANGES +backports.ssl-match-hostname==3.7.0.1 # required by tornado<5.0 patool>=1.12,<1.13 # https://github.com/wummel/patool/blob/master/doc/changelog.txt bcrypt>=3.1,<3.2 # https://github.com/pyca/bcrypt/ chardet>=3.0,<3.1 # https://pypi.python.org/pypi/chardet -babel>=2.6,<2.7 # http://babel.pocoo.org/en/latest/changelog.html +babel==2.12.1 # http://babel.pocoo.org/en/latest/changelog.html pyxdg>=0.26,<0.27 # https://freedesktop.org/wiki/Software/pyxdg/ Jinja2>=2.10,<2.11 # http://jinja.pocoo.org/docs/latest/changelog/ @@ -26,5 +24,5 @@ MarkupSafe==2.0.1 pyyaml>=5.3,<5.4 # http://pyyaml.org/wiki/PyYAML # Only for printing: -pycups>=1.9,<1.10 # https://pypi.python.org/pypi/pycups +pycups==2.0.4 # https://pypi.python.org/pypi/pycups PyPDF2>=1.26,<1.27 # https://github.com/mstamy2/PyPDF2/blob/master/CHANGELOG From 40c0ef48a89762aeb50d3e0338411673f4ebd9fa Mon Sep 17 00:00:00 2001 From: William Di Luigi Date: Sun, 17 Nov 2024 02:00:00 +0100 Subject: [PATCH 07/10] Use CODECOV_TOKEN to fix rate limit issue. (#1278) --- .github/workflows/main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1cc293e8c2..fc06496b04 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,12 +25,16 @@ jobs: run: | docker compose -p cms -f docker-compose.test.yml run --rm testcms - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: files: ./codecov/unittests.xml flags: unittests - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: files: ./codecov/functionaltests.xml flags: functionaltests From b12d0b172b4a6971252ce27bedbb8faf1f6135ec Mon Sep 17 00:00:00 2001 From: Filippo Casarin <55024474+Virv12@users.noreply.github.com> Date: Sun, 17 Nov 2024 02:12:27 +0100 Subject: [PATCH 08/10] Add support for c++20 language (#1275) --- cms/db/contest.py | 2 +- cms/grading/languages/cpp20_gpp.py | 63 +++++++++++++++++++++++ cmstestsuite/Tests.py | 8 +-- cmstestsuite/code/correct-stdio-cxx20.cpp | 9 ++++ setup.py | 1 + 5 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 cms/grading/languages/cpp20_gpp.py create mode 100644 cmstestsuite/code/correct-stdio-cxx20.cpp diff --git a/cms/db/contest.py b/cms/db/contest.py index 7543c58d68..2789e952bd 100644 --- a/cms/db/contest.py +++ b/cms/db/contest.py @@ -79,7 +79,7 @@ class Contest(Base): languages = Column( ARRAY(String), nullable=False, - default=["C11 / gcc", "C++17 / g++", "Pascal / fpc"]) + default=["C11 / gcc", "C++20 / g++", "Pascal / fpc"]) # Whether contestants allowed to download their submissions. submissions_download_allowed = Column( diff --git a/cms/grading/languages/cpp20_gpp.py b/cms/grading/languages/cpp20_gpp.py new file mode 100644 index 0000000000..3d70ce27fe --- /dev/null +++ b/cms/grading/languages/cpp20_gpp.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +# Contest Management System - http://cms-dev.github.io/ +# Copyright © 2024 Filippo Casarin +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""C++20 programming language definition.""" + +from cms.grading import CompiledLanguage + + +__all__ = ["Cpp20Gpp"] + + +class Cpp20Gpp(CompiledLanguage): + """This defines the C++ programming language, compiled with g++ (the + version available on the system) using the C++20 standard. + + """ + + @property + def name(self): + """See Language.name.""" + return "C++20 / g++" + + @property + def source_extensions(self): + """See Language.source_extensions.""" + return [".cpp", ".cc", ".cxx", ".c++", ".C"] + + @property + def header_extensions(self): + """See Language.header_extensions.""" + return [".h"] + + @property + def object_extensions(self): + """See Language.object_extensions.""" + return [".o"] + + def get_compilation_commands(self, + source_filenames, executable_filename, + for_evaluation=True): + """See Language.get_compilation_commands.""" + command = ["/usr/bin/g++"] + if for_evaluation: + command += ["-DEVAL"] + command += ["-std=gnu++20", "-O2", "-pipe", "-static", + "-s", "-o", executable_filename] + command += source_filenames + return [command] diff --git a/cmstestsuite/Tests.py b/cmstestsuite/Tests.py index 2cf440d4af..37947d39ef 100644 --- a/cmstestsuite/Tests.py +++ b/cmstestsuite/Tests.py @@ -43,6 +43,7 @@ LANG_CPP = "C++11 / g++" LANG_CPP14 = "C++14 / g++" LANG_CPP17 = "C++17 / g++" +LANG_CPP20 = "C++20 / g++" LANG_C = "C11 / gcc" LANG_HS = "Haskell / ghc" LANG_JAVA = "Java / JDK" @@ -52,14 +53,14 @@ LANG_RUST = "Rust" LANG_C_SHARP = "C# / Mono" ALL_LANGUAGES = ( - LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_C, LANG_HS, LANG_JAVA, LANG_PASCAL, + LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_CPP20, LANG_C, LANG_HS, LANG_JAVA, LANG_PASCAL, LANG_PHP, LANG_PYTHON3, LANG_RUST, LANG_C_SHARP ) NON_INTERPRETED_LANGUAGES = ( - LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_PASCAL + LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_CPP20, LANG_PASCAL ) COMPILED_LANGUAGES = ( - LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_PASCAL, LANG_JAVA, + LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_CPP20, LANG_PASCAL, LANG_JAVA, LANG_PYTHON3, LANG_HS, LANG_RUST, LANG_C_SHARP ) @@ -72,6 +73,7 @@ alt_filenames={ LANG_CPP14: ['correct-stdio-cxx14.%l'], LANG_CPP17: ['correct-stdio-cxx17.%l'], + LANG_CPP20: ['correct-stdio-cxx20.%l'], }, languages=ALL_LANGUAGES, checks=[CheckOverallScore(100, 100)]), diff --git a/cmstestsuite/code/correct-stdio-cxx20.cpp b/cmstestsuite/code/correct-stdio-cxx20.cpp new file mode 100644 index 0000000000..6577248216 --- /dev/null +++ b/cmstestsuite/code/correct-stdio-cxx20.cpp @@ -0,0 +1,9 @@ +#include + +static_assert(__cplusplus == 202002L, "C++20 expected"); + +int main() { + int n; + std::cin >> n; + std::cout << "correct " << n << std::endl; +} diff --git a/setup.py b/setup.py index 8ba0d4cc9c..9761864d6b 100755 --- a/setup.py +++ b/setup.py @@ -185,6 +185,7 @@ def run(self): "C++11 / g++=cms.grading.languages.cpp11_gpp:Cpp11Gpp", "C++14 / g++=cms.grading.languages.cpp14_gpp:Cpp14Gpp", "C++17 / g++=cms.grading.languages.cpp17_gpp:Cpp17Gpp", + "C++20 / g++=cms.grading.languages.cpp20_gpp:Cpp20Gpp", "C11 / gcc=cms.grading.languages.c11_gcc:C11Gcc", "C# / Mono=cms.grading.languages.csharp_mono:CSharpMono", "Haskell / ghc=cms.grading.languages.haskell_ghc:HaskellGhc", From 9ef11a8de09bf31b4bdcda17e42e04236dfa6485 Mon Sep 17 00:00:00 2001 From: Filippo Casarin <55024474+Virv12@users.noreply.github.com> Date: Sun, 17 Nov 2024 02:17:37 +0100 Subject: [PATCH 09/10] Add missing templates for submission export (#1279) --- cmscontrib/ExportSubmissions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmscontrib/ExportSubmissions.py b/cmscontrib/ExportSubmissions.py index b53fd5cb9b..44d5d5ee8b 100755 --- a/cmscontrib/ExportSubmissions.py +++ b/cmscontrib/ExportSubmissions.py @@ -57,6 +57,8 @@ TEMPLATE[".cpp"] = TEMPLATE[".c"] TEMPLATE[".java"] = TEMPLATE[".c"] TEMPLATE[".txt"] = TEMPLATE[".c"] +TEMPLATE[".cs"] = TEMPLATE[".c"] +TEMPLATE[".rs"] = TEMPLATE[".c"] def filter_top_scoring(results, unique): From 39d272b0bde1771a10d84b8aa8605e8f6945117c Mon Sep 17 00:00:00 2001 From: Hyun-seok Jeon <92401080+melongist@users.noreply.github.com> Date: Sun, 17 Nov 2024 10:26:37 +0900 Subject: [PATCH 10/10] Add support for PyPy flavor of Python3 (#1206) Co-authored-by: Luca Versari --- Dockerfile | 1 + cms/grading/languages/python3_pypy.py | 81 +++++++++++++++++++++++++++ cmstestsuite/Tests.py | 5 +- docs/Installation.rst | 4 +- setup.py | 1 + 5 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 cms/grading/languages/python3_pypy.py diff --git a/Dockerfile b/Dockerfile index 2c71a46068..05ea777079 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ openjdk-8-jdk-headless \ php-cli \ postgresql-client \ + pypy3 \ python3-pip \ python3.12 \ python3.12-dev \ diff --git a/cms/grading/languages/python3_pypy.py b/cms/grading/languages/python3_pypy.py new file mode 100644 index 0000000000..e87b29ab01 --- /dev/null +++ b/cms/grading/languages/python3_pypy.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +# Contest Management System - http://cms-dev.github.io/ +# Copyright © 2016-2018 Stefano Maggiolo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Python programming language, version 3, definition.""" + +import os + +from cms.grading import CompiledLanguage + + +__all__ = ["Python3PyPy"] + + +class Python3PyPy(CompiledLanguage): + """This defines the Python programming language, version 3 (more + precisely, the subversion of Python 3 available on the system) + using the default PyPy interpeter in the system. + + """ + + MAIN_FILENAME = "__main__.pyc" + + @property + def name(self): + """See Language.name.""" + return "Python 3 / PyPy" + + @property + def source_extensions(self): + """See Language.source_extensions.""" + return [".py"] + + @property + def executable_extension(self): + """See Language.executable.extension.""" + # Defined in PEP 441 (https://www.python.org/dev/peps/pep-0441/). + return ".pyz" + + def get_compilation_commands(self, + source_filenames, executable_filename, + for_evaluation=True): + """See Language.get_compilation_commands.""" + + commands = [] + files_to_package = [] + commands.append(["/usr/bin/pypy3", "-m", "compileall", "-b", "."]) + for idx, source_filename in enumerate(source_filenames): + basename = os.path.splitext(os.path.basename(source_filename))[0] + pyc_filename = "%s.pyc" % basename + # The file with the entry point must be in first position. + if idx == 0: + commands.append(["/bin/mv", pyc_filename, self.MAIN_FILENAME]) + files_to_package.append(self.MAIN_FILENAME) + else: + files_to_package.append(pyc_filename) + + commands.append(["/usr/bin/zip", executable_filename] + + files_to_package) + + return commands + + def get_evaluation_commands( + self, executable_filename, main=None, args=None): + """See Language.get_evaluation_commands.""" + args = args if args is not None else [] + return [["/usr/bin/pypy3", executable_filename] + args] diff --git a/cmstestsuite/Tests.py b/cmstestsuite/Tests.py index 37947d39ef..caa432fd76 100644 --- a/cmstestsuite/Tests.py +++ b/cmstestsuite/Tests.py @@ -50,18 +50,19 @@ LANG_PASCAL = "Pascal / fpc" LANG_PHP = "PHP" LANG_PYTHON3 = "Python 3 / CPython" +LANG_PYPY3 = "Python 3 / PyPy" LANG_RUST = "Rust" LANG_C_SHARP = "C# / Mono" ALL_LANGUAGES = ( LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_CPP20, LANG_C, LANG_HS, LANG_JAVA, LANG_PASCAL, - LANG_PHP, LANG_PYTHON3, LANG_RUST, LANG_C_SHARP + LANG_PHP, LANG_PYTHON3, LANG_PYPY3, LANG_RUST, LANG_C_SHARP ) NON_INTERPRETED_LANGUAGES = ( LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_CPP20, LANG_PASCAL ) COMPILED_LANGUAGES = ( LANG_C, LANG_CPP, LANG_CPP14, LANG_CPP17, LANG_CPP20, LANG_PASCAL, LANG_JAVA, - LANG_PYTHON3, LANG_HS, LANG_RUST, LANG_C_SHARP + LANG_PYTHON3, LANG_PYPY3, LANG_HS, LANG_RUST, LANG_C_SHARP ) ALL_TESTS = [ diff --git a/docs/Installation.rst b/docs/Installation.rst index 2f5936d2e1..9e079cf58f 100644 --- a/docs/Installation.rst +++ b/docs/Installation.rst @@ -64,7 +64,7 @@ On Ubuntu 24.04, one will need to run the following script to satisfy all depend # Optional sudo apt-get install nginx-full php-cli texlive-latex-base \ - a2ps ghc rustc mono-mcs + a2ps ghc rustc mono-mcs pypy3 The above commands provide a very essential Pascal environment. Consider installing the following packages for additional units: `fp-units-base`, `fp-units-fcl`, `fp-units-misc`, `fp-units-math` and `fp-units-rtl`. @@ -87,7 +87,7 @@ On Arch Linux, unofficial AUR packages can be found: `cms