AssertionError: Checksums were not equal. The new checksums have been written to /scratch/tm70/repro-ci/experiments/access-om3-configs/04639095b2fbf0538b26ca6f524420249b0e3a6f/checksum/historical-3hr-checksum.json.
assert {'output': {'...ion': '1-0-0'} == {'output': {'...ion': '1-0-0'}
  
  Omitting 1 identical items, use -vv to show
  Differing items:
  {'output': {'CFL': ['0.00000', '0.06867'], 'En': ['3.1034198253312566E-23', '5.2673113885059347E-04'], 'M': ['1.36404E+21', '1.36404E+21'], 'Me': ['0.00E+00', '2.92E-20'], ...}} != {'output': {'CFL': ['0.00000', '0.06876'], 'En': ['3.0906073728130855E-23', '5.1157582642758414E-04'], 'M': ['1.36404E+21', '1.36404E+21'], 'Me': ['0.00E+00', '-3.59E-10'], ...}}
  Use -v to get more diff
self = <model_config_tests.test_bit_reproducibility.TestBitReproducibility object at 0x7fc1ef6892a0>
output_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-om3-configs/04639095b2fbf0538b26ca6f524420249b0e3a6f')
control_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-om3-configs/04639095b2fbf0538b26ca6f524420249b0e3a6f/base-experiment')
checksum_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-om3-configs/04639095b2fbf0538b26ca6f524420249b0e3a6f/compared/testing/checksum/historical-3hr-checksum.json')
keep_archive = False

    @pytest.mark.checksum
    def test_bit_repro_historical(
        self,
        output_path: Path,
        control_path: Path,
        checksum_path: Optional[Path],
        keep_archive: Optional[bool],
    ):
        """
        Test that a run reproduces historical checksums
    
        Parameters (these are fixtures defined in conftest.py)
        ----------
        output_path: Path
            Output directory for test output and where the control and
            lab directories are stored for the payu experiments. Default is
            set in conftest.py
        control_path: Path
            Path to the model configuration to test. This is copied for
            for control directories in experiments. Default is set in
            conftests.py
        checksum_path: Optional[Path]
            Path to checksums to compare model output against. Default is
            set to checksums saved on model configuration (set in )
        keep_archive: Optional[bool]
            This flag is used in testing for test code to use a previous test
            archive, and to disable running the model with payu
        """
        # Setup checksum output directory
        checksum_output_dir = set_checksum_output_dir(output_path=output_path)
    
        # Setup experiment
        exp = setup_exp(
            control_path, output_path, "test_bit_repro_historical", keep_archive
        )
    
        # Set model runtime using the configured default
        exp.model.set_model_runtime()
    
        # Run the experiment using payu
        status, stdout, stderr, output_files = exp.setup_and_run()
    
        if status != 0 or not exp.model.output_exists():
            # Log the run information
            exp.print_run_logs(status, stdout, stderr, output_files)
    
        assert status == 0, (
            "There was an error running the experiment. "
            "See the logs for more infomation on the experiment run"
        )
    
        assert exp.model.output_exists(), (
            "Output file for the model does not exist. "
            "See the logs for more information on the experiment run"
        )
    
        # Set the checksum output filename using the model default runtime
        runtime_hours = exp.model.default_runtime_seconds // HOUR_IN_SECONDS
        checksum_filename = f"historical-{runtime_hours}hr-checksum.json"
    
        # Read the historical checksum file
        hist_checksums = read_historical_checksums(
            control_path, checksum_filename, checksum_path
        )
    
        # Use historical file checksums schema version for parsing checksum,
        # otherwise use the model default, if file does not exist
        schema_version = (
            hist_checksums["schema_version"]
            if hist_checksums
            else exp.model.default_schema_version
        )
    
        # Extract checksums
        checksums = exp.extract_checksums(schema_version=schema_version)
    
        # Write out checksums to output file
        checksum_output_file = checksum_output_dir / checksum_filename
        with open(checksum_output_file, "w") as file:
            json.dump(checksums, file, indent=2)
    
>       assert (
            hist_checksums == checksums
        ), f"Checksums were not equal. The new checksums have been written to {checksum_output_file}."
E       AssertionError: Checksums were not equal. The new checksums have been written to /scratch/tm70/repro-ci/experiments/access-om3-configs/04639095b2fbf0538b26ca6f524420249b0e3a6f/checksum/historical-3hr-checksum.json.
E       assert {'output': {'...ion': '1-0-0'} == {'output': {'...ion': '1-0-0'}
E         
E         Omitting 1 identical items, use -vv to show
E         Differing items:
E         {'output': {'CFL': ['0.00000', '0.06867'], 'En': ['3.1034198253312566E-23', '5.2673113885059347E-04'], 'M': ['1.36404E+21', '1.36404E+21'], 'Me': ['0.00E+00', '2.92E-20'], ...}} != {'output': {'CFL': ['0.00000', '0.06876'], 'En': ['3.0906073728130855E-23', '5.1157582642758414E-04'], 'M': ['1.36404E+21', '1.36404E+21'], 'Me': ['0.00E+00', '-3.59E-10'], ...}}
E         Use -v to get more diff

../test-venv/lib/python3.10/site-packages/model_config_tests/test_bit_reproducibility.py:131: AssertionError