From c6e1285dd641d001348c1ab3cc39f8c8e05df29c Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 8 Nov 2022 13:04:13 -0700 Subject: [PATCH 01/64] Remove mct tests. --- cime_config/testlist_allactive.xml | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/cime_config/testlist_allactive.xml b/cime_config/testlist_allactive.xml index d0802a00e..7b2c3996a 100644 --- a/cime_config/testlist_allactive.xml +++ b/cime_config/testlist_allactive.xml @@ -48,14 +48,6 @@ - - - - - - - - @@ -64,14 +56,6 @@ - - - - - - - - @@ -150,15 +134,6 @@ - - - - - - - - - From bf1bf7fbec28281f80130effe003c7f5f2db13e2 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Thu, 10 Nov 2022 10:16:48 -0700 Subject: [PATCH 02/64] Update for cesm2_3_alpha10c --- Externals.cfg | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index dae4c3689..a1272372c 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,12 +1,12 @@ [ccs_config] -tag = ccs_config_cesm0.0.47 +tag = ccs_config_cesm0.0.48 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config required = True [cam] -tag = cam6_3_078 +tag = cam6_3_082 protocol = git repo_url = https://github.com/ESCOMP/CAM local_path = components/cam @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.13.78 +tag = cmeps0.13.79 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.64 +tag = cime6.0.75 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev107 +tag = ctsm5.1.dev113 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm @@ -111,7 +111,7 @@ externals = Externals.cfg required = False [mosart] -tag = mosart1_0_45 +tag = mosart1_0_46 protocol = git repo_url = https://github.com/ESCOMP/MOSART local_path = components/mosart @@ -140,7 +140,7 @@ local_path = components/rtm required = True [ww3] -tag = ww3_220322 +tag = ww3_221108 protocol = git repo_url = https://github.com/ESCOMP/WW3-CESM local_path = components/ww3 From c4d300d729216f286de9654a18069aaa3026d880 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 16 Nov 2022 12:42:01 -0700 Subject: [PATCH 03/64] Update for cesm2_3_alpha10c --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index a1272372c..e2ba32a91 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,12 +1,12 @@ [ccs_config] -tag = ccs_config_cesm0.0.48 +tag = ccs_config_cesm0.0.49 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config required = True [cam] -tag = cam6_3_082 +tag = cam6_3_083 protocol = git repo_url = https://github.com/ESCOMP/CAM local_path = components/cam From 060ff47701b6db53b1c65f53d39c9846a429a55c Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 29 Nov 2022 16:36:06 -0700 Subject: [PATCH 04/64] Update for cesm2_3_alpha10c --- ChangeLog | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/ChangeLog b/ChangeLog index ecceb33f0..011b0832e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,163 @@ +============================================================== +Tag name: cesm2_3_alpha010c +Originator(s): CSEG +Date: 29 Novemberr 2022 +One-line Summary: CAM and CLM updates. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_083 ** +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_2_0_34 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.75 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.12 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.49 ** +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.13.79 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps0.12.65 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev113 ** +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_20220425 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_220624 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_46 ** +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 ** +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.4 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_9 -- + +cam + Cheryl Craig 2022-11-15 - cam6_3_083 - components/cam (cesm2_3_alpha10c) + https://github.com/ESCOMP/CAM/tags/cam6_3_083 + + Move emissions before CLUBB in cam_dev + Fix bad ozone (Sa_o3) values in first coupling time step + + - Answer changing for cam_dev runs + + + Francis Vitt 2022-11-01 - cam6_3_082 - components/cam (cesm2_3_alpha10c) + https://github.com/ESCOMP/CAM/tags/cam6_3_082 + + Refactor ice nucleation code to use generalized aerosol interfaces + + + Courtney Peverley 2022-10-27 - cam6_3_081 - components/cam (cesm2_3_alpha10c) + https://github.com/ESCOMP/CAM/tags/cam6_3_081 + + miscellaneous tag: + - waccmsc data reader fix + - interpolate_output fixes + - DAE test fix (first part of fix; test still FAILs) + - remove unnecessary addlfd call in convect_shallow + - fix CAM thermo indexing bug in get_dp + + Test DIFF for ERP_Ln9_Vnuopc.f19_f19_mg17.FWsc1850.cheyenne_intel.cam-outfrq9s due to WACCMSC fix. + + + Cheryl Craig 2022-10-14 - cam6_3_080 - components/cam (cesm2_3_alpha10c) + https://github.com/ESCOMP/CAM/tags/cam6_3_080 + + Bring in optional ALI-ARMS (an alternative method for computing non-LTE heating rates in upper atmosphere) + + + Cheryl Craig 2022-10-14 - cam6_3_079 - components/cam (cesm2_3_alpha10c) + https://github.com/ESCOMP/CAM/tags/cam6_3_079 + + Fix irreproducible results bug with variable resolution (fixed via external CAM updates) + + Non answer changing for CESM runs + + +ccs_config + Chris Fischer 2022-11-16 - ccs_config_cesm0.0.49 - ccs_config (cesm2_3_alpha10c) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm0.0.49 + + ESMF libraries on cheyenne updated to v8.4.0 release + + +cime + James Edwards 2022-06-15 - cime6.0.75 - cime (cesm2_3_alpha10c) + https://github.com/ESMCI/cime/tags/cime6.0.36 + + Reorder IO initialization in nuopc x components + + +clm + Erik Kluzek 2022-10-15 - ctsm5.1.dev112 - components/clm (cesm2_3_alpha10c) + https://github.com/ESCOMP/ctsm/tags/ctsm51.dev112 + + Update FATES + Add longer FATES tests, truncate some of the test names, make sure + there is a FatesSp test that just uses the compset + + + Erik Kluzek 2022-09-26 - ctsm5.1.dev109 - components/clm (cesm2_3_alpha10c) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1dev109 + + If not MIMICS, do not output certain MIMICS history fields + + (Field lists do differ -- but results are identical) + + + William Sacks 2022-10-28 - ctsm5.1.dev113 - components/clm (cesm2_3_alpha10c) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev113 + + Fix some compsets; add only clauses for ESMF use statements + + Changes answers for I1850Clm51BgcCrop compset and cases with DATM%CPLHIST + + SHAREDLIBBUILD will FAIL for gnu mct cases + + + William Sacks 2022-10-05 - ctsm5.1.dev111 - components/clm (cesm2_3_alpha10c) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev111 + + Fixes for NEON cases + + Just changes answers for NEON cases + + + Erik Kluzek 2022-09-26 - ctsm5.1.dev110 - components/clm (cesm2_3_alpha10c) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev110 + + Add python tool to modify mesh files. + + + William Sacks 2022-09-08 - ctsm5.1.dev108 - components/clm (cesm2_3_alpha10c) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev108 + + (From Adrianna Foster) Connect ozone from atmosphere to CTSM + + Changes answers for cases with ozone damage active in CTSM (none in prealpha / prebeta test suites) + + +cmeps + James Edwards 2022-06-15 - cmeps0.13.79 - src/drivers/nuopc/ (cesm2_3_alpha10c) + https://github.com/ESCOMP/CMEPS/tags/TBD + + ADD an IO initialization phase to the ensemble driver in preperation for asyncio capability. + https://github.com/ESCOMP/CMEPS/pull/305 + Depends on + https://github.com/ESCOMP/MOSART/pull/55 and https://github.com/ESCOMP/CICE/pull/18 + https://github.com/ESCOMP/WW3-CESM/pull/26 + + +mosart + James Edwards 2022-06-15 - mosart1_0_46 - components/mosart (cesm2_3_alpha10c) + https://github.com/ESCOMP/mosart/tags/TBD + + https://github.com/ESCOMP/MOSART/pull/55 + Reorder initialization to accommodate changes in cmeps for ayncio + + +ww3 + James Edwards 2022-06-15 - ww3_221108 - components/ww3 (cesm2_3_alpha10c) + https://github.com/ESCOMP/WW3-CESM/tags/TBD + + https://github.com/ESCOMP/WW3-CESM/pull/26 + + Reorders initialization to accommodate changes in IO initialization in cmeps + ============================================================== Tag name: cesm2_3_alpha010b Originator(s): CSEG From 83767a9684ac31d02831cde75d8d933b07f19e62 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 30 Nov 2022 16:50:35 -0700 Subject: [PATCH 05/64] Update for cesm2_3_alpha10d --- Externals.cfg | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index e2ba32a91..6b2cebefa 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -6,7 +6,7 @@ local_path = ccs_config required = True [cam] -tag = cam6_3_083 +tag = cam6_3_085 protocol = git repo_url = https://github.com/ESCOMP/CAM local_path = components/cam @@ -29,14 +29,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.13.79 +tag = cmeps0.14.0 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.65 +tag = cdeps0.12.67 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev113 +tag = ctsm5.1.dev114 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm @@ -103,7 +103,7 @@ externals = Externals_FMS.cfg required = False [mom] -tag = mi_220624 +tag = mi_221119 protocol = git repo_url = https://github.com/ESCOMP/MOM_interface local_path = components/mom @@ -111,7 +111,7 @@ externals = Externals.cfg required = False [mosart] -tag = mosart1_0_46 +tag = mosart1_0_47 protocol = git repo_url = https://github.com/ESCOMP/MOSART local_path = components/mosart @@ -147,7 +147,7 @@ local_path = components/ww3 required = True [ww3dev] -tag = main_0.0.4 +tag = main_0.0.5 protocol = git repo_url = https://github.com/ESCOMP/WW3_interface local_path = components/ww3dev From 0f2e66f3aeeaac4fb534f137cff4938312671efd Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 7 Dec 2022 10:55:34 -0700 Subject: [PATCH 06/64] Update for cesm2_3_alpha10d --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 6b2cebefa..38738395a 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -103,7 +103,7 @@ externals = Externals_FMS.cfg required = False [mom] -tag = mi_221119 +tag = mi_221206 protocol = git repo_url = https://github.com/ESCOMP/MOM_interface local_path = components/mom From 57176d2a6932e9f1ab75d9e4ec11e337bb3adcaa Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 7 Dec 2022 16:22:22 -0700 Subject: [PATCH 07/64] Update for cesm2_3_alpha10d --- ChangeLog | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/ChangeLog b/ChangeLog index 011b0832e..679859482 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,99 @@ ============================================================== +Tag name: cesm2_3_alpha010d +Originator(s): CSEG +Date: 7 December 2022 +One-line Summary: MOM changes + +components/cam https://github.com/ESCOMP/CAM/cam6_3_085 ** +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_2_0_34 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.75 -- +share https://github.com/ESCOMP/CESM_share/tree/share1.0.12 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.49 -- +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.0 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps0.12.67 ** +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev114 ** +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_20220425 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_221206 ** +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_47 ** +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 ** +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_9 -- + + +cam + Courtney Peverley 2022-11-28 - cam6_3_085 - components/cam (cesm2_3_alpha10d) + https://github.com/ESCOMP/CAM/tags/cam6_3_085 + + Update externals; fix PE layout issue + + + + + Courtney Peverley 2022-11-18 - cam6_3_084 - components/cam (cesm2_3_alpha10d) + https://github.com/ESCOMP/CAM/tags/cam6_3_084 + + 1. Fix secondary SE advection bug + 2. Increase max history field length from 24 to 32 + + Answer changes for SE tests due to bugfixes + + +cdeps + Chris Fischer 2022-11-28 - cdeps0.12.67 - components/cdeps (cesm2_3_alpha10d) + https://github.com/ESCOMP/CDEPS/tags/cdeps0.12.67 + + cdeps0.12.67: Fix 384x3 cam tests. + cdeps0.12.66: Fixes an issue in aux data streams + + +clm + Erik Kluzek 2022-11-19 - ctsm5.1.dev114 - components/clm (cesm2_3_alpha10d) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev114 + + Some NEON updates. Only NEON tests change answers. Some tools and testing updates + + +cmeps + Chris Fischer 2022-11-30 - cmeps0.14.0 - src/drivers/nuopc/ (cesm2_3_alpha10d) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.0 + + cmeps0.13.81/0.14.0: Make the med_fldList fields a linked list instead of an array. + cmeps0.13.80: Needed for using ESMF_AWARE_THREADING=TRUE. + + + +mom + Alper Altuntas 2022-11-15 - mi_221119 - components/mom (cesm2_3_alpha10d) + https://github.com/ESCOMP/MOM_interface/tags/mi_22011?? + + Latest changes from GFDL sync. + + + Chris Fischer 2022-12-06 - mi_221206 - components/mom (cesm2_3_alpha10d) + https://github.com/ESCOMP/MOM_interface/tags/ mi_221206 + + Fix debug output. + + +mosart + Erik Kluzek 2022-11-19 - mosart1_0_47 - components/mosart (cesm2_3_alpha10d) + https://github.com/ESCOMP/mosart/tags/mosart1_0_47 + + Fix some issues for direct_to_outlet mode. + + +ww3dev + Alper Altuntas 2022-11-15 - main_0.0.5 - components/ww3dev (cesm2_3_alpha10d) + https://github.com/ESCOMP/WW3_interface/tags/??? + + merge updates to esmf-weather-model +============================================================== Tag name: cesm2_3_alpha010c Originator(s): CSEG Date: 29 Novemberr 2022 From 6b5ea202dfa479f3c710b115188f4f2edbfe02a1 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 13 Dec 2022 16:25:27 -0700 Subject: [PATCH 08/64] Update for cesm2_3_beta10 --- ChangeLog | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/ChangeLog b/ChangeLog index 679859482..dbaac5fc1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,115 @@ +============================================================== +Tag name: cesm2_3_beta10 +Originator(s): CSEG +Date: 13 December 2022 +One-line Summary: clm,cam,mom and misc updates + +components/cam https://github.com/ESCOMP/CAM/cam6_3_085 +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_2_0_34 +cime https://github.com/ESMCI/cime/tree/cime6.0.75 +share https://github.com/ESCOMP/CESM_share/tree/share1.0.12 +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.49 +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.0 +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps0.12.67 +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev114 +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_20220425 +components/mom https://github.com/ESCOMP/MOM_interface/mi_221206 +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_47 +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_9 + +Answer Changes introduced with this tag when compared to cesm2_3_beta09: +CLM + Update to fates history names and machine configuration + answers for cases with an atmosphere / land model time step differing from 30 minutes. + answers for cases with winter wheat + answers for all Clm50 / Clm51 cases + answers for some Crop cases + answers for clm45 crop cases with C isotopes on + Changes answers for I1850Clm51BgcCrop compset and cases with DATM%CPLHIST + changes answers for NEON cases + answers for cases with ozone damage active in CTSM +CDEPS + coszen interpolation to work for CIAF and GIAF compsets +CAM + Answer changing for restarts when radiation is called on the restart time step + Roundoff differences for CCN diagnostics, otherwise BFB + Answer changing for WACCMX + Answer changing for CAMSE runs and select compsets which forced SSTICE namelist settings + Answer changing for runs using CLUBB (all CAM6) + Answer changing just for F2000dev compsets + Answer changing for FWsc1850 due to WACCMSC + Answer changing for SE. +MOM + Latest changes from GFDL sync. +WW3DEV + merge updates to esmf-weather-model + + +Problems identified after tag creation: +- All T31_g37_rx1.A tests fail with floating point exception. +- All T62_s11 nuopc tests fail because of missing mesh file. +- All ERS_N2 tests should be ERS_C2 tests. +Cheyenne intel + FAIL ERI_D.T31_g37_rx1.A.cheyenne_intel RUN time=48 + FAIL ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 MODEL_BUILD time=3 + FAIL ERS_IOP.T62_s11.G.cheyenne_intel.pop-cice SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_intel CREATE_NEWCASE + FAIL PET.T62_g16.G.cheyenne_intel.pop-cice COMPARE_base_single_thread + FAIL SMS_C80_P108x1_Lh1_Vnuopc.f09_f09_mg17.FHIST_DARTC6.cheyenne_intel.cam-dartcambigens RUN time=2427 + PEND ERP_D_Ln9_Vnuopc.C48_C48_mg17.QPC6.cheyenne_intel.cam-outfrq9s MODEL_BUILD RERUN + FAIL PET_PM.f19_g17.B1850.cheyenne_intel.allactive-defaultiomi COMPARE_base_single_thread + FAIL SMS_D_Ld1_PS.f09_g17.I1850Clm50BgcSpinup.cheyenne_intel.clm-cplhist RUN time=49 + PEND SMS_Ld1_P144_D.T62_g17.C.cheyenne_intel.pop-144blocks_320x384_spacecurve RUN + PEND SMS_Ld2_P80_D.T62_g37.C1850ECO.cheyenne_intel.pop-ecosys_81blocks_100x116_spacecurve RUN +Cheyenne gnu + FAIL ERI_D.T31_g37_rx1.A.cheyenne_gnu RUN time=29 + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_gnu CREATE_NEWCASE + FAIL IRT_C3_Ld7.f19_g17.BHIST.cheyenne_gnu.allactive-defaultio RUN time=2011 + FAIL PET.T62_g16.G.cheyenne_gnu.pop-cice COMPARE_base_single_thread + FAIL SMS_D.T62_s11.G.cheyenne_gnu.pop-cice SUBMIT + FAIL PET_PM.f19_g17.B1850.cheyenne_gnu.allactive-defaultiomi COMPARE_base_single_thread +Cheyenne nvhpc + FAIL ERI_D.T31_g37_rx1.A.cheyenne_nvhpc RUN time=18 + FAIL ERS_IOP.T62_s11.G.cheyenne_nvhpc.pop-cice SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_nvhpc CREATE_NEWCASE + FAIL ERS.T62_g16.C1850ECO.cheyenne_nvhpc.pop-ecosys RUN time=172 + FAIL PEA_P1_Ld3.f45_g37_rx1.A.cheyenne_nvhpc RUN time=93 + FAIL PEA_P1_Ld3.T31_g37.X.cheyenne_nvhpc RUN time=411 + FAIL PET.T62_g16.G.cheyenne_nvhpc.pop-cice RUN time=1111 + FAIL SMS_D.T62_s11.G.cheyenne_nvhpc.pop-cice RUN time=13 + FAIL ERS.T62_g16.G1850ECO.cheyenne_nvhpc.pop-cice_ecosys RUN time=177 + +Izumi intel + FAIL ERI.f19_g17_rx1.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default RUN time=30 + FAIL ERI_Vnuopc.f19_g17_rx1.2000_DATM%NYF_SLND_CICE_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice-default RUN time=45 + FAIL ERI_D.T31_g37_rx1.A.izumi_intel RUN time=45 + PEND ERS_D.f19_g16_rx1.G1850ECO.izumi_intel.pop-cice_ecosys RUN + FAIL ERS_Ld5.T62_s11.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.izumi_intel CREATE_NEWCASE + PEND ERS.T62_g16.C1850ECO.izumi_intel.pop-ecosys RUN + FAIL ERS.T62_g16.G.izumi_intel.pop-cice RUN time=112 + +Izumi nag + FAIL DAE.ww3a.ADWAV.izumi_nag RUN time=28 + PEND ERP_D_Ld5.ne30_g17.I1850Clm50BgcCrop.izumi_nag.clm-default RUN + FAIL ERS_Ld5.f19_g17.2000_CAM60_CLM50%SP_CICE5_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice5-default SHAREDLIB_BUILD time=50 + FAIL ERS_Ld5_Vnuopc.f19_g17.2000_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice-default SHAREDLIB_BUILD time=12 + FAIL PRE.f19_f19.ADESP_TEST.izumi_nag CREATE_NEWCASE + FAIL SMS_Ld1_P136_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN time=259 + PEND SMS_Ld1_P144_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN + FAIL SMS_Ld2_P72_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN time=59 + PEND SMS_Ld2_P80_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN + FAIL SMS.T62_g37.C1850ECO.izumi_nag.pop-ecosys RUN time=74 + FAIL ERS.T62_s11.G.izumi_nag.pop-cice SUBMIT + ============================================================== Tag name: cesm2_3_alpha010d Originator(s): CSEG From 3257b3d87bcb9a1c2330ddda50d158d679faa4d1 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 20 Dec 2022 10:56:16 -0700 Subject: [PATCH 09/64] Update for cesm2_3_alpha11a --- Externals.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 38738395a..de114cbf0 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.0 +tag = cmeps0.14.5 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -51,7 +51,7 @@ local_path = components/cpl7 required = True [share] -tag = share1.0.13 +tag = share1.0.15 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.75 +tag = cime6.0.82 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime From 3fdcac4a7833e78670eba7372e3b92dcbb6eceef Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Thu, 22 Dec 2022 12:33:38 -0700 Subject: [PATCH 10/64] Update for cesm2_3_alpha11a --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index de114cbf0..762d001c4 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.5 +tag = cmeps0.14.2 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -51,7 +51,7 @@ local_path = components/cpl7 required = True [share] -tag = share1.0.15 +tag = share1.0.16 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share From 9817d6f562a998c68d8f67e154bb753c34b1d975 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 23 Dec 2022 10:47:57 -0700 Subject: [PATCH 11/64] Update for cesm2_3_alpha11a --- ChangeLog | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/ChangeLog b/ChangeLog index dbaac5fc1..b1cd8d5c2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,65 @@ +============================================================== +Tag name: cesm2_3_alpha011a +Originator(s): CSEG +Date: 23 December 2022 +One-line Summary: NUOPC restart fix. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_085 -- +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_2_0_34 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.82 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 ** +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.49 -- +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.2 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps0.12.67 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev114 -- +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_20220425 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_221206 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_47 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_9 -- + +cime + Chris Fischer 2022-12-19 - cime6.0.82 - cime (cesm2_3_alpha11a) + https://github.com/ESMCI/cime/tags/cime6.0.?? + + cime6.0.82: Handle time interval issue, fix submit arguments + cime6.0.81: Update's CIME to be more flexible when using aprun. + cime6.0.80: Update doc build workflow for forks. + cime6.0.79: Enables pio asyncio in cmeps. + cime6.0.78: Add ww3dev testlist file in config_files.xml. + cime6.0.77: Improve float detection in argument resolution of the batch environment + cime6.0.76: Use the jobid_pattern xml variable in determining jobid. + + +cmeps + Chris Fischer 2022-12-19 - cmeps0.14.2 - src/drivers/nuopc/ (cesm2_3_alpha11a) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.?? + + cmeps0.14.2: Move shr_file_getLogUnit to shr_log_getLogUnit. Depends on ESCOMP/CESM_share#36 + cmeps0.14.1: Update flds_exchange_nems for wave coupling changes. + + +share + Chris Fischer 2022-12-22 - share1.0.16 - share (cesm2_3_alpha11a) + https://github.com/ESCOMP/CESM_share/tags/share1.0.16 + + asyncio + + + Chris Fischer 2022-12-20 - share1.0.15 - share (cesm2_3_alpha11a) + https://github.com/ESCOMP/CESM_share/tags/share1.0.15 + + Move shr_file_setlogunit to shr_log_setlogunit + + + ============================================================== Tag name: cesm2_3_beta10 Originator(s): CSEG From 755f1ad718ae450aa00d4295333e3d6843ea8cb6 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 23 Dec 2022 12:06:09 -0700 Subject: [PATCH 12/64] Update for cesm2_3_beta11 --- ChangeLog | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/ChangeLog b/ChangeLog index b1cd8d5c2..9a98ccdde 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,91 @@ +============================================================== +Tag name: cesm2_3_beta11 +Originator(s): CSEG +Date: 23 December 2022 +One-line Summary: NUOPC restart fix. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_085 +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_2_0_34 +cime https://github.com/ESMCI/cime/tree/cime6.0.82 +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.49 +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.2 +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps0.12.67 +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev114 +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_20220425 +components/mom https://github.com/ESCOMP/MOM_interface/mi_221206 +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_47 +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_9 + +Answer Changes introduced with this tag when compared to cesm2_3_beta10: + NONE + +Problems identified after tag creation: +- Multi-instance init is being done serially. +- All T31_g37_rx1.A tests fail with floating point exception. +- All T62_s11 nuopc tests fail because of missing mesh file. +- All ERS_N2 tests should be ERS_C2 tests. +Cheyenne intel + FAIL ERI_D.T31_g37_rx1.A.cheyenne_intel RUN time=48 + FAIL ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 MODEL_BUILD time=3 + FAIL ERS_IOP.T62_s11.G.cheyenne_intel.pop-cice SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_intel CREATE_NEWCASE + FAIL PET.T62_g16.G.cheyenne_intel.pop-cice COMPARE_base_single_thread + FAIL SMS_C80_P108x1_Lh1_Vnuopc.f09_f09_mg17.FHIST_DARTC6.cheyenne_intel.cam-dartcambigens RUN time=2427 + PEND ERP_D_Ln9_Vnuopc.C48_C48_mg17.QPC6.cheyenne_intel.cam-outfrq9s MODEL_BUILD RERUN + FAIL PET_PM.f19_g17.B1850.cheyenne_intel.allactive-defaultiomi COMPARE_base_single_thread + FAIL SMS_D_Ld1_PS.f09_g17.I1850Clm50BgcSpinup.cheyenne_intel.clm-cplhist RUN time=49 + PEND SMS_Ld1_P144_D.T62_g17.C.cheyenne_intel.pop-144blocks_320x384_spacecurve RUN + PEND SMS_Ld2_P80_D.T62_g37.C1850ECO.cheyenne_intel.pop-ecosys_81blocks_100x116_spacecurve RUN +Cheyenne gnu + FAIL ERI_D.T31_g37_rx1.A.cheyenne_gnu RUN time=29 + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_gnu CREATE_NEWCASE + FAIL IRT_C3_Ld7.f19_g17.BHIST.cheyenne_gnu.allactive-defaultio RUN time=2011 + FAIL PET.T62_g16.G.cheyenne_gnu.pop-cice COMPARE_base_single_thread + FAIL SMS_D.T62_s11.G.cheyenne_gnu.pop-cice SUBMIT + FAIL PET_PM.f19_g17.B1850.cheyenne_gnu.allactive-defaultiomi COMPARE_base_single_thread +Cheyenne nvhpc + FAIL ERI_D.T31_g37_rx1.A.cheyenne_nvhpc RUN time=18 + FAIL ERS_IOP.T62_s11.G.cheyenne_nvhpc.pop-cice SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_nvhpc CREATE_NEWCASE + FAIL ERS.T62_g16.C1850ECO.cheyenne_nvhpc.pop-ecosys RUN time=172 + FAIL PEA_P1_Ld3.f45_g37_rx1.A.cheyenne_nvhpc RUN time=93 + FAIL PEA_P1_Ld3.T31_g37.X.cheyenne_nvhpc RUN time=411 + FAIL PET.T62_g16.G.cheyenne_nvhpc.pop-cice RUN time=1111 + FAIL SMS_D.T62_s11.G.cheyenne_nvhpc.pop-cice RUN time=13 + FAIL ERS.T62_g16.G1850ECO.cheyenne_nvhpc.pop-cice_ecosys RUN time=177 + +Izumi intel + FAIL ERI.f19_g17_rx1.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default RUN time=30 + FAIL ERI_Vnuopc.f19_g17_rx1.2000_DATM%NYF_SLND_CICE_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice-default RUN time=45 + FAIL ERI_D.T31_g37_rx1.A.izumi_intel RUN time=45 + PEND ERS_D.f19_g16_rx1.G1850ECO.izumi_intel.pop-cice_ecosys RUN + FAIL ERS_Ld5.T62_s11.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.izumi_intel CREATE_NEWCASE + PEND ERS.T62_g16.C1850ECO.izumi_intel.pop-ecosys RUN + FAIL ERS.T62_g16.G.izumi_intel.pop-cice RUN time=112 + +Izumi nag + FAIL DAE.ww3a.ADWAV.izumi_nag RUN time=28 + PEND ERP_D_Ld5.ne30_g17.I1850Clm50BgcCrop.izumi_nag.clm-default RUN + FAIL ERS_Ld5.f19_g17.2000_CAM60_CLM50%SP_CICE5_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice5-default SHAREDLIB_BUILD time=50 + FAIL ERS_Ld5_Vnuopc.f19_g17.2000_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice-default SHAREDLIB_BUILD time=12 + FAIL PRE.f19_f19.ADESP_TEST.izumi_nag CREATE_NEWCASE + FAIL SMS_Ld1_P136_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN time=259 + PEND SMS_Ld1_P144_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN + FAIL SMS_Ld2_P72_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN time=59 + PEND SMS_Ld2_P80_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN + FAIL SMS.T62_g37.C1850ECO.izumi_nag.pop-ecosys RUN time=74 + FAIL ERS.T62_s11.G.izumi_nag.pop-cice SUBMIT + ============================================================== Tag name: cesm2_3_alpha011a Originator(s): CSEG From 16eb854f9414238c240b69a94358f327135d585c Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 6 Jan 2023 15:08:46 -0700 Subject: [PATCH 13/64] update to manic-v1.2.0 --- manage_externals/README.md | 5 + manage_externals/checkout_externals | 2 +- manage_externals/manic/checkout.py | 87 +++--- .../manic/externals_description.py | 24 +- manage_externals/manic/externals_status.py | 30 +-- manage_externals/manic/repository_factory.py | 1 + manage_externals/manic/repository_git.py | 29 +- manage_externals/manic/repository_svn.py | 9 +- manage_externals/manic/sourcetree.py | 253 +++++++++++------- manage_externals/manic/utils.py | 2 +- manage_externals/test/test_sys_checkout.py | 2 +- .../test/test_sys_repository_git.py | 2 +- .../test/test_unit_externals_description.py | 2 +- .../test/test_unit_externals_status.py | 2 +- manage_externals/test/test_unit_repository.py | 2 +- .../test/test_unit_repository_git.py | 4 +- .../test/test_unit_repository_svn.py | 4 +- manage_externals/test/test_unit_utils.py | 2 +- 18 files changed, 271 insertions(+), 191 deletions(-) mode change 100644 => 100755 manage_externals/test/test_unit_repository_svn.py diff --git a/manage_externals/README.md b/manage_externals/README.md index c931c8e21..9475301b5 100644 --- a/manage_externals/README.md +++ b/manage_externals/README.md @@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. externals description file pointed 'useful_library/sub-xternals.cfg', Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + Note that by default, `checkout_externals` will clone an external's + submodules. As a special case, the entry, `externals = None`, will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the `from_submodule` + configuration entry below). * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in diff --git a/manage_externals/checkout_externals b/manage_externals/checkout_externals index a0698baef..48bce2401 100755 --- a/manage_externals/checkout_externals +++ b/manage_externals/checkout_externals @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Main driver wrapper around the manic/checkout utility. diff --git a/manage_externals/manic/checkout.py b/manage_externals/manic/checkout.py index 8dd1798d7..dff11b661 100755 --- a/manage_externals/manic/checkout.py +++ b/manage_externals/manic/checkout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Tool to assemble repositories represented in a model-description file. @@ -227,6 +227,12 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. + Note that by default, checkout_externals will clone an external's + submodules. As a special case, the entry, "externals = None", will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the from_submodule + configuration entry below). + * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in this repository. Note that the section name (the entry in square @@ -332,7 +338,34 @@ def commandline_arguments(args=None): options = parser.parse_args() return options - +def _dirty_local_repo_msg(program_name, config_file): + return """The external repositories labeled with 'M' above are not in a clean state. +The following are four options for how to proceed: +(1) Go into each external that is not in a clean state and issue either a 'git status' or + an 'svn status' command (depending on whether the external is managed by git or + svn). Either revert or commit your changes so that all externals are in a clean + state. (To revert changes in git, follow the instructions given when you run 'git + status'.) (Note, though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. +(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually + update out-of-sync externals (labeled with 's' above) as described in the + configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' + commands to checkout the appropriate tags for each external, as given in + {config_file}.) +(3) You can also use {program_name} to manage most, but not all externals: You can specify + one or more externals to ignore using the '-x' or '--exclude' argument to + {program_name}. Excluding externals labeled with 'M' will allow {program_name} to + update the other, non-excluded externals. +(4) As a last resort, if you are confident that there is no work that needs to be saved + from a given external, you can remove that external (via "rm -rf [directory]") and + then rerun the {program_name} tool. This option is mainly useful as a workaround for + issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before rerunning the +{program_name} tool. +""".format(program_name=program_name, config_file=config_file) # --------------------------------------------------------------------- # # main @@ -363,19 +396,23 @@ def main(args): load_all = True root_dir = os.path.abspath(os.getcwd()) - external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description( - external_data, components=args.components, exclude=args.exclude) + model_data = read_externals_description_file(root_dir, args.externals) + ext_description = create_externals_description( + model_data, components=args.components, exclude=args.exclude) for comp in args.components: - if comp not in external.keys(): + if comp not in ext_description.keys(): fatal_error( "No component {} found in {}".format( comp, args.externals)) - source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) - printlog('Checking status of externals: ', end='') - tree_status = source_tree.status() + source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry) + if args.components: + components_str = 'specified components' + else: + components_str = 'required & optional components' + printlog('Checking local status of ' + components_str + ': ', end='') + tree_status = source_tree.status(print_progress=True) printlog('') if args.status: @@ -390,38 +427,8 @@ def main(args): for comp in sorted(tree_status): tree_status[comp].log_status_message(args.verbose) # exit gracefully - msg = """The external repositories labeled with 'M' above are not in a clean state. - -The following are three options for how to proceed: - -(1) Go into each external that is not in a clean state and issue either a 'git status' or - an 'svn status' command (depending on whether the external is managed by git or - svn). Either revert or commit your changes so that all externals are in a clean - state. (To revert changes in git, follow the instructions given when you run 'git - status'.) (Note, though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. - -(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually - update out-of-sync externals (labeled with 's' above) as described in the - configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' - commands to checkout the appropriate tags for each external, as given in - {config_file}.) - -(3) You can also use {program_name} to manage most, but not all externals: You can specify - one or more externals to ignore using the '-x' or '--exclude' argument to - {program_name}. Excluding externals labeled with 'M' will allow {program_name} to - update the other, non-excluded externals. - - -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before re-running the -checkout_externals tool. -""".format(program_name=program_name, config_file=args.externals) - printlog('-' * 70) - printlog(msg) + printlog(_dirty_local_repo_msg(program_name, args.externals)) printlog('-' * 70) else: if not args.components: diff --git a/manage_externals/manic/externals_description.py b/manage_externals/manic/externals_description.py index 918d616e3..f5615b673 100644 --- a/manage_externals/manic/externals_description.py +++ b/manage_externals/manic/externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Model description @@ -71,7 +71,8 @@ def read_externals_description_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing externals description file : {0}'.format(file_name)) + printlog('Processing externals description file : {0} ({1})'.format(file_name, + root_dir)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): @@ -193,6 +194,9 @@ def parse_submodules_desc_section(section_items, file_path): def read_gitmodules_file(root_dir, file_name): # pylint: disable=deprecated-method # Disabling this check because the method is only used for python2 + # pylint: disable=too-many-locals + # pylint: disable=too-many-branches + # pylint: disable=too-many-statements """Read a .gitmodules file and convert it to be compatible with an externals description. """ @@ -278,6 +282,10 @@ def read_gitmodules_file(root_dir, file_name): def create_externals_description( model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data + + components: list of component names to include, None to include all. If a + name isn't found, it is silently omitted from the return value. + exclude: list of component names to skip. """ externals_description = None if model_format == 'dict': @@ -354,8 +362,9 @@ class ExternalsDescription(dict): input value. """ - # keywords defining the interface into the externals description data - EXTERNALS = 'externals' + # keywords defining the interface into the externals description data; these + # are brought together by the schema below. + EXTERNALS = 'externals' # path to externals file. BRANCH = 'branch' SUBMODULE = 'from_submodule' HASH = 'hash' @@ -381,6 +390,8 @@ class ExternalsDescription(dict): _V1_BRANCH = 'BRANCH' _V1_REQ_SOURCE = 'REQ_SOURCE' + # Dictionary keys are component names. The corresponding values are laid out + # according to this schema. _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', @@ -757,6 +768,8 @@ def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 @@ -780,6 +793,9 @@ def _remove_metadata(model_data): def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. + + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ def list_to_dict(input_list, convert_to_lower_case=True): """Convert a list of key-value pairs into a dictionary. diff --git a/manage_externals/manic/externals_status.py b/manage_externals/manic/externals_status.py index d3d238f28..4900e4125 100644 --- a/manage_externals/manic/externals_status.py +++ b/manage_externals/manic/externals_status.py @@ -29,16 +29,16 @@ class ExternalStatus(object): transactions (e.g. add, remove, rename, untracked files). """ - DEFAULT = '-' + # sync_state and clean_state can be one of the following: + DEFAULT = '-' # aka not set yet. UNKNOWN = '?' EMPTY = 'e' - MODEL_MODIFIED = 's' # a.k.a. out-of-sync - DIRTY = 'M' - - STATUS_OK = ' ' + MODEL_MODIFIED = 's' # repo version != externals (sync_state only) + DIRTY = 'M' # repo is dirty (clean_state only) + STATUS_OK = ' ' # repo is clean/matches externals. STATUS_ERROR = '!' - # source types + # source_type can be one of the following: OPTIONAL = 'o' STANDALONE = 's' MANAGED = ' ' @@ -55,19 +55,21 @@ def __init__(self): def log_status_message(self, verbosity): """Write status message to the screen and log file """ - self._default_status_message() + printlog(self._default_status_message()) if verbosity >= VERBOSITY_VERBOSE: - self._verbose_status_message() + printlog(self._verbose_status_message()) if verbosity >= VERBOSITY_DUMP: - self._dump_status_message() + printlog(self._dump_status_message()) + + def __repr__(self): + return self._default_status_message() def _default_status_message(self): """Return the default terse status message string """ - msg = '{sync}{clean}{src_type} {path}'.format( + return '{sync}{clean}{src_type} {path}'.format( sync=self.sync_state, clean=self.clean_state, src_type=self.source_type, path=self.path) - printlog(msg) def _verbose_status_message(self): """Return the verbose status message string @@ -82,14 +84,12 @@ def _verbose_status_message(self): if self.sync_state != self.STATUS_OK: sync_str = '{current} --> {expected}'.format( current=self.current_version, expected=self.expected_version) - msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) - printlog(msg) + return ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) def _dump_status_message(self): """Return the dump status message string """ - msg = indent_string(self.status_output, 12) - printlog(msg) + return indent_string(self.status_output, 12) def safe_to_update(self): """Report if it is safe to update a repository. Safe is defined as: diff --git a/manage_externals/manic/repository_factory.py b/manage_externals/manic/repository_factory.py index 80a92a9d8..18c73ffc4 100644 --- a/manage_externals/manic/repository_factory.py +++ b/manage_externals/manic/repository_factory.py @@ -15,6 +15,7 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False): """Determine what type of repository we have, i.e. git or svn, and create the appropriate object. + Can return None (e.g. if protocol is 'externals_only'). """ protocol = repo_info[ExternalsDescription.PROTOCOL].lower() if protocol == 'git': diff --git a/manage_externals/manic/repository_git.py b/manage_externals/manic/repository_git.py index f98605100..3a6a0f171 100644 --- a/manage_externals/manic/repository_git.py +++ b/manage_externals/manic/repository_git.py @@ -109,27 +109,21 @@ def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): def _current_ref(self): """Determine the *name* associated with HEAD. - If we're on a branch, then returns the branch name; otherwise, - if we're on a tag, then returns the tag name; otherwise, returns + If we're on a tag, then returns the tag name; otherwise, returns the current hash. Returns an empty string if no reference can be determined (e.g., if we're not actually in a git repository). + + If we're on a branch, then the branch name is also included in + the returned string (in addition to the tag / hash). """ ref_found = False - # If we're on a branch, then use that as the current ref - branch_found, branch_name = self._git_current_branch() - if branch_found: - current_ref = branch_name + # If we're exactly at a tag, use that as the current ref + tag_found, tag_name = self._git_current_tag() + if tag_found: + current_ref = tag_name ref_found = True - if not ref_found: - # Otherwise, if we're exactly at a tag, use that as the - # current ref - tag_found, tag_name = self._git_current_tag() - if tag_found: - current_ref = tag_name - ref_found = True - if not ref_found: # Otherwise, use current hash as the current ref hash_found, hash_name = self._git_current_hash() @@ -137,7 +131,12 @@ def _current_ref(self): current_ref = hash_name ref_found = True - if not ref_found: + if ref_found: + # If we're on a branch, include branch name in current ref + branch_found, branch_name = self._git_current_branch() + if branch_found: + current_ref = "{} (branch {})".format(current_ref, branch_name) + else: # If we still can't find a ref, return empty string. This # can happen if we're not actually in a git repo current_ref = '' diff --git a/manage_externals/manic/repository_svn.py b/manage_externals/manic/repository_svn.py index 408ed8467..922855d34 100644 --- a/manage_externals/manic/repository_svn.py +++ b/manage_externals/manic/repository_svn.py @@ -43,10 +43,15 @@ def __init__(self, component_name, repo, ignore_ancestry=False): """ Repository.__init__(self, component_name, repo) self._ignore_ancestry = ignore_ancestry + if self._url.endswith('/'): + # there is already a '/' separator in the URL; no need to add another + url_sep = '' + else: + url_sep = '/' if self._branch: - self._url = os.path.join(self._url, self._branch) + self._url = self._url + url_sep + self._branch elif self._tag: - self._url = os.path.join(self._url, self._tag) + self._url = self._url + url_sep + self._tag else: msg = "DEV_ERROR in svn repository. Shouldn't be here!" fatal_error(msg) diff --git a/manage_externals/manic/sourcetree.py b/manage_externals/manic/sourcetree.py index 54de763c3..61e7a6c72 100644 --- a/manage_externals/manic/sourcetree.py +++ b/manage_externals/manic/sourcetree.py @@ -19,7 +19,7 @@ class _External(object): """ - _External represents an external object inside a SourceTree + A single component hosted in an external repository (and any children). """ # pylint: disable=R0902 @@ -41,31 +41,40 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): """ self._name = name - self._repo = None - self._externals = EMPTY_STR + self._repo = None # Repository object. + + # Subcomponent externals file and data object, if any. + self._externals_path = EMPTY_STR # Can also be "none" self._externals_sourcetree = None - self._stat = ExternalStatus() + + self._stat = None # Populated in status() self._sparse = None # Parse the sub-elements - # _path : local path relative to the containing source tree + # _local_path : local path relative to the containing source tree, e.g. + # "components/mom" self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir : full repository directory + # _repo_dir_path : full repository directory, e.g. + # "/components/mom" repo_dir = os.path.join(root_dir, self._local_path) self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir : base directory *containing* the repository + # _base_dir_path : base directory *containing* the repository, e.g. + # "/components" self._base_dir_path = os.path.dirname(self._repo_dir_path) - # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # _repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # e.g., "mom" self._repo_dir_name = os.path.basename(self._repo_dir_path) assert(os.path.join(self._base_dir_path, self._repo_dir_name) == self._repo_dir_path) self._required = ext_description[ExternalsDescription.REQUIRED] - self._externals = ext_description[ExternalsDescription.EXTERNALS] + + # Does this component have subcomponents aka an externals config? + self._externals_path = ext_description[ExternalsDescription.EXTERNALS] # Treat a .gitmodules file as a backup externals config - if not self._externals: + if not self._externals_path: if GitRepository.has_submodules(self._repo_dir_path): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME repo = create_repository( name, ext_description[ExternalsDescription.REPO], @@ -73,7 +82,8 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): if repo: self._repo = repo - if self._externals and (self._externals.lower() != 'none'): + # Recurse into subcomponents, if any. + if self._externals_path and (self._externals_path.lower() != 'none'): self._create_externals_sourcetree() def get_name(self): @@ -88,81 +98,86 @@ def get_local_path(self): """ return self._local_path - def status(self): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. + def status(self, force=False): """ + Returns status of this component and all subcomponents (if available). - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_externals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_externals - self._stat.source_type = ExternalStatus.MANAGED + Returns a dict mapping our local path to an ExternalStatus dict. Any + subcomponents will have their own top-level key. - ext_stats = {} + Side-effect: If self._stat is empty or force is True, calculates _stat. + """ + calc_stat = force or not self._stat + + if calc_stat: + self._stat = ExternalStatus() + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_subexternals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_subexternals + self._stat.source_type = ExternalStatus.MANAGED + subcomponent_stats = {} if not os.path.exists(self._repo_dir_path): - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() + if calc_stat: + # No local repository. + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() else: - if self._repo: + # Merge local repository state (e.g. clean/dirty) into self._stat. + if calc_stat and self._repo: self._repo.status(self._stat, self._repo_dir_path) - if self._externals and self._externals_sourcetree: - # we expect externals and they exist + # Status of subcomponents, if any. + if self._externals_path and self._externals_sourcetree: cwd = os.getcwd() - # SourceTree expects to be called from the correct + # SourceTree.status() expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - ext_stats = self._externals_sourcetree.status(self._local_path) + subcomponent_stats = self._externals_sourcetree.status(self._local_path, force=force) os.chdir(cwd) + # Merge our status + subcomponent statuses into one return dict keyed + # by component path. all_stats = {} # don't add the root component because we don't manage it # and can't provide useful info about it. if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under tha local_path, not comp name so + # store the stats under the local_path, not comp name so # it will be sorted correctly all_stats[self._stat.path] = self._stat - if ext_stats: - all_stats.update(ext_stats) + if subcomponent_stats: + all_stats.update(subcomponent_stats) return all_stats - def checkout(self, verbosity, load_all): + def checkout(self, verbosity): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the external. If the repo destination directory does not exist, checkout the correct branch or tag. - If load_all is True, also load all of the the externals sub-externals. + Does not check out sub-externals, see checkout_subexternals(). """ - if load_all: - pass # Make sure we are in correct location - if not os.path.exists(self._repo_dir_path): # repository directory doesn't exist. Need to check it # out, and for that we need the base_dir_path to exist @@ -174,6 +189,10 @@ def checkout(self, verbosity, load_all): self._base_dir_path) fatal_error(msg) + if not self._stat: + self.status() + assert self._stat + if self._stat.source_type != ExternalStatus.STANDALONE: if verbosity >= VERBOSITY_VERBOSE: # NOTE(bja, 2018-01) probably do not want to pass @@ -194,8 +213,10 @@ def checkout(self, verbosity, load_all): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_externals(self, verbosity, load_all): - """Checkout the sub-externals for this object + def checkout_subexternals(self, verbosity, load_all): + """Recursively checkout the sub-externals for this component, if any. + + See load_all documentation in SourceTree.checkout(). """ if self.load_externals(): if self._externals_sourcetree: @@ -210,25 +231,26 @@ def checkout_externals(self, verbosity, load_all): self._externals_sourcetree.checkout(verbosity, load_all) def load_externals(self): - 'Return True iff an externals file should be loaded' + 'Return True iff an externals file exists (and therefore should be loaded)' load_ex = False if os.path.exists(self._repo_dir_path): - if self._externals: - if self._externals.lower() != 'none': + if self._externals_path: + if self._externals_path.lower() != 'none': load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals)) + self._externals_path)) return load_ex def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' - # Try recursive unless there is an externals entry - recursive = not self._externals + # Try recursive .gitmodules unless there is an externals entry + recursive = not self._externals_path return recursive def _create_externals_sourcetree(self): """ + Note this only creates an object, it doesn't write to the file system. """ if not os.path.exists(self._repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out @@ -239,29 +261,31 @@ def _create_externals_sourcetree(self): cwd = os.getcwd() os.chdir(self._repo_dir_path) - if self._externals.lower() == 'none': + if self._externals_path.lower() == 'none': msg = ('Internal: Attempt to create source tree for ' 'externals = none in {}'.format(self._repo_dir_path)) fatal_error(msg) - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): if GitRepository.has_submodules(): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): # NOTE(bja, 2017-10) this check is redundent with the one # in read_externals_description_file! msg = ('External externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals, self._repo_dir_path)) + self._externals_path, self._repo_dir_path)) fatal_error(msg) externals_root = self._repo_dir_path + # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals) - externals = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, externals) + self._externals_path) + # ext_description is another dict-like object (see ExternalsDescription) + ext_description = create_externals_description(model_data, + parent_repo=self._repo) + self._externals_sourcetree = SourceTree(externals_root, ext_description) os.chdir(cwd) class SourceTree(object): @@ -269,45 +293,45 @@ class SourceTree(object): SourceTree represents a group of managed externals """ - def __init__(self, root_dir, model, svn_ignore_ancestry=False): + def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): """ - Build a SourceTree object from a model description + Build a SourceTree object from an ExternalDescription. """ self._root_dir = os.path.abspath(root_dir) - self._all_components = {} + self._all_components = {} # component_name -> _External self._required_compnames = [] - for comp in model: - src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) + for comp in ext_description: + src = _External(self._root_dir, comp, ext_description[comp], + svn_ignore_ancestry) self._all_components[comp] = src - if model[comp][ExternalsDescription.REQUIRED]: + if ext_description[comp][ExternalsDescription.REQUIRED]: self._required_compnames.append(comp) - def status(self, relative_path_base=LOCAL_PATH_INDICATOR): - """Report the status components + def status(self, relative_path_base=LOCAL_PATH_INDICATOR, + force=False, print_progress=False): + """Return a dictionary of local path->ExternalStatus. - FIXME(bja, 2017-10) what do we do about situations where the - user checked out the optional components, but didn't add - optional for running status? What do we do where the user - didn't add optional to the checkout but did add it to the - status. -- For now, we run status on all components, and try - to do the right thing based on the results.... + Note that all traversed components, whether recursive or top-level, have + a top-level key in the returned dictionary. + Note that all components that are checked out locally, whether required or + optional, ar included in the returned status. """ load_comps = self._all_components.keys() - summary = {} + summary = {} # Holds merged statuses from all components. for comp in load_comps: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status() + if print_progress: + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status(force=force) + + # Returned status dictionary is keyed by local path; prepend + # relative_path_base if not already there. stat_final = {} for name in stat.keys(): - # check if we need to append the relative_path_base to - # the path so it will be sorted in the correct order. if stat[name].path.startswith(relative_path_base): - # use as is, without any changes to path stat_final[name] = stat[name] else: - # append relative_path_base to path and store under key = updated path modified_path = os.path.join(relative_path_base, stat[name].path) stat_final[modified_path] = stat[name] @@ -316,15 +340,36 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): return summary + def _find_installed_optional_components(self): + """Returns a list of installed optional component names, if any.""" + installed_comps = set() + for comp_name, ext in self._all_components.items(): + if comp_name in self._required_compnames: + continue + # Note that in practice we expect this status to be cached. + stat = ext.status() + installed_comps.update(stat.keys()) + return list(installed_comps) + def checkout(self, verbosity, load_all, load_comp=None): """ - Checkout or update indicated components into the the configured - subdirs. + Checkout or update indicated components into the configured subdirs. - If load_all is True, recursively checkout all externals. - If load_all is False, load_comp is an optional set of components to load. - If load_all is True and load_comp is None, only load the required externals. + If load_all is True, checkout all externals (required + optional), recursively. + If load_all is False and load_comp is set, checkout load_comp (and any required subexternals, plus any optional subexternals that are already checked out, recursively) + If load_all is False and load_comp is None, checkout all required externals, plus any optionals that are already checked out, recursively. """ + if load_all: + tmp_comps = self._all_components.keys() + elif load_comp is not None: + tmp_comps = [load_comp] + else: + local_optional_compnames = self._find_installed_optional_components() + tmp_comps = self._required_compnames + local_optional_compnames + if local_optional_compnames: + printlog('Found locally installed optional components: ' + + ', '.join(local_optional_compnames)) + if verbosity >= VERBOSITY_VERBOSE: printlog('Checking out externals: ') else: @@ -347,7 +392,9 @@ def checkout(self, verbosity, load_all, load_comp=None): # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) - self._all_components[comp].checkout(verbosity, load_all) - # now give each external an opportunitity to checkout it's externals. - self._all_components[comp].checkout_externals(verbosity, load_all) + # Does not recurse. + self._all_components[comp].checkout(verbosity) + # Recursively check out subexternals, if any. + self._all_components[comp].checkout_subexternals(verbosity, + load_all) printlog('') diff --git a/manage_externals/manic/utils.py b/manage_externals/manic/utils.py index f57f43930..9c63ffe65 100644 --- a/manage_externals/manic/utils.py +++ b/manage_externals/manic/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Common public utilities for manic package diff --git a/manage_externals/test/test_sys_checkout.py b/manage_externals/test/test_sys_checkout.py index 118bee530..176228050 100644 --- a/manage_externals/test/test_sys_checkout.py +++ b/manage_externals/test/test_sys_checkout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_sys_repository_git.py b/manage_externals/test/test_sys_repository_git.py index f6dbf8428..29d5433b9 100644 --- a/manage_externals/test/test_sys_repository_git.py +++ b/manage_externals/test/test_sys_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Tests of some of the functionality in repository_git.py that actually interacts with git repositories. diff --git a/manage_externals/test/test_unit_externals_description.py b/manage_externals/test/test_unit_externals_description.py index 0b1248f67..30e528849 100644 --- a/manage_externals/test/test_unit_externals_description.py +++ b/manage_externals/test/test_unit_externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_unit_externals_status.py b/manage_externals/test/test_unit_externals_status.py index f8e953f75..f019514e9 100644 --- a/manage_externals/test/test_unit_externals_status.py +++ b/manage_externals/test/test_unit_externals_status.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for the manic external status reporting module. diff --git a/manage_externals/test/test_unit_repository.py b/manage_externals/test/test_unit_repository.py index 5b9c242fd..1b9386183 100644 --- a/manage_externals/test/test_unit_repository.py +++ b/manage_externals/test/test_unit_repository.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_unit_repository_git.py b/manage_externals/test/test_unit_repository_git.py index 4a0a334bb..a6ad9f100 100644 --- a/manage_externals/test/test_unit_repository_git.py +++ b/manage_externals/test/test_unit_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals @@ -101,7 +101,7 @@ def test_ref_branch(self): True, 'feature3') self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') - expected = 'feature3' + expected = 'foo_tag (branch feature3)' result = self._repo._current_ref() self.assertEqual(result, expected) diff --git a/manage_externals/test/test_unit_repository_svn.py b/manage_externals/test/test_unit_repository_svn.py old mode 100644 new mode 100755 index 7ff31c421..d9309df7f --- a/manage_externals/test/test_unit_repository_svn.py +++ b/manage_externals/test/test_unit_repository_svn.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals @@ -60,7 +60,7 @@ def setUp(self): self._name = 'component' rdata = {ExternalsDescription.PROTOCOL: 'svn', ExternalsDescription.REPO_URL: - 'https://svn-ccsm-models.cgd.ucar.edu/', + 'https://svn-ccsm-models.cgd.ucar.edu', ExternalsDescription.TAG: 'mosart/trunk_tags/mosart1_0_26', } diff --git a/manage_externals/test/test_unit_utils.py b/manage_externals/test/test_unit_utils.py index c994e58eb..80e163664 100644 --- a/manage_externals/test/test_unit_utils.py +++ b/manage_externals/test/test_unit_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals From aa15ad9d3eb702309856e8899332467dd5329105 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 6 Jan 2023 17:51:57 -0700 Subject: [PATCH 14/64] Revert "update to manic-v1.2.0" --- manage_externals/README.md | 5 - manage_externals/checkout_externals | 2 +- manage_externals/manic/checkout.py | 87 +++--- .../manic/externals_description.py | 24 +- manage_externals/manic/externals_status.py | 30 +-- manage_externals/manic/repository_factory.py | 1 - manage_externals/manic/repository_git.py | 29 +- manage_externals/manic/repository_svn.py | 9 +- manage_externals/manic/sourcetree.py | 253 +++++++----------- manage_externals/manic/utils.py | 2 +- manage_externals/test/test_sys_checkout.py | 2 +- .../test/test_sys_repository_git.py | 2 +- .../test/test_unit_externals_description.py | 2 +- .../test/test_unit_externals_status.py | 2 +- manage_externals/test/test_unit_repository.py | 2 +- .../test/test_unit_repository_git.py | 4 +- .../test/test_unit_repository_svn.py | 4 +- manage_externals/test/test_unit_utils.py | 2 +- 18 files changed, 191 insertions(+), 271 deletions(-) mode change 100755 => 100644 manage_externals/test/test_unit_repository_svn.py diff --git a/manage_externals/README.md b/manage_externals/README.md index 9475301b5..c931c8e21 100644 --- a/manage_externals/README.md +++ b/manage_externals/README.md @@ -201,11 +201,6 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. externals description file pointed 'useful_library/sub-xternals.cfg', Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. - Note that by default, `checkout_externals` will clone an external's - submodules. As a special case, the entry, `externals = None`, will - prevent this behavior. For more control over which externals are - checked out, create an externals file (and see the `from_submodule` - configuration entry below). * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in diff --git a/manage_externals/checkout_externals b/manage_externals/checkout_externals index 48bce2401..a0698baef 100755 --- a/manage_externals/checkout_externals +++ b/manage_externals/checkout_externals @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Main driver wrapper around the manic/checkout utility. diff --git a/manage_externals/manic/checkout.py b/manage_externals/manic/checkout.py index dff11b661..8dd1798d7 100755 --- a/manage_externals/manic/checkout.py +++ b/manage_externals/manic/checkout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """ Tool to assemble repositories represented in a model-description file. @@ -227,12 +227,6 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. - Note that by default, checkout_externals will clone an external's - submodules. As a special case, the entry, "externals = None", will - prevent this behavior. For more control over which externals are - checked out, create an externals file (and see the from_submodule - configuration entry below). - * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in this repository. Note that the section name (the entry in square @@ -338,34 +332,7 @@ def commandline_arguments(args=None): options = parser.parse_args() return options -def _dirty_local_repo_msg(program_name, config_file): - return """The external repositories labeled with 'M' above are not in a clean state. -The following are four options for how to proceed: -(1) Go into each external that is not in a clean state and issue either a 'git status' or - an 'svn status' command (depending on whether the external is managed by git or - svn). Either revert or commit your changes so that all externals are in a clean - state. (To revert changes in git, follow the instructions given when you run 'git - status'.) (Note, though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. -(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually - update out-of-sync externals (labeled with 's' above) as described in the - configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' - commands to checkout the appropriate tags for each external, as given in - {config_file}.) -(3) You can also use {program_name} to manage most, but not all externals: You can specify - one or more externals to ignore using the '-x' or '--exclude' argument to - {program_name}. Excluding externals labeled with 'M' will allow {program_name} to - update the other, non-excluded externals. -(4) As a last resort, if you are confident that there is no work that needs to be saved - from a given external, you can remove that external (via "rm -rf [directory]") and - then rerun the {program_name} tool. This option is mainly useful as a workaround for - issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before rerunning the -{program_name} tool. -""".format(program_name=program_name, config_file=config_file) + # --------------------------------------------------------------------- # # main @@ -396,23 +363,19 @@ def main(args): load_all = True root_dir = os.path.abspath(os.getcwd()) - model_data = read_externals_description_file(root_dir, args.externals) - ext_description = create_externals_description( - model_data, components=args.components, exclude=args.exclude) + external_data = read_externals_description_file(root_dir, args.externals) + external = create_externals_description( + external_data, components=args.components, exclude=args.exclude) for comp in args.components: - if comp not in ext_description.keys(): + if comp not in external.keys(): fatal_error( "No component {} found in {}".format( comp, args.externals)) - source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry) - if args.components: - components_str = 'specified components' - else: - components_str = 'required & optional components' - printlog('Checking local status of ' + components_str + ': ', end='') - tree_status = source_tree.status(print_progress=True) + source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) + printlog('Checking status of externals: ', end='') + tree_status = source_tree.status() printlog('') if args.status: @@ -427,8 +390,38 @@ def main(args): for comp in sorted(tree_status): tree_status[comp].log_status_message(args.verbose) # exit gracefully + msg = """The external repositories labeled with 'M' above are not in a clean state. + +The following are three options for how to proceed: + +(1) Go into each external that is not in a clean state and issue either a 'git status' or + an 'svn status' command (depending on whether the external is managed by git or + svn). Either revert or commit your changes so that all externals are in a clean + state. (To revert changes in git, follow the instructions given when you run 'git + status'.) (Note, though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. + +(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually + update out-of-sync externals (labeled with 's' above) as described in the + configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' + commands to checkout the appropriate tags for each external, as given in + {config_file}.) + +(3) You can also use {program_name} to manage most, but not all externals: You can specify + one or more externals to ignore using the '-x' or '--exclude' argument to + {program_name}. Excluding externals labeled with 'M' will allow {program_name} to + update the other, non-excluded externals. + + +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before re-running the +checkout_externals tool. +""".format(program_name=program_name, config_file=args.externals) + printlog('-' * 70) - printlog(_dirty_local_repo_msg(program_name, args.externals)) + printlog(msg) printlog('-' * 70) else: if not args.components: diff --git a/manage_externals/manic/externals_description.py b/manage_externals/manic/externals_description.py index f5615b673..918d616e3 100644 --- a/manage_externals/manic/externals_description.py +++ b/manage_externals/manic/externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Model description @@ -71,8 +71,7 @@ def read_externals_description_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing externals description file : {0} ({1})'.format(file_name, - root_dir)) + printlog('Processing externals description file : {0}'.format(file_name)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): @@ -194,9 +193,6 @@ def parse_submodules_desc_section(section_items, file_path): def read_gitmodules_file(root_dir, file_name): # pylint: disable=deprecated-method # Disabling this check because the method is only used for python2 - # pylint: disable=too-many-locals - # pylint: disable=too-many-branches - # pylint: disable=too-many-statements """Read a .gitmodules file and convert it to be compatible with an externals description. """ @@ -282,10 +278,6 @@ def read_gitmodules_file(root_dir, file_name): def create_externals_description( model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data - - components: list of component names to include, None to include all. If a - name isn't found, it is silently omitted from the return value. - exclude: list of component names to skip. """ externals_description = None if model_format == 'dict': @@ -362,9 +354,8 @@ class ExternalsDescription(dict): input value. """ - # keywords defining the interface into the externals description data; these - # are brought together by the schema below. - EXTERNALS = 'externals' # path to externals file. + # keywords defining the interface into the externals description data + EXTERNALS = 'externals' BRANCH = 'branch' SUBMODULE = 'from_submodule' HASH = 'hash' @@ -390,8 +381,6 @@ class ExternalsDescription(dict): _V1_BRANCH = 'BRANCH' _V1_REQ_SOURCE = 'REQ_SOURCE' - # Dictionary keys are component names. The corresponding values are laid out - # according to this schema. _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', @@ -768,8 +757,6 @@ def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects - components: list of component names to include, None to include all. - exclude: list of component names to skip. """ ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 @@ -793,9 +780,6 @@ def _remove_metadata(model_data): def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. - - components: list of component names to include, None to include all. - exclude: list of component names to skip. """ def list_to_dict(input_list, convert_to_lower_case=True): """Convert a list of key-value pairs into a dictionary. diff --git a/manage_externals/manic/externals_status.py b/manage_externals/manic/externals_status.py index 4900e4125..d3d238f28 100644 --- a/manage_externals/manic/externals_status.py +++ b/manage_externals/manic/externals_status.py @@ -29,16 +29,16 @@ class ExternalStatus(object): transactions (e.g. add, remove, rename, untracked files). """ - # sync_state and clean_state can be one of the following: - DEFAULT = '-' # aka not set yet. + DEFAULT = '-' UNKNOWN = '?' EMPTY = 'e' - MODEL_MODIFIED = 's' # repo version != externals (sync_state only) - DIRTY = 'M' # repo is dirty (clean_state only) - STATUS_OK = ' ' # repo is clean/matches externals. + MODEL_MODIFIED = 's' # a.k.a. out-of-sync + DIRTY = 'M' + + STATUS_OK = ' ' STATUS_ERROR = '!' - # source_type can be one of the following: + # source types OPTIONAL = 'o' STANDALONE = 's' MANAGED = ' ' @@ -55,21 +55,19 @@ def __init__(self): def log_status_message(self, verbosity): """Write status message to the screen and log file """ - printlog(self._default_status_message()) + self._default_status_message() if verbosity >= VERBOSITY_VERBOSE: - printlog(self._verbose_status_message()) + self._verbose_status_message() if verbosity >= VERBOSITY_DUMP: - printlog(self._dump_status_message()) - - def __repr__(self): - return self._default_status_message() + self._dump_status_message() def _default_status_message(self): """Return the default terse status message string """ - return '{sync}{clean}{src_type} {path}'.format( + msg = '{sync}{clean}{src_type} {path}'.format( sync=self.sync_state, clean=self.clean_state, src_type=self.source_type, path=self.path) + printlog(msg) def _verbose_status_message(self): """Return the verbose status message string @@ -84,12 +82,14 @@ def _verbose_status_message(self): if self.sync_state != self.STATUS_OK: sync_str = '{current} --> {expected}'.format( current=self.current_version, expected=self.expected_version) - return ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) + msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) + printlog(msg) def _dump_status_message(self): """Return the dump status message string """ - return indent_string(self.status_output, 12) + msg = indent_string(self.status_output, 12) + printlog(msg) def safe_to_update(self): """Report if it is safe to update a repository. Safe is defined as: diff --git a/manage_externals/manic/repository_factory.py b/manage_externals/manic/repository_factory.py index 18c73ffc4..80a92a9d8 100644 --- a/manage_externals/manic/repository_factory.py +++ b/manage_externals/manic/repository_factory.py @@ -15,7 +15,6 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False): """Determine what type of repository we have, i.e. git or svn, and create the appropriate object. - Can return None (e.g. if protocol is 'externals_only'). """ protocol = repo_info[ExternalsDescription.PROTOCOL].lower() if protocol == 'git': diff --git a/manage_externals/manic/repository_git.py b/manage_externals/manic/repository_git.py index 3a6a0f171..f98605100 100644 --- a/manage_externals/manic/repository_git.py +++ b/manage_externals/manic/repository_git.py @@ -109,21 +109,27 @@ def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): def _current_ref(self): """Determine the *name* associated with HEAD. - If we're on a tag, then returns the tag name; otherwise, returns + If we're on a branch, then returns the branch name; otherwise, + if we're on a tag, then returns the tag name; otherwise, returns the current hash. Returns an empty string if no reference can be determined (e.g., if we're not actually in a git repository). - - If we're on a branch, then the branch name is also included in - the returned string (in addition to the tag / hash). """ ref_found = False - # If we're exactly at a tag, use that as the current ref - tag_found, tag_name = self._git_current_tag() - if tag_found: - current_ref = tag_name + # If we're on a branch, then use that as the current ref + branch_found, branch_name = self._git_current_branch() + if branch_found: + current_ref = branch_name ref_found = True + if not ref_found: + # Otherwise, if we're exactly at a tag, use that as the + # current ref + tag_found, tag_name = self._git_current_tag() + if tag_found: + current_ref = tag_name + ref_found = True + if not ref_found: # Otherwise, use current hash as the current ref hash_found, hash_name = self._git_current_hash() @@ -131,12 +137,7 @@ def _current_ref(self): current_ref = hash_name ref_found = True - if ref_found: - # If we're on a branch, include branch name in current ref - branch_found, branch_name = self._git_current_branch() - if branch_found: - current_ref = "{} (branch {})".format(current_ref, branch_name) - else: + if not ref_found: # If we still can't find a ref, return empty string. This # can happen if we're not actually in a git repo current_ref = '' diff --git a/manage_externals/manic/repository_svn.py b/manage_externals/manic/repository_svn.py index 922855d34..408ed8467 100644 --- a/manage_externals/manic/repository_svn.py +++ b/manage_externals/manic/repository_svn.py @@ -43,15 +43,10 @@ def __init__(self, component_name, repo, ignore_ancestry=False): """ Repository.__init__(self, component_name, repo) self._ignore_ancestry = ignore_ancestry - if self._url.endswith('/'): - # there is already a '/' separator in the URL; no need to add another - url_sep = '' - else: - url_sep = '/' if self._branch: - self._url = self._url + url_sep + self._branch + self._url = os.path.join(self._url, self._branch) elif self._tag: - self._url = self._url + url_sep + self._tag + self._url = os.path.join(self._url, self._tag) else: msg = "DEV_ERROR in svn repository. Shouldn't be here!" fatal_error(msg) diff --git a/manage_externals/manic/sourcetree.py b/manage_externals/manic/sourcetree.py index 61e7a6c72..54de763c3 100644 --- a/manage_externals/manic/sourcetree.py +++ b/manage_externals/manic/sourcetree.py @@ -19,7 +19,7 @@ class _External(object): """ - A single component hosted in an external repository (and any children). + _External represents an external object inside a SourceTree """ # pylint: disable=R0902 @@ -41,40 +41,31 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): """ self._name = name - self._repo = None # Repository object. - - # Subcomponent externals file and data object, if any. - self._externals_path = EMPTY_STR # Can also be "none" + self._repo = None + self._externals = EMPTY_STR self._externals_sourcetree = None - - self._stat = None # Populated in status() + self._stat = ExternalStatus() self._sparse = None # Parse the sub-elements - # _local_path : local path relative to the containing source tree, e.g. - # "components/mom" + # _path : local path relative to the containing source tree self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir_path : full repository directory, e.g. - # "/components/mom" + # _repo_dir : full repository directory repo_dir = os.path.join(root_dir, self._local_path) self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir_path : base directory *containing* the repository, e.g. - # "/components" + # _base_dir : base directory *containing* the repository self._base_dir_path = os.path.dirname(self._repo_dir_path) - # _repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path - # e.g., "mom" + # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path self._repo_dir_name = os.path.basename(self._repo_dir_path) assert(os.path.join(self._base_dir_path, self._repo_dir_name) == self._repo_dir_path) self._required = ext_description[ExternalsDescription.REQUIRED] - - # Does this component have subcomponents aka an externals config? - self._externals_path = ext_description[ExternalsDescription.EXTERNALS] + self._externals = ext_description[ExternalsDescription.EXTERNALS] # Treat a .gitmodules file as a backup externals config - if not self._externals_path: + if not self._externals: if GitRepository.has_submodules(self._repo_dir_path): - self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME repo = create_repository( name, ext_description[ExternalsDescription.REPO], @@ -82,8 +73,7 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): if repo: self._repo = repo - # Recurse into subcomponents, if any. - if self._externals_path and (self._externals_path.lower() != 'none'): + if self._externals and (self._externals.lower() != 'none'): self._create_externals_sourcetree() def get_name(self): @@ -98,86 +88,81 @@ def get_local_path(self): """ return self._local_path - def status(self, force=False): + def status(self): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the external. + If the repo destination directory does not exist, checkout the correce + branch or tag. + If load_all is True, also load all of the the externals sub-externals. """ - Returns status of this component and all subcomponents (if available). - Returns a dict mapping our local path to an ExternalStatus dict. Any - subcomponents will have their own top-level key. + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_externals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_externals + self._stat.source_type = ExternalStatus.MANAGED - Side-effect: If self._stat is empty or force is True, calculates _stat. - """ - calc_stat = force or not self._stat - - if calc_stat: - self._stat = ExternalStatus() - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_subexternals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_subexternals - self._stat.source_type = ExternalStatus.MANAGED + ext_stats = {} - subcomponent_stats = {} if not os.path.exists(self._repo_dir_path): - if calc_stat: - # No local repository. - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() else: - # Merge local repository state (e.g. clean/dirty) into self._stat. - if calc_stat and self._repo: + if self._repo: self._repo.status(self._stat, self._repo_dir_path) - # Status of subcomponents, if any. - if self._externals_path and self._externals_sourcetree: + if self._externals and self._externals_sourcetree: + # we expect externals and they exist cwd = os.getcwd() - # SourceTree.status() expects to be called from the correct + # SourceTree expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - subcomponent_stats = self._externals_sourcetree.status(self._local_path, force=force) + ext_stats = self._externals_sourcetree.status(self._local_path) os.chdir(cwd) - # Merge our status + subcomponent statuses into one return dict keyed - # by component path. all_stats = {} # don't add the root component because we don't manage it # and can't provide useful info about it. if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under the local_path, not comp name so + # store the stats under tha local_path, not comp name so # it will be sorted correctly all_stats[self._stat.path] = self._stat - if subcomponent_stats: - all_stats.update(subcomponent_stats) + if ext_stats: + all_stats.update(ext_stats) return all_stats - def checkout(self, verbosity): + def checkout(self, verbosity, load_all): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the external. If the repo destination directory does not exist, checkout the correct branch or tag. - Does not check out sub-externals, see checkout_subexternals(). + If load_all is True, also load all of the the externals sub-externals. """ + if load_all: + pass # Make sure we are in correct location + if not os.path.exists(self._repo_dir_path): # repository directory doesn't exist. Need to check it # out, and for that we need the base_dir_path to exist @@ -189,10 +174,6 @@ def checkout(self, verbosity): self._base_dir_path) fatal_error(msg) - if not self._stat: - self.status() - assert self._stat - if self._stat.source_type != ExternalStatus.STANDALONE: if verbosity >= VERBOSITY_VERBOSE: # NOTE(bja, 2018-01) probably do not want to pass @@ -213,10 +194,8 @@ def checkout(self, verbosity): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_subexternals(self, verbosity, load_all): - """Recursively checkout the sub-externals for this component, if any. - - See load_all documentation in SourceTree.checkout(). + def checkout_externals(self, verbosity, load_all): + """Checkout the sub-externals for this object """ if self.load_externals(): if self._externals_sourcetree: @@ -231,26 +210,25 @@ def checkout_subexternals(self, verbosity, load_all): self._externals_sourcetree.checkout(verbosity, load_all) def load_externals(self): - 'Return True iff an externals file exists (and therefore should be loaded)' + 'Return True iff an externals file should be loaded' load_ex = False if os.path.exists(self._repo_dir_path): - if self._externals_path: - if self._externals_path.lower() != 'none': + if self._externals: + if self._externals.lower() != 'none': load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals_path)) + self._externals)) return load_ex def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' - # Try recursive .gitmodules unless there is an externals entry - recursive = not self._externals_path + # Try recursive unless there is an externals entry + recursive = not self._externals return recursive def _create_externals_sourcetree(self): """ - Note this only creates an object, it doesn't write to the file system. """ if not os.path.exists(self._repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out @@ -261,31 +239,29 @@ def _create_externals_sourcetree(self): cwd = os.getcwd() os.chdir(self._repo_dir_path) - if self._externals_path.lower() == 'none': + if self._externals.lower() == 'none': msg = ('Internal: Attempt to create source tree for ' 'externals = none in {}'.format(self._repo_dir_path)) fatal_error(msg) - if not os.path.exists(self._externals_path): + if not os.path.exists(self._externals): if GitRepository.has_submodules(): - self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME - if not os.path.exists(self._externals_path): + if not os.path.exists(self._externals): # NOTE(bja, 2017-10) this check is redundent with the one # in read_externals_description_file! msg = ('External externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals_path, self._repo_dir_path)) + self._externals, self._repo_dir_path)) fatal_error(msg) externals_root = self._repo_dir_path - # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals_path) - # ext_description is another dict-like object (see ExternalsDescription) - ext_description = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, ext_description) + self._externals) + externals = create_externals_description(model_data, + parent_repo=self._repo) + self._externals_sourcetree = SourceTree(externals_root, externals) os.chdir(cwd) class SourceTree(object): @@ -293,45 +269,45 @@ class SourceTree(object): SourceTree represents a group of managed externals """ - def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): + def __init__(self, root_dir, model, svn_ignore_ancestry=False): """ - Build a SourceTree object from an ExternalDescription. + Build a SourceTree object from a model description """ self._root_dir = os.path.abspath(root_dir) - self._all_components = {} # component_name -> _External + self._all_components = {} self._required_compnames = [] - for comp in ext_description: - src = _External(self._root_dir, comp, ext_description[comp], - svn_ignore_ancestry) + for comp in model: + src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) self._all_components[comp] = src - if ext_description[comp][ExternalsDescription.REQUIRED]: + if model[comp][ExternalsDescription.REQUIRED]: self._required_compnames.append(comp) - def status(self, relative_path_base=LOCAL_PATH_INDICATOR, - force=False, print_progress=False): - """Return a dictionary of local path->ExternalStatus. + def status(self, relative_path_base=LOCAL_PATH_INDICATOR): + """Report the status components - Note that all traversed components, whether recursive or top-level, have - a top-level key in the returned dictionary. + FIXME(bja, 2017-10) what do we do about situations where the + user checked out the optional components, but didn't add + optional for running status? What do we do where the user + didn't add optional to the checkout but did add it to the + status. -- For now, we run status on all components, and try + to do the right thing based on the results.... - Note that all components that are checked out locally, whether required or - optional, ar included in the returned status. """ load_comps = self._all_components.keys() - summary = {} # Holds merged statuses from all components. + summary = {} for comp in load_comps: - if print_progress: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status(force=force) - - # Returned status dictionary is keyed by local path; prepend - # relative_path_base if not already there. + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status() stat_final = {} for name in stat.keys(): + # check if we need to append the relative_path_base to + # the path so it will be sorted in the correct order. if stat[name].path.startswith(relative_path_base): + # use as is, without any changes to path stat_final[name] = stat[name] else: + # append relative_path_base to path and store under key = updated path modified_path = os.path.join(relative_path_base, stat[name].path) stat_final[modified_path] = stat[name] @@ -340,36 +316,15 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR, return summary - def _find_installed_optional_components(self): - """Returns a list of installed optional component names, if any.""" - installed_comps = set() - for comp_name, ext in self._all_components.items(): - if comp_name in self._required_compnames: - continue - # Note that in practice we expect this status to be cached. - stat = ext.status() - installed_comps.update(stat.keys()) - return list(installed_comps) - def checkout(self, verbosity, load_all, load_comp=None): """ - Checkout or update indicated components into the configured subdirs. + Checkout or update indicated components into the the configured + subdirs. - If load_all is True, checkout all externals (required + optional), recursively. - If load_all is False and load_comp is set, checkout load_comp (and any required subexternals, plus any optional subexternals that are already checked out, recursively) - If load_all is False and load_comp is None, checkout all required externals, plus any optionals that are already checked out, recursively. + If load_all is True, recursively checkout all externals. + If load_all is False, load_comp is an optional set of components to load. + If load_all is True and load_comp is None, only load the required externals. """ - if load_all: - tmp_comps = self._all_components.keys() - elif load_comp is not None: - tmp_comps = [load_comp] - else: - local_optional_compnames = self._find_installed_optional_components() - tmp_comps = self._required_compnames + local_optional_compnames - if local_optional_compnames: - printlog('Found locally installed optional components: ' + - ', '.join(local_optional_compnames)) - if verbosity >= VERBOSITY_VERBOSE: printlog('Checking out externals: ') else: @@ -392,9 +347,7 @@ def checkout(self, verbosity, load_all, load_comp=None): # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) - # Does not recurse. - self._all_components[comp].checkout(verbosity) - # Recursively check out subexternals, if any. - self._all_components[comp].checkout_subexternals(verbosity, - load_all) + self._all_components[comp].checkout(verbosity, load_all) + # now give each external an opportunitity to checkout it's externals. + self._all_components[comp].checkout_externals(verbosity, load_all) printlog('') diff --git a/manage_externals/manic/utils.py b/manage_externals/manic/utils.py index 9c63ffe65..f57f43930 100644 --- a/manage_externals/manic/utils.py +++ b/manage_externals/manic/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """ Common public utilities for manic package diff --git a/manage_externals/test/test_sys_checkout.py b/manage_externals/test/test_sys_checkout.py index 176228050..118bee530 100644 --- a/manage_externals/test/test_sys_checkout.py +++ b/manage_externals/test/test_sys_checkout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_sys_repository_git.py b/manage_externals/test/test_sys_repository_git.py index 29d5433b9..f6dbf8428 100644 --- a/manage_externals/test/test_sys_repository_git.py +++ b/manage_externals/test/test_sys_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Tests of some of the functionality in repository_git.py that actually interacts with git repositories. diff --git a/manage_externals/test/test_unit_externals_description.py b/manage_externals/test/test_unit_externals_description.py index 30e528849..0b1248f67 100644 --- a/manage_externals/test/test_unit_externals_description.py +++ b/manage_externals/test/test_unit_externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_unit_externals_status.py b/manage_externals/test/test_unit_externals_status.py index f019514e9..f8e953f75 100644 --- a/manage_externals/test/test_unit_externals_status.py +++ b/manage_externals/test/test_unit_externals_status.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Unit test driver for the manic external status reporting module. diff --git a/manage_externals/test/test_unit_repository.py b/manage_externals/test/test_unit_repository.py index 1b9386183..5b9c242fd 100644 --- a/manage_externals/test/test_unit_repository.py +++ b/manage_externals/test/test_unit_repository.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_unit_repository_git.py b/manage_externals/test/test_unit_repository_git.py index a6ad9f100..4a0a334bb 100644 --- a/manage_externals/test/test_unit_repository_git.py +++ b/manage_externals/test/test_unit_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Unit test driver for checkout_externals @@ -101,7 +101,7 @@ def test_ref_branch(self): True, 'feature3') self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') - expected = 'foo_tag (branch feature3)' + expected = 'feature3' result = self._repo._current_ref() self.assertEqual(result, expected) diff --git a/manage_externals/test/test_unit_repository_svn.py b/manage_externals/test/test_unit_repository_svn.py old mode 100755 new mode 100644 index d9309df7f..7ff31c421 --- a/manage_externals/test/test_unit_repository_svn.py +++ b/manage_externals/test/test_unit_repository_svn.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Unit test driver for checkout_externals @@ -60,7 +60,7 @@ def setUp(self): self._name = 'component' rdata = {ExternalsDescription.PROTOCOL: 'svn', ExternalsDescription.REPO_URL: - 'https://svn-ccsm-models.cgd.ucar.edu', + 'https://svn-ccsm-models.cgd.ucar.edu/', ExternalsDescription.TAG: 'mosart/trunk_tags/mosart1_0_26', } diff --git a/manage_externals/test/test_unit_utils.py b/manage_externals/test/test_unit_utils.py index 80e163664..c994e58eb 100644 --- a/manage_externals/test/test_unit_utils.py +++ b/manage_externals/test/test_unit_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python """Unit test driver for checkout_externals From 72da587d358dbcfe080fe81aa5064bacd7cdc54a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Mon, 9 Jan 2023 09:15:26 -0700 Subject: [PATCH 15/64] try again to update manic --- .../.github/workflows/bumpversion.yml | 19 ++ manage_externals/.github/workflows/tests.yml | 30 ++ manage_externals/README.md | 5 + manage_externals/checkout_externals | 2 +- manage_externals/manic/checkout.py | 89 +++--- .../manic/externals_description.py | 24 +- manage_externals/manic/externals_status.py | 30 +- manage_externals/manic/repository_factory.py | 1 + manage_externals/manic/repository_git.py | 29 +- manage_externals/manic/repository_svn.py | 9 +- manage_externals/manic/sourcetree.py | 275 +++++++++++------- manage_externals/manic/utils.py | 2 +- manage_externals/test/README.md | 20 +- manage_externals/test/test_sys_checkout.py | 2 +- .../test/test_sys_repository_git.py | 2 +- .../test/test_unit_externals_description.py | 2 +- .../test/test_unit_externals_status.py | 2 +- manage_externals/test/test_unit_repository.py | 2 +- .../test/test_unit_repository_git.py | 4 +- .../test/test_unit_repository_svn.py | 4 +- manage_externals/test/test_unit_utils.py | 2 +- 21 files changed, 337 insertions(+), 218 deletions(-) create mode 100644 manage_externals/.github/workflows/bumpversion.yml create mode 100644 manage_externals/.github/workflows/tests.yml mode change 100644 => 100755 manage_externals/test/test_unit_repository_svn.py diff --git a/manage_externals/.github/workflows/bumpversion.yml b/manage_externals/.github/workflows/bumpversion.yml new file mode 100644 index 000000000..f4dc9b7ca --- /dev/null +++ b/manage_externals/.github/workflows/bumpversion.yml @@ -0,0 +1,19 @@ +name: Bump version +on: + push: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + create_annotated_tag: true + default_bump: patch + dry_run: false + tag_prefix: manic- diff --git a/manage_externals/.github/workflows/tests.yml b/manage_externals/.github/workflows/tests.yml new file mode 100644 index 000000000..dd75b91b4 --- /dev/null +++ b/manage_externals/.github/workflows/tests.yml @@ -0,0 +1,30 @@ +# This is a workflow to compile the cmeps source without cime +name: Test Manic + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + test-manic: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Test Manic + run: | + pushd test + git config --global user.email "devnull@example.com" + git config --global user.name "GITHUB tester" + git config --global protocol.file.allow always + make utest + make stest + popd + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/manage_externals/README.md b/manage_externals/README.md index c931c8e21..9475301b5 100644 --- a/manage_externals/README.md +++ b/manage_externals/README.md @@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. externals description file pointed 'useful_library/sub-xternals.cfg', Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + Note that by default, `checkout_externals` will clone an external's + submodules. As a special case, the entry, `externals = None`, will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the `from_submodule` + configuration entry below). * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in diff --git a/manage_externals/checkout_externals b/manage_externals/checkout_externals index a0698baef..48bce2401 100755 --- a/manage_externals/checkout_externals +++ b/manage_externals/checkout_externals @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Main driver wrapper around the manic/checkout utility. diff --git a/manage_externals/manic/checkout.py b/manage_externals/manic/checkout.py index 8dd1798d7..ac30f3a5d 100755 --- a/manage_externals/manic/checkout.py +++ b/manage_externals/manic/checkout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Tool to assemble repositories represented in a model-description file. @@ -227,6 +227,12 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. + Note that by default, checkout_externals will clone an external's + submodules. As a special case, the entry, "externals = None", will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the from_submodule + configuration entry below). + * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in this repository. Note that the section name (the entry in square @@ -332,7 +338,34 @@ def commandline_arguments(args=None): options = parser.parse_args() return options - +def _dirty_local_repo_msg(program_name, config_file): + return """The external repositories labeled with 'M' above are not in a clean state. +The following are four options for how to proceed: +(1) Go into each external that is not in a clean state and issue either a 'git status' or + an 'svn status' command (depending on whether the external is managed by git or + svn). Either revert or commit your changes so that all externals are in a clean + state. (To revert changes in git, follow the instructions given when you run 'git + status'.) (Note, though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. +(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually + update out-of-sync externals (labeled with 's' above) as described in the + configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' + commands to checkout the appropriate tags for each external, as given in + {config_file}.) +(3) You can also use {program_name} to manage most, but not all externals: You can specify + one or more externals to ignore using the '-x' or '--exclude' argument to + {program_name}. Excluding externals labeled with 'M' will allow {program_name} to + update the other, non-excluded externals. +(4) As a last resort, if you are confident that there is no work that needs to be saved + from a given external, you can remove that external (via "rm -rf [directory]") and + then rerun the {program_name} tool. This option is mainly useful as a workaround for + issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before rerunning the +{program_name} tool. +""".format(program_name=program_name, config_file=config_file) # --------------------------------------------------------------------- # # main @@ -363,19 +396,25 @@ def main(args): load_all = True root_dir = os.path.abspath(os.getcwd()) - external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description( - external_data, components=args.components, exclude=args.exclude) + model_data = read_externals_description_file(root_dir, args.externals) + ext_description = create_externals_description( + model_data, components=args.components, exclude=args.exclude) for comp in args.components: - if comp not in external.keys(): + if comp not in ext_description.keys(): + # Note we can't print out the list of found externals because + # they were filtered in create_externals_description above. fatal_error( "No component {} found in {}".format( comp, args.externals)) - source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) - printlog('Checking status of externals: ', end='') - tree_status = source_tree.status() + source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry) + if args.components: + components_str = 'specified components' + else: + components_str = 'required & optional components' + printlog('Checking local status of ' + components_str + ': ', end='') + tree_status = source_tree.status(print_progress=True) printlog('') if args.status: @@ -390,38 +429,8 @@ def main(args): for comp in sorted(tree_status): tree_status[comp].log_status_message(args.verbose) # exit gracefully - msg = """The external repositories labeled with 'M' above are not in a clean state. - -The following are three options for how to proceed: - -(1) Go into each external that is not in a clean state and issue either a 'git status' or - an 'svn status' command (depending on whether the external is managed by git or - svn). Either revert or commit your changes so that all externals are in a clean - state. (To revert changes in git, follow the instructions given when you run 'git - status'.) (Note, though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. - -(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually - update out-of-sync externals (labeled with 's' above) as described in the - configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' - commands to checkout the appropriate tags for each external, as given in - {config_file}.) - -(3) You can also use {program_name} to manage most, but not all externals: You can specify - one or more externals to ignore using the '-x' or '--exclude' argument to - {program_name}. Excluding externals labeled with 'M' will allow {program_name} to - update the other, non-excluded externals. - - -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before re-running the -checkout_externals tool. -""".format(program_name=program_name, config_file=args.externals) - printlog('-' * 70) - printlog(msg) + printlog(_dirty_local_repo_msg(program_name, args.externals)) printlog('-' * 70) else: if not args.components: diff --git a/manage_externals/manic/externals_description.py b/manage_externals/manic/externals_description.py index 918d616e3..f5615b673 100644 --- a/manage_externals/manic/externals_description.py +++ b/manage_externals/manic/externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Model description @@ -71,7 +71,8 @@ def read_externals_description_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing externals description file : {0}'.format(file_name)) + printlog('Processing externals description file : {0} ({1})'.format(file_name, + root_dir)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): @@ -193,6 +194,9 @@ def parse_submodules_desc_section(section_items, file_path): def read_gitmodules_file(root_dir, file_name): # pylint: disable=deprecated-method # Disabling this check because the method is only used for python2 + # pylint: disable=too-many-locals + # pylint: disable=too-many-branches + # pylint: disable=too-many-statements """Read a .gitmodules file and convert it to be compatible with an externals description. """ @@ -278,6 +282,10 @@ def read_gitmodules_file(root_dir, file_name): def create_externals_description( model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data + + components: list of component names to include, None to include all. If a + name isn't found, it is silently omitted from the return value. + exclude: list of component names to skip. """ externals_description = None if model_format == 'dict': @@ -354,8 +362,9 @@ class ExternalsDescription(dict): input value. """ - # keywords defining the interface into the externals description data - EXTERNALS = 'externals' + # keywords defining the interface into the externals description data; these + # are brought together by the schema below. + EXTERNALS = 'externals' # path to externals file. BRANCH = 'branch' SUBMODULE = 'from_submodule' HASH = 'hash' @@ -381,6 +390,8 @@ class ExternalsDescription(dict): _V1_BRANCH = 'BRANCH' _V1_REQ_SOURCE = 'REQ_SOURCE' + # Dictionary keys are component names. The corresponding values are laid out + # according to this schema. _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', @@ -757,6 +768,8 @@ def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 @@ -780,6 +793,9 @@ def _remove_metadata(model_data): def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. + + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ def list_to_dict(input_list, convert_to_lower_case=True): """Convert a list of key-value pairs into a dictionary. diff --git a/manage_externals/manic/externals_status.py b/manage_externals/manic/externals_status.py index d3d238f28..4900e4125 100644 --- a/manage_externals/manic/externals_status.py +++ b/manage_externals/manic/externals_status.py @@ -29,16 +29,16 @@ class ExternalStatus(object): transactions (e.g. add, remove, rename, untracked files). """ - DEFAULT = '-' + # sync_state and clean_state can be one of the following: + DEFAULT = '-' # aka not set yet. UNKNOWN = '?' EMPTY = 'e' - MODEL_MODIFIED = 's' # a.k.a. out-of-sync - DIRTY = 'M' - - STATUS_OK = ' ' + MODEL_MODIFIED = 's' # repo version != externals (sync_state only) + DIRTY = 'M' # repo is dirty (clean_state only) + STATUS_OK = ' ' # repo is clean/matches externals. STATUS_ERROR = '!' - # source types + # source_type can be one of the following: OPTIONAL = 'o' STANDALONE = 's' MANAGED = ' ' @@ -55,19 +55,21 @@ def __init__(self): def log_status_message(self, verbosity): """Write status message to the screen and log file """ - self._default_status_message() + printlog(self._default_status_message()) if verbosity >= VERBOSITY_VERBOSE: - self._verbose_status_message() + printlog(self._verbose_status_message()) if verbosity >= VERBOSITY_DUMP: - self._dump_status_message() + printlog(self._dump_status_message()) + + def __repr__(self): + return self._default_status_message() def _default_status_message(self): """Return the default terse status message string """ - msg = '{sync}{clean}{src_type} {path}'.format( + return '{sync}{clean}{src_type} {path}'.format( sync=self.sync_state, clean=self.clean_state, src_type=self.source_type, path=self.path) - printlog(msg) def _verbose_status_message(self): """Return the verbose status message string @@ -82,14 +84,12 @@ def _verbose_status_message(self): if self.sync_state != self.STATUS_OK: sync_str = '{current} --> {expected}'.format( current=self.current_version, expected=self.expected_version) - msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) - printlog(msg) + return ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) def _dump_status_message(self): """Return the dump status message string """ - msg = indent_string(self.status_output, 12) - printlog(msg) + return indent_string(self.status_output, 12) def safe_to_update(self): """Report if it is safe to update a repository. Safe is defined as: diff --git a/manage_externals/manic/repository_factory.py b/manage_externals/manic/repository_factory.py index 80a92a9d8..18c73ffc4 100644 --- a/manage_externals/manic/repository_factory.py +++ b/manage_externals/manic/repository_factory.py @@ -15,6 +15,7 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False): """Determine what type of repository we have, i.e. git or svn, and create the appropriate object. + Can return None (e.g. if protocol is 'externals_only'). """ protocol = repo_info[ExternalsDescription.PROTOCOL].lower() if protocol == 'git': diff --git a/manage_externals/manic/repository_git.py b/manage_externals/manic/repository_git.py index f98605100..3a6a0f171 100644 --- a/manage_externals/manic/repository_git.py +++ b/manage_externals/manic/repository_git.py @@ -109,27 +109,21 @@ def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): def _current_ref(self): """Determine the *name* associated with HEAD. - If we're on a branch, then returns the branch name; otherwise, - if we're on a tag, then returns the tag name; otherwise, returns + If we're on a tag, then returns the tag name; otherwise, returns the current hash. Returns an empty string if no reference can be determined (e.g., if we're not actually in a git repository). + + If we're on a branch, then the branch name is also included in + the returned string (in addition to the tag / hash). """ ref_found = False - # If we're on a branch, then use that as the current ref - branch_found, branch_name = self._git_current_branch() - if branch_found: - current_ref = branch_name + # If we're exactly at a tag, use that as the current ref + tag_found, tag_name = self._git_current_tag() + if tag_found: + current_ref = tag_name ref_found = True - if not ref_found: - # Otherwise, if we're exactly at a tag, use that as the - # current ref - tag_found, tag_name = self._git_current_tag() - if tag_found: - current_ref = tag_name - ref_found = True - if not ref_found: # Otherwise, use current hash as the current ref hash_found, hash_name = self._git_current_hash() @@ -137,7 +131,12 @@ def _current_ref(self): current_ref = hash_name ref_found = True - if not ref_found: + if ref_found: + # If we're on a branch, include branch name in current ref + branch_found, branch_name = self._git_current_branch() + if branch_found: + current_ref = "{} (branch {})".format(current_ref, branch_name) + else: # If we still can't find a ref, return empty string. This # can happen if we're not actually in a git repo current_ref = '' diff --git a/manage_externals/manic/repository_svn.py b/manage_externals/manic/repository_svn.py index 408ed8467..922855d34 100644 --- a/manage_externals/manic/repository_svn.py +++ b/manage_externals/manic/repository_svn.py @@ -43,10 +43,15 @@ def __init__(self, component_name, repo, ignore_ancestry=False): """ Repository.__init__(self, component_name, repo) self._ignore_ancestry = ignore_ancestry + if self._url.endswith('/'): + # there is already a '/' separator in the URL; no need to add another + url_sep = '' + else: + url_sep = '/' if self._branch: - self._url = os.path.join(self._url, self._branch) + self._url = self._url + url_sep + self._branch elif self._tag: - self._url = os.path.join(self._url, self._tag) + self._url = self._url + url_sep + self._tag else: msg = "DEV_ERROR in svn repository. Shouldn't be here!" fatal_error(msg) diff --git a/manage_externals/manic/sourcetree.py b/manage_externals/manic/sourcetree.py index 54de763c3..218b5febb 100644 --- a/manage_externals/manic/sourcetree.py +++ b/manage_externals/manic/sourcetree.py @@ -19,7 +19,7 @@ class _External(object): """ - _External represents an external object inside a SourceTree + A single component hosted in an external repository (and any children). """ # pylint: disable=R0902 @@ -41,31 +41,40 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): """ self._name = name - self._repo = None - self._externals = EMPTY_STR + self._repo = None # Repository object. + + # Subcomponent externals file and data object, if any. + self._externals_path = EMPTY_STR # Can also be "none" self._externals_sourcetree = None - self._stat = ExternalStatus() + + self._stat = None # Populated in status() self._sparse = None # Parse the sub-elements - # _path : local path relative to the containing source tree + # _local_path : local path relative to the containing source tree, e.g. + # "components/mom" self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir : full repository directory + # _repo_dir_path : full repository directory, e.g. + # "/components/mom" repo_dir = os.path.join(root_dir, self._local_path) self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir : base directory *containing* the repository + # _base_dir_path : base directory *containing* the repository, e.g. + # "/components" self._base_dir_path = os.path.dirname(self._repo_dir_path) - # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # _repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # e.g., "mom" self._repo_dir_name = os.path.basename(self._repo_dir_path) assert(os.path.join(self._base_dir_path, self._repo_dir_name) == self._repo_dir_path) self._required = ext_description[ExternalsDescription.REQUIRED] - self._externals = ext_description[ExternalsDescription.EXTERNALS] + + # Does this component have subcomponents aka an externals config? + self._externals_path = ext_description[ExternalsDescription.EXTERNALS] # Treat a .gitmodules file as a backup externals config - if not self._externals: + if not self._externals_path: if GitRepository.has_submodules(self._repo_dir_path): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME repo = create_repository( name, ext_description[ExternalsDescription.REPO], @@ -73,7 +82,8 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): if repo: self._repo = repo - if self._externals and (self._externals.lower() != 'none'): + # Recurse into subcomponents, if any. + if self._externals_path and (self._externals_path.lower() != 'none'): self._create_externals_sourcetree() def get_name(self): @@ -88,81 +98,88 @@ def get_local_path(self): """ return self._local_path - def status(self): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. + def status(self, force=False, print_progress=False): """ + Returns status of this component and all subcomponents. - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_externals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_externals - self._stat.source_type = ExternalStatus.MANAGED + Returns a dict mapping our local path (not component name!) to an + ExternalStatus dict. Any subcomponents will have their own top-level + path keys. Note the return value includes entries for this and all + subcomponents regardless of whether they are locally installed or not. - ext_stats = {} + Side-effect: If self._stat is empty or force is True, calculates _stat. + """ + calc_stat = force or not self._stat + + if calc_stat: + self._stat = ExternalStatus() + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_subexternals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_subexternals + self._stat.source_type = ExternalStatus.MANAGED + subcomponent_stats = {} if not os.path.exists(self._repo_dir_path): - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() + if calc_stat: + # No local repository. + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() else: - if self._repo: + # Merge local repository state (e.g. clean/dirty) into self._stat. + if calc_stat and self._repo: self._repo.status(self._stat, self._repo_dir_path) - if self._externals and self._externals_sourcetree: - # we expect externals and they exist + # Status of subcomponents, if any. + if self._externals_path and self._externals_sourcetree: cwd = os.getcwd() - # SourceTree expects to be called from the correct + # SourceTree.status() expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - ext_stats = self._externals_sourcetree.status(self._local_path) + subcomponent_stats = self._externals_sourcetree.status(self._local_path, force=force, print_progress=print_progress) os.chdir(cwd) + # Merge our status + subcomponent statuses into one return dict keyed + # by component path. all_stats = {} # don't add the root component because we don't manage it # and can't provide useful info about it. if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under tha local_path, not comp name so + # store the stats under the local_path, not comp name so # it will be sorted correctly all_stats[self._stat.path] = self._stat - if ext_stats: - all_stats.update(ext_stats) + if subcomponent_stats: + all_stats.update(subcomponent_stats) return all_stats - def checkout(self, verbosity, load_all): + def checkout(self, verbosity): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the external. If the repo destination directory does not exist, checkout the correct branch or tag. - If load_all is True, also load all of the the externals sub-externals. + Does not check out sub-externals, see checkout_subexternals(). """ - if load_all: - pass # Make sure we are in correct location - if not os.path.exists(self._repo_dir_path): # repository directory doesn't exist. Need to check it # out, and for that we need the base_dir_path to exist @@ -174,6 +191,10 @@ def checkout(self, verbosity, load_all): self._base_dir_path) fatal_error(msg) + if not self._stat: + self.status() + assert self._stat + if self._stat.source_type != ExternalStatus.STANDALONE: if verbosity >= VERBOSITY_VERBOSE: # NOTE(bja, 2018-01) probably do not want to pass @@ -194,8 +215,10 @@ def checkout(self, verbosity, load_all): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_externals(self, verbosity, load_all): - """Checkout the sub-externals for this object + def checkout_subexternals(self, verbosity, load_all): + """Recursively checkout the sub-externals for this component, if any. + + See load_all documentation in SourceTree.checkout(). """ if self.load_externals(): if self._externals_sourcetree: @@ -210,25 +233,26 @@ def checkout_externals(self, verbosity, load_all): self._externals_sourcetree.checkout(verbosity, load_all) def load_externals(self): - 'Return True iff an externals file should be loaded' + 'Return True iff an externals file exists (and therefore should be loaded)' load_ex = False if os.path.exists(self._repo_dir_path): - if self._externals: - if self._externals.lower() != 'none': + if self._externals_path: + if self._externals_path.lower() != 'none': load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals)) + self._externals_path)) return load_ex def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' - # Try recursive unless there is an externals entry - recursive = not self._externals + # Try recursive .gitmodules unless there is an externals entry + recursive = not self._externals_path return recursive def _create_externals_sourcetree(self): """ + Note this only creates an object, it doesn't write to the file system. """ if not os.path.exists(self._repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out @@ -239,29 +263,31 @@ def _create_externals_sourcetree(self): cwd = os.getcwd() os.chdir(self._repo_dir_path) - if self._externals.lower() == 'none': + if self._externals_path.lower() == 'none': msg = ('Internal: Attempt to create source tree for ' 'externals = none in {}'.format(self._repo_dir_path)) fatal_error(msg) - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): if GitRepository.has_submodules(): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME - if not os.path.exists(self._externals): + if not os.path.exists(self._externals_path): # NOTE(bja, 2017-10) this check is redundent with the one # in read_externals_description_file! msg = ('External externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals, self._repo_dir_path)) + self._externals_path, self._repo_dir_path)) fatal_error(msg) externals_root = self._repo_dir_path + # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals) - externals = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, externals) + self._externals_path) + # ext_description is another dict-like object (see ExternalsDescription) + ext_description = create_externals_description(model_data, + parent_repo=self._repo) + self._externals_sourcetree = SourceTree(externals_root, ext_description) os.chdir(cwd) class SourceTree(object): @@ -269,45 +295,48 @@ class SourceTree(object): SourceTree represents a group of managed externals """ - def __init__(self, root_dir, model, svn_ignore_ancestry=False): + def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): """ - Build a SourceTree object from a model description + Build a SourceTree object from an ExternalDescription. """ self._root_dir = os.path.abspath(root_dir) - self._all_components = {} + self._all_components = {} # component_name -> _External self._required_compnames = [] - for comp in model: - src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) + for comp in ext_description: + src = _External(self._root_dir, comp, ext_description[comp], + svn_ignore_ancestry) self._all_components[comp] = src - if model[comp][ExternalsDescription.REQUIRED]: + if ext_description[comp][ExternalsDescription.REQUIRED]: self._required_compnames.append(comp) - def status(self, relative_path_base=LOCAL_PATH_INDICATOR): - """Report the status components - - FIXME(bja, 2017-10) what do we do about situations where the - user checked out the optional components, but didn't add - optional for running status? What do we do where the user - didn't add optional to the checkout but did add it to the - status. -- For now, we run status on all components, and try - to do the right thing based on the results.... - - """ + def status(self, relative_path_base=LOCAL_PATH_INDICATOR, + force=False, print_progress=False): + """Return a dictionary of local path->ExternalStatus. + + Notes about the returned dictionary: + * It is keyed by local path (e.g. 'components/mom'), not by + component name (e.g. 'mom'). + * It contains top-level keys for all traversed components, whether + discovered by recursion or top-level. + * It contains entries for all components regardless of whether they + are locally installed or not, or required or optional. +x """ load_comps = self._all_components.keys() - summary = {} + summary = {} # Holds merged statuses from all components. for comp in load_comps: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status() + if print_progress: + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status(force=force, + print_progress=print_progress) + + # Returned status dictionary is keyed by local path; prepend + # relative_path_base if not already there. stat_final = {} for name in stat.keys(): - # check if we need to append the relative_path_base to - # the path so it will be sorted in the correct order. if stat[name].path.startswith(relative_path_base): - # use as is, without any changes to path stat_final[name] = stat[name] else: - # append relative_path_base to path and store under key = updated path modified_path = os.path.join(relative_path_base, stat[name].path) stat_final[modified_path] = stat[name] @@ -316,30 +345,48 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): return summary + def _find_installed_optional_components(self): + """Returns a list of installed optional component names, if any.""" + installed_comps = [] + for comp_name, ext in self._all_components.items(): + if comp_name in self._required_compnames: + continue + # Note that in practice we expect this status to be cached. + path_to_stat = ext.status() + if any(stat.sync_state != ExternalStatus.EMPTY + for stat in path_to_stat.values()): + installed_comps.append(comp_name) + return installed_comps + def checkout(self, verbosity, load_all, load_comp=None): """ - Checkout or update indicated components into the the configured - subdirs. + Checkout or update indicated components into the configured subdirs. - If load_all is True, recursively checkout all externals. - If load_all is False, load_comp is an optional set of components to load. - If load_all is True and load_comp is None, only load the required externals. + If load_all is True, checkout all externals (required + optional), recursively. + If load_all is False and load_comp is set, checkout load_comp (and any required subexternals, plus any optional subexternals that are already checked out, recursively) + If load_all is False and load_comp is None, checkout all required externals, plus any optionals that are already checked out, recursively. """ - if verbosity >= VERBOSITY_VERBOSE: - printlog('Checking out externals: ') - else: - printlog('Checking out externals: ', end='') - if load_all: tmp_comps = self._all_components.keys() elif load_comp is not None: tmp_comps = [load_comp] else: - tmp_comps = self._required_compnames + local_optional_compnames = self._find_installed_optional_components() + tmp_comps = self._required_compnames + local_optional_compnames + if local_optional_compnames: + printlog('Found locally installed optional components: ' + + ', '.join(local_optional_compnames)) + + if verbosity >= VERBOSITY_VERBOSE: + printlog('Checking out externals: ') + else: + printlog('Checking out externals: ', end='') + # Sort by path so that if paths are nested the # parent repo is checked out first. load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path()) - # checkout the primary externals + + # checkout. for comp in load_comps: if verbosity < VERBOSITY_VERBOSE: printlog('{0}, '.format(comp), end='') @@ -347,7 +394,9 @@ def checkout(self, verbosity, load_all, load_comp=None): # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) - self._all_components[comp].checkout(verbosity, load_all) - # now give each external an opportunitity to checkout it's externals. - self._all_components[comp].checkout_externals(verbosity, load_all) + # Does not recurse. + self._all_components[comp].checkout(verbosity) + # Recursively check out subexternals, if any. + self._all_components[comp].checkout_subexternals(verbosity, + load_all) printlog('') diff --git a/manage_externals/manic/utils.py b/manage_externals/manic/utils.py index f57f43930..9c63ffe65 100644 --- a/manage_externals/manic/utils.py +++ b/manage_externals/manic/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Common public utilities for manic package diff --git a/manage_externals/test/README.md b/manage_externals/test/README.md index 938a900ee..700bccf62 100644 --- a/manage_externals/test/README.md +++ b/manage_externals/test/README.md @@ -18,28 +18,18 @@ Development environments should be setup for python2 and python3: ## Unit tests -Tests should be run for both python2 and python3. It is recommended -that you have seperate terminal windows open python2 and python3 -testing to avoid errors activating and deactivating environments. - ```SH cd checkout_externals/test - . env_python2/bin/activate make utest - deactivate ``` +## System tests + ```SH cd checkout_externals/test - . env_python2/bin/activate - make utest - deactivate + make stest ``` -## System tests - -Not yet implemented. - ## Static analysis checkout_externals is difficult to test thoroughly because it relies @@ -51,9 +41,7 @@ regularly for automatic code formatting and linting. ```SH cd checkout_externals/test - . env_python2/bin/activate make lint - deactivate ``` The canonical formatting for the code is whatever autopep8 @@ -68,10 +56,8 @@ coverage, run the code coverage tool: ```SH cd checkout_externals/test - . env_python2/bin/activate make coverage open -a Firefox.app htmlcov/index.html - deactivate ``` diff --git a/manage_externals/test/test_sys_checkout.py b/manage_externals/test/test_sys_checkout.py index 118bee530..176228050 100644 --- a/manage_externals/test/test_sys_checkout.py +++ b/manage_externals/test/test_sys_checkout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_sys_repository_git.py b/manage_externals/test/test_sys_repository_git.py index f6dbf8428..29d5433b9 100644 --- a/manage_externals/test/test_sys_repository_git.py +++ b/manage_externals/test/test_sys_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Tests of some of the functionality in repository_git.py that actually interacts with git repositories. diff --git a/manage_externals/test/test_unit_externals_description.py b/manage_externals/test/test_unit_externals_description.py index 0b1248f67..30e528849 100644 --- a/manage_externals/test/test_unit_externals_description.py +++ b/manage_externals/test/test_unit_externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_unit_externals_status.py b/manage_externals/test/test_unit_externals_status.py index f8e953f75..f019514e9 100644 --- a/manage_externals/test/test_unit_externals_status.py +++ b/manage_externals/test/test_unit_externals_status.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for the manic external status reporting module. diff --git a/manage_externals/test/test_unit_repository.py b/manage_externals/test/test_unit_repository.py index 5b9c242fd..1b9386183 100644 --- a/manage_externals/test/test_unit_repository.py +++ b/manage_externals/test/test_unit_repository.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals diff --git a/manage_externals/test/test_unit_repository_git.py b/manage_externals/test/test_unit_repository_git.py index 4a0a334bb..a6ad9f100 100644 --- a/manage_externals/test/test_unit_repository_git.py +++ b/manage_externals/test/test_unit_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals @@ -101,7 +101,7 @@ def test_ref_branch(self): True, 'feature3') self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') - expected = 'feature3' + expected = 'foo_tag (branch feature3)' result = self._repo._current_ref() self.assertEqual(result, expected) diff --git a/manage_externals/test/test_unit_repository_svn.py b/manage_externals/test/test_unit_repository_svn.py old mode 100644 new mode 100755 index 7ff31c421..d9309df7f --- a/manage_externals/test/test_unit_repository_svn.py +++ b/manage_externals/test/test_unit_repository_svn.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals @@ -60,7 +60,7 @@ def setUp(self): self._name = 'component' rdata = {ExternalsDescription.PROTOCOL: 'svn', ExternalsDescription.REPO_URL: - 'https://svn-ccsm-models.cgd.ucar.edu/', + 'https://svn-ccsm-models.cgd.ucar.edu', ExternalsDescription.TAG: 'mosart/trunk_tags/mosart1_0_26', } diff --git a/manage_externals/test/test_unit_utils.py b/manage_externals/test/test_unit_utils.py index c994e58eb..80e163664 100644 --- a/manage_externals/test/test_unit_utils.py +++ b/manage_externals/test/test_unit_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals From da6f58bf97316ea0be3be0c85c42cce263853aff Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Thu, 12 Jan 2023 10:58:40 -0700 Subject: [PATCH 16/64] Update for cesm2_3_alpha12a --- Externals.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 762d001c4..65bf28e3a 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -6,7 +6,7 @@ local_path = ccs_config required = True [cam] -tag = cam6_3_085 +tag = cam6_3_086 protocol = git repo_url = https://github.com/ESCOMP/CAM local_path = components/cam @@ -29,14 +29,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.2 +tag = cmeps0.14.5 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.67 +tag = cdeps0.12.68 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev114 +tag = ctsm5.1.dev115 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm @@ -111,7 +111,7 @@ externals = Externals.cfg required = False [mosart] -tag = mosart1_0_47 +tag = mosart1_0_48 protocol = git repo_url = https://github.com/ESCOMP/MOSART local_path = components/mosart From 623ea46b9ec820d21021d2d6571d5db6d7039ccf Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 13 Jan 2023 11:35:44 -0700 Subject: [PATCH 17/64] Remove unnecessary import that isn't present in actual manage_externals This must have come in with some issue in a subtree merge --- manage_externals/test/test_sys_checkout.py | 1 - 1 file changed, 1 deletion(-) mode change 100644 => 100755 manage_externals/test/test_sys_checkout.py diff --git a/manage_externals/test/test_sys_checkout.py b/manage_externals/test/test_sys_checkout.py old mode 100644 new mode 100755 index 118bee530..9889feba0 --- a/manage_externals/test/test_sys_checkout.py +++ b/manage_externals/test/test_sys_checkout.py @@ -38,7 +38,6 @@ import os import os.path import shutil -import sys import unittest from manic.externals_description import ExternalsDescription From 17147044d3eac082c08fa87a1cc82cdac8f1a7a1 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 13 Jan 2023 11:38:39 -0700 Subject: [PATCH 18/64] Revert change in file mode --- manage_externals/test/test_sys_checkout.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 manage_externals/test/test_sys_checkout.py diff --git a/manage_externals/test/test_sys_checkout.py b/manage_externals/test/test_sys_checkout.py old mode 100755 new mode 100644 From cdca2fe59eeef8d65d2cba6f59c0f7ea55e07711 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 20 Jan 2023 15:10:18 -0700 Subject: [PATCH 19/64] Update for cesm2_3_alpha12a --- ChangeLog | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/ChangeLog b/ChangeLog index 9a98ccdde..30d0378f8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,71 @@ ============================================================== +Tag name: cesm2_3_alpha012a +Originator(s): CSEG +Date: 20 January 2023 +One-line Summary: CTSM answer changes + +components/cam https://github.com/ESCOMP/CAM/cam6_3_086 ** +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_2_0_34 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.82 -- +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.49 -- +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.5 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps0.12.68 ** +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev115 ** +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_20220425 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_221206 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 ** +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_9 -- + +cam + Cheryl Craig 2022-12-09 - cam6_3_086 - components/cam (cesm2_3_alpha12a) + https://github.com/ESCOMP/CAM/tags/cam6_3_086 + + Introduces PUMAS DDT + Fixes initialization for cam_dev + Update python paths and only build ALI-ARMS for WACCM + Reduce number of time orb_param is printed to log + Reduce number of processors needed for large grid regression tests + + +cdeps + James Edwards 2022-12-05 - cdeps0.12.68 - components/cdeps (cesm2_3_alpha12a) + https://github.com/ESCOMP/CDEPS/tags/cdeps0.12.68 + + Improve usage of shr_log_mod, replace shr_file_getLogUnit with shr_log_getLogUnit + depends on share1.0.15 + + +clm + Erik Kluzek 2022-12-02 - ctsm5.1.dev115 - components/clm (cesm2_3_alpha12a) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev115 + + FATES update for nutrient interface + + +cmeps + Chris Fischer 2022-12-19 - cmeps0.14.5 - src/drivers/nuopc/ (cesm2_3_alpha12a) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.5 + + cmeps0.14.5: remove multi_driver, add precommit config file + cmeps0.14.4: Fix documentation of DOUT_S_SAVE_INTERIM_RESTART_FILES + cmeps0.14.3: Fix issue with scaling over instances. + + +mosart + Erik Kluzek 2022-11-19 - mosart1_0_48 - components/mosart (cesm2_3_alpha12a) + https://github.com/ESCOMP/mosart/tags/mosart1_0_48 + + Change default for negative flow to go direct_to_outlet +============================================================== Tag name: cesm2_3_beta11 Originator(s): CSEG Date: 23 December 2022 From b68bac3cf105dee00b0343658c7e4b79b4332a44 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 23 Jan 2023 12:19:19 -0700 Subject: [PATCH 20/64] Update for cesm2_3_alpha12b --- Externals.cfg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 65bf28e3a..4f978f06d 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.49 +tag = ccs_config_cesm0.0.54 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -21,7 +21,7 @@ local_path = components/cice5 required = True [cice6] -tag = cesm_cice6_2_0_34 +tag = cesm_cice6_4_1_3 protocol = git repo_url = https://github.com/ESCOMP/CESM_CICE local_path = components/cice @@ -65,14 +65,14 @@ local_path = libraries/mct required = True [parallelio] -tag = pio2_5_9 +tag = pio2_5_10 protocol = git repo_url = https://github.com/NCAR/ParallelIO local_path = libraries/parallelio required = True [cime] -tag = cime6.0.82 +tag = cime6.0.85 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime @@ -95,7 +95,7 @@ externals = Externals_CLM.cfg required = True [fms] -tag = fi_20220425 +tag = fi_320121 protocol = git repo_url = https://github.com/ESCOMP/FMS_interface local_path = libraries/FMS @@ -103,7 +103,7 @@ externals = Externals_FMS.cfg required = False [mom] -tag = mi_221206 +tag = mi_230121 protocol = git repo_url = https://github.com/ESCOMP/MOM_interface local_path = components/mom From 3d59573f02b2a04a92db08d3a296205213aadc8e Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 23 Jan 2023 12:27:40 -0700 Subject: [PATCH 21/64] Update for cesm2_3_alpha12b --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 4f978f06d..672d9c2b7 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -95,7 +95,7 @@ externals = Externals_CLM.cfg required = True [fms] -tag = fi_320121 +tag = fi_230121 protocol = git repo_url = https://github.com/ESCOMP/FMS_interface local_path = libraries/FMS From edde799f7e6ad670b4a5b414cfa745af101f5380 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 24 Jan 2023 11:24:18 -0700 Subject: [PATCH 22/64] Update for cesm2_3_alpha12b --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 672d9c2b7..ef4db2811 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.85 +tag = cime6.0.84 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime From c38acd006d0879c09dc9296940d07b919935e82f Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 25 Jan 2023 15:02:49 -0700 Subject: [PATCH 23/64] Update for cesm2_3_alpha12b --- ChangeLog | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/ChangeLog b/ChangeLog index 30d0378f8..abbc8353a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,77 @@ +============================================================== +Tag name: cesm2_3_alpha012b +Originator(s): CSEG +Date: 25 January 2023 +One-line Summary: New CICE and WW3 tags for sea ice wave coupling. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_086 -- +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_3 ** +cime https://github.com/ESMCI/cime/tree/cime6.0.84 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.54 ** +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.5 -- +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps0.12.68 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev115 -- +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 ** +components/mom https://github.com/ESCOMP/MOM_interface/mi_230121 ** +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 ** + +ccs_config + Chris Fischer 2023-01-23 - ccs_config_cesm0.0.54 - ccs_config (cesm2_3_alpha12b) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm0.0.54 + + ccs_config_cesm0.0.54: pleiades machines updates + ccs_config_cesm0.0.53: Update Cheyenne esmf libraries to 8.4.1b01. + ccs_config_cesm0.0.52: Add some new grids for mizuRoute. + ccs_config_cesm0.0.51: Update frontera modules. + ccs_config_cesm0.0.50: Gust update. + + +cice6 + David Bailey 2022-12-07 - cesm_cice6_4_1_3 - components/cice6 (cesm2_3_alpha12b) + https://github.com/ESCOMP/CESM_CICE/tags/cice6_4_0_1 + + All active CICE cases will have different answers. + + +cime + James Edwards 2022-12-12 - cime6.0.84 - cime (cesm2_3_alpha12b) + https://github.com/ESMCI/cime/tags/cime6.0.79 + + cime6.0.84: Need to modify import of importlib. + cime6.0.83: Need to make sure that asyncio_interface is true. + + +fms + Alper Altuntas 2023-01-21 - fi_230121 - libraries/fms (cesm2_3_alpha12b) + https://github.com/ESCOMP/FMS_interface/tags/fi_230121 + + bump up FMS to 2021.03.01 and logging fixes. + + +mom + Alper Altuntas 2023-01-20 - mi_230121 - components/mom (cesm2_3_alpha12b) + https://github.com/ESCOMP/MOM_interface/tags/mi_2301XX + + misc features and bugfixes incl. logging. and changes for new FMS tag. + + +ParallelIO + James Edwards 2022-12-12 - pio2_5_10 - libraries/parallelio (cesm2_3_alpha12b) + https://github.com/NCAR/ParallelIO/tags/pio2_5_10 + + fix needed for mpich4.0.0, bug fix for open mode + + ============================================================== Tag name: cesm2_3_alpha012a Originator(s): CSEG From 5d480b3920b0999a3ce3c3de5e02a2d21fbea537 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 31 Jan 2023 15:27:25 -0700 Subject: [PATCH 24/64] Update for cesm2_3_alpha12c --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index ef4db2811..c14111fb5 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.54 +tag = ccs_config_cesm0.0.55 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.5 +tag = cmeps0.14.12 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps From 36edf01a0b71450c647748e7b44e75cb4a9f9593 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 1 Feb 2023 15:50:29 -0700 Subject: [PATCH 25/64] Update for cesm2_3_alpha12c --- Externals.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index c14111fb5..7d9e14b84 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,14 +29,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.12 +tag = cmeps0.14.15 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps0.12.68 +tag = cdeps1.0.4 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.84 +tag = cime6.0.90 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime From da279ba03ff73d540be6a06597e36ed5748142d9 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 27 Feb 2023 16:22:32 -0700 Subject: [PATCH 26/64] Update for cesm2_3_alpha12c --- Externals.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 7d9e14b84..69a27109a 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.55 +tag = ccs_config_cesm0.0.57 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -29,14 +29,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.15 +tag = cmeps0.14.16 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.4 +tag = cdeps1.0.7 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.90 +tag = cime6.0.94 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime From 6b9366ead1345e621f3c3d1acfea448df0c7e73a Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 3 Mar 2023 16:14:22 -0700 Subject: [PATCH 27/64] Squashed 'manage_externals/' changes from fde04e4..7b6d92e MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7b6d92e Merge pull request #198 from johnpaulalex/gitdir 927ce3a Merge pull request #197 from johnpaulalex/testpath a04f114 Merge pull request #196 from johnpaulalex/readmod d9c14bf Change the rest of the methods to use -C. Still some usage of getcwd in test_unit_repository_git. 332b106 Fix incorrect logged path of checkout_externals in test_sys_checkout: it was basically the parent of the current directory, which varies throughout the test. (it called abspath with '{0}/../../', which adds arbitrary and not-interpolated subdir '{0}' to the path, then removes it and removes one more level). 932a749 Remove printlog from read_gitmodules_file since read_externals_description_file() already has a nearly-the-same printlog (but add it to the other caller). 5d13719 Merge pull request #195 from johnpaulalex/check_repo 4233954 Update utest to mock _git_remote_verbose in a new way, since it is now called via the GitRepository class rather than on the specific GitRepository instance. d7a42ae Check that desired repo was actually checked out. 71596bb Merge pull request #194 from johnpaulalex/manic2 4c96e82 Make the MANIC_TEST_BARE_REPO_ROOT env var special - give it a constant for easy tracking, and automatically tear it down after each test. 259bfc0 test_sys_checkout: use actual paths in on-the-fly configs rather than MANIC_TEST_BARE_REPO_ROOT env var. This will make it easier to test (in the near future) that checkout_externals actually checked out the desired repo dir. 557bbd6 Merge pull request #193 from johnpaulalex/manic 5314eed Remove MANIC_TEST_TMP_REPO_ROOT environment variable in favor of module-level variable. 345fc1e Merge pull request #191 from johnpaulalex/test_doc12 2117b84 test_sys_checkout: verify that basic by-tag/branch/hash tests actually take us to the correct git tag/branch/hash. 94d6e5f Merge pull request #190 from johnpaulalex/test_doc11 3ff33a6 Inline local-path-creation methods 47dea7f Merge pull request #189 from johnpaulalex/test_doc10 9ea75cb Grab-bag of renamings: Remove redundant _NAME from repo constants, and consistently add _REPO suffix (This causes the majority of diffs). c0c847e Merge pull request #188 from johnpaulalex/test_doc9 2dd5ce0 test_sys_checkout.py: only check for correct 'required' or 'optional' state in the test that exercises required vs optional behavior. Removed a lot of boilerplate. eb30859 Merge pull request #187 from johnpaulalex/test_doc8 1832e1f test_sys_checkout: Simplify many tests to only use a single external. 8689d61 Merge pull request #186 from johnpaulalex/test_doc7 fbee425 Grab bag of test_sys_checkout cleanups: Doc inside of each test more clearly/consistently. TestSysCheckoutSVN didn’t get the inlining-of-helper-methods treatment, now it has that. Move various standalone repo helper methods (like create_branch) into a RepoUtils class. README.md was missing newlines when rendered as markdown. Doc the return value of checkout.main Fix test_container_exclude_component - it was looking for the wrong key (which is never present); now it looks for the correct key. f0ed44a Merge pull request #185 from johnpaulalex/test_doc6 a3d59f5 Merge pull request #184 from johnpaulalex/test_doc5 5329c8b test_sys_checkout: Inline config generation functions that are only called once. 464f2c7 test_sys_checkout: Inline another layer (per-config-file checks). Rename the 4 methods that are used multiple times, to reflect what they do rather than what they're called. 8872c0d Merge pull request #183 from johnpaulalex/doc_test4 c045335 Merge pull request #182 from johnpaulalex/doc_test3 c583b95 Merge pull request #181 from johnpaulalex/doc_test2 e01cfe2 test_sys_checkout: less confusing handling of return values from checkout_externals. Specifically, when doing a checkout, don't return tree_status from _before_ the checkout. Make a new wrapper to call checkout_externals a second time, to calculate the new status after a checkout (very frequent pattern). 2328681 test_sys_checkout: Remove another layer (which generates test component names) c3717b6 Merge pull request #180 from johnpaulalex/doc_test 36d7a44 test_sys_checkout.py: remove one layer of functions (that check for local status enums). No-op. 2c4584b More documentation about tests: * contents of test repositories (n a new README.md) * various constants in test_sys_checkout.py that point to those contents, and terminology like container/simple/mixed. * in each test method, the scenarios being tested. * The coupling between test methods. 55e74bd Merge pull request #179 from johnpaulalex/circ 66be842 Remove circular dependency by making _External stop doing tricky things with sourcetrees. 82d3b24 Merge pull request #178 from johnpaulalex/test_doc 3223f49 Additional documentation of system tests - global variables, method descriptions. 45b7c01 Merge pull request #177 from jedwards4b/git_workflow ace90b2 try setting credentials this way f4d6aa9 try setting credentials this way 1d61a69 use this to set git credentials 7f9d330 use this to set git credentials 5ac731b add tmate code 836847b get git workflow working dcd462d Merge pull request #176 from jedwards4b/add_github_testing 2d2479e Merge pull request #175 from johnpaulalex/fix 711a53f add github testing of prs and automatic tagging of main cfe0f88 fix typos 5665d61 Fix broken checkout behavior introduced by PR #172. 27909e2 Merge pull request #173 from johnpaulalex/readall 00ad044 Further tiny refactorings and docs of checkout API (no-op). Remove unused load_all param in _External.checkout(). Rename _External.checkout_externals() to checkout_subexternals(), to remove the ambiguity about whether the main external pointed to by the _External is itelf checked out (it is not) Clarify load_all documentation - it’s always recursive, but applies different criteria at each level. Rename variables in checkout.py (e.g. ext_description) to match the equivalent code in sourcetree.py. 2ea3d1a Merge pull request #172 from johnpaulalex/fixit 43bf809 Merge pull request #171 from johnpaulalex/docstatus e6aa7d2 Merge pull request #170 from johnpaulalex/printdir adbd715 On checkout, refresh locally installed optional packages regardless of whether -o is passed in. add0745 Comment tweaks, and fix 'ppath' typo 696527c Document the format of various status dictionaries, and the various paths and path components within an _External. c677b94 When processing an external, print out its path in addition to the base filename (to disambiguate all the externals.cfg's) 975d7fd Merge pull request #169 from johnpaulalex/docfix_branch 09709e3 Document _Externals.status(). The original comment was apparently copy-pasted from checkout(). 1d880e0 Merge pull request #167 from billsacks/fix_svn_on_windows 3510da8 Tweak a unit test to improve coverage eb7fc13 Handle the possibility that the URL already ends with '/' 02ea87e Fix svn URLs on Windows b1c02ab Merge pull request #165 from gold2718/doc_fix 9f4be8c Add documentation about externals = None feature a3b3a03 Merge pull request #162 from ESMCI/fischer/python3 d4f1b1e Change shebang lines to python3 2fd941a Merge pull request #158 from billsacks/modified_solution de08dc2 Add another option for when an external is in a modified state e954582 Merge pull request #156 from billsacks/onbranch_show_hash 952e44d Change output: put tag/hash before branch name 1028843 Fix pre-existing pylint issues 01b13f7 When on a branch, show tag/hash, too 39ad532 Merge pull request #150 from gold2718/fix_combo_config 75f8f02 Merge pull request #152 from jedwards4b/sort_by_local_path 42687bd remove commented code 29e26af fix pylint issues 7c9f3c6 add a test for nested repo checkout 75c5353 fix spacing 24a3726 improve sorting, checkout externals with each comp 29f45b0 remove py2 test and fix super call 880a4e7 remove decode 1c53be8 no need for set call 36c56db simplier fix for issue dc67cc6 simpler solution b32c6fc fix to allow submodule name different from path 5b5e1c2 Merge pull request #144 from billsacks/improve_errmsg c983863 Add another option for dealing with modified externals 59ce252 Add some details to the error message when externals are modified be5a1a4 Merge pull request #143 from jedwards4b/add_exclude 2aa014a fix lint issue 49cd5e8 fix lint issues 418173f Added tests for ExternalsDescriptionDict afab352 fix lint issue be85b7d fix the test a580a57 push test d437108 add a test 21affe3 fix formatting issue 72e6b64 add an exclude option c33a3bd Merge pull request #139 from jedwards4b/ignore_branch_prop b124a9a ignore this silently, we use a hash so it does not matter git-subtree-dir: manage_externals git-subtree-split: 7b6d92ef689e2f65733e27f8635ab91fb341356b --- .github/workflows/bumpversion.yml | 19 + .github/workflows/tests.yml | 30 + .gitignore | 3 + .travis.yml | 3 +- README.md | 5 + checkout_externals | 2 +- manic/checkout.py | 96 +- manic/externals_description.py | 88 +- manic/externals_status.py | 30 +- manic/repository_factory.py | 1 + manic/repository_git.py | 340 ++-- manic/repository_svn.py | 9 +- manic/sourcetree.py | 442 +++-- manic/utils.py | 2 +- test/README.md | 38 +- test/repos/README.md | 33 + test/test_sys_checkout.py | 2173 +++++++++++------------ test/test_sys_repository_git.py | 60 +- test/test_unit_externals_description.py | 79 +- test/test_unit_externals_status.py | 2 +- test/test_unit_repository.py | 2 +- test/test_unit_repository_git.py | 137 +- test/test_unit_repository_svn.py | 4 +- test/test_unit_utils.py | 2 +- 24 files changed, 1944 insertions(+), 1656 deletions(-) create mode 100644 .github/workflows/bumpversion.yml create mode 100644 .github/workflows/tests.yml create mode 100644 test/repos/README.md mode change 100644 => 100755 test/test_unit_repository_svn.py diff --git a/.github/workflows/bumpversion.yml b/.github/workflows/bumpversion.yml new file mode 100644 index 000000000..f4dc9b7ca --- /dev/null +++ b/.github/workflows/bumpversion.yml @@ -0,0 +1,19 @@ +name: Bump version +on: + push: + branches: + - main +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Bump version and push tag + id: tag_version + uses: mathieudutour/github-tag-action@v5.5 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + create_annotated_tag: true + default_bump: patch + dry_run: false + tag_prefix: manic- diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..dd75b91b4 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,30 @@ +# This is a workflow to compile the cmeps source without cime +name: Test Manic + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + test-manic: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Test Manic + run: | + pushd test + git config --global user.email "devnull@example.com" + git config --global user.name "GITHUB tester" + git config --global protocol.file.allow always + make utest + make stest + popd + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/.gitignore b/.gitignore index 411de5d96..a71ac0cd7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ components/ # generated python files *.pyc + +# test tmp file +test/tmp diff --git a/.travis.yml b/.travis.yml index 1990cb960..d9b24c584 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python os: linux -python: - - "2.7" +python: - "3.4" - "3.5" - "3.6" diff --git a/README.md b/README.md index c931c8e21..9475301b5 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. externals description file pointed 'useful_library/sub-xternals.cfg', Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + Note that by default, `checkout_externals` will clone an external's + submodules. As a special case, the entry, `externals = None`, will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the `from_submodule` + configuration entry below). * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in diff --git a/checkout_externals b/checkout_externals index a0698baef..48bce2401 100755 --- a/checkout_externals +++ b/checkout_externals @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Main driver wrapper around the manic/checkout utility. diff --git a/manic/checkout.py b/manic/checkout.py index edc565595..3f5537adc 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Tool to assemble repositories represented in a model-description file. @@ -227,6 +227,12 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. + Note that by default, checkout_externals will clone an external's + submodules. As a special case, the entry, "externals = None", will + prevent this behavior. For more control over which externals are + checked out, create an externals file (and see the from_submodule + configuration entry below). + * from_submodule (True / False) : used to pull the repo_url, local_path, and hash properties for this external from the .gitmodules file in this repository. Note that the section name (the entry in square @@ -279,6 +285,9 @@ def commandline_arguments(args=None): help='The externals description filename. ' 'Default: %(default)s.') + parser.add_argument('-x', '--exclude', nargs='*', + help='Component(s) listed in the externals file which should be ignored.') + parser.add_argument('-o', '--optional', action='store_true', default=False, help='By default only the required externals ' 'are checked out. This flag will also checkout the ' @@ -329,7 +338,34 @@ def commandline_arguments(args=None): options = parser.parse_args() return options - +def _dirty_local_repo_msg(program_name, config_file): + return """The external repositories labeled with 'M' above are not in a clean state. +The following are four options for how to proceed: +(1) Go into each external that is not in a clean state and issue either a 'git status' or + an 'svn status' command (depending on whether the external is managed by git or + svn). Either revert or commit your changes so that all externals are in a clean + state. (To revert changes in git, follow the instructions given when you run 'git + status'.) (Note, though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. +(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually + update out-of-sync externals (labeled with 's' above) as described in the + configuration file {config_file}. (For example, run 'git fetch' and 'git checkout' + commands to checkout the appropriate tags for each external, as given in + {config_file}.) +(3) You can also use {program_name} to manage most, but not all externals: You can specify + one or more externals to ignore using the '-x' or '--exclude' argument to + {program_name}. Excluding externals labeled with 'M' will allow {program_name} to + update the other, non-excluded externals. +(4) As a last resort, if you are confident that there is no work that needs to be saved + from a given external, you can remove that external (via "rm -rf [directory]") and + then rerun the {program_name} tool. This option is mainly useful as a workaround for + issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157). +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before rerunning the +{program_name} tool. +""".format(program_name=program_name, config_file=config_file) # --------------------------------------------------------------------- # # main @@ -342,9 +378,9 @@ def main(args): the --all option is passed. Returns a tuple (overall_status, tree_status). overall_status is 0 - on success, non-zero on failure. tree_status gives the full status - *before* executing the checkout command - i.e., the status that it - used to determine if it's safe to proceed with the checkout. + on success, non-zero on failure. tree_status is a dict mapping local path + to ExternalStatus -- if no checkout is happening. If checkout is happening, tree_status + is None. """ if args.do_logging: logging.basicConfig(filename=LOG_FILE_NAME, @@ -360,57 +396,41 @@ def main(args): load_all = True root_dir = os.path.abspath(os.getcwd()) - external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description( - external_data, components=args.components) + model_data = read_externals_description_file(root_dir, args.externals) + ext_description = create_externals_description( + model_data, components=args.components, exclude=args.exclude) for comp in args.components: - if comp not in external.keys(): + if comp not in ext_description.keys(): + # Note we can't print out the list of found externals because + # they were filtered in create_externals_description above. fatal_error( "No component {} found in {}".format( comp, args.externals)) - source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) - printlog('Checking status of externals: ', end='') - tree_status = source_tree.status() + source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry) + if args.components: + components_str = 'specified components' + else: + components_str = 'required & optional components' + printlog('Checking local status of ' + components_str + ': ', end='') + tree_status = source_tree.status(print_progress=True) printlog('') if args.status: # user requested status-only - for comp in sorted(tree_status.keys()): + for comp in sorted(tree_status): tree_status[comp].log_status_message(args.verbose) else: # checkout / update the external repositories. safe_to_update = check_safe_to_update_repos(tree_status) if not safe_to_update: # print status - for comp in sorted(tree_status.keys()): + for comp in sorted(tree_status): tree_status[comp].log_status_message(args.verbose) # exit gracefully - msg = """The external repositories labeled with 'M' above are not in a clean state. - -The following are two options for how to proceed: - -(1) Go into each external that is not in a clean state and issue either - an 'svn status' or a 'git status' command. Either revert or commit - your changes so that all externals are in a clean state. (Note, - though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. - -(2) Alternatively, you do not have to rely on {program_name}. Instead, you - can manually update out-of-sync externals (labeled with 's' above) - as described in the configuration file {config_file}. - - -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before re-running the -checkout_externals tool. -""".format(program_name=program_name, config_file=args.externals) - printlog('-' * 70) - printlog(msg) + printlog(_dirty_local_repo_msg(program_name, args.externals)) printlog('-' * 70) else: if not args.components: @@ -418,6 +438,8 @@ def main(args): for comp in args.components: source_tree.checkout(args.verbose, load_all, load_comp=comp) printlog('') + # New tree status is unknown, don't return anything. + tree_status = None logging.info('%s completed without exceptions.', program_name) # NOTE(bja, 2017-11) tree status is used by the systems tests diff --git a/manic/externals_description.py b/manic/externals_description.py index b0c4f736a..546e7fdcb 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Model description @@ -71,7 +71,8 @@ def read_externals_description_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing externals description file : {0}'.format(file_name)) + printlog('Processing externals description file : {0} ({1})'.format(file_name, + root_dir)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): @@ -87,7 +88,7 @@ def read_externals_description_file(root_dir, file_name): externals_description = None if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: - externals_description = read_gitmodules_file(root_dir, file_name) + externals_description = _read_gitmodules_file(root_dir, file_name) else: try: config = config_parser() @@ -150,9 +151,8 @@ def git_submodule_status(repo_dir): """Run the git submodule status command to obtain submodule hashes. """ # This function is here instead of GitRepository to avoid a dependency loop - cwd = os.getcwd() - os.chdir(repo_dir) - cmd = ['git', 'submodule', 'status'] + cmd = 'git -C {repo_dir} submodule status'.format( + repo_dir=repo_dir).split() git_output = execute_subprocess(cmd, output_to_caller=True) submodules = {} submods = git_output.split('\n') @@ -167,7 +167,6 @@ def git_submodule_status(repo_dir): submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} - os.chdir(cwd) return submodules def parse_submodules_desc_section(section_items, file_path): @@ -180,6 +179,9 @@ def parse_submodules_desc_section(section_items, file_path): path = item[1].strip() elif name == 'url': url = item[1].strip() + elif name == 'branch': + # We do not care about branch since we have a hash - silently ignore + pass else: msg = 'WARNING: Ignoring unknown {} property, in {}' msg = msg.format(item[0], file_path) # fool pylint @@ -187,21 +189,23 @@ def parse_submodules_desc_section(section_items, file_path): return path, url -def read_gitmodules_file(root_dir, file_name): +def _read_gitmodules_file(root_dir, file_name): # pylint: disable=deprecated-method # Disabling this check because the method is only used for python2 + # pylint: disable=too-many-locals + # pylint: disable=too-many-branches + # pylint: disable=too-many-statements """Read a .gitmodules file and convert it to be compatible with an externals description. """ root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing submodules description file : {0}'.format(file_name)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): msg = ('ERROR: submodules description file, "{0}", does not ' - 'exist at path:\n {1}'.format(file_name, file_path)) + 'exist in dir:\n {1}'.format(file_name, root_dir)) fatal_error(msg) submodules_description = None @@ -250,9 +254,21 @@ def read_gitmodules_file(root_dir, file_name): ExternalsDescription.REPO_URL, url) externals_description.set(sec_name, ExternalsDescription.REQUIRED, 'True') - git_hash = submods[sec_name]['hash'] - externals_description.set(sec_name, - ExternalsDescription.HASH, git_hash) + if sec_name in submods: + submod_name = sec_name + else: + # The section name does not have to match the path + submod_name = path + + if submod_name in submods: + git_hash = submods[submod_name]['hash'] + externals_description.set(sec_name, + ExternalsDescription.HASH, + git_hash) + else: + emsg = "submodule status has no section, '{}'" + emsg += "\nCheck section names in externals config file" + fatal_error(emsg.format(submod_name)) # Required items externals_description.add_section(DESCRIPTION_SECTION) @@ -261,18 +277,22 @@ def read_gitmodules_file(root_dir, file_name): return externals_description def create_externals_description( - model_data, model_format='cfg', components=None, parent_repo=None): + model_data, model_format='cfg', components=None, exclude=None, parent_repo=None): """Create the a externals description object from the provided data + + components: list of component names to include, None to include all. If a + name isn't found, it is silently omitted from the return value. + exclude: list of component names to skip. """ externals_description = None if model_format == 'dict': externals_description = ExternalsDescriptionDict( - model_data, components=components) + model_data, components=components, exclude=exclude) elif model_format == 'cfg': major, _, _ = get_cfg_schema_version(model_data) if major == 1: externals_description = ExternalsDescriptionConfigV1( - model_data, components=components, parent_repo=parent_repo) + model_data, components=components, exclude=exclude, parent_repo=parent_repo) else: msg = ('Externals description file has unsupported schema ' 'version "{0}".'.format(major)) @@ -339,8 +359,9 @@ class ExternalsDescription(dict): input value. """ - # keywords defining the interface into the externals description data - EXTERNALS = 'externals' + # keywords defining the interface into the externals description data; these + # are brought together by the schema below. + EXTERNALS = 'externals' # path to externals file. BRANCH = 'branch' SUBMODULE = 'from_submodule' HASH = 'hash' @@ -366,6 +387,8 @@ class ExternalsDescription(dict): _V1_BRANCH = 'BRANCH' _V1_REQ_SOURCE = 'REQ_SOURCE' + # Dictionary keys are component names. The corresponding values are laid out + # according to this schema. _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', @@ -614,8 +637,11 @@ def _repo_config_from_submodule(self, field, submod_desc): ' Parent repo, "{1}" does not have submodules') fatal_error(msg.format(field, self._parent_repo.name())) - submod_file = read_gitmodules_file(repo_path, submod_file) - submod_desc = create_externals_description(submod_file) + printlog( + 'Processing submodules description file : {0} ({1})'.format( + submod_file, repo_path)) + submod_model_data= _read_gitmodules_file(repo_path, submod_file) + submod_desc = create_externals_description(submod_model_data) # Can we find our external? repo_url = None @@ -707,7 +733,7 @@ class ExternalsDescriptionDict(ExternalsDescription): """ - def __init__(self, model_data, components=None): + def __init__(self, model_data, components=None, exclude=None): """Parse a native dictionary into a externals description. """ ExternalsDescription.__init__(self) @@ -719,10 +745,15 @@ def __init__(self, model_data, components=None): self._input_patch = 0 self._verify_schema_version() if components: - for key in model_data.items(): + for key in list(model_data.keys()): if key not in components: del model_data[key] + if exclude: + for key in list(model_data.keys()): + if key in exclude: + del model_data[key] + self.update(model_data) self._check_user_input() @@ -733,10 +764,12 @@ class ExternalsDescriptionConfigV1(ExternalsDescription): """ - def __init__(self, model_data, components=None, parent_repo=None): + def __init__(self, model_data, components=None, exclude=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 @@ -746,7 +779,7 @@ def __init__(self, model_data, components=None, parent_repo=None): get_cfg_schema_version(model_data) self._verify_schema_version() self._remove_metadata(model_data) - self._parse_cfg(model_data, components=components) + self._parse_cfg(model_data, components=components, exclude=exclude) self._check_user_input() @staticmethod @@ -758,8 +791,11 @@ def _remove_metadata(model_data): """ model_data.remove_section(DESCRIPTION_SECTION) - def _parse_cfg(self, cfg_data, components=None): + def _parse_cfg(self, cfg_data, components=None, exclude=None): """Parse a config_parser object into a externals description. + + components: list of component names to include, None to include all. + exclude: list of component names to skip. """ def list_to_dict(input_list, convert_to_lower_case=True): """Convert a list of key-value pairs into a dictionary. @@ -775,7 +811,7 @@ def list_to_dict(input_list, convert_to_lower_case=True): for section in cfg_data.sections(): name = config_string_cleaner(section.lower().strip()) - if components and name not in components: + if (components and name not in components) or (exclude and name in exclude): continue self[name] = {} self[name].update(list_to_dict(cfg_data.items(section))) diff --git a/manic/externals_status.py b/manic/externals_status.py index d3d238f28..6bc29e973 100644 --- a/manic/externals_status.py +++ b/manic/externals_status.py @@ -29,16 +29,16 @@ class ExternalStatus(object): transactions (e.g. add, remove, rename, untracked files). """ - DEFAULT = '-' + # sync_state and clean_state can be one of the following: + DEFAULT = '-' # not set yet (sync_state). clean_state can be this if sync_state is EMPTY. UNKNOWN = '?' EMPTY = 'e' - MODEL_MODIFIED = 's' # a.k.a. out-of-sync - DIRTY = 'M' - - STATUS_OK = ' ' + MODEL_MODIFIED = 's' # repo version != externals (sync_state only) + DIRTY = 'M' # repo is dirty (clean_state only) + STATUS_OK = ' ' # repo is clean (clean_state) or matches externals version (sync_state) STATUS_ERROR = '!' - # source types + # source_type can be one of the following: OPTIONAL = 'o' STANDALONE = 's' MANAGED = ' ' @@ -55,19 +55,21 @@ def __init__(self): def log_status_message(self, verbosity): """Write status message to the screen and log file """ - self._default_status_message() + printlog(self._default_status_message()) if verbosity >= VERBOSITY_VERBOSE: - self._verbose_status_message() + printlog(self._verbose_status_message()) if verbosity >= VERBOSITY_DUMP: - self._dump_status_message() + printlog(self._dump_status_message()) + + def __repr__(self): + return self._default_status_message() def _default_status_message(self): """Return the default terse status message string """ - msg = '{sync}{clean}{src_type} {path}'.format( + return '{sync}{clean}{src_type} {path}'.format( sync=self.sync_state, clean=self.clean_state, src_type=self.source_type, path=self.path) - printlog(msg) def _verbose_status_message(self): """Return the verbose status message string @@ -82,14 +84,12 @@ def _verbose_status_message(self): if self.sync_state != self.STATUS_OK: sync_str = '{current} --> {expected}'.format( current=self.current_version, expected=self.expected_version) - msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) - printlog(msg) + return ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) def _dump_status_message(self): """Return the dump status message string """ - msg = indent_string(self.status_output, 12) - printlog(msg) + return indent_string(self.status_output, 12) def safe_to_update(self): """Report if it is safe to update a repository. Safe is defined as: diff --git a/manic/repository_factory.py b/manic/repository_factory.py index 80a92a9d8..18c73ffc4 100644 --- a/manic/repository_factory.py +++ b/manic/repository_factory.py @@ -15,6 +15,7 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False): """Determine what type of repository we have, i.e. git or svn, and create the appropriate object. + Can return None (e.g. if protocol is 'externals_only'). """ protocol = repo_info[ExternalsDescription.PROTOCOL].lower() if protocol == 'git': diff --git a/manic/repository_git.py b/manic/repository_git.py index f98605100..adc666cc5 100644 --- a/manic/repository_git.py +++ b/manic/repository_git.py @@ -25,7 +25,7 @@ class GitRepository(Repository): * be isolated in separate functions with no application logic * of the form: - - cmd = ['git', ...] + - cmd = 'git -C {dirname} ...'.format(dirname=dirname).split() - value = execute_subprocess(cmd, output_to_caller={T|F}, status_to_caller={T|F}) - return value @@ -39,7 +39,7 @@ class GitRepository(Repository): def __init__(self, component_name, repo): """ - Parse repo (a XML element). + repo: ExternalsDescription. """ Repository.__init__(self, component_name, repo) self._gitmodules = None @@ -99,45 +99,42 @@ def submodules_file(self, repo_path=None): # # ---------------------------------------------------------------- def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): - """Prepare to execute the clone by managing directory location + """Clones repo_dir_name into base_dir_path. """ - cwd = os.getcwd() - os.chdir(base_dir_path) - self._git_clone(self._url, repo_dir_name, verbosity) - os.chdir(cwd) + self._git_clone(self._url, os.path.join(base_dir_path, repo_dir_name), + verbosity=verbosity) - def _current_ref(self): - """Determine the *name* associated with HEAD. + def _current_ref(self, dirname): + """Determine the *name* associated with HEAD at dirname. - If we're on a branch, then returns the branch name; otherwise, - if we're on a tag, then returns the tag name; otherwise, returns + If we're on a tag, then returns the tag name; otherwise, returns the current hash. Returns an empty string if no reference can be determined (e.g., if we're not actually in a git repository). + + If we're on a branch, then the branch name is also included in + the returned string (in addition to the tag / hash). """ ref_found = False - # If we're on a branch, then use that as the current ref - branch_found, branch_name = self._git_current_branch() - if branch_found: - current_ref = branch_name + # If we're exactly at a tag, use that as the current ref + tag_found, tag_name = self._git_current_tag(dirname) + if tag_found: + current_ref = tag_name ref_found = True - if not ref_found: - # Otherwise, if we're exactly at a tag, use that as the - # current ref - tag_found, tag_name = self._git_current_tag() - if tag_found: - current_ref = tag_name - ref_found = True - if not ref_found: # Otherwise, use current hash as the current ref - hash_found, hash_name = self._git_current_hash() + hash_found, hash_name = self._git_current_hash(dirname) if hash_found: current_ref = hash_name ref_found = True - if not ref_found: + if ref_found: + # If we're on a branch, include branch name in current ref + branch_found, branch_name = self._git_current_branch(dirname) + if branch_found: + current_ref = "{} (branch {})".format(current_ref, branch_name) + else: # If we still can't find a ref, return empty string. This # can happen if we're not actually in a git repo current_ref = '' @@ -185,17 +182,15 @@ def compare_refs(current_ref, expected_ref): status = ExternalStatus.MODEL_MODIFIED return status - cwd = os.getcwd() - os.chdir(repo_dir_path) - # get the full hash of the current commit - _, current_ref = self._git_current_hash() + _, current_ref = self._git_current_hash(repo_dir_path) if self._branch: if self._url == LOCAL_PATH_INDICATOR: expected_ref = self._branch else: - remote_name = self._determine_remote_name() + remote_name = self._remote_name_for_url(self._url, + repo_dir_path) if not remote_name: # git doesn't know about this remote. by definition # this is a modified state. @@ -212,7 +207,7 @@ def compare_refs(current_ref, expected_ref): fatal_error(msg) # record the *names* of the current and expected branches - stat.current_version = self._current_ref() + stat.current_version = self._current_ref(repo_dir_path) stat.expected_version = copy.deepcopy(expected_ref) if current_ref == EMPTY_STR: @@ -220,7 +215,7 @@ def compare_refs(current_ref, expected_ref): else: # get the underlying hash of the expected ref revparse_status, expected_ref_hash = self._git_revparse_commit( - expected_ref) + expected_ref, repo_dir_path) if revparse_status: # We failed to get the hash associated with # expected_ref. Maybe we should assign this to some special @@ -231,18 +226,13 @@ def compare_refs(current_ref, expected_ref): # compare the underlying hashes stat.sync_state = compare_refs(current_ref, expected_ref_hash) - os.chdir(cwd) - - def _determine_remote_name(self): - """Return the remote name. - - Note that this is for the *future* repo url and branch, not - the current working copy! + @classmethod + def _remote_name_for_url(cls, remote_url, dirname): + """Return the remote name matching remote_url (or None) """ - git_output = self._git_remote_verbose() + git_output = cls._git_remote_verbose(dirname) git_output = git_output.splitlines() - remote_name = '' for line in git_output: data = line.strip() if not data: @@ -250,10 +240,9 @@ def _determine_remote_name(self): data = data.split() name = data[0].strip() url = data[1].strip() - if self._url == url: - remote_name = name - break - return remote_name + if remote_url == url: + return name + return None def _create_remote_name(self): """The url specified in the externals description file was not known @@ -309,19 +298,16 @@ def _checkout_ref(self, repo_dir, verbosity, submodules): the repo's submodules """ # import pdb; pdb.set_trace() - cwd = os.getcwd() - os.chdir(repo_dir) if self._url.strip() == LOCAL_PATH_INDICATOR: - self._checkout_local_ref(verbosity, submodules) + self._checkout_local_ref(verbosity, submodules, repo_dir) else: - self._checkout_external_ref(verbosity, submodules) + self._checkout_external_ref(verbosity, submodules, repo_dir) if self._sparse: self._sparse_checkout(repo_dir, verbosity) - os.chdir(cwd) - def _checkout_local_ref(self, verbosity, submodules): + def _checkout_local_ref(self, verbosity, submodules, dirname): """Checkout the reference considering the local repo only. Do not fetch any additional remotes or specify the remote when checkout out the ref. @@ -335,13 +321,18 @@ def _checkout_local_ref(self, verbosity, submodules): else: ref = self._hash - self._check_for_valid_ref(ref) - self._git_checkout_ref(ref, verbosity, submodules) + self._check_for_valid_ref(ref, remote_name=None, + dirname=dirname) + self._git_checkout_ref(ref, verbosity, submodules, dirname) - def _checkout_external_ref(self, verbosity, submodules): - """Checkout the reference from a remote repository + def _checkout_external_ref(self, verbosity, submodules, dirname): + """Checkout the reference from a remote repository into dirname. if is True, recursively initialize and update - the repo's submodules + the repo's submodules. + Note that this results in a 'detached HEAD' state if checking out + a branch, because we check out the remote branch rather than the + local. See https://github.com/ESMCI/manage_externals/issues/34 for + more discussion. """ if self._tag: ref = self._tag @@ -350,44 +341,45 @@ def _checkout_external_ref(self, verbosity, submodules): else: ref = self._hash - remote_name = self._determine_remote_name() + remote_name = self._remote_name_for_url(self._url, dirname) if not remote_name: remote_name = self._create_remote_name() - self._git_remote_add(remote_name, self._url) - self._git_fetch(remote_name) + self._git_remote_add(remote_name, self._url, dirname) + self._git_fetch(remote_name, dirname) # NOTE(bja, 2018-03) we need to send separate ref and remote # name to check_for_vaild_ref, but the combined name to # checkout_ref! - self._check_for_valid_ref(ref, remote_name) + self._check_for_valid_ref(ref, remote_name, dirname) if self._branch: + # Prepend remote name to branch. This means we avoid various + # special cases if the local branch is not tracking the remote or + # cannot be trivially fast-forwarded to match; but, it also + # means we end up in a 'detached HEAD' state. ref = '{0}/{1}'.format(remote_name, ref) - self._git_checkout_ref(ref, verbosity, submodules) + self._git_checkout_ref(ref, verbosity, submodules, dirname) def _sparse_checkout(self, repo_dir, verbosity): """Use git read-tree to thin the working tree.""" - cwd = os.getcwd() - - cmd = ['cp', self._sparse, os.path.join(repo_dir, - '.git/info/sparse-checkout')] + cmd = ['cp', os.path.join(repo_dir, self._sparse), + os.path.join(repo_dir, + '.git/info/sparse-checkout')] if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - os.chdir(repo_dir) - self._git_sparse_checkout(verbosity) - - os.chdir(cwd) + self._git_sparse_checkout(verbosity, repo_dir) - def _check_for_valid_ref(self, ref, remote_name=None): + def _check_for_valid_ref(self, ref, remote_name, dirname): """Try some basic sanity checks on the user supplied reference so we can provide a more useful error message than calledprocess error... + remote_name can be NOne """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) + is_tag = self._ref_is_tag(ref, dirname) + is_branch = self._ref_is_branch(ref, remote_name, dirname) + is_hash = self._ref_is_hash(ref, dirname) is_valid = is_tag or is_branch or is_hash if not is_valid: @@ -398,7 +390,8 @@ def _check_for_valid_ref(self, ref, remote_name=None): fatal_error(msg) if is_tag: - is_unique_tag, msg = self._is_unique_tag(ref, remote_name) + is_unique_tag, msg = self._is_unique_tag(ref, remote_name, + dirname) if not is_unique_tag: msg = ('In repo "{0}": tag "{1}" {2}'.format( self._name, self._tag, msg)) @@ -406,7 +399,7 @@ def _check_for_valid_ref(self, ref, remote_name=None): return is_valid - def _is_unique_tag(self, ref, remote_name): + def _is_unique_tag(self, ref, remote_name, dirname): """Verify that a reference is a valid tag and is unique (not a branch) Tags may be tag names, or SHA id's. It is also possible that a @@ -417,9 +410,9 @@ def _is_unique_tag(self, ref, remote_name): error! """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) + is_tag = self._ref_is_tag(ref, dirname) + is_branch = self._ref_is_branch(ref, remote_name, dirname) + is_hash = self._ref_is_hash(ref, dirname) msg = '' is_unique_tag = False @@ -450,7 +443,7 @@ def _is_unique_tag(self, ref, remote_name): return is_unique_tag, msg - def _ref_is_tag(self, ref): + def _ref_is_tag(self, ref, dirname): """Verify that a reference is a valid tag according to git. Note: values returned by git_showref_* and git_revparse are @@ -458,28 +451,30 @@ def _ref_is_tag(self, ref): error! """ is_tag = False - value = self._git_showref_tag(ref) + value = self._git_showref_tag(ref, dirname) if value == 0: is_tag = True return is_tag - def _ref_is_branch(self, ref, remote_name=None): + def _ref_is_branch(self, ref, remote_name, dirname): """Verify if a ref is any kind of branch (local, tracked remote, untracked remote). + remote_name can be None. """ local_branch = False remote_branch = False if remote_name: - remote_branch = self._ref_is_remote_branch(ref, remote_name) - local_branch = self._ref_is_local_branch(ref) + remote_branch = self._ref_is_remote_branch(ref, remote_name, + dirname) + local_branch = self._ref_is_local_branch(ref, dirname) is_branch = False if local_branch or remote_branch: is_branch = True return is_branch - def _ref_is_local_branch(self, ref): + def _ref_is_local_branch(self, ref, dirname): """Verify that a reference is a valid branch according to git. show-ref branch returns local branches that have been @@ -492,12 +487,12 @@ def _ref_is_local_branch(self, ref): """ is_branch = False - value = self._git_showref_branch(ref) + value = self._git_showref_branch(ref, dirname) if value == 0: is_branch = True return is_branch - def _ref_is_remote_branch(self, ref, remote_name): + def _ref_is_remote_branch(self, ref, remote_name, dirname): """Verify that a reference is a valid branch according to git. show-ref branch returns local branches that have been @@ -510,12 +505,12 @@ def _ref_is_remote_branch(self, ref, remote_name): """ is_branch = False - value = self._git_lsremote_branch(ref, remote_name) + value = self._git_lsremote_branch(ref, remote_name, dirname) if value == 0: is_branch = True return is_branch - def _ref_is_commit(self, ref): + def _ref_is_commit(self, ref, dirname): """Verify that a reference is a valid commit according to git. This could be a tag, branch, sha1 id, HEAD and potentially others... @@ -525,12 +520,12 @@ def _ref_is_commit(self, ref): error! """ is_commit = False - value, _ = self._git_revparse_commit(ref) + value, _ = self._git_revparse_commit(ref, dirname) if value == 0: is_commit = True return is_commit - def _ref_is_hash(self, ref): + def _ref_is_hash(self, ref, dirname): """Verify that a reference is a valid hash according to git. Git doesn't seem to provide an exact way to determine if user @@ -545,7 +540,7 @@ def _ref_is_hash(self, ref): """ is_hash = False - status, git_output = self._git_revparse_commit(ref) + status, git_output = self._git_revparse_commit(ref, dirname) if status == 0: if git_output.strip().startswith(ref): is_hash = True @@ -555,9 +550,7 @@ def _status_summary(self, stat, repo_dir_path): """Determine the clean/dirty status of a git repository """ - cwd = os.getcwd() - os.chdir(repo_dir_path) - git_output = self._git_status_porcelain_v1z() + git_output = self._git_status_porcelain_v1z(repo_dir_path) is_dirty = self._status_v1z_is_dirty(git_output) if is_dirty: stat.clean_state = ExternalStatus.DIRTY @@ -566,8 +559,7 @@ def _status_summary(self, stat, repo_dir_path): # Now save the verbose status output incase the user wants to # see it. - stat.status_output = self._git_status_verbose() - os.chdir(cwd) + stat.status_output = self._git_status_verbose(repo_dir_path) @staticmethod def _status_v1z_is_dirty(git_output): @@ -602,7 +594,7 @@ def _status_v1z_is_dirty(git_output): # # ---------------------------------------------------------------- @staticmethod - def _git_current_hash(): + def _git_current_hash(dirname): """Return the full hash of the currently checked-out version. Returns a tuple, (hash_found, hash), where hash_found is a @@ -610,21 +602,51 @@ def _git_current_hash(): could mean we're not in a git repository at all). (If hash_found is False, then hash is ''.) """ - status, git_output = GitRepository._git_revparse_commit("HEAD") + status, git_output = GitRepository._git_revparse_commit("HEAD", + dirname) hash_found = not status if not hash_found: git_output = '' return hash_found, git_output @staticmethod - def _git_current_branch(): - """Determines the name of the current branch. + def _git_current_remote_branch(dirname): + """Determines the name of the current remote branch, if any. + + if dir is None, uses the cwd. + + Returns a tuple, (branch_found, branch_name), where branch_found + is a bool specifying whether a branch name was found for + HEAD. (If branch_found is False, then branch_name is ''). + branch_name is in the format '$remote/$branch', e.g. 'origin/foo'. + """ + branch_found = False + branch_name = '' + + cmd = 'git -C {dirname} log -n 1 --pretty=%d HEAD'.format( + dirname=dirname).split() + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + branch_found = 'HEAD,' in git_output + if branch_found: + # git_output is of the form " (HEAD, origin/blah)" + branch_name = git_output.split(',')[1].strip()[:-1] + return branch_found, branch_name + + @staticmethod + def _git_current_branch(dirname): + """Determines the name of the current local branch. Returns a tuple, (branch_found, branch_name), where branch_found - is a logical specifying whether a branch name was found for + is a bool specifying whether a branch name was found for HEAD. (If branch_found is False, then branch_name is ''.) + Note that currently we check out the remote branch rather than + the local, so this command does not return the just-checked-out + branch. See _git_current_remote_branch. """ - cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] + cmd = 'git -C {dirname} symbolic-ref --short -q HEAD'.format( + dirname=dirname).split() status, git_output = execute_subprocess(cmd, output_to_caller=True, status_to_caller=True) @@ -636,15 +658,17 @@ def _git_current_branch(): return branch_found, git_output @staticmethod - def _git_current_tag(): + def _git_current_tag(dirname): """Determines the name tag corresponding to HEAD (if any). + if dirname is None, uses the cwd. + Returns a tuple, (tag_found, tag_name), where tag_found is a - logical specifying whether we found a tag name corresponding to + bool specifying whether we found a tag name corresponding to HEAD. (If tag_found is False, then tag_name is ''.) """ - # git describe --exact-match --tags HEAD - cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] + cmd = 'git -C {dirname} describe --exact-match --tags HEAD'.format( + dirname=dirname).split() status, git_output = execute_subprocess(cmd, output_to_caller=True, status_to_caller=True) @@ -656,53 +680,54 @@ def _git_current_tag(): return tag_found, git_output @staticmethod - def _git_showref_tag(ref): + def _git_showref_tag(ref, dirname): """Run git show-ref check if the user supplied ref is a tag. could also use git rev-parse --quiet --verify tagname^{tag} """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/tags/{0}'.format(ref), ] + cmd = ('git -C {dirname} show-ref --quiet --verify refs/tags/{ref}' + .format(dirname=dirname, ref=ref).split()) status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_showref_branch(ref): + def _git_showref_branch(ref, dirname): """Run git show-ref check if the user supplied ref is a local or tracked remote branch. """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/heads/{0}'.format(ref), ] + cmd = ('git -C {dirname} show-ref --quiet --verify refs/heads/{ref}' + .format(dirname=dirname, ref=ref).split()) status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_lsremote_branch(ref, remote_name): + def _git_lsremote_branch(ref, remote_name, dirname): """Run git ls-remote to check if the user supplied ref is a remote branch that is not being tracked """ - cmd = ['git', 'ls-remote', '--exit-code', '--heads', - remote_name, ref, ] + cmd = ('git -C {dirname} ls-remote --exit-code --heads ' + '{remote_name} {ref}').format( + dirname=dirname, remote_name=remote_name, ref=ref).split() status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_revparse_commit(ref): + def _git_revparse_commit(ref, dirname): """Run git rev-parse to detect if a reference is a SHA, HEAD or other valid commit. """ - cmd = ['git', 'rev-parse', '--quiet', '--verify', - '{0}^{1}'.format(ref, '{commit}'), ] + cmd = ('git -C {dirname} rev-parse --quiet --verify {ref}^{commit}' + .format(dirname=dirname, ref=ref, commit='{commit}').split()) status, git_output = execute_subprocess(cmd, status_to_caller=True, output_to_caller=True) git_output = git_output.strip() return status, git_output @staticmethod - def _git_status_porcelain_v1z(): + def _git_status_porcelain_v1z(dirname): """Run git status to obtain repository information. This is run with '--untracked=no' to ignore untracked files. @@ -711,36 +736,38 @@ def _git_status_porcelain_v1z(): between git versions or *user configuration*. """ - cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] + cmd = ('git -C {dirname} status --untracked-files=no --porcelain -z' + .format(dirname=dirname)).split() git_output = execute_subprocess(cmd, output_to_caller=True) return git_output @staticmethod - def _git_status_verbose(): + def _git_status_verbose(dirname): """Run the git status command to obtain repository information. """ - cmd = ['git', 'status'] + cmd = 'git -C {dirname} status'.format(dirname=dirname).split() git_output = execute_subprocess(cmd, output_to_caller=True) return git_output @staticmethod - def _git_remote_verbose(): + def _git_remote_verbose(dirname): """Run the git remote command to obtain repository information. + + Returned string is of the form: + myfork git@github.com:johnpaulalex/manage_externals_jp.git (fetch) + myfork git@github.com:johnpaulalex/manage_externals_jp.git (push) """ - cmd = ['git', 'remote', '--verbose'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output + cmd = 'git -C {dirname} remote --verbose'.format( + dirname=dirname).split() + return execute_subprocess(cmd, output_to_caller=True) @staticmethod - def has_submodules(repo_dir_path=None): - """Return True iff the repository at (or the current - directory if is None) has a '.gitmodules' file + def has_submodules(repo_dir_path): + """Return True iff the repository at has a + '.gitmodules' file """ - if repo_dir_path is None: - fname = ExternalsDescription.GIT_SUBMODULES_FILENAME - else: - fname = os.path.join(repo_dir_path, - ExternalsDescription.GIT_SUBMODULES_FILENAME) + fname = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) return os.path.exists(fname) @@ -751,68 +778,71 @@ def has_submodules(repo_dir_path=None): # ---------------------------------------------------------------- @staticmethod def _git_clone(url, repo_dir_name, verbosity): - """Run git clone for the side effect of creating a repository. + """Clones url into repo_dir_name. """ - cmd = ['git', 'clone', '--quiet'] - subcmd = None - - cmd.extend([url, repo_dir_name]) + cmd = 'git clone --quiet {url} {repo_dir_name}'.format( + url=url, repo_dir_name=repo_dir_name).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - if subcmd is not None: - os.chdir(repo_dir_name) - execute_subprocess(subcmd) @staticmethod - def _git_remote_add(name, url): + def _git_remote_add(name, url, dirname): """Run the git remote command for the side effect of adding a remote """ - cmd = ['git', 'remote', 'add', name, url] + cmd = 'git -C {dirname} remote add {name} {url}'.format( + dirname=dirname, name=name, url=url).split() execute_subprocess(cmd) @staticmethod - def _git_fetch(remote_name): + def _git_fetch(remote_name, dirname): """Run the git fetch command for the side effect of updating the repo """ - cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] + cmd = 'git -C {dirname} fetch --quiet --tags {remote_name}'.format( + dirname=dirname, remote_name=remote_name).split() execute_subprocess(cmd) @staticmethod - def _git_checkout_ref(ref, verbosity, submodules): + def _git_checkout_ref(ref, verbosity, submodules, dirname): """Run the git checkout command for the side effect of updating the repo Param: ref is a reference to a local or remote object in the form 'origin/my_feature', or 'tag1'. """ - cmd = ['git', 'checkout', '--quiet', ref] + cmd = 'git -C {dirname} checkout --quiet {ref}'.format( + dirname=dirname, ref=ref).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) if submodules: - GitRepository._git_update_submodules(verbosity) + GitRepository._git_update_submodules(verbosity, dirname) @staticmethod - def _git_sparse_checkout(verbosity): + def _git_sparse_checkout(verbosity, dirname): """Configure repo via read-tree.""" - cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + cmd = 'git -C {dirname} config core.sparsecheckout true'.format( + dirname=dirname).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - cmd = ['git', 'read-tree', '-mu', 'HEAD'] + cmd = 'git -C {dirname} read-tree -mu HEAD'.format( + dirname=dirname).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) @staticmethod - def _git_update_submodules(verbosity): + def _git_update_submodules(verbosity, dirname): """Run git submodule update for the side effect of updating this repo's submodules. """ # First, verify that we have a .gitmodules file - if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): - cmd = ['git', 'submodule', 'update', '--init', '--recursive'] + if os.path.exists( + os.path.join(dirname, + ExternalsDescription.GIT_SUBMODULES_FILENAME)): + cmd = ('git -C {dirname} submodule update --init --recursive' + .format(dirname=dirname)).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) diff --git a/manic/repository_svn.py b/manic/repository_svn.py index 408ed8467..922855d34 100644 --- a/manic/repository_svn.py +++ b/manic/repository_svn.py @@ -43,10 +43,15 @@ def __init__(self, component_name, repo, ignore_ancestry=False): """ Repository.__init__(self, component_name, repo) self._ignore_ancestry = ignore_ancestry + if self._url.endswith('/'): + # there is already a '/' separator in the URL; no need to add another + url_sep = '' + else: + url_sep = '/' if self._branch: - self._url = os.path.join(self._url, self._branch) + self._url = self._url + url_sep + self._branch elif self._tag: - self._url = os.path.join(self._url, self._tag) + self._url = self._url + url_sep + self._tag else: msg = "DEV_ERROR in svn repository. Shouldn't be here!" fatal_error(msg) diff --git a/manic/sourcetree.py b/manic/sourcetree.py index b9c9c2108..cf2a5b756 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -1,6 +1,6 @@ """ - -FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! +Classes to represent an externals config file (SourceTree) and the components +within it (_External). """ import errno @@ -19,62 +19,54 @@ class _External(object): """ - _External represents an external object inside a SourceTree - """ + A single component hosted in an external repository (and any children). + The component may or may not be checked-out upon construction. + """ # pylint: disable=R0902 - def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): - """Parse an external description file into a dictionary of externals. + def __init__(self, root_dir, name, local_path, required, subexternals_path, + repo, svn_ignore_ancestry, subexternal_sourcetree): + """Create a single external component (checked out or not). Input: + root_dir : string - the (checked-out) parent repo's root dir. + local_path : string - this external's (checked-out) subdir relative + to root_dir, e.g. "components/mom" + repo: Repository - the repo object for this external. Can be None (e.g. if this external just refers to another external file). - root_dir : string - the root directory path where - 'local_path' is relative to. - - name : string - name of the ext_description object. may or may not - correspond to something in the path. + name : string - name of this external (as named by the parent + reference). May or may not correspond to something in the path. ext_description : dict - source ExternalsDescription object svn_ignore_ancestry : bool - use --ignore-externals with svn switch + subexternals_path: string - path to sub-externals config file, if any. Relative to local_path, or special value 'none'. + subexternal_sourcetree: SourceTree - corresponding to subexternals_path, if subexternals_path exists (it might not, if it is not checked out yet). """ self._name = name - self._repo = None - self._externals = EMPTY_STR - self._externals_sourcetree = None - self._stat = ExternalStatus() - self._sparse = None - # Parse the sub-elements - - # _path : local path relative to the containing source tree - self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir : full repository directory - repo_dir = os.path.join(root_dir, self._local_path) + self._required = required + + self._stat = None # Populated in status() + + self._local_path = local_path + # _repo_dir_path : full repository directory, e.g. + # "/components/mom" + repo_dir = os.path.join(root_dir, local_path) self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir : base directory *containing* the repository + # _base_dir_path : base directory *containing* the repository, e.g. + # "/components" self._base_dir_path = os.path.dirname(self._repo_dir_path) - # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # _repo_dir_name : base_dir_path + repo_dir_name = repo_dir_path + # e.g., "mom" self._repo_dir_name = os.path.basename(self._repo_dir_path) - assert(os.path.join(self._base_dir_path, self._repo_dir_name) - == self._repo_dir_path) + self._repo = repo - self._required = ext_description[ExternalsDescription.REQUIRED] - self._externals = ext_description[ExternalsDescription.EXTERNALS] - # Treat a .gitmodules file as a backup externals config - if not self._externals: - if GitRepository.has_submodules(self._repo_dir_path): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME - - repo = create_repository( - name, ext_description[ExternalsDescription.REPO], - svn_ignore_ancestry=svn_ignore_ancestry) - if repo: - self._repo = repo - - if self._externals and (self._externals.lower() != 'none'): - self._create_externals_sourcetree() + # Does this component have subcomponents aka an externals config? + self._subexternals_path = subexternals_path + self._subexternal_sourcetree = subexternal_sourcetree + def get_name(self): """ @@ -88,81 +80,97 @@ def get_local_path(self): """ return self._local_path - def status(self): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. + def get_repo_dir_path(self): + return self._repo_dir_path + + def get_subexternals_path(self): + return self._subexternals_path + + def get_repo(self): + return self._repo + + def status(self, force=False, print_progress=False): """ + Returns status of this component and all subcomponents. - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_externals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_externals - self._stat.source_type = ExternalStatus.MANAGED + Returns a dict mapping our local path (not component name!) to an + ExternalStatus dict. Any subcomponents will have their own top-level + path keys. Note the return value includes entries for this and all + subcomponents regardless of whether they are locally installed or not. - ext_stats = {} + Side-effect: If self._stat is empty or force is True, calculates _stat. + """ + calc_stat = force or not self._stat + + if calc_stat: + self._stat = ExternalStatus() + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_subexternals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_subexternals + self._stat.source_type = ExternalStatus.MANAGED + subcomponent_stats = {} if not os.path.exists(self._repo_dir_path): - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() + if calc_stat: + # No local repository. + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() else: - if self._repo: + # Merge local repository state (e.g. clean/dirty) into self._stat. + if calc_stat and self._repo: self._repo.status(self._stat, self._repo_dir_path) - if self._externals and self._externals_sourcetree: - # we expect externals and they exist + # Status of subcomponents, if any. + if self._subexternals_path and self._subexternal_sourcetree: cwd = os.getcwd() - # SourceTree expects to be called from the correct + # SourceTree.status() expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - ext_stats = self._externals_sourcetree.status(self._local_path) + subcomponent_stats = self._subexternal_sourcetree.status(self._local_path, force=force, print_progress=print_progress) os.chdir(cwd) + # Merge our status + subcomponent statuses into one return dict keyed + # by component path. all_stats = {} # don't add the root component because we don't manage it # and can't provide useful info about it. if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under tha local_path, not comp name so + # store the stats under the local_path, not comp name so # it will be sorted correctly all_stats[self._stat.path] = self._stat - if ext_stats: - all_stats.update(ext_stats) + if subcomponent_stats: + all_stats.update(subcomponent_stats) return all_stats - def checkout(self, verbosity, load_all): + def checkout(self, verbosity): """ If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. + correct URL, correct branch or tag), and possibly updateit. If the repo destination directory does not exist, checkout the correct branch or tag. - If load_all is True, also load all of the the externals sub-externals. + Does not check out sub-externals, see SourceTree.checkout(). """ - if load_all: - pass # Make sure we are in correct location - if not os.path.exists(self._repo_dir_path): # repository directory doesn't exist. Need to check it # out, and for that we need the base_dir_path to exist @@ -174,6 +182,10 @@ def checkout(self, verbosity, load_all): self._base_dir_path) fatal_error(msg) + if not self._stat: + self.status() + assert self._stat + if self._stat.source_type != ExternalStatus.STANDALONE: if verbosity >= VERBOSITY_VERBOSE: # NOTE(bja, 2018-01) probably do not want to pass @@ -194,120 +206,147 @@ def checkout(self, verbosity, load_all): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_externals(self, verbosity, load_all): - """Checkout the sub-externals for this object - """ - if self.load_externals(): - if self._externals_sourcetree: - # NOTE(bja, 2018-02): the subtree externals objects - # were created during initial status check. Updating - # the external may have changed which sub-externals - # are needed. We need to delete those objects and - # re-read the potentially modified externals - # description file. - self._externals_sourcetree = None - self._create_externals_sourcetree() - self._externals_sourcetree.checkout(verbosity, load_all) - - def load_externals(self): - 'Return True iff an externals file should be loaded' - load_ex = False - if os.path.exists(self._repo_dir_path): - if self._externals: - if self._externals.lower() != 'none': - load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals)) - - return load_ex + def replace_subexternal_sourcetree(self, sourcetree): + self._subexternal_sourcetree = sourcetree def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' - # Try recursive unless there is an externals entry - recursive = not self._externals + # Try recursive .gitmodules unless there is an externals entry + recursive = not self._subexternals_path return recursive - def _create_externals_sourcetree(self): - """ + +class SourceTree(object): + """ + SourceTree represents a group of managed externals. + + Those externals may not be checked out locally yet, they might only + have Repository objects pointing to their respective repositories. + """ + + @classmethod + def from_externals_file(cls, parent_repo_dir_path, parent_repo, + externals_path): + """Creates a SourceTree representing the given externals file. + + Looks up a git submodules file as an optional backup if there is no + externals file specified. + + Returns None if there is no externals file (i.e. it's None or 'none'), + or if the externals file hasn't been checked out yet. + + parent_repo_dir_path: parent repo root dir + parent_repo: parent repo. + externals_path: path to externals file, relative to parent_repo_dir_path. """ - if not os.path.exists(self._repo_dir_path): + if not os.path.exists(parent_repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out # yet, can't process the externals file. Assume we are # checking status before code is checkoud out and this # will be handled correctly later. - return + return None - cwd = os.getcwd() - os.chdir(self._repo_dir_path) - if self._externals.lower() == 'none': - msg = ('Internal: Attempt to create source tree for ' - 'externals = none in {}'.format(self._repo_dir_path)) - fatal_error(msg) + if externals_path.lower() == 'none': + # With explicit 'none', do not look for git submodules file. + return None - if not os.path.exists(self._externals): - if GitRepository.has_submodules(): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + cwd = os.getcwd() + os.chdir(parent_repo_dir_path) + + if not externals_path: + if GitRepository.has_submodules(parent_repo_dir_path): + externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + else: + return None - if not os.path.exists(self._externals): - # NOTE(bja, 2017-10) this check is redundent with the one + if not os.path.exists(externals_path): + # NOTE(bja, 2017-10) this check is redundant with the one # in read_externals_description_file! - msg = ('External externals description file "{0}" ' + msg = ('Externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals, self._repo_dir_path)) + externals_path, parent_repo_dir_path)) fatal_error(msg) - externals_root = self._repo_dir_path + externals_root = parent_repo_dir_path + # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals) - externals = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, externals) + externals_path) + # ext_description is another dict-like object (see ExternalsDescription) + ext_description = create_externals_description(model_data, + parent_repo=parent_repo) + externals_sourcetree = SourceTree(externals_root, ext_description) os.chdir(cwd) - -class SourceTree(object): - """ - SourceTree represents a group of managed externals - """ - - def __init__(self, root_dir, model, svn_ignore_ancestry=False): + return externals_sourcetree + + def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): """ - Build a SourceTree object from a model description + Build a SourceTree object from an ExternalDescription. + + root_dir: the (checked-out) parent repo root dir. """ self._root_dir = os.path.abspath(root_dir) - self._all_components = {} + self._all_components = {} # component_name -> _External self._required_compnames = [] - for comp in model: - src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) + for comp, desc in ext_description.items(): + local_path = desc[ExternalsDescription.PATH] + required = desc[ExternalsDescription.REQUIRED] + repo_info = desc[ExternalsDescription.REPO] + subexternals_path = desc[ExternalsDescription.EXTERNALS] + + repo = create_repository(comp, + repo_info, + svn_ignore_ancestry=svn_ignore_ancestry) + + sourcetree = None + # Treat a .gitmodules file as a backup externals config + if not subexternals_path: + parent_repo_dir_path = os.path.abspath(os.path.join(root_dir, + local_path)) + if GitRepository.has_submodules(parent_repo_dir_path): + subexternals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + + # Might return None (if the subexternal isn't checked out yet, or subexternal is None or 'none') + subexternal_sourcetree = SourceTree.from_externals_file( + os.path.join(self._root_dir, local_path), + repo, + subexternals_path) + src = _External(self._root_dir, comp, local_path, required, + subexternals_path, repo, svn_ignore_ancestry, + subexternal_sourcetree) + self._all_components[comp] = src - if model[comp][ExternalsDescription.REQUIRED]: + if required: self._required_compnames.append(comp) - def status(self, relative_path_base=LOCAL_PATH_INDICATOR): - """Report the status components - - FIXME(bja, 2017-10) what do we do about situations where the - user checked out the optional components, but didn't add - optional for running status? What do we do where the user - didn't add optional to the checkout but did add it to the - status. -- For now, we run status on all components, and try - to do the right thing based on the results.... - - """ + def status(self, relative_path_base=LOCAL_PATH_INDICATOR, + force=False, print_progress=False): + """Return a dictionary of local path->ExternalStatus. + + Notes about the returned dictionary: + * It is keyed by local path (e.g. 'components/mom'), not by + component name (e.g. 'mom'). + * It contains top-level keys for all traversed components, whether + discovered by recursion or top-level. + * It contains entries for all components regardless of whether they + are locally installed or not, or required or optional. +x """ load_comps = self._all_components.keys() - summary = {} + summary = {} # Holds merged statuses from all components. for comp in load_comps: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status() + if print_progress: + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status(force=force, + print_progress=print_progress) + + # Returned status dictionary is keyed by local path; prepend + # relative_path_base if not already there. stat_final = {} for name in stat.keys(): - # check if we need to append the relative_path_base to - # the path so it will be sorted in the correct order. if stat[name].path.startswith(relative_path_base): - # use as is, without any changes to path stat_final[name] = stat[name] else: - # append relative_path_base to path and store under key = updated path modified_path = os.path.join(relative_path_base, stat[name].path) stat_final[modified_path] = stat[name] @@ -316,38 +355,71 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): return summary + def _find_installed_optional_components(self): + """Returns a list of installed optional component names, if any.""" + installed_comps = [] + for comp_name, ext in self._all_components.items(): + if comp_name in self._required_compnames: + continue + # Note that in practice we expect this status to be cached. + path_to_stat = ext.status() + + # If any part of this component exists locally, consider it + # installed and therefore eligible for updating. + if any(s.sync_state != ExternalStatus.EMPTY + for s in path_to_stat.values()): + installed_comps.append(comp_name) + return installed_comps + def checkout(self, verbosity, load_all, load_comp=None): """ - Checkout or update indicated components into the the configured - subdirs. + Checkout or update indicated components into the configured subdirs. - If load_all is True, recursively checkout all externals. - If load_all is False, load_comp is an optional set of components to load. - If load_all is True and load_comp is None, only load the required externals. + If load_all is True, checkout all externals (required + optional), recursively. + If load_all is False and load_comp is set, checkout load_comp (and any required subexternals, plus any optional subexternals that are already checked out, recursively) + If load_all is False and load_comp is None, checkout all required externals, plus any optionals that are already checked out, recursively. """ + if load_all: + tmp_comps = self._all_components.keys() + elif load_comp is not None: + tmp_comps = [load_comp] + else: + local_optional_compnames = self._find_installed_optional_components() + tmp_comps = self._required_compnames + local_optional_compnames + if local_optional_compnames: + printlog('Found locally installed optional components: ' + + ', '.join(local_optional_compnames)) + bad_compnames = set(local_optional_compnames) - set(self._all_components.keys()) + if bad_compnames: + printlog('Internal error: found locally installed components that are not in the global list of all components: ' + ','.join(bad_compnames)) + if verbosity >= VERBOSITY_VERBOSE: printlog('Checking out externals: ') else: printlog('Checking out externals: ', end='') - if load_all: - load_comps = self._all_components.keys() - elif load_comp is not None: - load_comps = [load_comp] - else: - load_comps = self._required_compnames + # Sort by path so that if paths are nested the + # parent repo is checked out first. + load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path()) - # checkout the primary externals - for comp in load_comps: + # checkout. + for comp_name in load_comps: if verbosity < VERBOSITY_VERBOSE: - printlog('{0}, '.format(comp), end='') + printlog('{0}, '.format(comp_name), end='') else: # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) - self._all_components[comp].checkout(verbosity, load_all) + c = self._all_components[comp_name] + # Does not recurse. + c.checkout(verbosity) + # Recursively check out subexternals, if any. Returns None + # if there's no subexternals path. + component_subexternal_sourcetree = SourceTree.from_externals_file( + c.get_repo_dir_path(), + c.get_repo(), + c.get_subexternals_path()) + c.replace_subexternal_sourcetree(component_subexternal_sourcetree) + if component_subexternal_sourcetree: + component_subexternal_sourcetree.checkout(verbosity, load_all) printlog('') - - # now give each external an opportunitity to checkout it's externals. - for comp in load_comps: - self._all_components[comp].checkout_externals(verbosity, load_all) diff --git a/manic/utils.py b/manic/utils.py index f57f43930..9c63ffe65 100644 --- a/manic/utils.py +++ b/manic/utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Common public utilities for manic package diff --git a/test/README.md b/test/README.md index 938a900ee..1e8f2eaa7 100644 --- a/test/README.md +++ b/test/README.md @@ -1,45 +1,25 @@ # Testing for checkout_externals -NOTE: Python2 is the supported runtime environment. Python3 compatibility is -in progress, complicated by the different proposed input methods -(yaml, xml, cfg/ini, json) and their different handling of strings -(unicode vs byte) in python2. Full python3 compatibility will be -possible once the number of possible input formats has been narrowed. - -## Setup development environment - -Development environments should be setup for python2 and python3: +## Unit tests ```SH cd checkout_externals/test - make python=python2 env - make python=python3 env + make utest ``` -## Unit tests - -Tests should be run for both python2 and python3. It is recommended -that you have seperate terminal windows open python2 and python3 -testing to avoid errors activating and deactivating environments. +## System tests ```SH cd checkout_externals/test - . env_python2/bin/activate - make utest - deactivate + make stest ``` +Example to run a single test: ```SH - cd checkout_externals/test - . env_python2/bin/activate - make utest - deactivate + cd checkout_externals + python -m unittest test.test_sys_checkout.TestSysCheckout.test_container_simple_required ``` -## System tests - -Not yet implemented. - ## Static analysis checkout_externals is difficult to test thoroughly because it relies @@ -51,9 +31,7 @@ regularly for automatic code formatting and linting. ```SH cd checkout_externals/test - . env_python2/bin/activate make lint - deactivate ``` The canonical formatting for the code is whatever autopep8 @@ -68,10 +46,8 @@ coverage, run the code coverage tool: ```SH cd checkout_externals/test - . env_python2/bin/activate make coverage open -a Firefox.app htmlcov/index.html - deactivate ``` diff --git a/test/repos/README.md b/test/repos/README.md new file mode 100644 index 000000000..8a3502c35 --- /dev/null +++ b/test/repos/README.md @@ -0,0 +1,33 @@ +Git repositories for testing git-related behavior. For usage and terminology notes, see test/test_sys_checkout.py. + +To list files and view file contents at HEAD: +``` +cd +git ls-tree --full-tree -r --name-only HEAD +git cat-file -p HEAD: +``` + +File contents at a glance: +``` +container.git/ + readme.txt + +simple-ext.git/ + (has branches: feature2, feature3) + (has tags: tag1, tag2) + readme.txt + simple_subdir/subdir_file.txt + +simple-ext-fork.git/ + (has tags: abandoned-feature, forked-feature-v1, tag1) + (has branch: feature2) + readme.txt + +mixed-cont-ext.git/ + (has branch: new-feature) + readme.txt + sub-externals.cfg ('simp_branch' section refers to 'feature2' branch in simple-ext.git/ repo) + +error/ + (no git repo here, just a readme.txt in the clear) +``` diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index df726f2b7..ab4f77e88 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -1,7 +1,15 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals +Terminology: + * 'container': a repo that has externals + * 'simple': a repo that has no externals, but is referenced as an external by another repo. + * 'mixed': a repo that both has externals and is referenced as an external by another repo. + + * 'clean': the local repo matches the version in the externals and has no local modifications. + * 'empty': the external isn't checked out at all. + Note: this script assume the path to the manic and checkout_externals module is already in the python path. This is usually handled by the makefile. If you call it directly, you may need @@ -21,7 +29,6 @@ * Erase any existing repos at the begining of the module in setUpModule. - """ # NOTE(bja, 2017-11) pylint complains that the module is too big, but @@ -66,24 +73,51 @@ # # --------------------------------------------------------------------- -# environment variable names -MANIC_TEST_BARE_REPO_ROOT = 'MANIC_TEST_BARE_REPO_ROOT' -MANIC_TEST_TMP_REPO_ROOT = 'MANIC_TEST_TMP_REPO_ROOT' -# directory names -TMP_REPO_DIR_NAME = 'tmp' +# Module-wide root directory for all the per-test subdirs we'll create on +# the fly (which are placed under wherever $CWD is when the test runs). +# Set by setupModule(). +module_tmp_root_dir = None +TMP_REPO_DIR_NAME = 'tmp' # subdir under $CWD + +# subdir under test/ that holds all of our checked-in repositories (which we +# will clone for these tests). BARE_REPO_ROOT_NAME = 'repos' -CONTAINER_REPO_NAME = 'container.git' -MIXED_REPO_NAME = 'mixed-cont-ext.git' -SIMPLE_REPO_NAME = 'simple-ext.git' -SIMPLE_FORK_NAME = 'simple-ext-fork.git' + +# Environment var referenced by checked-in externals file in mixed-cont-ext.git, +# which should be pointed to the fully-resolved BARE_REPO_ROOT_NAME directory. +# We explicitly clear this after every test, via tearDown(). +MIXED_CONT_EXT_ROOT_ENV_VAR = 'MANIC_TEST_BARE_REPO_ROOT' + +# Subdirs under bare repo root, each holding a repository. For more info +# on the contents of these repositories, see test/repos/README.md. In these +# tests the 'parent' repos are cloned as a starting point, whereas the 'child' +# repos are checked out when the tests run checkout_externals. +CONTAINER_REPO = 'container.git' # Parent repo +SIMPLE_REPO = 'simple-ext.git' # Child repo +SIMPLE_FORK_REPO = 'simple-ext-fork.git' # Child repo +MIXED_REPO = 'mixed-cont-ext.git' # Both parent and child + +# Standard (arbitrary) external names for test configs +TAG_SECTION = 'simp_tag' +BRANCH_SECTION = 'simp_branch' +HASH_SECTION = 'simp_hash' + +# All the configs we construct check out their externals into these local paths. +EXTERNALS_PATH = 'externals' +SUB_EXTERNALS_PATH = 'src' # For mixed test repos, + +# For testing behavior with '.' instead of an explicit paths. SIMPLE_LOCAL_ONLY_NAME = '.' -ERROR_REPO_NAME = 'error' -EXTERNALS_NAME = 'externals' -SUB_EXTERNALS_PATH = 'src' -CFG_NAME = 'externals.cfg' -CFG_SUB_NAME = 'sub-externals.cfg' -README_NAME = 'readme.txt' + +# Externals files. +CFG_NAME = 'externals.cfg' # We construct this on a per-test basis. +CFG_SUB_NAME = 'sub-externals.cfg' # Already exists in mixed-cont-ext repo. + +# Arbitrary text file in all the test repos. +README_NAME = 'readme.txt' + +# Branch that exists in both the simple and simple-fork repos. REMOTE_BRANCH_FEATURE2 = 'feature2' SVN_TEST_REPO = 'https://github.com/escomp/cesm' @@ -107,136 +141,113 @@ def setUpModule(): # pylint: disable=C0103 pass # create clean dir for this run os.mkdir(repo_root) - # set into the environment so var will be expanded in externals - # filess when executables are run - os.environ[MANIC_TEST_TMP_REPO_ROOT] = repo_root - - -class GenerateExternalsDescriptionCfgV1(object): - """Class to provide building blocks to create - ExternalsDescriptionCfgV1 files. - - Includes predefined files used in tests. - """ + # Make available to all tests in this file. + global module_tmp_root_dir + assert module_tmp_root_dir == None, module_tmp_root_dir + module_tmp_root_dir = repo_root - def __init__(self): - self._schema_version = '1.1.0' - self._config = None - - def container_full(self, dest_dir): - """Create the full container config file with simple and mixed use - externals - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) - - self.create_section(SIMPLE_REPO_NAME, 'simp_opt', - tag='tag1', required=False) - - self.create_section(MIXED_REPO_NAME, 'mixed_req', - branch='master', externals=CFG_SUB_NAME) - - self.write_config(dest_dir) - - def container_simple_required(self, dest_dir): - """Create a container externals file with only simple externals. - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) - - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', - ref_hash='60b1cc1a38d63') - - self.write_config(dest_dir) - - def container_simple_optional(self, dest_dir): - """Create a container externals file with optional simple externals +class RepoUtils(object): + """Convenience methods for interacting with git repos.""" + @staticmethod + def create_branch(repo_base_dir, external_name, branch, with_commit=False): + """Create branch and optionally (with_commit) add a single commit. """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_req', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_opt', - tag='tag1', required=False) - - self.write_config(dest_dir) - - def container_simple_svn(self, dest_dir): - """Create a container externals file with only simple externals. + # pylint: disable=R0913 + cwd = os.getcwd() + repo_root = os.path.join(repo_base_dir, EXTERNALS_PATH, external_name) + os.chdir(repo_root) + cmd = ['git', 'checkout', '-b', branch, ] + execute_subprocess(cmd) + if with_commit: + msg = 'start work on {0}'.format(branch) + with open(README_NAME, 'a') as handle: + handle.write(msg) + cmd = ['git', 'add', README_NAME, ] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-m', msg, ] + execute_subprocess(cmd) + os.chdir(cwd) + @staticmethod + def create_commit(repo_base_dir, external_name): + """Make a commit to the given external. + + This is used to test sync state changes from local commits on + detached heads and tracking branches. """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', tag='tag1') - - self.create_svn_external('svn_branch', branch='trunk') - self.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') + cwd = os.getcwd() + repo_root = os.path.join(repo_base_dir, EXTERNALS_PATH, external_name) + os.chdir(repo_root) - self.write_config(dest_dir) + msg = 'work on great new feature!' + with open(README_NAME, 'a') as handle: + handle.write(msg) + cmd = ['git', 'add', README_NAME, ] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-m', msg, ] + execute_subprocess(cmd) + os.chdir(cwd) - def container_sparse(self, dest_dir): - """Create a container with a full external and a sparse external + @staticmethod + def clone_test_repo(bare_root, test_id, parent_repo_name, dest_dir_in): + """Clone repo at / into dest_dir_in or local per-test-subdir. + Returns output dir. """ - # Create a file for a sparse pattern match - sparse_filename = 'sparse_checkout' - with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile: - sfile.write('readme.txt') - - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag2') - - sparse_relpath = '../../{}'.format(sparse_filename) - self.create_section(SIMPLE_REPO_NAME, 'simp_sparse', - tag='tag2', sparse=sparse_relpath) + parent_repo_dir = os.path.join(bare_root, parent_repo_name) + if dest_dir_in is None: + # create unique subdir for this test + test_dir_name = test_id + print("Test repository name: {0}".format(test_dir_name)) + dest_dir = os.path.join(module_tmp_root_dir, test_dir_name) + else: + dest_dir = dest_dir_in - self.write_config(dest_dir) + # pylint: disable=W0212 + GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) + return dest_dir - def mixed_simple_base(self, dest_dir): - """Create a mixed-use base externals file with only simple externals. + @staticmethod + def add_file_to_repo(under_test_dir, filename, tracked): + """Add a file to the repository so we can put it into a dirty state """ - self.create_config() - self.create_section_ext_only('mixed_base') - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) + cwd = os.getcwd() + os.chdir(under_test_dir) + with open(filename, 'w') as tmp: + tmp.write('Hello, world!') - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', - ref_hash='60b1cc1a38d63') + if tracked: + # NOTE(bja, 2018-01) brittle hack to obtain repo dir and + # file name + path_data = filename.split('/') + repo_dir = os.path.join(path_data[0], path_data[1]) + os.chdir(repo_dir) + tracked_file = path_data[2] + cmd = ['git', 'add', tracked_file] + execute_subprocess(cmd) - self.write_config(dest_dir) + os.chdir(cwd) - def mixed_simple_sub(self, dest_dir): - """Create a mixed-use sub externals file with only simple externals. +class GenerateExternalsDescriptionCfgV1(object): + """Building blocks to create ExternalsDescriptionCfgV1 files. - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1', path=SUB_EXTERNALS_PATH) + Basic usage: create_config() multiple create_*(), then write_config(). + Optionally after that: write_with_*(). + """ - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2, - path=SUB_EXTERNALS_PATH) + def __init__(self, bare_root): + self._schema_version = '1.1.0' + self._config = None - self.write_config(dest_dir, filename=CFG_SUB_NAME) + # directory where we have test repositories (which we will clone for + # tests) + self._bare_root = bare_root def write_config(self, dest_dir, filename=CFG_NAME): - """Write the configuration file to disk + """Write self._config to disk """ dest_path = os.path.join(dest_dir, filename) @@ -258,27 +269,39 @@ def create_metadata(self): self._config.set(DESCRIPTION_SECTION, VERSION_ITEM, self._schema_version) - def create_section(self, repo_type, name, tag='', branch='', - ref_hash='', required=True, path=EXTERNALS_NAME, - externals='', repo_path=None, from_submodule=False, - sparse=''): + def url_for_repo_path(self, repo_path, repo_path_abs=None): + if repo_path_abs is not None: + return repo_path_abs + else: + return os.path.join(self._bare_root, repo_path) + + def create_section(self, repo_path, name, tag='', branch='', + ref_hash='', required=True, path=EXTERNALS_PATH, + sub_externals='', repo_path_abs=None, from_submodule=False, + sparse='', nested=False): # pylint: disable=too-many-branches - """Create a config section with autofilling some items and handling - optional items. + """Create a config ExternalsDescription section with the given name. + Autofills some items and handles some optional items. + + repo_path_abs overrides repo_path (which is relative to the bare repo) + path is a subdir under repo_path to check out to. """ # pylint: disable=R0913 self._config.add_section(name) if not from_submodule: - self._config.set(name, ExternalsDescription.PATH, - os.path.join(path, name)) + if nested: + self._config.set(name, ExternalsDescription.PATH, path) + else: + self._config.set(name, ExternalsDescription.PATH, + os.path.join(path, name)) self._config.set(name, ExternalsDescription.PROTOCOL, ExternalsDescription.PROTOCOL_GIT) # from_submodules is incompatible with some other options, turn them off if (from_submodule and - ((repo_path is not None) or tag or ref_hash or branch)): + ((repo_path_abs is not None) or tag or ref_hash or branch)): printlog('create_section: "from_submodule" is incompatible with ' '"repo_url", "tag", "hash", and "branch" options;\n' 'Ignoring those options for {}'.format(name)) @@ -287,10 +310,7 @@ def create_section(self, repo_type, name, tag='', branch='', ref_hash = '' branch = '' - if repo_path is not None: - repo_url = repo_path - else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + repo_url = self.url_for_repo_path(repo_path, repo_path_abs) if not from_submodule: self._config.set(name, ExternalsDescription.REPO_URL, repo_url) @@ -306,8 +326,9 @@ def create_section(self, repo_type, name, tag='', branch='', if ref_hash: self._config.set(name, ExternalsDescription.HASH, ref_hash) - if externals: - self._config.set(name, ExternalsDescription.EXTERNALS, externals) + if sub_externals: + self._config.set(name, ExternalsDescription.EXTERNALS, + sub_externals) if sparse: self._config.set(name, ExternalsDescription.SPARSE, sparse) @@ -315,10 +336,8 @@ def create_section(self, repo_type, name, tag='', branch='', if from_submodule: self._config.set(name, ExternalsDescription.SUBMODULE, "True") - def create_section_ext_only(self, name, - required=True, externals=CFG_SUB_NAME): - """Create a config section with autofilling some items and handling - optional items. + def create_section_reference_to_subexternal(self, name): + """Just a reference to another externals file. """ # pylint: disable=R0913 @@ -331,10 +350,9 @@ def create_section_ext_only(self, name, self._config.set(name, ExternalsDescription.REPO_URL, LOCAL_PATH_INDICATOR) - self._config.set(name, ExternalsDescription.REQUIRED, str(required)) + self._config.set(name, ExternalsDescription.REQUIRED, str(True)) - if externals: - self._config.set(name, ExternalsDescription.EXTERNALS, externals) + self._config.set(name, ExternalsDescription.EXTERNALS, CFG_SUB_NAME) def create_svn_external(self, name, tag='', branch=''): """Create a config section for an svn repository. @@ -342,7 +360,7 @@ def create_svn_external(self, name, tag='', branch=''): """ self._config.add_section(name) self._config.set(name, ExternalsDescription.PATH, - os.path.join(EXTERNALS_NAME, name)) + os.path.join(EXTERNALS_PATH, name)) self._config.set(name, ExternalsDescription.PROTOCOL, ExternalsDescription.PROTOCOL_SVN) @@ -357,65 +375,19 @@ def create_svn_external(self, name, tag='', branch=''): if branch: self._config.set(name, ExternalsDescription.BRANCH, branch) - @staticmethod - def create_branch(dest_dir, repo_name, branch, with_commit=False): - """Update a repository branch, and potentially the remote. - """ - # pylint: disable=R0913 - cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME) - repo_root = os.path.join(repo_root, repo_name) - os.chdir(repo_root) - cmd = ['git', 'checkout', '-b', branch, ] - execute_subprocess(cmd) - if with_commit: - msg = 'start work on {0}'.format(branch) - with open(README_NAME, 'a') as handle: - handle.write(msg) - cmd = ['git', 'add', README_NAME, ] - execute_subprocess(cmd) - cmd = ['git', 'commit', '-m', msg, ] - execute_subprocess(cmd) - os.chdir(cwd) - - @staticmethod - def create_commit(dest_dir, repo_name, local_tracking_branch=None): - """Make a commit on whatever is currently checked out. - - This is used to test sync state changes from local commits on - detached heads and tracking branches. + def write_with_git_branch(self, dest_dir, name, branch, new_remote_repo_path=None): + """Update fields in our config and write it to disk. - """ - cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME) - repo_root = os.path.join(repo_root, repo_name) - os.chdir(repo_root) - if local_tracking_branch: - cmd = ['git', 'checkout', '-b', local_tracking_branch, ] - execute_subprocess(cmd) - - msg = 'work on great new feature!' - with open(README_NAME, 'a') as handle: - handle.write(msg) - cmd = ['git', 'add', README_NAME, ] - execute_subprocess(cmd) - cmd = ['git', 'commit', '-m', msg, ] - execute_subprocess(cmd) - os.chdir(cwd) - - def update_branch(self, dest_dir, name, branch, repo_type=None, - filename=CFG_NAME): - """Update a repository branch, and potentially the remote. + name is the key of the ExternalsDescription in self._config to update. """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.BRANCH, branch) - if repo_type: - if repo_type == SIMPLE_LOCAL_ONLY_NAME: + if new_remote_repo_path: + if new_remote_repo_path == SIMPLE_LOCAL_ONLY_NAME: repo_url = SIMPLE_LOCAL_ONLY_NAME else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', - repo_type) + repo_url = os.path.join(self._bare_root, new_remote_repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -424,9 +396,9 @@ def update_branch(self, dest_dir, name, branch, repo_type=None, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): + def write_with_svn_branch(self, dest_dir, name, branch): """Update a repository branch, and potentially the remote. """ # pylint: disable=R0913 @@ -438,11 +410,11 @@ def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def update_tag(self, dest_dir, name, tag, repo_type=None, - filename=CFG_NAME, remove_branch=True): - """Update a repository tag, and potentially the remote + def write_with_tag_and_remote_repo(self, dest_dir, name, tag, new_remote_repo_path, + remove_branch=True): + """Update a repository tag and the remote. NOTE(bja, 2017-11) remove_branch=False should result in an overspecified external with both a branch and tag. This is @@ -452,8 +424,8 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, # pylint: disable=R0913 self._config.set(name, ExternalsDescription.TAG, tag) - if repo_type: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + if new_remote_repo_path: + repo_url = os.path.join(self._bare_root, new_remote_repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -463,10 +435,9 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def update_underspecify_branch_tag(self, dest_dir, name, - filename=CFG_NAME): + def write_without_branch_tag(self, dest_dir, name): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -482,10 +453,9 @@ def update_underspecify_branch_tag(self, dest_dir, name, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def update_underspecify_remove_url(self, dest_dir, name, - filename=CFG_NAME): + def write_without_repo_url(self, dest_dir, name): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -495,22 +465,59 @@ def update_underspecify_remove_url(self, dest_dir, name, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def update_protocol(self, dest_dir, name, protocol, repo_type=None, - filename=CFG_NAME): + def write_with_protocol(self, dest_dir, name, protocol, repo_path=None): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.PROTOCOL, protocol) - if repo_type: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + if repo_path: + repo_url = os.path.join(self._bare_root, repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) - self.write_config(dest_dir, filename) + self.write_config(dest_dir) + + +def _execute_checkout_in_dir(dirname, args, debug_env=''): + """Execute the checkout command in the appropriate repo dir with the + specified additional args. + + args should be a list of strings. + debug_env shuld be a string of the form 'FOO=bar' or the empty string. + Note that we are calling the command line processing and main + routines and not using a subprocess call so that we get code + coverage results! Note this means that environment variables are passed + to checkout_externals via os.environ; debug_env is just used to aid + manual reproducibility of a given call. + Returns (overall_status, tree_status) + where overall_status is 0 for success, nonzero otherwise. + and tree_status is set if --status was passed in, None otherwise. + + Note this command executes the checkout command, it doesn't + necessarily do any checking out (e.g. if --status is passed in). + """ + cwd = os.getcwd() + + # Construct a command line for reproducibility; this command is not + # actually executed in the test. + os.chdir(dirname) + cmdline = ['--externals', CFG_NAME, ] + cmdline += args + manual_cmd = ('Running equivalent of:\n' + 'pushd {dirname}; ' + '{debug_env} /path/to/checkout_externals {args}'.format( + dirname=dirname, debug_env=debug_env, + args=' '.join(cmdline))) + printlog(manual_cmd) + options = checkout.commandline_arguments(cmdline) + overall_status, tree_status = checkout.main(options) + os.chdir(cwd) + return overall_status, tree_status + class BaseTestSysCheckout(unittest.TestCase): """Base class of reusable systems level test setup for checkout_externals @@ -521,6 +528,7 @@ class BaseTestSysCheckout(unittest.TestCase): # cryptic. # pylint: disable=invalid-name + # Command-line args for checkout_externals, used in execute_checkout_in_dir() status_args = ['--status'] checkout_args = [] optional_args = ['--optional'] @@ -535,384 +543,74 @@ def setUp(self): self._test_id = self.id().split('.')[-1] - # path to the executable - self._checkout = os.path.join('../checkout_externals') - self._checkout = os.path.abspath(self._checkout) + # find root + if os.path.exists(os.path.join(os.getcwd(), 'checkout_externals')): + root_dir = os.path.abspath(os.getcwd()) + else: + # maybe we are in a subdir, search up + root_dir = os.path.abspath(os.path.join(os.getcwd(), os.pardir)) + while os.path.basename(root_dir): + if os.path.exists(os.path.join(root_dir, 'checkout_externals')): + break + root_dir = os.path.dirname(root_dir) + + if not os.path.exists(os.path.join(root_dir, 'checkout_externals')): + raise RuntimeError('Cannot find checkout_externals') - # directory where we have test repositories - self._bare_root = os.path.join(os.getcwd(), BARE_REPO_ROOT_NAME) - self._bare_root = os.path.abspath(self._bare_root) + # path to the executable + self._checkout = os.path.join(root_dir, 'checkout_externals') - # set into the environment so var will be expanded in externals files - os.environ[MANIC_TEST_BARE_REPO_ROOT] = self._bare_root + # directory where we have test repositories (which we will clone for + # tests) + self._bare_root = os.path.abspath( + os.path.join(root_dir, 'test', BARE_REPO_ROOT_NAME)) # set the input file generator - self._generator = GenerateExternalsDescriptionCfgV1() + self._generator = GenerateExternalsDescriptionCfgV1(self._bare_root) # set the input file generator for secondary externals - self._sub_generator = GenerateExternalsDescriptionCfgV1() + self._sub_generator = GenerateExternalsDescriptionCfgV1(self._bare_root) def tearDown(self): """Tear down for individual tests """ - # remove the env var we added in setup - del os.environ[MANIC_TEST_BARE_REPO_ROOT] - # return to our common starting point os.chdir(self._return_dir) - - def setup_test_repo(self, parent_repo_name, dest_dir_in=None): - """Setup the paths and clone the base test repo - - """ - # unique repo for this test - test_dir_name = self._test_id - print("Test repository name: {0}".format(test_dir_name)) - - parent_repo_dir = os.path.join(self._bare_root, parent_repo_name) - if dest_dir_in is None: - dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - test_dir_name) - else: - dest_dir = dest_dir_in - - # pylint: disable=W0212 - GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) - return dest_dir - - @staticmethod - def _add_file_to_repo(under_test_dir, filename, tracked): - """Add a file to the repository so we can put it into a dirty state - - """ - cwd = os.getcwd() - os.chdir(under_test_dir) - with open(filename, 'w') as tmp: - tmp.write('Hello, world!') - - if tracked: - # NOTE(bja, 2018-01) brittle hack to obtain repo dir and - # file name - path_data = filename.split('/') - repo_dir = os.path.join(path_data[0], path_data[1]) - os.chdir(repo_dir) - tracked_file = path_data[2] - cmd = ['git', 'add', tracked_file] - execute_subprocess(cmd) - - os.chdir(cwd) + + # (in case this was set) Don't pollute environment of other tests. + os.environ.pop(MIXED_CONT_EXT_ROOT_ENV_VAR, + None) # Don't care if key wasn't set. + + def clone_test_repo(self, parent_repo_name, dest_dir_in=None): + """Clones repo under self._bare_root""" + return RepoUtils.clone_test_repo(self._bare_root, self._test_id, + parent_repo_name, dest_dir_in) + + def execute_checkout_in_dir(self, dirname, args, debug_env=''): + overall_status, tree_status = _execute_checkout_in_dir(dirname, args, + debug_env=debug_env) + self.assertEqual(overall_status, 0) + return tree_status + + def execute_checkout_with_status(self, dirname, args, debug_env=''): + """Calls checkout a second time to get status if needed.""" + tree_status = self.execute_checkout_in_dir( + dirname, args, debug_env=debug_env) + if tree_status is None: + tree_status = self.execute_checkout_in_dir(dirname, + self.status_args, + debug_env=debug_env) + self.assertNotEqual(tree_status, None) + return tree_status + + def _check_sync_clean(self, ext_status, expected_sync_state, + expected_clean_state): + self.assertEqual(ext_status.sync_state, expected_sync_state) + self.assertEqual(ext_status.clean_state, expected_clean_state) @staticmethod - def execute_cmd_in_dir(under_test_dir, args): - """Extecute the checkout command in the appropriate repo dir with the - specified additional args - - Note that we are calling the command line processing and main - routines and not using a subprocess call so that we get code - coverage results! - - """ - cwd = os.getcwd() - checkout_path = os.path.abspath('{0}/../../checkout_externals') - os.chdir(under_test_dir) - cmdline = ['--externals', CFG_NAME, ] - cmdline += args - repo_root = 'MANIC_TEST_BARE_REPO_ROOT={root}'.format( - root=os.environ[MANIC_TEST_BARE_REPO_ROOT]) - manual_cmd = ('Test cmd:\npushd {cwd}; {env} {checkout} {args}'.format( - cwd=under_test_dir, env=repo_root, checkout=checkout_path, - args=' '.join(cmdline))) - printlog(manual_cmd) - options = checkout.commandline_arguments(cmdline) - overall_status, tree_status = checkout.main(options) - os.chdir(cwd) - return overall_status, tree_status - - # ---------------------------------------------------------------- - # - # Check results for generic perturbation of states - # - # ---------------------------------------------------------------- - def _check_generic_empty_default_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) - self.assertEqual(tree[name].clean_state, ExternalStatus.DEFAULT) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_ok_clean_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_ok_dirty_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.DIRTY) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_modified_ok_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.MODEL_MODIFIED) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_empty_default_optional(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) - self.assertEqual(tree[name].clean_state, ExternalStatus.DEFAULT) - self.assertEqual(tree[name].source_type, ExternalStatus.OPTIONAL) - - def _check_generic_ok_clean_optional(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.OPTIONAL) - - # ---------------------------------------------------------------- - # - # Check results for individual named externals - # - # ---------------------------------------------------------------- - def _check_simple_tag_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_tag_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_tag_dirty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_ok_dirty_required(tree, name) - - def _check_simple_tag_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_branch_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_branch_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_hash_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_hash_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_hash_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_req_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_req'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_req_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_req'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_opt_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_opt'.format(directory) - self._check_generic_empty_default_optional(tree, name) - - def _check_simple_opt_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_opt'.format(directory) - self._check_generic_ok_clean_optional(tree, name) - - def _check_mixed_ext_branch_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_mixed_ext_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_sparse'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_sparse'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - # ---------------------------------------------------------------- - # - # Check results for groups of externals under specific conditions - # - # ---------------------------------------------------------------- - def _check_container_simple_required_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_hash_empty(tree) - - def _check_container_simple_required_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_hash_empty(tree) - - def _check_container_simple_required_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_hash_ok(tree) - - def _check_container_simple_required_out_of_sync(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_modified(tree) - self._check_simple_branch_modified(tree) - self._check_simple_hash_modified(tree) - - def _check_container_simple_optional_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_empty(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_empty(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_ok(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_post_optional(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_ok(tree) - self._check_simple_opt_ok(tree) - - def _check_container_simple_required_sb_modified(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_modified(tree) - self._check_simple_hash_ok(tree) - - def _check_container_simple_optional_st_dirty(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_dirty(tree) - self._check_simple_branch_ok(tree) - - def _check_container_full_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_pre_checkout(overall, tree) - - def _check_container_component_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_opt_ok(tree) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - - def _check_container_component_post_checkout2(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_opt_ok(tree) - self._check_simple_tag_empty(tree) - self._check_simple_branch_ok(tree) - - def _check_container_full_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_post_checkout(overall, tree) - - def _check_container_full_pre_checkout_ext_change(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_pre_checkout_ext_change( - overall, tree) - - def _check_container_full_post_checkout_subext_modified( - self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_post_checkout_subext_modified( - overall, tree) - - def _check_mixed_ext_branch_required_pre_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_empty(tree, directory=EXTERNALS_NAME) - # NOTE: externals/mixed_req/src should not exist in the tree - # since this is the status before checkout of mixed_req. - - def _check_mixed_ext_branch_required_post_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_ok(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_ok(tree, directory=check_dir) - - def _check_mixed_ext_branch_required_pre_checkout_ext_change( - self, overall, tree): - # Note, this is the internal tree status just after change the - # externals description file, but before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_modified(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_ok(tree, directory=check_dir) - - def _check_mixed_ext_branch_required_post_checkout_subext_modified( - self, overall, tree): - # Note, this is the internal tree status just after change the - # externals description file, but before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_ok(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_modified(tree, directory=check_dir) - - def _check_mixed_cont_simple_required_pre_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=SUB_EXTERNALS_PATH) - - def _check_mixed_cont_simple_required_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=SUB_EXTERNALS_PATH) - - def _check_mixed_cont_simple_required_post_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH) - - def _check_container_sparse_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_sparse_empty(tree) - - def _check_container_sparse_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_sparse_ok(tree) - + def _external_path(section_name, base_path=EXTERNALS_PATH): + return './{0}/{1}'.format(base_path, section_name) + def _check_file_exists(self, repo_dir, pathname): "Check that exists in " self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname))) @@ -921,9 +619,9 @@ def _check_file_absent(self, repo_dir, pathname): "Check that does not exist in " self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname))) + class TestSysCheckout(BaseTestSysCheckout): """Run systems level tests of checkout_externals - """ # NOTE(bja, 2017-11) pylint complains about long method names, but # it is hard to differentiate tests without making them more @@ -935,214 +633,431 @@ class TestSysCheckout(BaseTestSysCheckout): # Run systems tests # # ---------------------------------------------------------------- - def test_container_simple_required(self): - """Verify that a container with simple subrepos - generates the correct initial status. - + def test_required_bytag(self): + """Check out a required external pointing to a git tag.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(TAG_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the external is 'clean' aka at the correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired tag. + (tag_found, tag_name) = GitRepository._git_current_tag(local_path_abs) + self.assertEqual(tag_name, 'tag1') + + # Check existence of some simp_tag files + tag_path = os.path.join('externals', TAG_SECTION) + self._check_file_exists(cloned_repo_dir, + os.path.join(tag_path, README_NAME)) + # Subrepo should not exist (not referenced by configs). + self._check_file_absent(cloned_repo_dir, os.path.join(tag_path, + 'simple_subdir', + 'subdir_file.txt')) + + def test_required_bybranch(self): + """Check out a required external pointing to a git branch.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(BRANCH_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the external is 'clean' aka at the correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertTrue(os.path.exists(local_path_abs)) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired branch. + (branch_found, branch_name) = GitRepository._git_current_remote_branch( + local_path_abs) + self.assertEquals(branch_name, 'origin/' + REMOTE_BRANCH_FEATURE2) + + def test_required_byhash(self): + """Check out a required external pointing to a git hash.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, HASH_SECTION, + ref_hash='60b1cc1a38d63') + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(HASH_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the externals are 'clean' aka at their correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired hash. + (hash_found, hash_name) = GitRepository._git_current_hash( + local_path_abs) + self.assertTrue(hash_name.startswith('60b1cc1a38d63'), + msg=hash_name) + + def test_container_nested_required(self): + """Verify that a container with nested subrepos generates the correct initial status. + Tests over all possible permutations """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) - - # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, + # Output subdirs for each of the externals, to test that one external can be + # checked out in a subdir of another. + NESTED_SUBDIR = ['./fred', './fred/wilma', './fred/wilma/barney'] + + # Assert that each type of external (e.g. tag vs branch) can be at any parent level + # (e.g. child/parent/grandparent). + orders = [[0, 1, 2], [1, 2, 0], [2, 0, 1], + [0, 2, 1], [2, 1, 0], [1, 0, 2]] + for n, order in enumerate(orders): + dest_dir = os.path.join(module_tmp_root_dir, self._test_id, + "test"+str(n)) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO, + dest_dir_in=dest_dir) + self._generator.create_config() + # We happen to check out each section via a different reference (tag/branch/hash) but + # those don't really matter, we just need to check out three repos into a nested set of + # directories. + self._generator.create_section( + SIMPLE_REPO, TAG_SECTION, nested=True, + tag='tag1', path=NESTED_SUBDIR[order[0]]) + self._generator.create_section( + SIMPLE_REPO, BRANCH_SECTION, nested=True, + branch=REMOTE_BRANCH_FEATURE2, path=NESTED_SUBDIR[order[1]]) + self._generator.create_section( + SIMPLE_REPO, HASH_SECTION, nested=True, + ref_hash='60b1cc1a38d63', path=NESTED_SUBDIR[order[2]]) + self._generator.write_config(cloned_repo_dir) + + # all externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) - self._check_container_simple_required_pre_checkout(overall, tree) - - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # status clean checked out - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) - + self._check_sync_clean(tree[NESTED_SUBDIR[order[0]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[NESTED_SUBDIR[order[1]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[NESTED_SUBDIR[order[2]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + + # after checkout, all the repos are 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[NESTED_SUBDIR[order[0]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[NESTED_SUBDIR[order[1]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[NESTED_SUBDIR[order[2]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + def test_container_simple_optional(self): - """Verify that container with an optional simple subrepos - generates the correct initial status. + """Verify that container with an optional simple subrepos generates + the correct initial status. """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_optional(under_test_dir) - - # check status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_optional_pre_checkout(overall, tree) - - # checkout required - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_optional_checkout(overall, tree) - - # status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_optional_post_checkout(overall, tree) + # create repo and externals config. + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, 'simp_req', + tag='tag1') - # checkout optional - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.optional_args) - self._check_container_simple_optional_post_checkout(overall, tree) + self._generator.create_section(SIMPLE_REPO, 'simp_opt', + tag='tag1', required=False) - # status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_optional_post_optional(overall, tree) + self._generator.write_config(cloned_repo_dir) + + # all externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) + + # after checkout, required external is clean, optional is still empty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) + + # after checking out optionals, the optional external is also clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.optional_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) def test_container_simple_verbose(self): - """Verify that container with simple subrepos runs with verbose status - output and generates the correct initial status. - + """Verify that verbose status matches non-verbose. """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) - - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # check verbose status - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.verbose_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # after checkout, all externals should be 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # 'Verbose' status should tell the same story. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_simple_dirty(self): - """Verify that a container with simple subrepos - and a dirty status exits gracefully. - + """Verify that a container with a new tracked file is marked dirty. """ - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) - - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # add a file to the repo - tracked = True - self._add_file_to_repo(under_test_dir, 'externals/simp_tag/tmp.txt', - tracked) - - # checkout: pre-checkout status should be dirty, did not - # modify working copy. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_optional_st_dirty(overall, tree) - - # verify status is still dirty - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_optional_st_dirty(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # checkout, should start out clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # add a tracked file to the simp_tag external, should be dirty. + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/{0}/tmp.txt'.format(TAG_SECTION), + tracked=True) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.DIRTY) + + # Re-checkout; simp_tag should still be dirty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.DIRTY) def test_container_simple_untracked(self): """Verify that a container with simple subrepos and a untracked files is not considered 'dirty' and will attempt an update. """ - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) - - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # add a file to the repo - tracked = False - self._add_file_to_repo(under_test_dir, 'externals/simp_tag/tmp.txt', - tracked) - - # checkout: pre-checkout status should be clean, ignoring the - # untracked file. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_post_checkout(overall, tree) - - # verify status is still clean - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # checkout, should start out clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # add an untracked file to the simp_tag external, should stay clean. + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/{0}/tmp.txt'.format(TAG_SECTION), + tracked=False) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # After checkout, the external should still be 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_simple_detached_sync(self): """Verify that a container with simple subrepos generates the correct out of sync status when making commits from a detached head - state. + state. + For more info about 'detached head' state: https://www.cloudbees.com/blog/git-detached-head """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) - - # status of empty repo - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_pre_checkout(overall, tree) - - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # make a commit on the detached head of the tag and hash externals - self._generator.create_commit(under_test_dir, 'simp_tag') - self._generator.create_commit(under_test_dir, 'simp_hash') - self._generator.create_commit(under_test_dir, 'simp_branch') - - # status of repo, branch, tag and hash should all be out of sync! - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_out_of_sync(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + self._generator.create_section(SIMPLE_REPO, 'simp_hash', + ref_hash='60b1cc1a38d63') + + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - # same pre-checkout out of sync status - self._check_container_simple_required_out_of_sync(overall, tree) - - # now status should be in-sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) + + # Commit on top of the tag and hash (creating the detached head state in those two + # externals' repos) + # The branch commit does not create the detached head state, but here for completeness. + RepoUtils.create_commit(cloned_repo_dir, TAG_SECTION) + RepoUtils.create_commit(cloned_repo_dir, HASH_SECTION) + RepoUtils.create_commit(cloned_repo_dir, BRANCH_SECTION) + + # sync status of all three should be 'modified' (uncommitted changes) + # clean status is 'ok' (matches externals version) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # after checkout, all externals should be totally clean (no uncommitted changes, + # and matches externals version). + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_branch(self): """Verify that a container with remote branch change works """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) - - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # update the config file to point to a different remote with - # the same branch - self._generator.update_branch(under_test_dir, 'simp_branch', - REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) + + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) + + # update the branch external to point to a different remote with the same branch, + # then simp_branch should be out of sync + self._generator.write_with_git_branch(cloned_repo_dir, + name=BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2, + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, now simp_branch should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_tag_same_branch(self): """Verify that a container with remote tag change works. The new tag @@ -1151,258 +1066,324 @@ def test_container_remote_tag_same_branch(self): the branch. """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the tag instead of branch. Tag MUST NOT be in the original - # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', - 'forked-feature-v1', SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the new tag replacing the old branch. Tag MUST NOT be in the original + # repo! status of simp_branch should then be out of sync + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='forked-feature-v1', + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, then should be synced. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_tag_fetch_all(self): """Verify that a container with remote tag change works. The new tag should not be in the original repo, only the new remote - fork. It should also not be on a branch that will be fetch, + fork. It should also not be on a branch that will be fetched, and therefore not fetched by default with 'git fetch'. It will - only be retreived by 'git fetch --tags' - + only be retrieved by 'git fetch --tags' """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the tag instead of branch. Tag MUST NOT be in the original - # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', - 'abandoned-feature', SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the new tag instead of the old branch. Tag MUST NOT be in the original + # repo! status of simp_branch should then be out of sync. + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='abandoned-feature', + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_preserve_dot(self): """Verify that after inital checkout, modifying an external git repo url to '.' and the current branch will leave it unchanged. """ - # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the same branch - self._generator.update_branch(under_test_dir, 'simp_branch', - REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - - # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the same branch. + self._generator.write_with_git_branch(cloned_repo_dir, name=BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2, + new_remote_repo_path=SIMPLE_FORK_REPO) + # after checkout, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) # update branch to point to a new branch that only exists in # the local fork - self._generator.create_branch(under_test_dir, 'simp_branch', - 'private-feature', with_commit=True) - self._generator.update_branch(under_test_dir, 'simp_branch', - 'private-feature', - SIMPLE_LOCAL_ONLY_NAME) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - - # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) - - def test_container_full(self): - """Verify that 'full' container with simple and mixed subrepos - generates the correct initial status. + RepoUtils.create_branch(cloned_repo_dir, external_name=BRANCH_SECTION, + branch='private-feature', with_commit=True) + self._generator.write_with_git_branch(cloned_repo_dir, name=BRANCH_SECTION, + branch='private-feature', + new_remote_repo_path=SIMPLE_LOCAL_ONLY_NAME) + # after checkout, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + def test_container_mixed_subrepo(self): + """Verify container with mixed subrepo. The mixed subrepo has a sub-externals file with different sub-externals on different branches. """ - # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - - # create the top level externals file - self._generator.container_full(under_test_dir) - - # inital checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_full_pre_checkout(overall, tree) - - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_full_post_checkout(overall, tree) - - # Check existance of some files - subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(under_test_dir, - os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(under_test_dir, os.path.join(subrepo_path, - 'simple_subdir', - 'subdir_file.txt')) - - # update the mixed-use repo to point to different branch - self._generator.update_branch(under_test_dir, 'mixed_req', - 'new-feature', MIXED_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # check status out of sync for mixed_req, but sub-externals + self._generator.create_config() + self._generator.create_section(MIXED_REPO, 'mixed_req', + branch='master', sub_externals=CFG_SUB_NAME) + self._generator.write_config(cloned_repo_dir) + + # The subrepo has a repo_url that uses this environment variable. + # It'll be cleared in tearDown(). + os.environ[MIXED_CONT_EXT_ROOT_ENV_VAR] = self._bare_root + debug_env = MIXED_CONT_EXT_ROOT_ENV_VAR + '=' + self._bare_root + + # inital checkout: all requireds are clean, and optional is empty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args, + debug_env=debug_env) + mixed_req_path = self._external_path('mixed_req') + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + sub_ext_base_path = "{0}/{1}/{2}".format(EXTERNALS_PATH, 'mixed_req', SUB_EXTERNALS_PATH) + # The already-checked-in subexternals file has a 'simp_branch' section + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # update the mixed-use external to point to different branch + # status should become out of sync for mixed_req, but sub-externals # are still in sync - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_full_pre_checkout_ext_change(overall, tree) - - # run the checkout. Now the mixed use external and it's - # sub-exterals should be changed. Returned status is - # pre-checkout! - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_full_pre_checkout_ext_change(overall, tree) - - # check status out of sync for mixed_req, and sub-externals - # are in sync. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_full_post_checkout(overall, tree) - + self._generator.write_with_git_branch(cloned_repo_dir, name='mixed_req', + branch='new-feature', + new_remote_repo_path=MIXED_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args, + debug_env=debug_env) + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # run the checkout. Now the mixed use external and its sub-externals should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args, + debug_env=debug_env) + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + def test_container_component(self): """Verify that optional component checkout works """ - # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) # create the top level externals file - self._generator.container_full(under_test_dir) - - # inital checkout, first try a nonexistant component argument noref + self._generator.create_config() + # Optional external, by tag. + self._generator.create_section(SIMPLE_REPO, 'simp_opt', + tag='tag1', required=False) + + # Required external, by branch. + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + # Required external, by hash. + self._generator.create_section(SIMPLE_REPO, HASH_SECTION, + ref_hash='60b1cc1a38d63') + self._generator.write_config(cloned_repo_dir) + + # inital checkout, first try a nonexistent component argument noref checkout_args = ['simp_opt', 'noref'] checkout_args.extend(self.checkout_args) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) + # Now explicitly check out one optional component.. + # Explicitly listed component (opt) should be present, the other two not. checkout_args = ['simp_opt'] checkout_args.extend(self.checkout_args) - - overall, tree = self.execute_cmd_in_dir(under_test_dir, - checkout_args) - - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_component_post_checkout(overall, tree) - checkout_args.append('simp_branch') - overall, tree = self.execute_cmd_in_dir(under_test_dir, - checkout_args) - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_component_post_checkout2(overall, tree) - - def test_mixed_simple(self): - """Verify that a mixed use repo can serve as a 'full' container, - pulling in a set of externals and a seperate set of sub-externals. - + tree = self.execute_checkout_with_status(cloned_repo_dir, + checkout_args) + self._check_sync_clean(tree[self._external_path('simp_opt')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + + # Check out a second component, this one required. + # Explicitly listed component (branch) should be present, the still-unlisted one (tag) not. + checkout_args.append(BRANCH_SECTION) + tree = self.execute_checkout_with_status(cloned_repo_dir, + checkout_args) + self._check_sync_clean(tree[self._external_path('simp_opt')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + + + def test_container_exclude_component(self): + """Verify that exclude component checkout works """ - #import pdb; pdb.set_trace() - # create repository - under_test_dir = self.setup_test_repo(MIXED_REPO_NAME) - # create top level externals file - self._generator.mixed_simple_base(under_test_dir) - # NOTE: sub-externals file is already in the repo so we can - # switch branches during testing. Since this is a mixed-repo - # serving as the top level container repo, we can't switch - # during this test. + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + self._generator.create_section(SIMPLE_REPO, 'simp_hash', + ref_hash='60b1cc1a38d63') + + self._generator.write_config(cloned_repo_dir) + + # inital checkout should result in all externals being clean except excluded TAG_SECTION. + checkout_args = ['--exclude', TAG_SECTION] + checkout_args.extend(self.checkout_args) + tree = self.execute_checkout_with_status(cloned_repo_dir, checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + def test_subexternal(self): + """Verify that an externals file can be brought in as a reference. - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_mixed_cont_simple_required_checkout(overall, tree) + """ + cloned_repo_dir = self.clone_test_repo(MIXED_REPO) - # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_mixed_cont_simple_required_post_checkout(overall, tree) + self._generator.create_config() + self._generator.create_section_reference_to_subexternal('mixed_base') + self._generator.write_config(cloned_repo_dir) + + # The subrepo has a repo_url that uses this environment variable. + # It'll be cleared in tearDown(). + os.environ[MIXED_CONT_EXT_ROOT_ENV_VAR] = self._bare_root + debug_env = MIXED_CONT_EXT_ROOT_ENV_VAR + '=' + self._bare_root + + # After checkout, confirm required's are clean and the referenced + # subexternal's contents are also clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args, + debug_env=debug_env) + + self._check_sync_clean( + tree[self._external_path(BRANCH_SECTION, base_path=SUB_EXTERNALS_PATH)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_sparse(self): """Verify that 'full' container with simple subrepo can run a sparse checkout and generate the correct initial status. """ - # create the test repository - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # create the top level externals file - self._generator.container_sparse(under_test_dir) - - # inital checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_sparse_pre_checkout(overall, tree) - - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_sparse_post_checkout(overall, tree) + # Create a file to list filenames to checkout. + sparse_filename = 'sparse_checkout' + with open(os.path.join(cloned_repo_dir, sparse_filename), 'w') as sfile: + sfile.write(README_NAME) - # Check existance of some files - subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(under_test_dir, - os.path.join(subrepo_path, 'readme.txt')) - self._check_file_exists(under_test_dir, os.path.join(subrepo_path, + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag2') + + # Same tag as above, but with a sparse file too. + sparse_relpath = '../../' + sparse_filename + self._generator.create_section(SIMPLE_REPO, 'simp_sparse', + tag='tag2', sparse=sparse_relpath) + + self._generator.write_config(cloned_repo_dir) + + # inital checkout, confirm required's are clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_sparse')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Check existence of some files - full set in TAG_SECTION, and sparse set + # in 'simp_sparse'. + subrepo_path = os.path.join('externals', TAG_SECTION) + self._check_file_exists(cloned_repo_dir, + os.path.join(subrepo_path, README_NAME)) + self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) subrepo_path = os.path.join('externals', 'simp_sparse') - self._check_file_exists(under_test_dir, - os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + self._check_file_exists(cloned_repo_dir, + os.path.join(subrepo_path, README_NAME)) + self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) @@ -1438,42 +1419,27 @@ class TestSysCheckoutSVN(BaseTestSysCheckout): """ - def _check_svn_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_branch'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_svn_branch_dirty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_branch'.format(directory) - self._check_generic_ok_dirty_required(tree, name) - - def _check_svn_tag_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_tag'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_svn_tag_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_tag'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_container_simple_svn_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_branch_ok(tree) - self._check_svn_tag_ok(tree) - - def _check_container_simple_svn_sb_dirty_st_mod(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_tag_modified(tree) - self._check_svn_branch_dirty(tree) + @staticmethod + def _svn_branch_name(): + return './{0}/svn_branch'.format(EXTERNALS_PATH) - def _check_container_simple_svn_sb_clean_st_mod(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_tag_modified(tree) - self._check_svn_branch_ok(tree) + @staticmethod + def _svn_tag_name(): + return './{0}/svn_tag'.format(EXTERNALS_PATH) + + def _check_tag_branch_svn_tag_clean(self, tree): + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._svn_branch_name()], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._svn_tag_name()], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) @staticmethod - def have_svn_access(): + def _have_svn_access(): """Check if we have svn access so we can enable tests that use svn. """ @@ -1486,10 +1452,10 @@ def have_svn_access(): pass return have_svn - def skip_if_no_svn_access(self): + def _skip_if_no_svn_access(self): """Function decorator to disable svn tests when svn isn't available """ - have_svn = self.have_svn_access() + have_svn = self._have_svn_access() if not have_svn: raise unittest.SkipTest("No svn access") @@ -1497,60 +1463,55 @@ def test_container_simple_svn(self): """Verify that a container repo can pull in an svn branch and svn tag. """ - self.skip_if_no_svn_access() + self._skip_if_no_svn_access() # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_svn(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) + self._generator.create_config() + # Git repo. + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, tag='tag1') - # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_svn_post_checkout(overall, tree) + # Svn repos. + self._generator.create_svn_external('svn_branch', branch='trunk') + self._generator.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') + + self._generator.write_config(cloned_repo_dir) + + # checkout, make sure all sections are clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_tag_branch_svn_tag_clean(tree) # update description file to make the tag into a branch and # trigger a switch - self._generator.update_svn_branch(under_test_dir, 'svn_tag', 'trunk') + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', + 'trunk') - # checkout - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - - # verify status is clean and unmodified - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.status_args) - self._check_container_simple_svn_post_checkout(overall, tree) + # checkout, again the results should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_tag_branch_svn_tag_clean(tree) # add an untracked file to the repo tracked = False - self._add_file_to_repo(under_test_dir, - 'externals/svn_branch/tmp.txt', tracked) + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/svn_branch/tmp.txt', tracked) - # run a no-op checkout: pre-checkout status should be clean, - # ignoring the untracked file. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_svn_post_checkout(overall, tree) + # run a no-op checkout. + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update description file to make the branch into a tag and # trigger a modified sync status - self._generator.update_svn_branch(under_test_dir, 'svn_tag', - 'tags/cesm2.0.beta07') + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', + 'tags/cesm2.0.beta07') - # checkout: pre-checkout status should be clean and modified, - # will modify working copy. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.checkout_args) - self._check_container_simple_svn_sb_clean_st_mod(overall, tree) + self.execute_checkout_in_dir(cloned_repo_dir,self.checkout_args) # verify status is still clean and unmodified, last # checkout modified the working dir state. - overall, tree = self.execute_cmd_in_dir(under_test_dir, - self.verbose_args) - self._check_container_simple_svn_post_checkout(overall, tree) + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) + self._check_tag_branch_svn_tag_clean(tree) class TestSubrepoCheckout(BaseTestSysCheckout): # Need to store information at setUp time for checking @@ -1569,7 +1530,7 @@ def setUp(self): """ # Run the basic setup - super(TestSubrepoCheckout, self).setUp() + super().setUp() # create test repo # We need to do this here (rather than have a static repo) because # git submodules do not allow for variables in .gitmodules files @@ -1577,19 +1538,19 @@ def setUp(self): self._bare_branch_name = 'subrepo_branch' self._config_branch_name = 'subrepo_config_branch' self._container_extern_name = 'externals_container.cfg' - self._my_test_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - self._test_id) + self._my_test_dir = os.path.join(module_tmp_root_dir, self._test_id) self._repo_dir = os.path.join(self._my_test_dir, self._test_repo_name) self._checkout_dir = 'repo_with_submodules' - check_dir = self.setup_test_repo(CONTAINER_REPO_NAME, + check_dir = self.clone_test_repo(CONTAINER_REPO, dest_dir_in=self._repo_dir) self.assertTrue(self._repo_dir == check_dir) # Add the submodules cwd = os.getcwd() - fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_NAME) - simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO_NAME) - self._simple_ext_fork_name = SIMPLE_FORK_NAME.split('.')[0] - self._simple_ext_name = SIMPLE_REPO_NAME.split('.')[0] + fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_REPO) + simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO) + self._simple_ext_fork_name = os.path.splitext(SIMPLE_FORK_REPO)[0] + self._simple_ext_name = os.path.join('sourc', + os.path.splitext(SIMPLE_REPO)[0]) os.chdir(self._repo_dir) # Add a branch with a subrepo cmd = ['git', 'branch', self._bare_branch_name, 'master'] @@ -1610,7 +1571,8 @@ def setUp(self): execute_subprocess(cmd) cmd = ['git', 'checkout', self._config_branch_name] execute_subprocess(cmd) - cmd = ['git', 'submodule', 'add', simple_repo_dir] + cmd = ['git', 'submodule', 'add', '--name', SIMPLE_REPO, + simple_repo_dir, self._simple_ext_name] execute_subprocess(cmd) # Checkout feature2 os.chdir(self._simple_ext_name) @@ -1621,8 +1583,8 @@ def setUp(self): # Save the fork repo hash for comparison self._simple_hash_check = self.get_git_hash() os.chdir(self._repo_dir) - self.create_externals_file(filename=self._container_extern_name, - dest_dir=self._repo_dir, from_submodule=True) + self.write_externals_config(filename=self._container_extern_name, + dest_dir=self._repo_dir, from_submodule=True) cmd = ['git', 'add', self._container_extern_name] execute_subprocess(cmd) cmd = ['git', 'commit', '-am', "'Added simple-ext as a submodule'"] @@ -1639,9 +1601,10 @@ def get_git_hash(revision="HEAD"): git_out = execute_subprocess(cmd, output_to_caller=True) return git_out.strip() - def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, - branch_name=None, sub_externals=None, - from_submodule=False): + def write_externals_config(self, name='', dest_dir=None, + filename=CFG_NAME, + branch_name=None, sub_externals=None, + from_submodule=False): # pylint: disable=too-many-arguments """Create a container externals file with only simple externals. @@ -1652,10 +1615,10 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, dest_dir = self._my_test_dir if from_submodule: - self._generator.create_section(SIMPLE_FORK_NAME, + self._generator.create_section(SIMPLE_FORK_REPO, self._simple_ext_fork_name, from_submodule=True) - self._generator.create_section(SIMPLE_REPO_NAME, + self._generator.create_section(SIMPLE_REPO, self._simple_ext_name, branch='feature3', path='', from_submodule=False) @@ -1666,8 +1629,8 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, self._generator.create_section(self._test_repo_name, self._checkout_dir, branch=branch_name, - path=name, externals=sub_externals, - repo_path=self._repo_dir) + path=name, sub_externals=sub_externals, + repo_path_abs=self._repo_dir) self._generator.write_config(dest_dir, filename=filename) @@ -1676,12 +1639,10 @@ def idempotence_check(self, checkout_dir): checkout_externals --status does not cause errors""" cwd = os.getcwd() os.chdir(checkout_dir) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.status_args) - self.assertTrue(overall == 0) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) + self.execute_checkout_in_dir(self._my_test_dir, + self.status_args) os.chdir(cwd) def test_submodule_checkout_bare(self): @@ -1693,17 +1654,17 @@ def test_submodule_checkout_bare(self): """ simple_ext_fork_tag = "(tag1)" simple_ext_fork_status = " " - self.create_externals_file(branch_name=self._bare_branch_name) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._bare_branch_name) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, self._simple_ext_fork_name, "readme.txt") self.assertTrue(os.path.exists(fork_file)) - os.chdir(checkout_dir) + submods = git_submodule_status(checkout_dir) + print('checking status of', checkout_dir, ':', submods) self.assertEqual(len(submods.keys()), 1) self.assertTrue(self._simple_ext_fork_name in submods) submod = submods[self._simple_ext_fork_name] @@ -1713,7 +1674,6 @@ def test_submodule_checkout_bare(self): self.assertEqual(submod['status'], simple_ext_fork_status) self.assertTrue('tag' in submod) self.assertEqual(submod['tag'], simple_ext_fork_tag) - os.chdir(cwd) self.idempotence_check(checkout_dir) def test_submodule_checkout_none(self): @@ -1722,11 +1682,10 @@ def test_submodule_checkout_none(self): externals cfg file. Correct behavior is the submodle is not checked out. """ - self.create_externals_file(branch_name=self._bare_branch_name, - sub_externals="none") - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._bare_branch_name, + sub_externals="none") + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, @@ -1744,11 +1703,10 @@ def test_submodule_checkout_config(self): # pylint: disable=too-many-locals """ tag_check = None # Not checked out as submodule status_check = "-" # Not checked out as submodule - self.create_externals_file(branch_name=self._config_branch_name, - sub_externals=self._container_extern_name) - overall, _ = self.execute_cmd_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._config_branch_name, + sub_externals=self._container_extern_name) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, @@ -1810,17 +1768,20 @@ def test_error_unknown_protocol(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_protocol(under_test_dir, 'simp_branch', - 'this-protocol-does-not-exist') + self._generator.write_with_protocol(cloned_repo_dir, BRANCH_SECTION, + 'this-protocol-does-not-exist') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_switch_protocol(self): """Verify that a runtime error is raised when the user switches @@ -1831,15 +1792,18 @@ def test_error_switch_protocol(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_protocol(under_test_dir, 'simp_branch', 'svn') + self._generator.write_with_protocol(cloned_repo_dir, BRANCH_SECTION, 'svn') with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_unknown_tag(self): """Verify that a runtime error is raised when the user specified tag @@ -1847,17 +1811,21 @@ def test_error_unknown_tag(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME) + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='this-tag-does-not-exist', + new_remote_repo_path=SIMPLE_REPO) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_overspecify_tag_branch(self): """Verify that a runtime error is raised when the user specified both @@ -1865,18 +1833,22 @@ def test_error_overspecify_tag_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_tag(under_test_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME, - remove_branch=False) + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='this-tag-does-not-exist', + new_remote_repo_path=SIMPLE_REPO, + remove_branch=False) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_underspecify_tag_branch(self): """Verify that a runtime error is raised when the user specified @@ -1884,17 +1856,19 @@ def test_error_underspecify_tag_branch(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_underspecify_branch_tag(under_test_dir, - 'simp_branch') + self._generator.write_without_branch_tag(cloned_repo_dir, BRANCH_SECTION) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) def test_error_missing_url(self): """Verify that a runtime error is raised when the user specified @@ -1902,17 +1876,20 @@ def test_error_missing_url(self): """ # create repo - under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(under_test_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.update_underspecify_remove_url(under_test_dir, - 'simp_branch') + self._generator.write_without_repo_url(cloned_repo_dir, + BRANCH_SECTION) with self.assertRaises(RuntimeError): - self.execute_cmd_in_dir(under_test_dir, self.checkout_args) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) if __name__ == '__main__': diff --git a/test/test_sys_repository_git.py b/test/test_sys_repository_git.py index f6dbf8428..7e5fb5020 100644 --- a/test/test_sys_repository_git.py +++ b/test/test_sys_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Tests of some of the functionality in repository_git.py that actually interacts with git repositories. @@ -131,12 +131,12 @@ def tearDown(self): shutil.rmtree(self._tmpdir, ignore_errors=True) @staticmethod - def make_git_repo(): + def make_cwd_git_repo(): """Turn the current directory into an empty git repository""" execute_subprocess(['git', 'init']) @staticmethod - def add_git_commit(): + def add_cwd_git_commit(): """Add a git commit in the current directory""" with open('README', 'a') as myfile: myfile.write('more info') @@ -144,17 +144,17 @@ def add_git_commit(): execute_subprocess(['git', 'commit', '-m', 'my commit message']) @staticmethod - def checkout_git_branch(branchname): + def checkout_cwd_git_branch(branchname): """Checkout a new branch in the current directory""" execute_subprocess(['git', 'checkout', '-b', branchname]) @staticmethod - def make_git_tag(tagname): + def make_cwd_git_tag(tagname): """Make a lightweight tag at the current commit""" execute_subprocess(['git', 'tag', '-m', 'making a tag', tagname]) @staticmethod - def checkout_ref(refname): + def checkout_cwd_ref(refname): """Checkout the given refname in the current directory""" execute_subprocess(['git', 'checkout', refname]) @@ -164,72 +164,72 @@ def checkout_ref(refname): def test_currentHash_returnsHash(self): """Ensure that the _git_current_hash function returns a hash""" - self.make_git_repo() - self.add_git_commit() - hash_found, myhash = self._repo._git_current_hash() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + hash_found, myhash = self._repo._git_current_hash(os.getcwd()) self.assertTrue(hash_found) self.assertIsHash(myhash) def test_currentHash_outsideGitRepo(self): """Ensure that the _git_current_hash function returns False when outside a git repository""" - hash_found, myhash = self._repo._git_current_hash() + hash_found, myhash = self._repo._git_current_hash(os.getcwd()) self.assertFalse(hash_found) self.assertEqual('', myhash) def test_currentBranch_onBranch(self): """Ensure that the _git_current_branch function returns the name of the branch""" - self.make_git_repo() - self.add_git_commit() - self.checkout_git_branch('foo') - branch_found, mybranch = self._repo._git_current_branch() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.checkout_cwd_git_branch('foo') + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertTrue(branch_found) self.assertEqual('foo', mybranch) def test_currentBranch_notOnBranch(self): """Ensure that the _git_current_branch function returns False when not on a branch""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('mytag') - self.checkout_ref('mytag') - branch_found, mybranch = self._repo._git_current_branch() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('mytag') + self.checkout_cwd_ref('mytag') + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertFalse(branch_found) self.assertEqual('', mybranch) def test_currentBranch_outsideGitRepo(self): """Ensure that the _git_current_branch function returns False when outside a git repository""" - branch_found, mybranch = self._repo._git_current_branch() + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertFalse(branch_found) self.assertEqual('', mybranch) def test_currentTag_onTag(self): """Ensure that the _git_current_tag function returns the name of the tag""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('some_tag') - tag_found, mytag = self._repo._git_current_tag() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('some_tag') + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertTrue(tag_found) self.assertEqual('some_tag', mytag) def test_currentTag_notOnTag(self): """Ensure tha the _git_current_tag function returns False when not on a tag""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('some_tag') - self.add_git_commit() - tag_found, mytag = self._repo._git_current_tag() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('some_tag') + self.add_cwd_git_commit() + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertFalse(tag_found) self.assertEqual('', mytag) def test_currentTag_outsideGitRepo(self): """Ensure that the _git_current_tag function returns False when outside a git repository""" - tag_found, mytag = self._repo._git_current_tag() + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertFalse(tag_found) self.assertEqual('', mytag) diff --git a/test/test_unit_externals_description.py b/test/test_unit_externals_description.py index 637f760ee..30e528849 100644 --- a/test/test_unit_externals_description.py +++ b/test/test_unit_externals_description.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals @@ -342,6 +342,40 @@ def setup_config(self): # NOTE(goldy, 2019-03) Should test other possible keywords such as # fetchRecurseSubmodules, ignore, and shallow + @staticmethod + def setup_dict_config(): + """Create the full container dictionary with simple and mixed use + externals + + """ + rdatat = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'simple-ext.git', + ExternalsDescription.TAG: 'tag1'} + rdatab = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'simple-ext.git', + ExternalsDescription.BRANCH: 'feature2'} + rdatam = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: 'mixed-cont-ext.git', + ExternalsDescription.BRANCH: 'master'} + desc = {'simp_tag': {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'simp_tag', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatat}, + 'simp_branch' : {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'simp_branch', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatab}, + 'simp_opt': {ExternalsDescription.REQUIRED: False, + ExternalsDescription.PATH: 'simp_opt', + ExternalsDescription.EXTERNALS: EMPTY_STR, + ExternalsDescription.REPO: rdatat}, + 'mixed_req': {ExternalsDescription.REQUIRED: True, + ExternalsDescription.PATH: 'mixed_req', + ExternalsDescription.EXTERNALS: 'sub-ext.cfg', + ExternalsDescription.REPO: rdatam}} + + return desc + def test_cfg_v1_ok(self): """Test that a correct cfg v1 object is created by create_externals_description @@ -379,6 +413,49 @@ def test_dict(self): ext = create_externals_description(desc, model_format='dict') self.assertIsInstance(ext, ExternalsDescriptionDict) + def test_cfg_component_dict(self): + """Verify that create_externals_description works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Check external with all repos + external = create_externals_description(desc, model_format='dict') + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertTrue('simp_tag' in external) + self.assertTrue('simp_branch' in external) + self.assertTrue('simp_opt' in external) + self.assertTrue('mixed_req' in external) + + def test_cfg_exclude_component_dict(self): + """Verify that exclude component checkout works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Test an excluded repo + external = create_externals_description(desc, model_format='dict', + exclude=['simp_tag', + 'simp_opt']) + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertFalse('simp_tag' in external) + self.assertTrue('simp_branch' in external) + self.assertFalse('simp_opt' in external) + self.assertTrue('mixed_req' in external) + + def test_cfg_opt_component_dict(self): + """Verify that exclude component checkout works with a dictionary + """ + # create the top level externals file + desc = self.setup_dict_config() + # Test an excluded repo + external = create_externals_description(desc, model_format='dict', + components=['simp_tag', + 'simp_opt']) + self.assertIsInstance(external, ExternalsDescriptionDict) + self.assertTrue('simp_tag' in external) + self.assertFalse('simp_branch' in external) + self.assertTrue('simp_opt' in external) + self.assertFalse('mixed_req' in external) + def test_cfg_unknown_version(self): """Test that a runtime error is raised when an unknown file version is received diff --git a/test/test_unit_externals_status.py b/test/test_unit_externals_status.py index f8e953f75..f019514e9 100644 --- a/test/test_unit_externals_status.py +++ b/test/test_unit_externals_status.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for the manic external status reporting module. diff --git a/test/test_unit_repository.py b/test/test_unit_repository.py index 5b9c242fd..1b9386183 100644 --- a/test/test_unit_repository.py +++ b/test/test_unit_repository.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals diff --git a/test/test_unit_repository_git.py b/test/test_unit_repository_git.py index 4a0a334bb..1c01098ac 100644 --- a/test/test_unit_repository_git.py +++ b/test/test_unit_repository_git.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals @@ -67,7 +67,7 @@ def setUp(self): def _git_current_branch(branch_found, branch_name): """Return a function that takes the place of repo._git_current_branch, which returns the given output.""" - def my_git_current_branch(): + def my_git_current_branch(dirname): """mock function that can take the place of repo._git_current_branch""" return branch_found, branch_name return my_git_current_branch @@ -76,7 +76,7 @@ def my_git_current_branch(): def _git_current_tag(tag_found, tag_name): """Return a function that takes the place of repo._git_current_tag, which returns the given output.""" - def my_git_current_tag(): + def my_git_current_tag(dirname): """mock function that can take the place of repo._git_current_tag""" return tag_found, tag_name return my_git_current_tag @@ -85,7 +85,7 @@ def my_git_current_tag(): def _git_current_hash(hash_found, hash_name): """Return a function that takes the place of repo._git_current_hash, which returns the given output.""" - def my_git_current_hash(): + def my_git_current_hash(dirname): """mock function that can take the place of repo._git_current_hash""" return hash_found, hash_name return my_git_current_hash @@ -101,8 +101,8 @@ def test_ref_branch(self): True, 'feature3') self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') - expected = 'feature3' - result = self._repo._current_ref() + expected = 'foo_tag (branch feature3)' + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_detached_tag(self): @@ -112,7 +112,7 @@ def test_ref_detached_tag(self): self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'foo_tag' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_detached_hash(self): @@ -123,7 +123,7 @@ def test_ref_detached_hash(self): self._repo._git_current_tag = self._git_current_tag(False, '') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'abc123' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_none(self): @@ -132,7 +132,7 @@ def test_ref_none(self): self._repo._git_current_branch = self._git_current_branch(False, '') self._repo._git_current_tag = self._git_current_tag(False, '') self._repo._git_current_hash = self._git_current_hash(False, '') - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, EMPTY_STR) @@ -206,11 +206,19 @@ def setUp(self): self._repo._current_ref = self._current_ref_empty self._create_tmp_git_dir() + # We have to override this class method rather than the self._repo + # instance method because it is called via + # GitRepository._remote_name_for_url, which is itself a @classmethod + # calls cls._git_remote_verbose(). + self._orignal_git_remote_verbose = GitRepository._git_remote_verbose + GitRepository._git_remote_verbose = self._git_remote_origin_upstream def tearDown(self): """Cleanup tmp stuff on the file system """ self._remove_tmp_git_dir() + GitRepository._git_remote_verbose = self._orignal_git_remote_verbose + def _create_tmp_git_dir(self): """Create a temporary fake git directory for testing purposes. """ @@ -227,29 +235,27 @@ def _remove_tmp_git_dir(self): # mock methods replacing git system calls # @staticmethod - def _current_ref_empty(): + def _current_ref_empty(dirname): """Return an empty string. + + Drop-in for GitRepository._current_ref """ return EMPTY_STR @staticmethod - def _git_remote_origin_upstream(): - """Return an info string that is a checkout hash - """ - return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM + def _git_remote_origin_upstream(dirname): + """Return an info string that is a checkout hash. - @staticmethod - def _git_remote_none(): - """Return an info string that is a checkout hash + Drop-in for GitRepository._git_remote_verbose. """ - return EMPTY_STR + return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM @staticmethod def _git_current_hash(myhash): """Return a function that takes the place of repo._git_current_hash, which returns the given hash """ - def my_git_current_hash(): + def my_git_current_hash(dirname): """mock function that can take the place of repo._git_current_hash""" return 0, myhash return my_git_current_hash @@ -263,7 +269,7 @@ def _git_revparse_commit(self, expected_ref, mystatus, myhash): status = 0 implies success, non-zero implies failure """ - def my_git_revparse_commit(ref): + def my_git_revparse_commit(ref, dirname): """mock function that can take the place of repo._git_revparse_commit""" self.assertEqual(expected_ref, ref) return mystatus, myhash @@ -291,9 +297,6 @@ def test_sync_dir_exist_no_git_info(self): """Test that a non-existent git repo returns an unknown status """ stat = ExternalStatus() - # Now we over-ride the _git_remote_verbose method on the repo to return - # a known value without requiring access to git. - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -313,7 +316,6 @@ def test_sync_invalid_reference(self): """Test that an invalid reference returns out-of-sync """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('abc123') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -333,7 +335,6 @@ def test_sync_tag_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('abc123') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -348,7 +349,6 @@ def test_sync_tag_on_different_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('def456') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -368,7 +368,6 @@ def test_sync_hash_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = '' self._repo._hash = 'abc' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -384,7 +383,6 @@ def test_sync_hash_on_different_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = '' self._repo._hash = 'abc' self._repo._git_current_hash = self._git_current_hash('def456') @@ -405,7 +403,6 @@ def test_sync_branch_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -421,7 +418,6 @@ def test_sync_branch_on_diff_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -433,11 +429,10 @@ def test_sync_branch_on_diff_hash(self): self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) def test_sync_branch_diff_remote(self): - """Test _determine_remote_name with a different remote + """Test _remote_name_for_url with a different remote """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/other/repo' @@ -449,11 +444,10 @@ def test_sync_branch_diff_remote(self): # expected argument def test_sync_branch_diff_remote2(self): - """Test _determine_remote_name with a different remote + """Test _remote_name_for_url with a different remote """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/local/repo2' @@ -469,7 +463,6 @@ def test_sync_branch_on_unknown_remote(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/unknown/repo' @@ -491,7 +484,6 @@ def test_sync_branch_on_untracked_local(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature3' self._repo._tag = '' self._repo._url = '.' @@ -611,24 +603,20 @@ def setUp(self): self._repo = GitRepository('test', repo) @staticmethod - def _shell_true(url, remote=None): - _ = url - _ = remote + def _shell_true(*args, **kwargs): return 0 @staticmethod - def _shell_false(url, remote=None): - _ = url - _ = remote + def _shell_false(*args, **kwargs): return 1 @staticmethod - def _mock_function_true(ref): + def _mock_revparse_commit(ref, dirname): _ = ref return (TestValidRef._shell_true, '97ebc0e0deadc0de') @staticmethod - def _mock_function_false(ref): + def _mock_revparse_commit_false(ref, dirname): _ = ref return (TestValidRef._shell_false, '97ebc0e0deadc0de') @@ -638,10 +626,11 @@ def test_tag_not_tag_branch_commit(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'something' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_not_tag(self): @@ -650,10 +639,11 @@ def test_tag_not_tag(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_true - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'tag1' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_indeterminant(self): @@ -662,10 +652,11 @@ def test_tag_indeterminant(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_true - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = 'something' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_is_unique(self): @@ -674,10 +665,11 @@ def test_tag_is_unique(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = 'tag1' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertTrue(received) def test_tag_is_not_hash(self): @@ -686,10 +678,11 @@ def test_tag_is_not_hash(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = '97ebc0e0' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_hash_is_commit(self): @@ -698,10 +691,11 @@ def test_hash_is_commit(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = '97ebc0e0' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) @@ -746,13 +740,14 @@ def _shell_false(url, remote=None): return 1 @staticmethod - def _mock_function_false(ref): + def _mock_revparse_commit_false(ref, dirname): _ = ref return (TestValidRef._shell_false, '') @staticmethod - def _mock_function_true(ref): + def _mock_revparse_commit_true(ref, dirname): _ = ref + _ = dirname return (TestValidRef._shell_true, '') def test_valid_ref_is_invalid(self): @@ -761,10 +756,12 @@ def test_valid_ref_is_invalid(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'invalid_ref' with self.assertRaises(RuntimeError): - self._repo._check_for_valid_ref(self._repo._tag) + self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) def test_valid_tag(self): """Verify a valid tag return true @@ -772,9 +769,11 @@ def test_valid_tag(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit_true self._repo._tag = 'tag1' - received = self._repo._check_for_valid_ref(self._repo._tag) + received = self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) def test_valid_branch(self): @@ -783,24 +782,28 @@ def test_valid_branch(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit_true self._repo._tag = 'tag1' - received = self._repo._check_for_valid_ref(self._repo._tag) + received = self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) def test_valid_hash(self): """Verify a valid hash return true """ - def _mock_revparse_commit(ref): + def _mock_revparse_commit_true(ref, dirname): _ = ref return (0, '56cc0b539426eb26810af9e') self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = _mock_revparse_commit + self._repo._git_revparse_commit = _mock_revparse_commit_true self._repo._hash = '56cc0b5394' - received = self._repo._check_for_valid_ref(self._repo._hash) + received = self._repo._check_for_valid_ref(self._repo._hash, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) diff --git a/test/test_unit_repository_svn.py b/test/test_unit_repository_svn.py old mode 100644 new mode 100755 index 7ff31c421..d9309df7f --- a/test/test_unit_repository_svn.py +++ b/test/test_unit_repository_svn.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals @@ -60,7 +60,7 @@ def setUp(self): self._name = 'component' rdata = {ExternalsDescription.PROTOCOL: 'svn', ExternalsDescription.REPO_URL: - 'https://svn-ccsm-models.cgd.ucar.edu/', + 'https://svn-ccsm-models.cgd.ucar.edu', ExternalsDescription.TAG: 'mosart/trunk_tags/mosart1_0_26', } diff --git a/test/test_unit_utils.py b/test/test_unit_utils.py index c994e58eb..80e163664 100644 --- a/test/test_unit_utils.py +++ b/test/test_unit_utils.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Unit test driver for checkout_externals From d5b146c3b0b04598c56fa28c8fba04db73d4eab5 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 6 Mar 2023 16:04:23 -0700 Subject: [PATCH 28/64] Update for cesm2_3_alpha12c --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 69a27109a..f46c68b30 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.16 +tag = cmeps0.14.14 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps From f6db9fb7bcf22e3a179599226c03aa5918f8f406 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 8 Mar 2023 16:17:14 -0700 Subject: [PATCH 29/64] Update for cesm2_3_alpha12c --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index f46c68b30..f8f3b5554 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.57 +tag = ccs_config_cesm0.0.58 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config From 55b180be0da246c6c5418dfeca23e3e1c6ee4415 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 14 Mar 2023 08:52:22 -0600 Subject: [PATCH 30/64] Update for cesm2_3_alpha12c --- ChangeLog | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/ChangeLog b/ChangeLog index abbc8353a..5d5295583 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,116 @@ +============================================================== +Tag name: cesm2_3_alpha012c +Originator(s): CSEG +Date: 14 March 2023 +One-line Summary: CDEPS and CMEPS updates. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_086 -- +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_3 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.94 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.58 ** +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.14 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.7 ** +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev115 -- +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230121 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20220322 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + +ccs_config + Chris Fischer 2023-02-27 - ccs_config_cesm0.0.58 - ccs_config (cesm2_3_alpha12c) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm0.0.58 + + ccs_config_cesm0.0.58: Update esmf libraries on remove pgi support on izumi. + ccs_config_cesm0.0.57: Add ne3pg3_ne3pg3_mg37 grid. + ccs_config_cesm0.0.56: Update ESMF and PIO libraries for fontera. Also add mpasa3p75_mpasa3p75_mt13 grid alias. + + + Chris Fischer 2023-01-26 - ccs_config_cesm0.0.55 - ccs_config (cesm2_3_alpha12c) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm0.0.55 + + ccs_config_cesm0.0.55: Update cheyenne esmf library to 8.4.1b02 + + +cdeps + Chris Fischer 2023-02-27 - cdeps1.0.7 - components/cdeps (cesm2_3_alpha12c) + https://github.com/ESCOMP/CDEPS/tags/cdeps1.0.7 + + cdeps1.0.7: get_standard_cmake_args no longer takes a shared_lib argument + cdeps1.0.6: update doc by adding NUOPC cap phases + cdeps1.0.5: allow non-existant include directories + cdeps1.0.4: New docn features and update to documentation + + + Chris Fischer 2023-02-01 - cdeps1.0.4 - components/cdeps (cesm2_3_alpha12c) + https://github.com/ESCOMP/CDEPS/tags/cdeps1.0.4 + + cdeps1.0.4: New docn features and update to documentation. + cdeps1.0.3: The build in github was not consistant with that outside github. + cdeps1.0.2: Rename master to main. + cdeps1.0.1: Code cleanup. + cdeps1.0.0: Fix documentation. + cdeps0.12.72: Fix documentation. + cdeps0.12.71: Update CDEPS documentation. + cdeps0.12.70: Remove shr_mpi_mod. + cdeps0.12.69: Create reusable workflows. + + +cime + Chris Fischer 2023-02-27 - cime6.0.94 - cime (cesm2_3_alpha12c) + https://github.com/ESMCI/cime/tags/cime6.0.94 + + cime6.0.94: bless_test_results: Fix mistake in re match counting + cime6.0.93: Improve cmake build system for sharedlibs + cime6.0.92: bless_test_results: improve error checking of bless_tests args + cime6.0.91: Updates testing worflow to upload logs on failure + + + Chris Fischer 2023-02-01 - cime6.0.90 - cime (cesm2_3_alpha12c) + https://github.com/ESMCI/cime/tags/cime6.0.90 + + cime6.0.90: Add instance to drv logs. + cime6.0.89: Switch file compare count mismatch to a warning. + cime6.0.88: Fix exception chain in import_and_run_sub_or_cmd. + cime6.0.87: Have sys tests use same config. + cime6.0.86: jenkins_generic_job: Add a bit more archiving output + cime6.0.85: Check number of hist file comparisons in system_test_common + + +cmeps + James Edwards 2023-01-03 - cmeps0.14.7 - src/drivers/nuopc/ (cesm2_3_alpha12c) + https://github.com/ESCOMP/CMEPS/tags/TBD + + Change default ocean atmosphere flux grid to xgrid + + + Chris Fischer 2023-02-01 - cmeps0.14.14 - src/drivers/nuopc/ (cesm2_3_alpha12c) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.15 + + For some reason tag 0.14.14 was created after 0.14.15 + + cmeps0.14.14: Changes needed for CDEPS PR #213 + cmeps0.14.15: Fix the multi instance initialization. + cmeps0.14.13: Enable asyncio using pio. + + + Chris Fischer 2023-01-30 - cmeps0.14.12 - src/drivers/nuopc/ (cesm2_3_alpha12c) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.12 + + cmeps0.14.12: Updates to the MEGAN specifier string parser. + cmeps0.14.11: Update esmFldsExchange_nems for ungridded wave fields. + cmeps0.14.10: Redo multiinstance support. + cmeps0.14.09: Replace use of master with main. + cmeps0.14.08: Add werror to extbuild. + ============================================================== Tag name: cesm2_3_alpha012b Originator(s): CSEG From 5b3de96c3cf4fedb9de87f5c4a52b91f340b1456 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 29 Mar 2023 15:35:12 -0600 Subject: [PATCH 31/64] Update for cesm2_3_alph12d --- Externals.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index f8f3b5554..28f1e0900 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.58 +tag = ccs_config_cesm0.0.59 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -29,14 +29,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.14 +tag = cmeps0.14.18 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.7 +tag = cdeps1.0.8 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev115 +tag = ctsm5.1.dev119 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm @@ -118,7 +118,7 @@ local_path = components/mosart required = True [pop] -tag = cesm_pop_2_1_20220322 +tag = cesm_pop_2_1_20230209 protocol = git repo_url = https://github.com/ESCOMP/POP2-CESM local_path = components/pop From d390ba47b3034ecdb68363f11f17563e0b5ffb05 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Thu, 30 Mar 2023 16:09:48 -0600 Subject: [PATCH 32/64] Update for cesm2_3_alpha12d --- ChangeLog | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/ChangeLog b/ChangeLog index 5d5295583..0cc287b35 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,96 @@ +============================================================== +Tag name: cesm2_3_alpha012d +Originator(s): CSEG +Date: 30 March 2023 +One-line Summary: CLM answer changes. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_086 -- +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_3 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.94 -- +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.59 ** +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.18 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.8 ** +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev119 ** +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230121 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 ** +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + +ccs_config + Chris Fischer 2023-03-24 - ccs_config_cesm0.0.59 - ccs_config (cesm2_3_alpha12d) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm0.0.59 + + make 4x5 mesh cdf5 + + +cdeps + Chris Fischer 2023-03-24 - cdeps1.0.8 - components/cdeps (cesm2_3_alpha12d) + https://github.com/ESCOMP/CDEPS/tags/cdeps1.0.8 + + Fix iradsw logic for CPLHIST mode + + +clm + Erik Kluzek 2023-03-16 - ctsm5.1.dev119 - components/clm (cesm2_3_alpha12d) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev119 + + Allows for landuse.timeseries datasets with unrepresented land use change on them. + + + Erik Kluzek 2023-02-05 - ctsm5.1.dev118 - components/clm (cesm2_3_alpha12d) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev118 + + Update for some python tools to use conda rather than ncar_pylib + + + Erik Kluzek 2023-02-02 - ctsm5.1.dev117 - components/clm (cesm2_3_alpha12d) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev117 + + Adding capability to run FATES for NEON sites. @afoster did this tag. + + The only test that changes answers is a NEON site test on izumi. + + + Erik Kluzek 2023-01-24 - ctsm5.1.dev116 - components/clm (cesm2_3_alpha12d) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev116 + + Answer changes with some small bug fixes, and setting of zetamaxstable stability parameter + to 2.0 for clm5_1 physics. Single point datasets are updated, so I cases that use them + have different answers as well. + + +cmeps + Chris Fischer 2023-03-23 - cmeps0.14.18 - src/drivers/nuopc/ (cesm2_3_alpha12d) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.18 + + cmeps0.14.18: Remove unnecessary deallocate. + cneps0.14.17: Revert cmeps0.14.16, but leave in multi-instance fix. + + + Chris Fischer 2023-02-22 - cmeps0.14.16 - src/drivers/nuopc/ (cesm2_3_alpha12d) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.16 + + Set xgrid as default. + + +pop + Michael Levy 2023-02-09 - cesm_pop_2_1_20230209 - components/pop (cesm2_3_alpha12d) + https://github.com/ESCOMP/POP2-CESM/tags/cesm_pop_2_1_20230209 + + NUOPC was relying on MCT to read stream files in some specific situations, but + now that can be handled by CDEPS instead. + + + ============================================================== Tag name: cesm2_3_alpha012c Originator(s): CSEG From 795fee76ca785b0466be18a93a4c6fe54a003177 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 7 Apr 2023 13:46:27 -0600 Subject: [PATCH 33/64] Update for cesm2_3_alpha12e --- Externals.cfg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 28f1e0900..5e6d9d0bb 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.59 +tag = ccs_config_cesm0.0.61 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -21,7 +21,7 @@ local_path = components/cice5 required = True [cice6] -tag = cesm_cice6_4_1_3 +tag = cesm_cice6_4_1_7 protocol = git repo_url = https://github.com/ESCOMP/CESM_CICE local_path = components/cice @@ -29,14 +29,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.18 +tag = cmeps0.14.21 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.8 +tag = cdeps1.0.9 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.94 +tag = cime6.0.105 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev119 +tag = ctsm5.1.dev121 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm From b44496ec29ad089f166671a031e0ad17f06b8997 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Thu, 13 Apr 2023 13:34:31 -0600 Subject: [PATCH 34/64] Update for cesm2_3_alpha12e --- ChangeLog | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/ChangeLog b/ChangeLog index 0cc287b35..e9ec8d938 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,94 @@ +============================================================== +Tag name: cesm2_3_alpha012e +Originator(s): CSEG +Date: 13 April 2023 +One-line Summary: Update cice6, clm, cmeps, cdeps, ccs_config, cime. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_086 -- +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_7 ** +cime https://github.com/ESMCI/cime/tree/cime6.0.105 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.61 ** +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.14 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.21 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.9 ** +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev121 ** +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230121 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + +ccs_config + Chris Fischer 2023-04-07 - ccs_config_cesm0.0.61 - ccs_config (cesm2_3_alpha12e) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm0.0.61 + + ccs_config_cesm0.0.61: Update CMake module for Cheyenne + ccs_config_cesm0.0.60: Include PRISM precipitation datm streams + ccs_config_cesm0.0.59: Make 4x5 mesh cdf5. + + +cdeps + Chris Fischer 2023-04-07 - cdeps1.0.9 - components/cdeps (cesm2_3_alpha12e) + https://github.com/ESCOMP/CDEPS/tags/cdeps1.0.9 + + Need if statement for advertising ndep + + +cice6 + David Bailey 2023-02-28 - cesm_cice6_4_1_7 - components/cice6 (cesm2_3_alpha12e) + https://github.com/ESCOMP/CESM_CICE/tags/cesm_cice6_4_1_4 + + Update CICE6 tag with small bug fixes that are answer changing, but not climate changing. + All active CICE6 cases will be different. Also adds some new namelist flags, so NLFAIL is + also expected. + + +cime + Chris Fischer 2023-03-24 - cime6.0.105 - cime (cesm2_3_alpha12e) + https://github.com/ESMCI/cime/tags/cime6.0.xxx + + cime6.0.105: Improve netcdf link step on systems with seperate c and fortran install which use esmf. + cime6.0.104: Check_input_data: Expand SharedArea coverage to entire implementation + cime6.0.103: Adding support for ADIOS for E3SM. + cime6.0.102: Should not test timestep of stub components. + cime6.0.101: Minor updates to allow moab driver. + cime6.0.100: Revert "Avoid reading in the same config tests file multiple times". + cime6.0.99: Loosen the XSD xml specification for config_machines.xsd to be consistent with env_mach_specific.xsd + cime6.0.98: Avoid reading in the same config tests file multiple times. + cime6.0.97: Fixes single_exe create_test option. + cime6.0.96: jenkins_generic_job: Add support for queue selection + cime6.0.95: Fixes restoring quotes to command arguments + + +clm + Erik Kluzek 2023-04-05 - ctsm5.1.dev121 - components/clm (cesm2_3_alpha12e) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev121 + + Update FATES and increase soil moisture for cold starts with FATES + + + William Sacks 2023-03-25 - ctsm5.1.dev120 - components/clm (cesm2_3_alpha12e) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev120 + + Just updates externals used in standalone checkouts and some aux_clm tests + - so no impact in CESM testing + + +cmeps + Chris Fischer 2023-04-07 - cmeps0.14.21 - src/drivers/nuopc/ (cesm2_3_alpha12e) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.21 + + cmeps0.14.21: Add CAM linked lbs to exe build + cmeps0.14.20: Send nitrogen deposition from atm to ocn + cmeps0.14.19: Replace aux_cam with aux_cmeps in testlist + ============================================================== Tag name: cesm2_3_alpha012d Originator(s): CSEG From 02ed0b8f0870651cacd8f50ccfd03f1e90ecb18e Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 14 Apr 2023 16:01:16 -0600 Subject: [PATCH 35/64] Update for cesm2_3_alpha12f --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 5e6d9d0bb..1ec0381ca 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -6,7 +6,7 @@ local_path = ccs_config required = True [cam] -tag = cam6_3_086 +tag = cam6_3_106 protocol = git repo_url = https://github.com/ESCOMP/CAM local_path = components/cam @@ -44,7 +44,7 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl7.0.14 +tag = cpl7.0.15 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 From 7ea4edd5a1e9026a70895060b5c860b9676596cc Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 28 Apr 2023 10:51:35 -0600 Subject: [PATCH 36/64] Update for cesm2_3_alpha12f --- ChangeLog | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/ChangeLog b/ChangeLog index e9ec8d938..5fb8fd166 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,174 @@ +============================================================== +Tag name: cesm2_3_alpha012f +Originator(s): CSEG +Date: 28 April 2023 +One-line Summary: Answer changes for cam + +components/cam https://github.com/ESCOMP/CAM/cam6_3_106 ** +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_7 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.105 -- +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.61 -- +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.15 ** +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.21 -- +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.9 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev121 -- +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230121 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + +cam + Kate Thayer-Calder 2023-01-24 - cam6_3_091 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_91 + + Update CLUBB external. Changes answers for all CAM6 and dev composts + + + Brian Eaton 2023-04-06 - cam6_3_106 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_106 + + Initialize CO2 when it's missing from initial file. This only impacts runs using the new ghg_mam4 chemistry. + + + Brian Eaton 2023-04-06 - cam6_3_105 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_105 + + Fix for COSP with cam_dev physics. + + + Francis Vitt 2023-04-05 - cam6_3_104 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_104 + + Misc bug fixes + + + Francis Vitt 2023-04-04 - cam6_3_102 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_102 + + Updates to TEM diagnostics + + + Cheryl Craig 2023-03-31 - cam6_3_101 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_101 + + Introduce FLTHIST_v0a compset (low top 58 level compset using cam_dev) + + CLUBB bug fix and new topo file for ne30pg3. + + WACCM ne30pg3 runs are broken with this tag - will be fixed in future tag + + + Francis Vitt 2023-03-20 - cam6_3_100 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_100 + + Introduce prognostic GHG chemistry mechanism for CAM7 + + + Brian Eaton 2023-03-16 - cam6_3_099 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_099 + + Fix drydep emissions for cam_dev physics. + + Changes answers for configurations using 'cam_dev' version of physics. Otherwise BFB. + + + Brian Eaton 2023-03-15 - cam6_3_098 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_098 + + always pass NDEP from CAM and remove sst specs + + + Cheryl Craig 2023-03-15 - cam6_3_097 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_097 + + For Francis Vitt + + Updates to heterogenous freezing code (answer changing mods) + + + Francis Vitt 2023-03-08 - cam6_3_096 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_096 + + Aqueous chemistry bug fix + + + Francis Vitt 2023-02-15 - cam6_3_095 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_095 + + Chemistry updates + + + Cheryl Craig 2023-02-09 - cam6_3_094 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_094 + + modify CLUBB to operate on dry density / mixing ratios + + Answer changing for all CAM6 and dev runs (which use CLUBB) + + + Francis Vitt 2023-02-07 - cam6_3_093 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_093 + + Update SOA scheme in simplified chemistry configurations + DMS emissions in complex chemistry configurations are changed + + + Francis Vitt 2023-01-25 - cam6_3_092 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_092 + + Add capability for zonal mean nudging + + + Francis Vitt 2023-01-12 - cam6_3_090 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_090 + + Functional support for FHIST on SE refined grids + + + Francis Vitt 2023-01-10 - cam6_3_089 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_089 + + Introduce zonal mean and Transformed Eulerian Mean diagnostics capabilities on arbitrary grids + + + Cheryl Craig 2022-12-21 - cam6_3_088 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_088 + + Update externals to match (almost) cesm2_3_alpha11a + + Add missing ./xmlchange ROF_NCPL and GLC_NCPL to various tests + + + Cheryl Craig 2022-12-21 - cam6_3_087 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/cam6_3_087 + + For Francis Vitt: + + Add MAM5 chemistry and change coarse mode size parameters for MAM4 used by cam_dev physics + + + Cheryl Craig 2023-04-05 - cam6_3_103 - components/cam (cesm2_3_alpha12f) + https://github.com/ESCOMP/CAM/tags/ cam6_3_102 + + Update to cesm2_3_alpha12d externals (almost) + + Answer changing for CAM checkouts, but not for CESM ones + + +cpl7 + Chris Fischer 2023-04-14 - cpl7.0.15 - components/cpl7 (cesm2_3_alpha12f) + https://github.com/ESCOMP/CESM_CPL7andDataComps/tags/cpl7.0.15 + + Updates to MEGAN namelist parser + ============================================================== Tag name: cesm2_3_alpha012e Originator(s): CSEG From 094e455a0ad7a783a1768e806d85112988bf745a Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 28 Apr 2023 18:02:39 -0600 Subject: [PATCH 37/64] Update for cesm2_3_alpha12g --- Externals.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 1ec0381ca..9e47f49aa 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,12 +1,12 @@ [ccs_config] -tag = ccs_config_cesm0.0.61 +tag = ccs_config_cesm0.0.67 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config required = True [cam] -tag = cam6_3_106 +tag = cam6_3_109 protocol = git repo_url = https://github.com/ESCOMP/CAM local_path = components/cam @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.21 +tag = cmeps0.14.24 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev121 +tag = ctsm5.1.dev122 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm From 068b4882ab1015c71416beeb6caf2adebeb0e1cb Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 1 May 2023 16:04:47 -0600 Subject: [PATCH 38/64] Update for cesm2_3_alpha12g --- ChangeLog | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/ChangeLog b/ChangeLog index 5fb8fd166..ed960c29b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,83 @@ +============================================================== +Tag name: cesm2_3_alpha012g +Originator(s): CSEG +Date: 1 May 2023 +One-line Summary: Tag for coupled run. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_109 ** +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_7 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.105 -- +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.61 -- +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.15 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.24 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.9 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev122 ** +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230121 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + + +cam + Cheryl Craig 2023-04-26 - cam6_3_109 - components/cam (cesm2_3_alpha12g) + https://github.com/ESCOMP/CAM/tags/cam6_3_109 + + For John Truesdale + + Energy fixer and mass budget reporting + + + Cheryl Craig 2023-04-19 - cam6_3_108 - components/cam (cesm2_3_alpha12g) + https://github.com/ESCOMP/CAM/tags/cam6_3_108 + + For Francis Vitt + + Ocean emissions fix and lightning flash freq passed to land model + + Needs cmeps0.14.24 + + + Brian Eaton 2023-04-18 - cam6_3_107 - components/cam (cesm2_3_alpha12g) + https://github.com/ESCOMP/CAM/tags/cam6_3_107 + + Reimplement zonal_mean_mod::Invert_Matrix using LAPACK DGESV. + + Answer changes for runs with nudging and Nudge_ZonalFilter set to true. + Other configurations are BFB, but the diagnostic zonal mean fields only agree to a couple of significant figures due to fixing a bug in a matrix inversion routine. + + +cesm + Chris Fischer 2023-04-28 - cesm2_3_alpha12g - (cesm2_3_alpha12g) + https://github.com/ESCOMP/cesm/tags/cesm2_3_alpha12g + + Update manage externals. + + +clm + William Sacks 2023-04-26 - ctsm5.1.dev122 - components/clm (cesm2_3_alpha12g) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev122 + + Rework handling of evaporation constraint in SoilFluxes + + Roundoff-level answer changes appear occasionally + + +cmeps + Chris Fischer 2023-04-28 - cmeps0.14.24 - src/drivers/nuopc/ (cesm2_3_alpha12g) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.24 + + cmeps0.14.24: Cloud-to-ground lightning flash frequency coupling + cmeps0.14.23: Fix issue with xgrid reproducibility + cmeps0.14.22: A fix for #346 so that LND2ROF_FMAPNAME will be used + ============================================================== Tag name: cesm2_3_alpha012f Originator(s): CSEG From b3507d36d40894b7d5c10b77c55e2bed7f4646bd Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 1 May 2023 16:35:11 -0600 Subject: [PATCH 39/64] Update for cesm2_3_beta12 --- ChangeLog | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/ChangeLog b/ChangeLog index ed960c29b..2c14f9015 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,101 @@ +============================================================== +Tag name: cesm2_3_beta12 +Originator(s): CSEG +Date: 1 May 2023 +One-line Summary: Beta tag for coupled run. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_109 +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_7 +cime https://github.com/ESMCI/cime/tree/cime6.0.105 +share https://github.com/ESCOMP/CESM_share/tree/share1.0.16 +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.61 +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl7.0.15 +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.24 +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.9 +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev122 +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 +components/mom https://github.com/ESCOMP/MOM_interface/mi_230121 +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 + +Answer Changes introduced with this tag when compared to cesm2_3_beta11: + +Change default ocean atmosphere flux grid to xgrid + +CICE +- All active CICE cases will have different answers. + +MOM +- Misc features and bugfixes including logging and changes for new FMS tag. + +CLM +- FATES update for nutrient interface +- NEON site test on izumi. +- Answer changes with some small bug fixes. +- Single point datasets are updated, so I cases that use them have different answers as well. + +CAM +- Only impacts runs using the new ghg_mam4 chemistry. +- Configurations using 'cam_dev' version of physics. +- Heterogenous freezing code. +- Aqueous chemistry bug fix. +- Chemistry updates. +- All CAM6 and dev runs (which use CLUBB). +- SOA scheme in simplified chemistry configurations. +- DMS emissions in complex chemistry configurations. +- MAM4 used by cam_dev physics. + + +Problems identified after tag creation: +- All T31_g37_rx1.A tests fail with floating point exception. +- All T62_s11 nuopc tests fail because of missing mesh file. +- All ERS_N2 tests should be ERS_C2 tests. + +Cheyenne Intel + FAIL ERP_D_Ln9_Vnuopc.C48_C48_mg17.QPC6.cheyenne_intel.cam-outfrq9s MODEL_BUILD time=3 + FAIL ERS.TL319_t061_wt061.GMOM_JRA_WD.cheyenne_intel MODEL_BUILD time=2 + FAIL SMS_D_Ld1_PS.f09_g17.I1850Clm50BgcSpinup.cheyenne_intel.clm-cplhist RUN time=45 + FAIL ERI_D.T31_g37_rx1.A.cheyenne_intel RUN time=45 + FAIL ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 MODEL_BUILD time=3 + FAIL ERS_IOP.T62_s11.G.cheyenne_intel.pop-cice SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_intel CREATE_NEWCASE + FAIL SMS_D_Ln9_Vnuopc.ne30pg3_ne30pg3_mg17.FWma2000climo.cheyenne_intel.cam-outfrq9s RUN time=519 + FAIL SMS_D.T62_s11.G.cheyenne_intel.pop-cice RUN time=4 +Cheyenne gnu + FAIL ERI_D.T31_g37_rx1.A.cheyenne_gnu RUN time=40 + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_gnu CREATE_NEWCASE + FAIL IRT_C3_Ld7.f19_g17.BHIST.cheyenne_gnu.allactive-defaultio RUN time=307 + FAIL SMS_D.T62_s11.G.cheyenne_gnu.pop-cice SUBMIT +Izumi Intel + FAIL ERI.f19_g17_rx1.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default RUN time=40 + FAIL ERI_Vnuopc.f19_g17_rx1.2000_DATM%NYF_SLND_CICE_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice-default RUN time=41 + FAIL ERI_D.T31_g37_rx1.A.izumi_intel RUN time=50 + PEND ERS_D.f19_g16_rx1.G1850ECO.izumi_intel.pop-cice_ecosys RUN + FAIL ERS_Ld5.T62_s11.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.izumi_intel CREATE_NEWCASE + PEND ERS.T62_g16.C1850ECO.izumi_intel.pop-ecosys RUN + FAIL ERS.T62_g16.G.izumi_intel.pop-cice RUN time=84 +Izumi nag + FAIL DAE.ww3a.ADWAV.izumi_nag RUN time=26 + PEND ERP_D_Ld5.ne30_g17.I1850Clm50BgcCrop.izumi_nag.clm-default RUN + FAIL ERS_Ld5.f19_g17.2000_CAM60_CLM50%SP_CICE5_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice5-default SHAREDLIB_BUILD time=73 + FAIL ERS_Ld5_Vnuopc.f19_g17.2000_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice-default SHAREDLIB_BUILD time=13 + FAIL PRE.f19_f19.ADESP_TEST.izumi_nag CREATE_NEWCASE + FAIL SMS_D_Ln9_Vnuopc.f10_f10_mg37.QPWmadC4.izumi_nag.cam-outfrq9s BASELINE cesm2_3_alpha12f: DIFF + FAIL SMS_Ld1_P136_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN time=245 + PEND SMS_Ld1_P144_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN + FAIL SMS_Ld2_P72_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN time=62 + PEND SMS_Ld2_P80_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN + FAIL SMS.T62_g37.C1850ECO.izumi_nag.pop-ecosys RUN time=76 + FAIL ERS.T62_s11.G.izumi_nag.pop-cice SUBMIT + ============================================================== Tag name: cesm2_3_alpha012g Originator(s): CSEG From c152e211c488d5bc65a198ce44cfc4d1806b69ca Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Thu, 4 May 2023 16:11:39 -0600 Subject: [PATCH 40/64] Update for cesm2_3_alpha13a --- Externals.cfg | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 9e47f49aa..a410cf59b 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,5 +1,5 @@ [ccs_config] -tag = ccs_config_cesm0.0.67 +tag = ccs_config_cesm0.0.70 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -21,7 +21,7 @@ local_path = components/cice5 required = True [cice6] -tag = cesm_cice6_4_1_7 +tag = cesm_cice6_4_1_8 protocol = git repo_url = https://github.com/ESCOMP/CESM_CICE local_path = components/cice @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.24 +tag = cmeps0.14.26 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -44,14 +44,14 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl7.0.15 +tag = cpl77.0.5 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 required = True [share] -tag = share1.0.16 +tag = share1.0.17 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.105 +tag = cime6.0.113 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev122 +tag = ctsm5.1.dev123 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm @@ -103,7 +103,7 @@ externals = Externals_FMS.cfg required = False [mom] -tag = mi_230121 +tag = mi_230504 protocol = git repo_url = https://github.com/ESCOMP/MOM_interface local_path = components/mom From 3be34db212a5f07eae1c0cb2b21103968b822c4b Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 8 May 2023 10:09:18 -0600 Subject: [PATCH 41/64] Add BLT1850_v0b BMT1850_v0b --- cime_config/config_compsets.xml | 10 ++++++++++ cime_config/testlist_allactive.xml | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml index 25463d6db..7928e1fda 100644 --- a/cime_config/config_compsets.xml +++ b/cime_config/config_compsets.xml @@ -43,6 +43,16 @@ 1850_CAM60_CLM50%BGC-CROP_CICE_POP2%ECO_MOSART_CISM2%GRIS-NOEVOLVE_WW3_BGC%BDRD + + BLT1850_v0b + 1850_CAM%DEV%LT%GHGMAM4_CLM51%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_WW3 + + + + BMT1850_v0b + 1850_CAM%DEV%MT%GHGMAM4_CLM51%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_WW3 + + BW1850 1850_CAM60%WCTS_CLM50%BGC-CROP_CICE_POP2%ECO%NDEP_MOSART_CISM2%GRIS-NOEVOLVE_WW3 diff --git a/cime_config/testlist_allactive.xml b/cime_config/testlist_allactive.xml index 7b2c3996a..848934548 100644 --- a/cime_config/testlist_allactive.xml +++ b/cime_config/testlist_allactive.xml @@ -24,6 +24,22 @@ + + + + + + + + + + + + + + + + From 4cb8cadf9141a5a805c055084273698ec7428070 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 9 May 2023 12:26:58 -0600 Subject: [PATCH 42/64] Change v0b to v0c --- cime_config/config_compsets.xml | 4 ++-- cime_config/testlist_allactive.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml index 7928e1fda..941d96b64 100644 --- a/cime_config/config_compsets.xml +++ b/cime_config/config_compsets.xml @@ -44,12 +44,12 @@ - BLT1850_v0b + BLT1850_v0c 1850_CAM%DEV%LT%GHGMAM4_CLM51%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_WW3 - BMT1850_v0b + BMT1850_v0c 1850_CAM%DEV%MT%GHGMAM4_CLM51%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_WW3 diff --git a/cime_config/testlist_allactive.xml b/cime_config/testlist_allactive.xml index 848934548..055104780 100644 --- a/cime_config/testlist_allactive.xml +++ b/cime_config/testlist_allactive.xml @@ -24,7 +24,7 @@ - + @@ -32,7 +32,7 @@ - + From 9511406063365e80331584e154a1ff6600a6b29f Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 9 May 2023 13:00:01 -0600 Subject: [PATCH 43/64] Update for cesm2_3_alpha13a --- ChangeLog | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/ChangeLog b/ChangeLog index 2c14f9015..3069c75b7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,99 @@ +============================================================== +Tag name: cesm2_3_alpha013a +Originator(s): CSEG +Date: 9 May 2023 +One-line Summary: MOM workhorse grid, updates for pFUnit 4. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_109 -- +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_8 ** +cime https://github.com/ESMCI/cime/tree/cime6.0.113 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.17 ** +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.70 ** +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl77.0.5 ** +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.26 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.9 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev123 ** +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230504 ** +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.5 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + +ccs_config + Alper Altuntas 2023-05-02 - ccs_config_cesm0.0.70 - ccs_config (cesm2_3_alpha13a) + https://github.com/ESMCI/ccs_config_cesm/tags/??? + + ccs_config_cesm0.0.70: new MOM6 workhorse grid + ccs_config_cesm0.0.69: Update module versions on Casper and add settings for the RRTMGP GPU code + ccs_config_cesm0.0.68: Gust update + + +cice6 + James Edwards 2023-05-03 - cesm_cice6_4_1_8 - components/cice6 (cesm2_3_alpha13a) + https://github.com/ESCOMP/CESM_CICE/tags/cesm_cice6_4_1_8 + + new mom grid added + + +cime + James Edwards 2023-05-02 - cime6.0.113 - cime (cesm2_3_alpha13a) + https://github.com/ESMCI/cime/tags/cime6.0.113 + + cime6.0.117: Use compiler info when looking for test status files + cime6.0.116: case_clone: Fix bug in srcroot determination + cime6.0.115: cce/15.0.1 compiler did not link properly without this update + cime6.0.114: BATCH_COMMAND_FLAGS was not in env_batch, change self to case + cime6.0.113: port to python 3.10 + cime6.0.112: Improve the xmllint not found error message + cime6.0.111: Add the code to handle stale issues. + cime6.0.110: Fix get_all_hist _files regex + cime6.0.109: jenkins_script: Remove from CIME + cime6.0.108: Update pFUnit support to version 4 + cime6.0.107: Fix short term archiving submit on sigma2 betzy + cime6.0.106: Add a note in SourceMods/src.cdeps/README + + +clm + William Sacks 2023-05-01 - ctsm5.1.dev123 - components/clm (cesm2_3_alpha13a) + https://github.com/ESCOMP/ctsm/tags/ctsm5.1.dev123 + + Updates needed for pFUnit 4 and other externals updates + + +cmeps + James Edwards 2023-05-02 - cmeps0.14.26 - src/drivers/nuopc/ (cesm2_3_alpha13a) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.26 + + cmeps0.14.26: refactors salt formating in diag output + cmeps0.14.25: Turn off HierarchyProtocol. + + +cpl7 + William Sacks 2023-05-03 - cpl77.0.5 - components/cpl7 (cesm2_3_alpha13a) + https://github.com/ESCOMP/CESM_CPL7andDataComps/tags/cpl77.0.5 + + Update pFUnit support to version 4 + + +mom + Alper Altuntas 2023-05-02 - mi_230504 - components/mom (cesm2_3_alpha13a) + https://github.com/ESCOMP/MOM_interface/tags/??? + + new MOM6 workhorse grid + + +share + William Sacks 2023-05-03 - share1.0.17 - share (cesm2_3_alpha13a) + https://github.com/ESCOMP/CESM_share/tags/share1.0.17 + + Update pFUnit support to version 4 + ============================================================== Tag name: cesm2_3_beta12 Originator(s): CSEG From adae188eec3e945c3b9470e8e1a8ff4d8a9d8dc6 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 17 May 2023 13:46:26 -0600 Subject: [PATCH 44/64] alpha13b candidate --- Externals.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index a410cf59b..782468d68 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,12 +1,12 @@ [ccs_config] -tag = ccs_config_cesm0.0.70 +tag = ccs_config_cesm0.0.71 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config required = True [cam] -tag = cam6_3_109 +tag = cam6_3_111 protocol = git repo_url = https://github.com/ESCOMP/CAM local_path = components/cam @@ -29,14 +29,14 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.26 +tag = cmeps0.14.30 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.9 +tag = cdeps1.0.14 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.113 +tag = cime6.0.121 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime From 919b2869468648ab2bd02cfeb10c8ee36b1efc59 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 18 May 2023 07:13:11 -0600 Subject: [PATCH 45/64] update ww3dev tag --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 782468d68..92a48329b 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -147,7 +147,7 @@ local_path = components/ww3 required = True [ww3dev] -tag = main_0.0.5 +tag = main_0.0.6 protocol = git repo_url = https://github.com/ESCOMP/WW3_interface local_path = components/ww3dev From 049f5ff2037809aee1b1ca58ace81488198640a7 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 19 May 2023 10:52:41 -0600 Subject: [PATCH 46/64] Remove BMT1850_v0c compset --- cime_config/config_compsets.xml | 5 ----- cime_config/testlist_allactive.xml | 8 -------- 2 files changed, 13 deletions(-) diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml index 941d96b64..b234e3736 100644 --- a/cime_config/config_compsets.xml +++ b/cime_config/config_compsets.xml @@ -48,11 +48,6 @@ 1850_CAM%DEV%LT%GHGMAM4_CLM51%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_WW3 - - BMT1850_v0c - 1850_CAM%DEV%MT%GHGMAM4_CLM51%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_WW3 - - BW1850 1850_CAM60%WCTS_CLM50%BGC-CROP_CICE_POP2%ECO%NDEP_MOSART_CISM2%GRIS-NOEVOLVE_WW3 diff --git a/cime_config/testlist_allactive.xml b/cime_config/testlist_allactive.xml index 055104780..8f7f692fd 100644 --- a/cime_config/testlist_allactive.xml +++ b/cime_config/testlist_allactive.xml @@ -32,14 +32,6 @@ - - - - - - - - From a64fd882f535470c333213a60c1c5a950fca5a8e Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 19 May 2023 13:57:59 -0600 Subject: [PATCH 47/64] Set working PE layout for BLT1850_v0c test --- cime_config/config_pes.xml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index edda21daf..bb32b0401 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -349,6 +349,43 @@ + + + + none + + 1800 + 724 + 724 + 1080 + 324 + 36 + 36 + 1800 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 724 + 1800 + 0 + 1 + 0 + + + + From 1b6da055e47d6332909fe9a5b8fa22590f0bced8 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Fri, 19 May 2023 14:01:41 -0600 Subject: [PATCH 48/64] Use t232 ocean grid for BLT1850_v0c test --- cime_config/testlist_allactive.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testlist_allactive.xml b/cime_config/testlist_allactive.xml index 8f7f692fd..e62307b05 100644 --- a/cime_config/testlist_allactive.xml +++ b/cime_config/testlist_allactive.xml @@ -24,7 +24,7 @@ - + From 7e768d8729af8a12ca7e74f9d0c4e359c5d491b7 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 22 May 2023 13:54:43 -0600 Subject: [PATCH 49/64] Update for cesm2_3_alpha13b --- ChangeLog | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/ChangeLog b/ChangeLog index 3069c75b7..8ca661507 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,103 @@ +============================================================== +Tag name: cesm2_3_alpha013b +Originator(s): CSEG +Date: 18 May 2023 +One-line Summary: Support for BLT1850_v0c compset + +components/cam https://github.com/ESCOMP/CAM/cam6_3_111 ** +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_8 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.121 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.17 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.71 ** +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl77.0.5 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.30 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.14 ** +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev123 -- +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230504 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.6 ** +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + +cam + Cheryl Craig 2023-05-08 - cam6_3_111 - components/cam (cesm2_3_alpha13b) + https://github.com/ESCOMP/CAM/tags/cam6_3_??? + + Add 1850_cam_lt.xml and 1850_cam_mt.xml use_cases + + + Cheryl Craig 2023-05-08 - cam6_3_110 - components/cam (cesm2_3_alpha13b) + https://github.com/ESCOMP/CAM/tags/cam6_3_110 + + For Brian Eaton- + + Misc changes for CAM + + Includes adding GHGMAM4 to FMTHIST compset + + +ccs_config + James Edwards 2023-05-18 - ccs_config_cesm0.0.71 - ccs_config (cesm2_3_alpha13b) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm0.0.71 + + Updates for gust + + +cdeps + James Edwards 2023-05-18 - cdeps1.0.14 - components/cdeps (cesm2_3_alpha13b) + https://github.com/ESCOMP/CDEPS/tags/cdeps1.0.14 + + Allow adding a day to streams' file name date specifier. + Include PRISM precipitation datm streams. + Modify stream_definition_datm.xml to generate a streams file (datm.streams.xml) with the new coupler history file format. + Black reformat python files. + + +cesm + Chris Fischer 2023-05-19 - cesm2_3_alpha13b - (cesm2_3_alpha13b) + https://github.com/ESCOMP/cesm/tags/cesm2_3_alpha13b + + Add BLT1850_v0c compset and test. + + +cime + James Edwards 2023-05-18 - cime6.0.121 - cime (cesm2_3_alpha13b) + https://github.com/ESMCI/cime/tags/cime6.0.121 + + Bug fix for BATCH_COMMAND_FLAGS + Fix for cce/15.0.1 compiler + SRCROOT should match the original case if possible. This bug was causing multi-build tests, like PEM, to not respect a custom --srcroot argument t + o create_test. + Use compiler info when looking for test status files. + Reduce newlines in append to user_nl_* files. + Fix caching test files in container. + Rename xml-driver to driver in create_test + Update docs and add error message to scripts_regression_tests.py if cprnc is missing. + + +cmeps + James Edwards 2023-05-18 - cmeps0.14.30 - src/drivers/nuopc/ (cesm2_3_alpha13b) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.30 + + remove unneeded xml variables + add check for nans in mediator fields + Fix issues with aux_cpl file writes + Update minimum ESMF version requirement + + +ww3dev + James Edwards 2023-05-18 - main_0.0.6 - components/ww3dev (cesm2_3_alpha13b) + https://github.com/ESCOMP/WW3_interface/tags/main_0.0.6 + + Fix build interface + + ============================================================== Tag name: cesm2_3_alpha013a Originator(s): CSEG From 9d790076a99e705c6100c217dd75299a22ff65de Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Mon, 22 May 2023 14:04:15 -0600 Subject: [PATCH 50/64] Update for cesm2_3_beta13 --- ChangeLog | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/ChangeLog b/ChangeLog index 8ca661507..f7307f581 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,77 @@ +============================================================== +Tag name: cesm2_3_beta13 +Originator(s): CSEG +Date: 18 May 2023 +One-line Summary: Beta tag for BLT1850_v0c coupled run. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_111 +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_8 +cime https://github.com/ESMCI/cime/tree/cime6.0.121 +share https://github.com/ESCOMP/CESM_share/tree/share1.0.17 +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.71 +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl77.0.5 +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.30 +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.14 +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev123 +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 +components/mom https://github.com/ESCOMP/MOM_interface/mi_230504 +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.6 +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 + +Answer Changes introduced with this tag when compared to cesm2_3_beta12: +- No answer changes. + +Problems identified after tag creation: +- CICE5 namelist file issues. +- All T31_g37_rx1.A tests fail with floating point exception. +- All T62_s11 nuopc tests fail because of missing mesh file. +- All ERS_N2 tests should be ERS_C2 tests. + +Cheyenne Intel + FAIL ERP_D_Ln9_Vnuopc.C48_C48_mg17.QPC6.cheyenne_intel.cam-outfrq9s MODEL_BUILD time=3 + FAIL SMS_D_Ld1_PS.f09_g17.I1850Clm50BgcSpinup.cheyenne_intel.clm-cplhist RUN time=45 + FAIL ERI_D.T31_g37_rx1.A.cheyenne_intel RUN time=45 + FAIL ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 MODEL_BUILD time=3 + FAIL ERS_IOP.T62_s11.G.cheyenne_intel.pop-cice SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_intel CREATE_NEWCASE + FAIL SMS_D_Ln9_Vnuopc.ne30pg3_ne30pg3_mg17.FWma2000climo.cheyenne_intel.cam-outfrq9s RUN time=519 + FAIL SMS_D.T62_s11.G.cheyenne_intel.pop-cice RUN time=4 +Cheyenne gnu + FAIL ERI_D.T31_g37_rx1.A.cheyenne_gnu RUN time=40 + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_gnu CREATE_NEWCASE + FAIL IRT_C3_Ld7.f19_g17.BHIST.cheyenne_gnu.allactive-defaultio RUN time=307 + FAIL SMS_D.T62_s11.G.cheyenne_gnu.pop-cice SUBMIT +Izumi Intel + FAIL ERI.f19_g17_rx1.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default RUN time=40 + FAIL ERI_Vnuopc.f19_g17_rx1.2000_DATM%NYF_SLND_CICE_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice-default RUN time=41 + FAIL ERI_D.T31_g37_rx1.A.izumi_intel RUN time=50 + PEND ERS_D.f19_g16_rx1.G1850ECO.izumi_intel.pop-cice_ecosys RUN + FAIL ERS_Ld5.T62_s11.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.izumi_intel CREATE_NEWCASE + PEND ERS.T62_g16.C1850ECO.izumi_intel.pop-ecosys RUN + FAIL ERS.T62_g16.G.izumi_intel.pop-cice RUN time=84 +Izumi nag + FAIL DAE.ww3a.ADWAV.izumi_nag RUN time=26 + PEND ERP_D_Ld5.ne30_g17.I1850Clm50BgcCrop.izumi_nag.clm-default RUN + FAIL ERS_Ld5.f19_g17.2000_CAM60_CLM50%SP_CICE5_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice5-default SHAREDLIB_BUILD time=73 + FAIL ERS_Ld5_Vnuopc.f19_g17.2000_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice-default SHAREDLIB_BUILD time=13 + FAIL PRE.f19_f19.ADESP_TEST.izumi_nag CREATE_NEWCASE + FAIL SMS_D_Ln9_Vnuopc.f10_f10_mg37.QPWmadC4.izumi_nag.cam-outfrq9s BASELINE cesm2_3_alpha12f: DIFF + FAIL SMS_Ld1_P136_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN time=245 + PEND SMS_Ld1_P144_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN + FAIL SMS_Ld2_P72_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN time=62 + PEND SMS_Ld2_P80_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN + FAIL SMS.T62_g37.C1850ECO.izumi_nag.pop-ecosys RUN time=76 + FAIL ERS.T62_s11.G.izumi_nag.pop-cice SUBMIT + + ============================================================== Tag name: cesm2_3_alpha013b Originator(s): CSEG From 27b332f33639b001bac1fc6a27621f04d07ff7e8 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 24 May 2023 12:21:09 -0600 Subject: [PATCH 51/64] Update for cesm2_3_alpha14a --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 92a48329b..c2e3ee560 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.30 +tag = cmeps0.14.32 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps @@ -72,7 +72,7 @@ local_path = libraries/parallelio required = True [cime] -tag = cime6.0.121 +tag = cime6.0.125 protocol = git repo_url = https://github.com/ESMCI/cime local_path = cime From 995399d0cba6c9e691c7f17c05da4f57dab4f555 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 26 May 2023 10:23:08 -0600 Subject: [PATCH 52/64] update cdeps to revert default aoflux grid to ogrid --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index c2e3ee560..24285f8a8 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,7 +29,7 @@ externals = Externals.cfg required = True [cmeps] -tag = cmeps0.14.32 +tag = cmeps0.14.34 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps From c0884c9750e7b8421e66675aa450ef2439b11b83 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Thu, 1 Jun 2023 16:12:15 -0600 Subject: [PATCH 53/64] Update for cesm2_3_beta14 --- ChangeLog | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/ChangeLog b/ChangeLog index f7307f581..e20b9fc08 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,130 @@ +============================================================== +Tag name: cesm2_3_beta14 +Originator(s): CSEG +Date: 1 June 2023 +One-line Summary: Misc. fixes. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_111 +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_8 +cime https://github.com/ESMCI/cime/tree/cime6.0.125 +share https://github.com/ESCOMP/CESM_share/tree/share1.0.17 +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.71 +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl77.0.5 +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.34 +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.14 +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev123 +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 +components/mom https://github.com/ESCOMP/MOM_interface/mi_230504 +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.6 +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 + + +Answer Changes introduced with this tag when compared to cesm2_3_beta13: +- Answer changes for sw flux to MOM ocn. + +Problems identified after tag creation: +- All T31_g37_rx1.A tests fail with floating point exception. +- All T62_s11 nuopc tests fail because of missing mesh file. +- All ERS_N2 tests should be ERS_C2 tests. + +Cheyenne Intel + FAIL ERP_D_Ln9_Vnuopc.C48_C48_mg17.QPC6.cheyenne_intel.cam-outfrq9s MODEL_BUILD time=3 + FAIL SMS_D_Ld1_PS.f09_g17.I1850Clm50BgcSpinup.cheyenne_intel.clm-cplhist RUN time=45 + FAIL ERI_D.T31_g37_rx1.A.cheyenne_intel RUN time=45 + FAIL ERP_Ln9_Vnuopc.C96_C96_mg17.F2000climo.cheyenne_intel.cam-outfrq9s_mg3 MODEL_BUILD time=3 + FAIL ERS_IOP.T62_s11.G.cheyenne_intel.pop-cice SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_intel CREATE_NEWCASE + FAIL SMS_D_Ln9_Vnuopc.ne30pg3_ne30pg3_mg17.FWma2000climo.cheyenne_intel.cam-outfrq9s RUN time=519 + FAIL SMS_D.T62_s11.G.cheyenne_intel.pop-cice RUN time=4 +Cheyenne gnu + FAIL ERI_D.T31_g37_rx1.A.cheyenne_gnu RUN time=40 + FAIL ERS_N2_Ld3.T31_g37_rx1.A.cheyenne_gnu CREATE_NEWCASE + FAIL IRT_C3_Ld7.f19_g17.BHIST.cheyenne_gnu.allactive-defaultio RUN time=307 + FAIL SMS_D.T62_s11.G.cheyenne_gnu.pop-cice SUBMIT +Izumi Intel + FAIL ERI.f19_g17_rx1.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default RUN time=40 + FAIL ERI_Vnuopc.f19_g17_rx1.2000_DATM%NYF_SLND_CICE_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice-default RUN time=41 + FAIL ERI_D.T31_g37_rx1.A.izumi_intel RUN time=50 + PEND ERS_D.f19_g16_rx1.G1850ECO.izumi_intel.pop-cice_ecosys RUN + FAIL ERS_Ld5.T62_s11.2000_DATM%NYF_SLND_CICE5_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST.izumi_intel.cice5-default SUBMIT + FAIL ERS_N2_Ld3.T31_g37_rx1.A.izumi_intel CREATE_NEWCASE + PEND ERS.T62_g16.C1850ECO.izumi_intel.pop-ecosys RUN + FAIL ERS.T62_g16.G.izumi_intel.pop-cice RUN time=84 +Izumi nag + FAIL DAE.ww3a.ADWAV.izumi_nag RUN time=26 + PEND ERP_D_Ld5.ne30_g17.I1850Clm50BgcCrop.izumi_nag.clm-default RUN + FAIL ERS_Ld5.f19_g17.2000_CAM60_CLM50%SP_CICE5_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice5-default SHAREDLIB_BUILD time=73 + FAIL ERS_Ld5_Vnuopc.f19_g17.2000_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST.izumi_nag.cice-default SHAREDLIB_BUILD time=13 + FAIL PRE.f19_f19.ADESP_TEST.izumi_nag CREATE_NEWCASE + FAIL SMS_D_Ln9_Vnuopc.f10_f10_mg37.QPWmadC4.izumi_nag.cam-outfrq9s BASELINE cesm2_3_alpha12f: DIFF + FAIL SMS_Ld1_P136_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN time=245 + PEND SMS_Ld1_P144_D.T62_g17.C.izumi_nag.pop-144blocks_320x384_spacecurve RUN + FAIL SMS_Ld2_P72_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN time=62 + PEND SMS_Ld2_P80_D.T62_g37.C1850ECO.izumi_nag.pop-ecosys_81blocks_100x116_spacecurve RUN + FAIL SMS.T62_g37.C1850ECO.izumi_nag.pop-ecosys RUN time=76 + FAIL ERS.T62_s11.G.izumi_nag.pop-cice SUBMIT + + + +============================================================== +Tag name: cesm2_3_alpha014a +Originator(s): CSEG +Date: 26 May 2023 +One-line Summary: Misc. fixes. + +components/cam https://github.com/ESCOMP/CAM/cam6_3_111 -- +components/cice5 https://github.com/ESCOMP/CESM_CICE5/tree/cice5_20220204 -- +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_4_1_8 -- +cime https://github.com/ESMCI/cime/tree/cime6.0.125 ** +share https://github.com/ESCOMP/CESM_share/tree/share1.0.17 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm0.0.71 -- +components/cpl7 https://github.com/ESCOMP/CESM_CPL7andDataComps/cpl77.0.5 -- +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps0.14.34 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.14 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cism_2_1_95 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm.1.dev123 -- +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_230121 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_230504 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_0_48 -- +components/pop https://github.com/ESCOMP/POP2-CESM/cesm_pop_2_1_20230209 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_78 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3_221108 -- +components/ww3dev https://github.com/ESCOMP/WW3_interface/tree/dev/main_0.0.6 -- +libraries/mct https://github.com/MCSclimate/MCT/tree/MCT_2.11.0 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_5_10 -- + +cime + Chris Fischer 2023-05-23 - cime6.0.125 - cime (cesm2_3_alpha14a) + https://github.com/ESMCI/cime/tags/cime6.0.125 + + cime6.0.125: Fix cice5 namelist file creation. + cime6.0.124: Add flag for ignoring memleaks. + cime6.0.123: Fix create_test --single-exe option. + cime6.0.122: Fixes docs workflow. + + +cmeps + Chris Fischer 2023-05-26 - cmeps0.14.34 - src/drivers/nuopc/ (cesm2_3_alpha14a) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.34 + + cmeps0.14.34: Set ogrid as default + cmeps0.14.33: Add check for nans config option + + + Chris Fischer 2023-05-23 - cmeps0.14.32 - src/drivers/nuopc/ (cesm2_3_alpha14a) + https://github.com/ESCOMP/CMEPS/tags/cmeps0.14.32 + + cmeps0.14.32: Make xgrid default and fix sw flux to mom ocn. + Answer changes for sw flux to MOM ocn. + cmeps0.14.31: Add new optional mapping of taux and tauy from ocean to wave + ============================================================== Tag name: cesm2_3_beta13 Originator(s): CSEG From a97a1086f59286426c85ddeeced8447ed468c4a4 Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Wed, 7 Jun 2023 11:41:56 -0600 Subject: [PATCH 54/64] Update for cesm2_3_alpha15a --- Externals.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 24285f8a8..6f67543d8 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -87,7 +87,7 @@ externals = Externals_CISM.cfg required = True [clm] -tag = ctsm5.1.dev123 +tag = ctsm5.1.dev128 protocol = git repo_url = https://github.com/ESCOMP/CTSM local_path = components/clm @@ -95,7 +95,7 @@ externals = Externals_CLM.cfg required = True [fms] -tag = fi_230121 +tag = fi_230601 protocol = git repo_url = https://github.com/ESCOMP/FMS_interface local_path = libraries/FMS @@ -103,7 +103,7 @@ externals = Externals_FMS.cfg required = False [mom] -tag = mi_230504 +tag = mi_230601 protocol = git repo_url = https://github.com/ESCOMP/MOM_interface local_path = components/mom @@ -118,7 +118,7 @@ local_path = components/mosart required = True [pop] -tag = cesm_pop_2_1_20230209 +tag = cesm_pop_2_1_20230511 protocol = git repo_url = https://github.com/ESCOMP/POP2-CESM local_path = components/pop From 6db482e93ea8e4a4aa0c63814c91d5f48c18b88c Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 22 Jun 2023 16:08:42 -0600 Subject: [PATCH 55/64] add derecho pelayouts remove retired systems --- cime_config/config_pes.xml | 795 ++++++------------------------------- 1 file changed, 125 insertions(+), 670 deletions(-) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index bb32b0401..81eb878dc 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -40,36 +40,85 @@ - - - - use 2 hyperthreads per core + + + + 20 ypd/ 7100 pe-hrs/simyr expected + + 5400 + 939 + 64 + 4397 + 488 + 64 + 64 + 5400 + - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + 0 + 0 + 0 + 1003 + 5400 + 0 + 0 + 0 + - - - - - - none + + 13.9 ypd/ 5300 pe-hrs/simyr expected - -4 - -2 - -2 - -2 - -2 - -1 - -1 - -4 + 2700 + 896 + 32 + 1772 + 372 + 32 + 32 + 2700 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 896 + 928 + 2700 + 896 + 896 + 0 + + + + 10 ypd/ 5100 pe-hrs/simyr expected + + 1920 + 640 + 64 + 1216 + 256 + 64 + 64 + 1920 1 @@ -84,9 +133,42 @@ 0 0 + 640 + 704 + 1920 + 640 + 640 + 0 + + + + 2.9 ypd/ 4185 pe-hrs/simyr expected + + 448 + 208 + 16 + 224 + 64 + 16 + 16 + 448 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 16 0 - -2 - -4 + 224 + 448 0 0 0 @@ -94,8 +176,9 @@ + - + none @@ -109,14 +192,14 @@ -4 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 0 @@ -201,43 +284,6 @@ - - - - none - - -16 - -9 - -9 - -7 - -1 - -1 - -1 - -16 - - - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - - - 0 - 0 - 0 - -9 - -16 - 0 - 0 - 0 - - - - @@ -351,7 +397,7 @@ - + none 1800 @@ -386,80 +432,6 @@ - - - - none - - 2600 - 144 - 144 - 2456 - 192 - 1 - 160 - 2600 - - - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - - - 0 - 0 - 144 - 0 - 2600 - 0 - 0 - 0 - - - - - - - - none - - 960 - 48 - 48 - 912 - 48 - 1 - 160 - 960 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 48 - 960 - 0 - 0 - 0 - - - - @@ -497,43 +469,6 @@ - - - - none - - 12384 - 96 - 96 - 12288 - 4000 - 1 - 160 - 12384 - - - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - - - 0 - 0 - 0 - 96 - 12384 - 0 - 0 - 0 - - - - @@ -571,43 +506,6 @@ - - - - none - - 9600 - 8640 - 600 - 960 - 960 - 960 - 160 - 960 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 8640 - 9600 - 0 - 0 - 0 - - - - @@ -643,43 +541,6 @@ - - - - none - - -64 - -16 - -16 - -48 - -64 - -1 - -1 - -16 - - - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - - - 0 - 0 - 0 - -16 - 0 - 0 - 0 - -64 - - - - @@ -900,120 +761,9 @@ - - - - none - - 96 - 96 - 96 - 96 - 96 - 96 - 96 - 96 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 640 - 96 - 96 - 544 - 96 - 180 - 160 - 640 - - - 2 - 1 - 1 - 2 - 1 - 2 - 2 - 2 - - - 0 - 544 - 544 - 0 - 640 - 0 - 0 - 0 - - - - - - - - none - - 96 - 96 - 96 - 96 - 96 - 96 - 96 - 96 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - + + + none -8 @@ -1048,43 +798,6 @@ - - - - none - - -8 - -4 - -4 - -4 - -1 - -8 - -8 - -8 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - -4 - -8 - 0 - 0 - 0 - - - - @@ -1398,153 +1111,6 @@ - - - about 17ypd expected - - -50 - -30 - -30 - -18 - -20 - -30 - -2 - -50 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - -30 - -50 - 0 - -48 - 0 - - - - - - - - none - - -16 - -9 - -9 - -7 - -1 - -1 - -1 - -16 - - - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - - - 0 - 0 - 0 - -9 - -16 - 0 - 0 - 0 - - - - - - - - - none - - -208 - -32 - -32 - -80 - -48 - -1 - -1 - -208 - - - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - - - 0 - 0 - -32 - -64 - -208 - 0 - 0 - 0 - - - - - - - - none - - 960 - 48 - 48 - 912 - 48 - 1 - 160 - 960 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 48 - 960 - 0 - 0 - 0 - - - @@ -1583,81 +1149,6 @@ - - - - none - - -8 - -6 - -6 - -2 - -2 - 1 - -4 - -8 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - -6 - -8 - 0 - 0 - 0 - - - - - - - - - none - - 192 - 192 - 192 - 192 - 192 - 192 - 160 - 192 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - @@ -1695,43 +1186,7 @@ - - - - none - - 3200 - 1600 - 1600 - 1600 - 32 - 3200 - 160 - 3200 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - 1600 - 3200 - 0 - 0 - 0 - - - - +