Merge pull request #155 from alan-turing-institute/fix-quarto-github-…

…deploy

.dockerignore

@@ -13,3 +13,4 @@ python/tests/data/test-run-results*
 **/*.tic
 **/*.aux.xml
 .**/*
+!.conda-linux-64.lock

.github/workflows/ci.yaml

@@ -86,7 +86,8 @@ jobs:
     needs: [linter, pytest]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - name: Check out repository
+        uses: actions/checkout@v4
 
       - name: Download coverage svg
         uses: actions/download-artifact@v3
@@ -107,6 +108,23 @@ jobs:
         run: |
           mamba run -n ${{ env.CONDA_ENV_NAME }} quartodoc build
 
+      - name: Set up Quarto
+        uses: quarto-dev/quarto-actions/setup@v2
+
+      - name: Install Python and Dependencies
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+      - run: pip install jupyter
+
+      - name: Render and Publish
+        uses: quarto-dev/quarto-actions/publish@v2
+        with:
+          target: gh-pages
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   # The below raises a code 137 error on `GitHub` Actions
 
   # docker:

_quarto.yml

@@ -13,7 +13,7 @@ project:
     - "R/misc/Identifying_Runs.md"
     - "R/comparing-r-and-python/HADs-reprojection/WIP-Comparing-HADs-grids.md"
     - "docs/reference"
-    - "docs/pipeline.qmd"
+    # - "docs/pipeline.qmd"
     - "docs/contributing.md"
     - "docs/pipeline_guidance.qmd"
     - "docs/docker-configurations.qmd"
@@ -40,12 +40,14 @@ website:
         href: "README.md"
       - text: "Install"
         href: "setup-instructions.md"
-      - section: "Pipeline"
-        contents:
-          - text: "Guidance"
-            href: "docs/pipeline_guidance.qmd"
-          - text: "Diagram"
-            href: "docs/pipeline.qmd"
+      - text: "Pipeline"
+        href: "docs/pipeline_guidance.qmd"
+      # - section: "Pipeline"
+      #   contents:
+      #     - text: "Guidance"
+      #       href: "docs/pipeline_guidance.qmd"
+      #     - text: "Diagram"
+      #       href: "docs/pipeline.qmd"
       - section: "R"
         href: "R/README.md"
         contents:

compose/docs/Dockerfile

@@ -8,7 +8,7 @@ ARG EXTERNAL_PORT=$LOCAL_PORT
 # an ARG
 ENV HOST_IP=0.0.0.0
 ENV LOCAL_PORT=${LOCAL_PORT}
-ARG py_ver=3.11
+ARG py_ver=3.12
 ENV DEBIAN_FRONTEND=noninteractive
 
 COPY . /app

docs/pipeline.qmd

@@ -15,177 +15,7 @@ Ideally this gets resolved to simply import that file here like so:
 -->
 
 ```{mermaid}
-graph TB
-
-subgraph Legend
-    direction RL
-    data_external[(external data)]
-    data_fileshare[path to fileshare]
-    script_r([R script])
-    script_py([Python script])
-    script_bash([Bash script])
-    var[parameter]:::var
-end
-
-%%% INPUT DATA
-subgraph CEDA
-    data_hads[(HADS)]
-    data_cpm[(UKCP2.2)]
-    data_hads --> script_load
-    data_cpm --> script_load
-    data_hads --> script_load
-end
-
-subgraph Core pipeline
-    subgraph Data Ingress
-        %%% Loading data to disk
-        script_load([ceda_ftp_download.py])
-        data_hads_raw[RAW/HadsUKgrid/../*.nc]
-        data_cpm_raw[RAW/UKCP2.2/../*.nc]
-        script_load --> data_hads_raw
-        script_load --> data_cpm_raw
-    end
-    subgraph Preprocessing
-        %% resampling & reprojecting
-        script_resampling([resampling_hads.py])
-        script_reproject([reproject_all.sh])
-
-        data_hads_res[Processed/HadsUKgrid/../*.nc]
-        data_cpm_rep[Reprojected/UKCP2.2/../*.tiff]
-
-        script_resampling --> data_hads_res
-        script_reproject --> data_cpm_rep
-
-        %% cropping
-        script_crop_city([Cropping_Rasters_to_three_cities.R])
-
-        data_cropped_cpm[Cropped/cpm/..]
-        data_cropped_hads[Cropped/hads/..]
-        script_crop_city --> data_cropped_cpm
-        script_crop_city --> data_cropped_hads
-
-
-    end
-
-    subgraph Data Splitting
-        data_outdir[Cropped/preprocessed/..]
-
-        script_preproc([preprocess_data.py])
-
-        data_out_train[../simh..]
-        data_out_calibrate[../simp..]
-        data_out_groundtruth_h[../obsh..]
-        data_out_groundtruth_p[../obsp..]
-
-        script_preproc --> data_outdir
-
-        data_outdir --> data_out_train
-        data_outdir --> data_out_calibrate
-        data_outdir --> data_out_groundtruth_h
-        data_outdir --> data_out_groundtruth_p
-    end
-
-    subgraph bc[Bias Correction]
-        script_bc_py([run_cmethods.py])
-        script_bc_r([run_cmethods.R])
-        function_bc_r[[fitQmapQUANT.R]]
-
-
-        data_out_py[Debiased/...]
-        data_out_r[Debiased/R/QuantileMapping/resultsL*]
-
-        data_out_train --> script_bc_py
-        data_out_calibrate --> script_bc_py
-        data_out_groundtruth_h --> script_bc_py
-        data_out_train --> script_bc_r
-        data_out_calibrate --> script_bc_r
-        data_out_groundtruth_h --> script_bc_r
-        script_bc_r --> function_bc_r
-
-        script_bc_py-->data_out_py
-        function_bc_r-->data_out_r
-    end
-
-    subgraph Assessment
-        script_asses[tbc]
-        data_out_groundtruth_p --> script_asses
-    end
-    data_out_py --> script_asses
-    data_out_r --> script_asses
-end
-
-
-subgraph nner_py[Execute Python pipeline for MO dataset]
-    data_shape_uk[(shape London)]
-    data_shape_gl[(shape Glasgow)]
-    data_shape_ma[(shape Manchester)]
-
-
-    script_BC_wrapper[three_cities_debiasing.sh]
-    param1["metric (eg tasmax)"]:::var
-    param2["runs (eg 05)"]:::var
-    param3["BC method (eg quantile_mapping)"]:::var
-    param4[city]:::var
-
-    script_BC_wrapper --> param1
-    param1 --> param2
-    param2 --> param3
-    param3 --> param4
-    param4 -- for loop --> script_preproc
-
-    %% Looping connections
-    param4 -.-> param3
-    param3 -.-> param2
-    param2 -.-> param1
-end
-
-subgraph nner_jupyter[Jupyter Notebook for Guidance]
-    direction BT
-    data_shape_gl2[(shape Glasgow)]
-    data_cpm2[(UKCP2.2_Monthly)]
-
-    param5["tasmax"]:::var
-    param6["quantile_mapping"]:::var
-    param7[Glasgow]:::var
-
-    script_BC_wrapper --> param1
-    param5 --> script_preproc
-    param6 --> script_preproc
-    param7 --> script_preproc
-
-    data_cpm2 --> script_load
-    data_shape_gl2 --> script_crop_city
-end
-
-%% between block connections
-%% input preproc 1
-data_hads_raw --> script_resampling
-data_cpm_raw --> script_reproject
-%% input cropping
-data_cpm_rep --> script_crop_city
-
-data_hads_res --> script_crop_city
-data_shape_uk --> script_crop_city
-data_shape_ma --> script_crop_city
-data_shape_gl --> script_crop_city
-
-%% input preproc2
-data_cropped_cpm --> script_preproc
-data_cropped_hads --> script_preproc
-
-param4 -- for loop --> script_bc_py
-
-
-%% class styles
-classDef python fill:#4CAF50;
-classDef r fill:#FF5722;
-classDef bash fill:#f9f
-classDef var fill:none,stroke:#0f0;
-classDef dashed stroke-dasharray: 5 5;
-
-class script_crop_city,script_crop_uk,function_bc_r,script_r,script_df_uk,function_bc,function_crop_bc,fn_crop_cpm,fn_crop_hads,fn_bc,script_bc_r r;
-class script_load,script_resampling,script_preproc,script_bc_py,script_py python;
-class script_reproject,script_BC_wrapper,script_bash bash;
-class inner_py dashed;
-class inner_r dashed;
+%%| label: fig-outdated-workflow
+%%| fig-cap: "A diagram of the previous workflow."
+%%| file: assets/pipeline.mermaid
 ```