Merge branch 'master' into RichardMN-test-json-2

RichardMN · Feb 5, 2022 · 2c6c101 · 2c6c101
2 parents f9672ac + f54b220
commit 2c6c101
Show file tree

Hide file tree

Showing 23 changed files with 235 additions and 114 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -2,12 +2,7 @@
 // https://github.com/microsoft/vscode-dev-containers/tree/v0.183.0/containers/r
 {
 	"name": "covidregionaldata",
-	"build": {
-		"dockerfile": "Dockerfile",
-		// Update VARIANT to pick a specific R version: latest, ... ,4.0.1 , 4.0.0
-		"args": { "VARIANT": "latest" },
-		"context": ".."
-	},
+	"image": "ghcr.io/epiforecasts/covidregionaldata:master",
 
 	// Set *default* container specific settings.json values on container create.
 	"settings": {
@@ -26,7 +21,8 @@
 		"shan.code-settings-sync",
 		"searking.preview-vscode",
 		"tht13.html-preview-vscode",
-		"formulahendry.code-runner"
+		"formulahendry.code-runner",
+		"github.vscode-pull-request-github"
 	],
 
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.

diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml
@@ -0,0 +1,50 @@
+
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches: ['release', 'master']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+
+      - name: Cancel previous builds if present
+        uses: n1hility/cancel-previous-runs@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v2
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./.devcontainer/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.github/workflows/check-full.yaml b/.github/workflows/check-full.yaml
@@ -270,27 +270,3 @@ jobs:
           git config --local user.name "GitHub Actions"
           Rscript -e "pkgdown::deploy_to_branch(new_process = FALSE, clean = TRUE)"
         shell: bash
-
-  push_to_registry:
-    if: github.ref == 'refs/heads/master' && github.repository == 'epiforecasts/covidregionaldata'
-    name: Push Docker image to GitHub Packages
-    runs-on: ubuntu-20.04
-
-    needs: [R-CMD-check, pkgdown, test-coverage]
-
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v2
-      - name: Login to GitHub Packages
-        uses: docker/login-action@v1
-        with:
-          registry: docker.pkg.github.com
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Push to GitHub Packages
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          file: .devcontainer/Dockerfile
-          push: true
-          tags: docker.pkg.github.com/epiforecasts/covidregionaldata/covidregionaldata:latest
diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml
@@ -0,0 +1,53 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/master/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, master]
+    tags: ['*']
+
+name: pkgdown
+
+jobs:
+  pkgdown:
+    runs-on: macOS-latest
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: n1hility/cancel-previous-runs@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/checkout@v2
+      - uses: r-lib/actions/setup-pandoc@v1
+      - uses: r-lib/actions/setup-r@v1
+
+      - name: Query dependencies
+        run: |
+          install.packages('remotes')
+          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
+          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
+        shell: Rscript {0}
+
+      - name: Restore R package cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.R_LIBS_USER }}
+          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
+
+      - name: Install dependencies
+        run: |
+          install.packages(c("remotes", "curl"))
+          remotes::install_deps(dependencies = TRUE)
+          remotes::install_github("r-lib/pkgdown")
+          remotes::install_github("amirmasoudabdol/preferably", type = "source")
+        shell: Rscript {0}
+
+      - name: Install package
+        run: R CMD INSTALL .
+
+      - name: Deploy package
+        run: |
+          git config --local user.name "$GITHUB_ACTOR"
+          git config --local user.email "[email protected]"
+          Rscript -e 'pkgdown::deploy_to_branch(new_process = FALSE)'
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 # R specific hooks: https://github.com/lorenzwalthert/precommit
 repos:
 -   repo: https://github.com/lorenzwalthert/precommit
-    rev: v0.1.3.9014
+    rev: v0.2.2.9009
     hooks: 
     -   id: style-files
         args: [--style_pkg=styler, --style_fun=tidyverse_style]    
@@ -45,7 +45,7 @@ repos:
     -   id: no-browser-statement
     -   id: deps-in-desc
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.1.0
     hooks: 
     -   id: check-added-large-files
         args: ['--maxkb=800']

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: covidregionaldata
 Title: Subnational Data for COVID-19 Epidemiology
-Version: 0.9.2.3000
+Version: 0.9.2.4000
 Authors@R:
     c(person(given = "Joseph",
              family = "Palmer",
@@ -25,6 +25,10 @@ Authors@R:
              role = "aut",
              email = "[email protected]",
              comment = c(ORCID = "0000-0003-4413-453X")),
+      person(given = "Hugo",
+             family = "Gruson",
+             role = "aut",
+             comment = c(ORCID = "0000-0002-4094-1476")),
       person(given = "Sophie",
              family = "Meakin",
              role = "ctb",
@@ -51,10 +55,6 @@ Authors@R:
              family = "Boyes",
              role = "ctb",
              comment = c(URL = "https://github.com/rboyes")),
-      person(given = "Hugo",
-             family = "Gruson",
-             role = "ctb",
-             comment = c(ORCID = "0000-0002-4094-1476")),
       person(given = "Vang",
              family = "Le",
              role = "ctb",
@@ -98,6 +98,7 @@ Imports:
     R6,
     readxl,
     rlang,
+    stringi,
     stringr,
     tidyr (>= 1.0.0),
     vroom,
@@ -107,7 +108,6 @@ Suggests:
     ggspatial,
     knitr,
     mockery,
-    pkgdown,
     rmarkdown,
     rvest,
     rworldmap,
@@ -118,6 +118,9 @@ Suggests:
 VignetteBuilder:
     knitr
 Config/testthat/edition: 3
+Config/Needs/website: 
+    r-lib/pkgdown,
+    amirmasoudabdol/preferably
 Encoding: UTF-8
 Language: en-gb
 LazyData: true

diff --git a/NAMESPACE b/NAMESPACE
@@ -62,6 +62,7 @@ importFrom(dplyr,filter)
 importFrom(dplyr,full_join)
 importFrom(dplyr,group_by)
 importFrom(dplyr,group_by_at)
+importFrom(dplyr,group_vars)
 importFrom(dplyr,if_else)
 importFrom(dplyr,lag)
 importFrom(dplyr,last_col)
@@ -118,12 +119,12 @@ importFrom(rlang,"!!")
 importFrom(rlang,":=")
 importFrom(rlang,.data)
 importFrom(rlang,syms)
-importFrom(stringr,str_conv)
+importFrom(stringi,stri_trans_general)
+importFrom(stringi,stri_trim_both)
 importFrom(stringr,str_detect)
 importFrom(stringr,str_replace_all)
 importFrom(stringr,str_to_sentence)
 importFrom(stringr,str_to_title)
-importFrom(stringr,str_trim)
 importFrom(tidyr,complete)
 importFrom(tidyr,drop_na)
 importFrom(tidyr,fill)

diff --git a/NEWS.md b/NEWS.md
@@ -16,6 +16,7 @@ This release is currently under development
 
 ## Bug fixes
 
+- Fixed a bug in `fill_empty_dates_with_na()` caused by changes made in version `1.2.0` of `tidyr`. 
 - Fixed a bug in the data sourced from Germany so that instead of treating it as a line list of individuals it is treated as a relatively finely resolved count data which needs to be summed up (by @sbfnk).
 
 # covidregionaldata 0.9.2

diff --git a/R/UK.R b/R/UK.R
@@ -450,7 +450,8 @@ UK <- R6::R6Class("UK",
     #'     Section 2, "2. Estimated new hospital cases"
     #' @importFrom lubridate year month
     #' @importFrom readxl read_excel cell_limits
-    #' @importFrom dplyr mutate select %>% group_by summarise left_join as_tibble
+    #' @importFrom dplyr mutate select %>% group_by summarise left_join
+    #' @importFrom dplyr as_tibble
     #' @importFrom tidyr pivot_longer
     #' @param clean_data Cleaned UK covid-19 data
     #' @param nhs_data NHS region data

diff --git a/R/Vietnam.R b/R/Vietnam.R
@@ -64,7 +64,8 @@ Vietnam <- R6::R6Class("Vietnam",
     #' @importFrom dplyr filter select mutate rename tibble as_tibble full_join
     #' @importFrom tidyr replace_na drop_na separate
     #' @importFrom purrr map
-    #' @importFrom stringr str_conv str_trim str_to_title str_replace_all
+    #' @importFrom stringr str_to_title str_replace_all
+    #' @importFrom stringi stri_trans_general stri_trim_both
     #' @importFrom lubridate dmy
     clean_common = function() {
       # The first three elements of self$data$raw are the data
@@ -77,31 +78,33 @@ Vietnam <- R6::R6Class("Vietnam",
           function(x) as_tibble(unlist(x),
                                 rownames = "date")),
         function(y) {
-          y %>% separate(date, sep = "[.]+", into = c(NA, "province", "date"))
+          separate(y, date, sep = "[.]+", into = c(NA, "province", "date"))
         }
       )
-      self$data$clean <- full_join(
-        full_join(
-          flat_all$case_by_time, flat_all$death_by_time,
-          by = c("province", "date"),
-          suffix = c(".cases", ".deaths"),
-          copy = TRUE
-        ),
-        flat_all$recovered_by_time,
-        by = c("province", "date"),
-        suffix = c("", ".recovered"),
-        copy = TRUE
-      ) %>%
+      index_cols <- bind_rows(
+        select(flat_all$case_by_time, "date", "province"),
+        select(flat_all$death_by_time, "date", "province"),
+        select(flat_all$recovered_by_time, "date", "province")) %>%
+        unique()
+
+      self$data$clean <- index_cols %>%
+        left_join(rename(flat_all$case_by_time, cases_total = value),
+                  by = c("province", "date") ) %>%
+        left_join(rename(flat_all$death_by_time, deaths_total = value),
+                  by = c("province", "date") ) %>%
+        left_join(rename(flat_all$recovered_by_time, recovered_total = value),
+                  by = c("province", "date") ) %>%
         # The api uses integer codes for provinces which do not
         # line up with ISO 3166-2 (some of which are not numbers)
         # so we use this as a temporary code to line names up
         # with data.
         select(
           ncsc_region_code = province,
           date,
-          cases_total = value.cases,
-          deaths_total = value.deaths,
-          recovered_total = value) %>%
+          cases_total,
+          deaths_total,
+          recovered_total
+          ) %>%
         mutate(ncsc_region_code = as.numeric(ncsc_region_code)) %>%
         left_join(
           self$data$raw$provinces %>%
@@ -119,8 +122,8 @@ Vietnam <- R6::R6Class("Vietnam",
         #
         #tidyr::drop_na(date, region_name) %>%
         mutate(
-          level_1_region = str_conv(level_1_region, "ASCII"),
-          level_1_region = str_trim(level_1_region, side = "both"),
+          level_1_region = stri_trans_general(level_1_region, "ASCII"),
+          level_1_region = stri_trim_both(level_1_region),
           level_1_region = str_replace_all(level_1_region,
                                            "\\(.*\\)|-| ", ""),
           level_1_region = str_to_title(level_1_region),

diff --git a/R/processing.R b/R/processing.R
@@ -51,16 +51,21 @@ set_negative_values_to_zero <- function(data) {
 #' @family compulsory_processing
 #' @concept compulsory_processing
 #' @importFrom tidyr complete full_seq nesting
-#' @importFrom dplyr starts_with
+#' @importFrom dplyr starts_with group_by ungroup group_vars
 #' @importFrom rlang !!! syms
 fill_empty_dates_with_na <- function(data) {
   regions <- select(data, starts_with("level_")) %>%
     names()
+
+  groups <- group_vars(data)
+
   data <- data %>%
+    ungroup() %>%
     complete(
       date = full_seq(data$date, period = 1),
       nesting(!!!syms(regions))
-    )
+    ) %>%
+    group_by(across(.cols = all_of(groups)))
   return(data)
 }
 
@@ -219,8 +224,8 @@ run_optional_processing_fns <- function(data, process_fns) {
 #' processing steps
 #' @concept utility
 #' @family processing
-#' @importFrom dplyr do group_by_at across ungroup select everything arrange all_of
-#' @importFrom dplyr rename
+#' @importFrom dplyr do group_by_at across ungroup select everything arrange
+#' @importFrom dplyr rename all_of
 #' @importFrom tidyr drop_na
 #' @importFrom rlang !!!
 process_internal <- function(clean_data, level, group_vars,
@@ -258,9 +263,9 @@ process_internal <- function(clean_data, level, group_vars,
           "hosp_new", "hosp_total", "tested_new", "tested_total"
         )),
         everything()
-      ) %>%
-      arrange(.data$date, all_of(group_vars_standard[1]))
+      )
   }
+
   dat <- ungroup(dat)
 
   if (localise) {

diff --git a/R/shared-methods.R b/R/shared-methods.R
@@ -640,7 +640,8 @@ CountryDataClass <- R6::R6Class("CountryDataClass",
         if (!is.null(self$target_regions)) {
           self$target_regions <- countryname(
             self$target_regions,
-            destination = "country.name.en"
+            destination = "country.name.en",
+            warn = FALSE
           )
           if (all(is.na(self$target_regions))) {
             stop("No countries found with target names")