diff --git a/.buildlibrary b/.buildlibrary index beaf099..54b2984 100644 --- a/.buildlibrary +++ b/.buildlibrary @@ -1,4 +1,4 @@ -ValidationKey: '63359856' +ValidationKey: '634347506094' AcceptedWarnings: - 'Warning: package ''.*'' was built under R version' - 'Warning: namespace ''.*'' is not available and has been replaced' diff --git a/CITATION.cff b/CITATION.cff index 3b35a61..a254e73 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,8 +2,8 @@ cff-version: 1.2.0 message: If you use this software, please cite it using the metadata from this file. type: software title: 'madrat: May All Data be Reproducible and Transparent (MADRaT) *' -version: 3.15.6 -date-released: '2024-12-19' +version: 3.15.6.9001 +date-released: '2025-01-06' abstract: Provides a framework which should improve reproducibility and transparency in data processing. It provides functionality such as automatic meta data creation and management, rudimentary quality management, data caching, work-flow management diff --git a/DESCRIPTION b/DESCRIPTION index 4bdc72f..80525b0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Type: Package Package: madrat Title: May All Data be Reproducible and Transparent (MADRaT) * -Version: 3.15.6 -Date: 2024-12-19 +Version: 3.15.6.9001 +Date: 2025-01-06 Authors@R: c( person("Jan Philipp", "Dietrich", , "dietrich@pik-potsdam.de", role = c("aut", "cre"), comment = c(affiliation = "Potsdam Institute for Climate Impact Research", ORCID = "0000-0002-4309-6431")), diff --git a/R/compareData.R b/R/compareData.R index 1a65a74..e47971a 100644 --- a/R/compareData.R +++ b/R/compareData.R @@ -14,7 +14,8 @@ #' @importFrom withr local_tempdir #' @export -compareData <- function(x, y, tolerance = 10^-5, yearLim = NULL) { +compareData <- function(x, y, tolerance = 10^-5, # nolint: cyclocomp_linter + yearLim = NULL) { tDir <- local_tempdir() .getDir <- function(tDir, file, name) { @@ -51,45 +52,65 @@ compareData <- function(x, y, tolerance = 10^-5, yearLim = NULL) { return(equal) } + .rmag <- function(f, yearLim) { + x <- try(read.magpie(f), silent = TRUE) + if (!is.magpie(x)) { + return(NULL) + } else { + if (!is.null(yearLim)) x <- x[, getYears(x, as.integer = TRUE) <= yearLim, ] + } + attr(x, "comment") <- NULL + return(x) + } + + .hashFile <- function(f) { + x <- system(paste("test -f", f, "&& grep -v '^\\*'", f, "| md5sum"), + intern = TRUE) + + if (is.null(attr(x, "status"))) x else NULL + } + i <- 1 for (f in out$files$inBoth) { counter <- format(paste0("(", i, "/", length(out$files$inBoth), ") "), width = 10) message(counter, format(f, width = maxchar), " ... ", appendLF = FALSE) i <- i + 1 - x <- .rmag(file.path(xDir, f), yearLim) - y <- .rmag(file.path(yDir, f), yearLim) - if (is.null(x) && is.null(y)) { - message("skipped") - out$skip <- out$skip + 1 + + xFile <- file.path(xDir, f) + yFile <- file.path(yDir, f) + + if (all(Sys.which(c("test", "grep", "md5sum")) != "") + && .hashFile(xFile) == .hashFile(yFile)) { + # checking hashes of all but the file header is much faster then checking + # all the data + message("OK") + out$ok <- out$ok + 1 } else { - if (!identical(dim(x), dim(y))) { - message("!= dim") - out$diff <- out$diff + 1 - } else if (!.dimEqual(x, y)) { - message("!= dimnames") - out$diff <- out$diff + 1 + x <- .rmag(xFile, yearLim) + y <- .rmag(yFile, yearLim) + if (is.null(x) && is.null(y)) { + message("skipped") + out$skip <- out$skip + 1 } else { - diff <- max(abs(x - y), na.rm = TRUE) - if (!identical(x, y) && diff > tolerance) { - message("!= values (max diff = ", round(diff, 8), ")") + if (!identical(dim(x), dim(y))) { + message("!= dim") + out$diff <- out$diff + 1 + } else if (!.dimEqual(x, y)) { + message("!= dimnames") out$diff <- out$diff + 1 } else { - message("OK") - out$ok <- out$ok + 1 + diff <- max(abs(x - y), na.rm = TRUE) + if (!identical(x, y) && diff > tolerance) { + message("!= values (max diff = ", round(diff, 8), ")") + out$diff <- out$diff + 1 + } else { + message("OK") + out$ok <- out$ok + 1 + } } } } } - message("[OK ", out$ok, " | DIFF ", out$diff, " | SKIP ", out$skip, " | MISS ", out$miss, "]") -} -.rmag <- function(f, yearLim) { - x <- try(read.magpie(f), silent = TRUE) - if (!is.magpie(x)) { - return(NULL) - } else { - if (!is.null(yearLim)) x <- x[, getYears(x, as.integer = TRUE) <= yearLim, ] - } - attr(x, "comment") <- NULL - return(x) + message("[OK ", out$ok, " | DIFF ", out$diff, " | SKIP ", out$skip, " | MISS ", out$miss, "]") } diff --git a/README.md b/README.md index 64b70a6..850652e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # May All Data be Reproducible and Transparent (MADRaT) * -R package **madrat**, version **3.15.6** +R package **madrat**, version **3.15.6.9001** [![CRAN status](https://www.r-pkg.org/badges/version/madrat)](https://cran.r-project.org/package=madrat) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1115490.svg)](https://doi.org/10.5281/zenodo.1115490) [![R build status](https://github.com/pik-piam/madrat/workflows/check/badge.svg)](https://github.com/pik-piam/madrat/actions) [![codecov](https://codecov.io/gh/pik-piam/madrat/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/madrat) [![r-universe](https://pik-piam.r-universe.dev/badges/madrat)](https://pik-piam.r-universe.dev/builds) @@ -55,7 +55,7 @@ In case of questions / problems please contact Jan Philipp Dietrich , Version: 3.15.6, . +Dietrich J, Baumstark L, Wirth S, Giannousakis A, Rodrigues R, Bodirsky B, Leip D, Kreidenweis U, Klein D, Sauer P (2025). "madrat: May All Data be Reproducible and Transparent (MADRaT) *." doi:10.5281/zenodo.1115490 , Version: 3.15.6.9001, . A BibTeX entry for LaTeX users is @@ -64,9 +64,9 @@ A BibTeX entry for LaTeX users is title = {madrat: May All Data be Reproducible and Transparent (MADRaT) *}, author = {Jan Philipp Dietrich and Lavinia Baumstark and Stephen Wirth and Anastasis Giannousakis and Renato Rodrigues and Benjamin Leon Bodirsky and Debbora Leip and Ulrich Kreidenweis and David Klein and Pascal Sauer}, doi = {10.5281/zenodo.1115490}, - date = {2024-12-19}, - year = {2024}, + date = {2025-01-06}, + year = {2025}, url = {https://github.com/pik-piam/madrat}, - note = {Version: 3.15.6}, + note = {Version: 3.15.6.9001}, } ```