From af01d84ff1734bf429fa5304f278fc38b0729f95 Mon Sep 17 00:00:00 2001 From: mpadge Date: Wed, 15 Jan 2025 17:41:57 +0100 Subject: [PATCH 01/10] use curl and not httr2 to download tarballs [ci skip] https://github.com/r-lib/httr2/issues/487 --- DESCRIPTION | 2 +- R/data-update-cran.R | 24 +++++++++++++++--------- codemeta.json | 2 +- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index df0462f..38d49db 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.025 +Version: 0.4.2.026 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/R/data-update-cran.R b/R/data-update-cran.R index fa4ccbf..0b869ed 100644 --- a/R/data-update-cran.R +++ b/R/data-update-cran.R @@ -79,21 +79,27 @@ dl_one_tarball <- function (results_path, tarball) { return (path) } - req <- httr2::request (url) |> - httr2::req_headers ("Accept" = "application/octet-stream") - resp <- tryCatch ( - httr2::req_perform (req), + # httr2 download keeps on failing because of + # https://github.com/r-lib/httr2/issues/487 + # req <- httr2::request (url) |> + # httr2::req_headers ("Accept" = "application/octet-stream") + # resp <- tryCatch ( + # httr2::req_perform (req), + # error = function (e) NULL + # ) + path <- tryCatch ( + curl::curl_download (url, destfile = path), error = function (e) NULL ) - if (is.null (resp)) { - return (NULL) - } - if (httr2::resp_is_error (resp)) { + if (is.null (path)) { return (NULL) } + # if (httr2::resp_is_error (resp)) { + # return (NULL) + # } - writeBin (httr2::resp_body_raw (resp), path) + # writeBin (httr2::resp_body_raw (resp), path) return (path) } diff --git a/codemeta.json b/codemeta.json index 7580752..865251d 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.025", + "version": "0.4.2.026", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", From fd33323e1b24f3706e276c7e55244f223fdb0d62 Mon Sep 17 00:00:00 2001 From: mpadge Date: Wed, 15 Jan 2025 17:54:32 +0100 Subject: [PATCH 02/10] all 'closeAllConnections' call to cran update loop for #92 [ci skip] --- DESCRIPTION | 2 +- R/data-update-cran.R | 32 +++++++++++++++++--------------- codemeta.json | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 38d49db..b9dffba 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.026 +Version: 0.4.2.027 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/R/data-update-cran.R b/R/data-update-cran.R index 0b869ed..57f52a7 100644 --- a/R/data-update-cran.R +++ b/R/data-update-cran.R @@ -47,6 +47,10 @@ pkgmatch_update_cran <- function () { fs::file_delete (tarball_path) } + # This is necessary because `utils::untar()` can create hanging + # connections: + closeAllConnections () + pkgmatch_update_progress_message (p, 1, npkgs, pt0) return (dat) @@ -79,27 +83,25 @@ dl_one_tarball <- function (results_path, tarball) { return (path) } - # httr2 download keeps on failing because of - # https://github.com/r-lib/httr2/issues/487 - # req <- httr2::request (url) |> - # httr2::req_headers ("Accept" = "application/octet-stream") - # resp <- tryCatch ( - # httr2::req_perform (req), - # error = function (e) NULL - # ) - path <- tryCatch ( - curl::curl_download (url, destfile = path), + req <- httr2::request (url) |> + httr2::req_headers ("Accept" = "application/octet-stream") + resp <- tryCatch ( + httr2::req_perform (req), error = function (e) NULL ) + # path <- tryCatch ( + # curl::curl_download (url, destfile = path), + # error = function (e) NULL + # ) - if (is.null (path)) { + if (is.null (resp)) { + return (NULL) + } + if (httr2::resp_is_error (resp)) { return (NULL) } - # if (httr2::resp_is_error (resp)) { - # return (NULL) - # } - # writeBin (httr2::resp_body_raw (resp), path) + writeBin (httr2::resp_body_raw (resp), path) return (path) } diff --git a/codemeta.json b/codemeta.json index 865251d..e8190d9 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.026", + "version": "0.4.2.027", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", From 3117d01d6d402b6f0fbe0618b981835fb0bacce4 Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 10:15:59 +0100 Subject: [PATCH 03/10] run update workflow on Mon and Thurs --- .github/workflows/update.yaml | 2 +- DESCRIPTION | 2 +- codemeta.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update.yaml b/.github/workflows/update.yaml index 2fe4704..614538a 100644 --- a/.github/workflows/update.yaml +++ b/.github/workflows/update.yaml @@ -3,7 +3,7 @@ on: branches: - main schedule: - - cron: 0 0 * * 1 + - cron: 0 0 * * 1,4 name: Update pkgmatch data diff --git a/DESCRIPTION b/DESCRIPTION index b9dffba..fe9b410 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.027 +Version: 0.4.2.028 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/codemeta.json b/codemeta.json index e8190d9..e1db7c5 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.027", + "version": "0.4.2.028", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", From afa5f66d23582558dc3e4c03f0a585adef0b6ca4 Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 11:30:00 +0100 Subject: [PATCH 04/10] update ropensci fn embeddings for #92 --- DESCRIPTION | 2 +- R/data-update-ropensci.R | 24 +++++++++++++++++++++++- codemeta.json | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index fe9b410..6bc1c86 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.028 +Version: 0.4.2.029 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/R/data-update-ropensci.R b/R/data-update-ropensci.R index 4af35ea..725d794 100644 --- a/R/data-update-ropensci.R +++ b/R/data-update-ropensci.R @@ -41,18 +41,26 @@ pkgmatch_update_ropensci <- function () { extract_data_from_local_dir (pkg_dir), error = function (e) NULL ) + fns <- tryCatch ( + pkgmatch_embeddings_from_pkgs (pkg_dir, functions_only = TRUE), + error = function (e) NULL + ) fs::dir_delete (pkg_dir) pkgmatch_update_progress_message (i, 1, nrow (reg_updated), pt0) - return (dat) + return (list (dat = dat, fns = fns)) }) + fns <- do.call (cbind, lapply (res, function (i) i$fns)) + res <- lapply (res, function (i) i$dat) names (res) <- reg_updated$name append_data_to_embeddings (res, flist, cran = FALSE) append_data_to_bm25 (res, flist, cran = FALSE) append_data_to_fn_calls (res, flist, cran = FALSE) + append_data_to_fn_embeddings (fns, flist) + options ("rlib_message_verbosity" = op) return (TRUE) @@ -94,3 +102,17 @@ days_in_this_month <- function (today = Sys.Date ()) { dates <- seq (this_month, next_month, by = "day") return (length (dates) - 1L) } + +append_data_to_fn_embeddings <- function (fns, flist) { + + pkgs <- unique (gsub ("::.*$", "", colnames (fns))) + f <- grep ("embeddings\\-fns\\.", flist, value = TRUE) + embeddings_fns <- readRDS (f) + ptn <- paste0 (paste0 (pkgs, "::"), collapse = "|") + index <- which (!grepl (ptn, colnames (embeddings_fns))) + embeddings_fns <- cbind (embeddings_fns [, index], fns) + + index <- order (colnames (embeddings_fns)) + embeddings_fns <- embeddings_fns [, index] + saveRDS (f, embeddings_fns) +} diff --git a/codemeta.json b/codemeta.json index e1db7c5..731b1a5 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.028", + "version": "0.4.2.029", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", From b22e8b89a5b21c83a571d550daf6eb141e071b71 Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 11:30:44 +0100 Subject: [PATCH 05/10] update workflow to daily cron runs --- .github/workflows/update.yaml | 2 +- DESCRIPTION | 2 +- codemeta.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update.yaml b/.github/workflows/update.yaml index 614538a..73f3c77 100644 --- a/.github/workflows/update.yaml +++ b/.github/workflows/update.yaml @@ -3,7 +3,7 @@ on: branches: - main schedule: - - cron: 0 0 * * 1,4 + - cron: 1 0 * * * name: Update pkgmatch data diff --git a/DESCRIPTION b/DESCRIPTION index 6bc1c86..5de9679 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.029 +Version: 0.4.2.030 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/codemeta.json b/codemeta.json index 731b1a5..5034eb2 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.029", + "version": "0.4.2.030", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", From 4f2ada1dbfe931a3d9d538061532d2850547ad72 Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 11:46:52 +0100 Subject: [PATCH 06/10] fix data save in append_data_to_fn_embeddings for #92 --- R/data-update-ropensci.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/data-update-ropensci.R b/R/data-update-ropensci.R index 725d794..247eb10 100644 --- a/R/data-update-ropensci.R +++ b/R/data-update-ropensci.R @@ -114,5 +114,5 @@ append_data_to_fn_embeddings <- function (fns, flist) { index <- order (colnames (embeddings_fns)) embeddings_fns <- embeddings_fns [, index] - saveRDS (f, embeddings_fns) + saveRDS (embeddings_fns, f) } From 3e4501b052b574fb15eef99fe1dc399f121a8849 Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 12:02:35 +0100 Subject: [PATCH 07/10] fix 'opt_is_quiet' reset in similar_fns fn --- DESCRIPTION | 2 +- R/similar-fns.R | 4 ++-- codemeta.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 5de9679..a582db6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.030 +Version: 0.4.2.031 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/R/similar-fns.R b/R/similar-fns.R index 3cc0f11..6b006c2 100644 --- a/R/similar-fns.R +++ b/R/similar-fns.R @@ -34,10 +34,10 @@ pkgmatch_similar_fns <- function (input, embeddings = NULL, n = 5L, browse = FAL stopifnot (!is.null (nms)) stopifnot (all (grepl ("\\:\\:", nms))) - op <- options () + op <- getOption ("rlib_message_verbosity") options (rlib_message_verbosity = "quiet") emb <- get_embeddings (input) - options (op) + options (rlib_message_verbosity = op) res <- cosine_similarity (emb [, 1], embeddings) res$rank <- seq_len (nrow (res)) diff --git a/codemeta.json b/codemeta.json index 5034eb2..5d68e5a 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.030", + "version": "0.4.2.031", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", From c406fa1a96945dad30fd9603071341d146b19d91 Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 12:15:46 +0100 Subject: [PATCH 08/10] fix another verbosity control reset option --- DESCRIPTION | 2 +- R/similar-pkgs.R | 4 ++-- codemeta.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index a582db6..56873ca 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.031 +Version: 0.4.2.032 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/R/similar-pkgs.R b/R/similar-pkgs.R index 4705934..b6469e2 100644 --- a/R/similar-pkgs.R +++ b/R/similar-pkgs.R @@ -186,12 +186,12 @@ similar_pkgs_from_pkg <- function (input, embeddings) { similar_pkgs_from_pkg_internal <- function (input, embeddings) { - op <- options () + op <- getOption ("rlib_message_verbosity") options (rlib_message_verbosity = "quiet") emb <- pkgmatch_embeddings_from_pkgs (input) - options (op) + options ("rlib_message_verbosity" = op) d_text <- lapply ( c ("text_with_fns", "text_wo_fns"), diff --git a/codemeta.json b/codemeta.json index 5d68e5a..a6279b7 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.031", + "version": "0.4.2.032", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", From 2d8bdf3ba004fbb87b4f27bf6cf00384dcb008bf Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 12:19:07 +0100 Subject: [PATCH 09/10] version bump -> 0.4.3 --- DESCRIPTION | 2 +- codemeta.json | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 56873ca..d950378 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.2.032 +Version: 0.4.3 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/codemeta.json b/codemeta.json index a6279b7..c33b8d4 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.2.032", + "version": "0.4.3", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", @@ -405,13 +405,10 @@ "name": "R", "version": ">= 3.5.0" }, - "SystemRequirements": {} + "SystemRequirements": null }, - "fileSize": "492.3KB", + "fileSize": "2839.819KB", "readme": "https://github.com/ropensci-review-tools/pkgmatch/blob/main/README.md", - "contIntegration": [ - "https://github.com/ropensci-review-tools/pkgmatch/actions?query=workflow%3AR-CMD-check", - "https://app.codecov.io/gh/ropensci-review-tools/pkgmatch" - ], + "contIntegration": ["https://github.com/ropensci-review-tools/pkgmatch/actions?query=workflow%3AR-CMD-check", "https://app.codecov.io/gh/ropensci-review-tools/pkgmatch"], "developmentStatus": "https://www.repostatus.org/#wip" } From 872ce02deeba509fa86a67098bc106c497b197a2 Mon Sep 17 00:00:00 2001 From: mpadge Date: Thu, 16 Jan 2025 12:24:17 +0100 Subject: [PATCH 10/10] only run update workflow on cron, not on push --- .github/workflows/update.yaml | 6 +++--- DESCRIPTION | 2 +- codemeta.json | 9 ++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/update.yaml b/.github/workflows/update.yaml index 73f3c77..c5d27b4 100644 --- a/.github/workflows/update.yaml +++ b/.github/workflows/update.yaml @@ -1,7 +1,7 @@ on: - push: - branches: - - main + # push: + # branches: + # - main schedule: - cron: 1 0 * * * diff --git a/DESCRIPTION b/DESCRIPTION index d950378..b9a305a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: pkgmatch Title: Find R Packages Matching Either Descriptions or Other R Packages -Version: 0.4.3 +Version: 0.4.3.001 Authors@R: c( person("Mark", "Padgham", , "mark.padgham@email.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0003-2172-5265")), diff --git a/codemeta.json b/codemeta.json index c33b8d4..93f3d2f 100644 --- a/codemeta.json +++ b/codemeta.json @@ -8,7 +8,7 @@ "codeRepository": "https://github.com/ropensci-review-tools/pkgmatch", "issueTracker": "https://github.com/ropensci-review-tools/pkgmatch/issues", "license": "https://spdx.org/licenses/MIT", - "version": "0.4.3", + "version": "0.4.3.001", "programmingLanguage": { "@type": "ComputerLanguage", "name": "R", @@ -405,10 +405,13 @@ "name": "R", "version": ">= 3.5.0" }, - "SystemRequirements": null + "SystemRequirements": {} }, "fileSize": "2839.819KB", "readme": "https://github.com/ropensci-review-tools/pkgmatch/blob/main/README.md", - "contIntegration": ["https://github.com/ropensci-review-tools/pkgmatch/actions?query=workflow%3AR-CMD-check", "https://app.codecov.io/gh/ropensci-review-tools/pkgmatch"], + "contIntegration": [ + "https://github.com/ropensci-review-tools/pkgmatch/actions?query=workflow%3AR-CMD-check", + "https://app.codecov.io/gh/ropensci-review-tools/pkgmatch" + ], "developmentStatus": "https://www.repostatus.org/#wip" }