#!/bin/bash
# A simple script to backup an organization's GitHub repositories.
# Initially from https://gist.github.com/rodw/3073987 snapshot at
# https://gist.githubusercontent.com/rodw/3073987/raw/d5e9ab4785647e558df488eb18623aa6c52af86b/backup-github.sh
# Continued afterwards 2023+ by Jim Klimov <gmail.com>
#-------------------------------------------------------------------------------
# NOTES:
#-------------------------------------------------------------------------------
# * Under the heading "CONFIG" below you'll find a number of configuration
# parameters that must be personalized for your GitHub account and org.
# Replace the `<CHANGE-ME>` strings with the value described in the comments
# (or overwrite those values at run-time by providing environment variables).
#
# * Your terminal/screen session used for backups would benefit from having
# your SSH key preloaded into the ssh-agent, e.g.:
# eval `ssh-agent`
# ssh-add ~/.ssh/id_rsa
# or otherwise made available to the git client (no passphrase? oh no!)
#
# * If you have more than 100 repositories, the script should be able to
# step thru the list of repos returned by GitHub one page at a time,
# beware API limits (server-side throttling); maybe support for HTTP-304
# cache would be beneficial (also to avoid fetches that bring no news?)
#
# * If you want to back up the repos for a USER rather than an ORGANIZATION,
# or a user's gists (and their comments), see GHBU_ORGMODE setting below.
#
# * Thanks to @rodw for the original script, and to @Calrion, @vnaum,
# @BartHaagdorens and other commenters in original gist for various fixes
# and updates.
#
# * Also see those comments (and related revisions and forks) for more
# information and general troubleshooting.
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
# CONFIG:
#-------------------------------------------------------------------------------
GHBU_ORG=${GHBU_ORG-"<CHANGE-ME>"} # the GitHub organization whose repos will be backed up
# # (if you're backing up a USER's repos, this should be your GitHub username; also see the note below about the `REPOLIST` definition)
GHBU_UNAME=${GHBU_UNAME-"<CHANGE-ME>"} # the username of a GitHub account (to use with the GitHub API)
GHBU_PASSWD=${GHBU_PASSWD-"<CHANGE-ME>"} # the password for that account
#-------------------------------------------------------------------------------
GHBU_ORGMODE=${GHBU_ORGMODE-"org"} # "org", "user" or "gists"?
GHBU_BACKUP_DIR=${GHBU_BACKUP_DIR-"github-backups"} # where to place the backup files; avoid using ":" in the name (confuses tar as a hostname; confuses Windows as a drive letter)
GHBU_GITHOST=${GHBU_GITHOST-"github.com"} # the GitHub hostname (see comments)
GHBU_REUSE_REPOS=${GHBU_REUSE_REPOS-false} # as part of backup process, we mirror-clone remote git repos; should we keep and reuse them for next backups (true), or always snatch from scratch (false)?
GHBU_TARBALL_REPOS=${GHBU_TARBALL_REPOS-true} # when `true`, tarballs for each backed-up repository would be made; when false (e.g. if persistent repos due to GHBU_REUSE_REPOS=true suffice), only non-git items would be tarballed (issues, comments, metadata)
GHBU_PRUNE_INCOMPLETE=${GHBU_PRUNE_INCOMPLETE-false} # when `true`, backups named like *.__WRITING__ will be deleted when script starts (set `false` if using same GHBU_BACKUP_DIR for several scripts running in parallel)
GHBU_PRUNE_PREV=${GHBU_PRUNE_PREV-false} # when `true`, only the "*.latest.tar.gz" backups will be tracked; when `false` also the `*.prev.tar.gz"
GHBU_PRUNE_OLD=${GHBU_PRUNE_OLD-true} # when `true`, old backups will be deleted
GHBU_PRUNE_AFTER_N_DAYS=${GHBU_PRUNE_AFTER_N_DAYS-3} # the min age (in days) of backup files to delete
GHBU_SILENT=${GHBU_SILENT-false} # when `true`, only show error messages
GHBU_API=${GHBU_API-"https://api.github.com"} # base URI for the GitHub API
GHBU_GIT_CLONE_CMD="GITCMD clone --quiet --mirror " # base command to use to clone GitHub repos from an URL (may need more info for SSH)
GHBU_GIT_CLONE_CMD_SSH="${GHBU_GIT_CLONE_CMD} git@${GHBU_GITHOST}:" # base command to use to clone GitHub repos over SSH
GHBU_FAILFAST_GETGIT="${GHBU_FAILFAST_GETGIT-true}" # if true, repeated failure of getgit() will fail the script; if false - go over other repos
TSTAMP="`TZ=UTC date "+%Y%m%dT%H%MZ"`" # format of timestamp suffix appended to archived files
#-------------------------------------------------------------------------------
# (end config)
#-------------------------------------------------------------------------------
# The function `check` will exit the script if the given command fails.
function check {
"$@"
status=$?
if [ $status -ne 0 ]; then
echo "ERROR: Encountered error (${status}) while running the following:" >&2
echo " $@" >&2
echo " (at line ${BASH_LINENO[0]} of file $0.)" >&2
echo " Aborting." >&2
exit $status
fi
}
# The function `check_unexpected_success` will exit the script if the given command DOES NOT FAIL.
function check_unexpected_success {
"$@"
status=$?
if [ $status -eq 0 ]; then
echo "ERROR: Encountered error (unexpected success) while running the following:" >&2
echo " $@" >&2
echo " (at line ${BASH_LINENO[0]} of file $0.)" >&2
echo " Aborting." >&2
exit 1
fi
}
# The function `tgz` will create a gzipped tar archive of the specified
# file ($1) and then optionally remove the original
function tgz {
$GHBU_TARBALL_REPOS || return 0
check tar zcf "$1.$TSTAMP.tar.gz.__WRITING__" "$1" \
&& check mv -f "$1.$TSTAMP.tar.gz.__WRITING__" "$1.$TSTAMP.tar.gz" \
|| return
if ! $GHBU_REUSE_REPOS ; then
check rm -rf "$1"
fi
# Let one copy (or two if "prev" is used) survive the auto-prune
if $GHBU_PRUNE_PREV ; then
rm -f "$1.prev.tar.gz" || true
rm -f "$1.latest.tar.gz" || true
else
if [ -e "$1.latest.tar.gz" ] ; then
mv -f "$1.latest.tar.gz" "$1.prev.tar.gz" || true
fi
fi
check ln "$1.$TSTAMP.tar.gz" "$1.latest.tar.gz"
}
# Shortcut for non-repo items (issues, comments, metadata JSONs...)
function tgz_nonrepo {
GHBU_TARBALL_REPOS=true \
GHBU_REUSE_REPOS=false \
tgz "$@"
}
# Optionally delete files with a __WRITING__ extension,
# likely abandoned due to run-time errors like a reboot,
# Ctrl+C, connection loss, disk space...
function prune_incomplete {
if $GHBU_PRUNE_INCOMPLETE ; then
$GHBU_SILENT || (echo "" && echo "=== PRUNING INCOMPLETE LEFTOVERS (if any) ===" && echo "")
$GHBU_SILENT || echo "Found `find $GHBU_BACKUP_DIR -maxdepth 1 -name '*.__WRITING__*' | wc -l` files to prune."
find $GHBU_BACKUP_DIR -maxdepth 1 -name '*.__WRITING__*' -exec rm -fv {} > /dev/null \;
$GHBU_SILENT || (echo "" && echo "=== PRUNING FINISHED ===" && echo "")
fi
}
# The function `getdir` will return the repo directory name on stdout
# if successful (it depends on GHBU_REUSE_REPOS value).
function getdir {
local REPOURI="$1"
local DIRNAME
REPOURI="$(echo "$REPOURI" | sed 's,^https://gist.github.com/\(.*\)$,gist-\1,')"
# Note our caller adds a ".comments" suffix; another may be from API, orig:
# https://api.github.com/gists/b92a4fe5bb8eab70e79d6f1581563863/comments
REPOURI="$(echo "$REPOURI" | sed 's,^'"$GHBU_API"'/gists/\([^/]*\)/comments\(\.comments\)*$,gist-\1-comments,')"
if $GHBU_REUSE_REPOS ; then
DIRNAME="${GHBU_BACKUP_DIR}/${GHBU_ORG}-${REPOURI}"
fi
if ! $GHBU_REUSE_REPOS ; then
DIRNAME="${DIRNAME}-${TSTAMP}"
fi
case "$REPOURI" in
*.git) ;;
*) DIRNAME="${DIRNAME}.git" ;;
esac
echo "$DIRNAME"
}
# See coments below
function GITCMD {
if [ -z "$CRED_HELPER" ] ; then
git "$@"
else
# Note the deletion of credential.helper first, it is multivalued
# Per https://stackoverflow.com/a/70963737 explanation:
# > The additional empty value for credential.helper causes any
# > existing credential helpers to be removed, preventing the
# > addition of this token into the user's credential helper.
# > If you'd like the user to be able to save it, then remove
# > that directive.
# In our case, the first HTTP(S) download would block asking for
# user/pass; however a "credential.helper=cache --timeout=360000"
# might help subsequent activities (and/or corrupt them, if we use
# different credentials for backups and/or interactive development).
git -c credential.helper= -c credential.helper="$CRED_HELPER" "$@"
fi
}
# The function `getgit` will clone (or update) specified repo ($1, without
# a `.git` suffix) into specified directory ($2)
function getgit (
# Sub-shelled to constrain "export" visibility of credentials
local REPOURI="$1"
local DIRNAME="$2"
case x"$1" in
xhttp://*|xhttps://*)
# Prepare HTTP(S) credential support for this git operation.
local CRED_HELPER='!f() { echo "username=$GHBU_UNAME"; echo "password=$GHBU_PASSWD"; }; f'
export CRED_HELPER GHBU_UNAME GHBU_PASSWD
;;
esac
if $GHBU_REUSE_REPOS && [ -d "${DIRNAME}" ] ; then
# Update an existing repo (if reusing)
$GHBU_SILENT || echo "... Updating $REPOURI clone in $DIRNAME"
# Note "fatal: fetch --all does not make sense with refspecs" likely in the mirror:
(cd "${DIRNAME}" && {
GITCMD fetch --quiet --tags \
&& GITCMD fetch --quiet --all \
|| GITCMD fetch --quiet
}) || return
else
$GHBU_SILENT || echo "... Cloning $REPOURI into $DIRNAME"
case x"$1" in
x*@*|x*://*) # URL already; .git suffix should be irrelevant
${GHBU_GIT_CLONE_CMD} "${REPOURI}" "${DIRNAME}" || return
;;
*) # Just a repo name - complete it with data we know of
${GHBU_GIT_CLONE_CMD_SSH}"${GHBU_ORG}/${REPOURI}.git" "${DIRNAME}" \
|| { # Errors were seen above, so no GHBU_SILENT here:
echo "..... Attempt a retry over HTTPS" >&2
# FIXME: Effectively we craft the clone_url
# here, rather than using one from metadata
${GHBU_GIT_CLONE_CMD} "https://${GHBU_GITHOST}/${GHBU_ORG}/${REPOURI}" "${DIRNAME}" \
&& echo "..... Attempt a retry over HTTPS: SUCCEEDED" >&2 ; \
} || return
;;
esac
fi
# Return a success either way here:
$GHBU_SILENT || echo "+++ Received $REPOURI into $DIRNAME"
)
function filter_user_org {
# Might be better off getting a "clone_url" here, but so far our
# directory naming etc. rely on the "REPONAME" value received here:
check grep '^ "name"' | check awk -F': "' '{print $2}' | check sed -e 's/",//g'
}
function filter_gist {
check sed -n 's/.*git_pull_url": "\(.*\)",/\1/p'
}
function filter_gist_comments {
check sed -n 's/.*comments_url": "\(.*\)",/\1/p'
}
function get_multipage_file {
# Uses caller envvars FILENAME, APIURL, APIQUERY_SUFFIX, ENTRYID_REGEX
# Returns MULTIPAGE_NUM, MULTIPAGE_OK, ENTRY_COUNT
# Assumes GNU date or compatible - try to avoid extra traffic for us
# and load for servers (and REST API quota hits), by asking GitHub
# REST API directly: "If-Modified-Since" we last fetched the document:
[ -s "$FILENAME" ] && FILEDATE="`date -R -u -r "$FILENAME" | sed 's,\+0000,GMT,'`" || FILEDATE=""
ETAGS="`dirname "$FILENAME"`/etags.cache"
[ -s "$ETAGS" ] && FILEETAG="`grep -E "\t$APIURL\$" < "$ETAGS" | awk '{print $1}'`" || FILEETAG=""
MULTIPAGE_NUM=1
MULTIPAGE_OK=true
ENTRY_COUNT=0
while : ; do
# ex. APIURL="${GHBU_API}/repos/${GHBU_ORG}/${REPO}/issues"
# ex. APIQUERY_SUFFIX="&state=all"
# Also note: https://docs.github.com/en/rest/overview/resources-in-the-rest-api?apiVersion=2022-11-28#user-agent-required
# hence `the User-Agent: username` however it causes JSON
# markup not prettified for humans (and line-based grep)
# Ordering defaults to showing newest issues first
# TODO: Can we impact ordering of comments so newest
# changed would be first and we know from the first page
# that we need to re-fetch this?..
CURLRES=0
rm -f "${FILENAME}.headers" || true
WANT_HEADER=""
if [ 1 = "$MULTIPAGE_NUM" ] && ( [ -n "$FILEETAG" ] || [ -n "$FILEDATE" ] ) ; then
WANT_HEADER="yes"
fi
curl --silent -u "${GHBU_UNAME}:${GHBU_PASSWD}" \
-H "User-Agent: ${GHBU_UNAME}" \
${FILEETAG+-H "If-None-Match: ${FILEETAG}"} \
${FILEDATE+-H "If-Modified-Since: ${FILEDATE}"} \
${WANT_HEADER+-D "${FILENAME}.headers"} \
"${APIURL}?per_page=100&page=${MULTIPAGE_NUM}${APIQUERY_SUFFIX}" -q \
> "${FILENAME}.__WRITING__.tmp" \
|| {
echo "FAILED to fetch '${APIURL}' once: will sleep in case it is about the usage quota and try again"
sleep 120
check curl --silent -u "${GHBU_UNAME}:${GHBU_PASSWD}" \
-H "User-Agent: ${GHBU_UNAME}" \
${FILEETAG+-H "If-None-Match: ${FILEETAG}"} \
${FILEDATE+-H "If-Modified-Since: ${FILEDATE}"} \
${WANT_HEADER+-D "${FILENAME}.headers"} \
"${APIURL}?per_page=100&page=${MULTIPAGE_NUM}${APIQUERY_SUFFIX}" -q \
> "${FILENAME}.__WRITING__.tmp"
} || CURLRES=$?
# NOTE: Value may include quotes; header is posted with them!
NEWETAG="`grep -i '^etag:' "${FILENAME}.headers" | sed 's,^[Ee][Tt][Aa][Gg]: *,,' | tr -d '\r' | tr -d '\n'`" || NEWETAG=""
if [ -n "$NEWETAG" ] && [ x"$NEWETAG" != x"$FILEETAG" ] ; then
printf '%s\t%s\n' "$NEWETAG" "$APIURL" > "$ETAGS.tmp"
if [ -s "$ETAGS" ]; then grep -v "$APIURL" "$ETAGS" >> "$ETAGS.tmp" ; fi
mv "$ETAGS.tmp" "$ETAGS"
fi
if head -1 "${FILENAME}.headers" | grep -E "HTTP.*304" > /dev/null ; then
# Let HTTP-304 skips be seen
$GHBU_SILENT || echo "SKIP: (First page of) the requested resource did not change"
rm -f "${FILENAME}.headers" "${FILENAME}.__WRITING__.tmp"
MULTIPAGE_OK=true
ENTRY_COUNT="`grep -Ec "$ENTRYID_REGEX" < "${FILENAME}"`"
break
fi
# Only ask for first page
unset FILEDATE
unset FILEETAG
if [ $CURLRES != 0 ] ; then
MULTIPAGE_OK=false
break
fi
if [ 1 = "$MULTIPAGE_NUM" ] ; then
# Not cached => replace
rm -f "$FILENAME" || true
fi
# Produce pretty JSON we can grep in, and otherwise manipulate below:
jq < "${FILENAME}.__WRITING__.tmp" > "${FILENAME}.__WRITING__"
rm -f "${FILENAME}.__WRITING__.tmp"
if [ 1024 > "`wc -c < "${FILENAME}.__WRITING__"`" ] \
&& grep -q '"message": "API rate limit exceeded' "${FILENAME}.__WRITING__" \
; then
echo "FAILED to fetch '${APIURL}': got some contents but they are short and only say that API rate limit exceeded"
MULTIPAGE_OK=false
break
fi
# ex. ENTRYID_REGEX='"url": "'"${GHBU_API}/repos/${GHBU_ORG}/${REPO}/issues/[0123456789]+"'"'
NUM="`grep -Ec "$ENTRYID_REGEX" < "${FILENAME}.__WRITING__"`"
ENTRY_COUNT="`expr $ENTRY_COUNT + $NUM`"
if [ ! -e "${FILENAME}" ] ; then
check mv -f "${FILENAME}.__WRITING__" "${FILENAME}"
else
(head -n -2 "${FILENAME}" && \
echo " }," && \
tail -n +2 "${FILENAME}.__WRITING__"
) > "${FILENAME}.__WRITING__.tmp" \
|| { MULTIPAGE_OK=false; break; }
mv -f "${FILENAME}.__WRITING__.tmp" "${FILENAME}"
rm -f "${FILENAME}.__WRITING__"
fi
if [ $NUM -lt 100 ] ; then
# Last page
break
fi
MULTIPAGE_NUM="`expr $MULTIPAGE_NUM + 1`"
done
rm -f "${FILENAME}.headers"
# Return status
$MULTIPAGE_OK
}
check command -v jq
$GHBU_SILENT || (echo "" && echo "=== INITIALIZING ===" && echo "")
$GHBU_SILENT || echo "Using backup directory $GHBU_BACKUP_DIR"
check mkdir -p $GHBU_BACKUP_DIR
prune_incomplete
$GHBU_SILENT || echo -n "Fetching list of repositories for ${GHBU_ORG}..."
case x"$GHBU_ORGMODE" in
x"gist"|x"gists")
GHBU_ORG_URI="/gists"
;;
x"user"|x"users")
# NOTE: if you're backing up a *user's* repos, not an organizations, use this instead:
if [ "${GHBU_ORG}" = "${GHBU_UNAME}" ] ; then
# Backing up ourselves
GHBU_ORG_URI="/user"
else
# Backing up a friend/alter-ego (using our login)
GHBU_ORG_URI="/users/${GHBU_ORG}"
fi
;;
x"org"|*) # Legacy default
GHBU_ORG_URI="/orgs/${GHBU_ORG}"
;;
esac
# Be sure to stash a fresh copy, even if previous backup was interrupted
rm -f "${GHBU_BACKUP_DIR}/${GHBU_ORG}-metadata.json.__WRITING__"
touch "${GHBU_BACKUP_DIR}/${GHBU_ORG}-metadata.json.__WRITING__"
GIST_COMMENTLIST=""
GIST_COMMENTLIST_PAGE=""
REPOLIST=""
REPOLIST_PAGE=""
PAGENUM=1
# TODO: Convert to get_multipage_file()?
while : ; do
JSON=""
case x"$GHBU_ORGMODE" in
xorg*|xuser*)
# hat tip to https://gist.github.com/rodw/3073987#gistcomment-3217943 for the license name workaround
# The "type=owner" should be default per https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repositories-for-a-user
# but with a powerful token a "user" backup may see all repos
# one has access to (collaborating in other orgs). Other than
# lots of noise and more time to get the listing, this leads
# to broken backup cycles when we try to fetch repo names that
# are not known under this user's personal namespace.
# Note: user-agent causes compacted JSON we can not grep in => jq
JSON="$(curl --silent -u "${GHBU_UNAME}:${GHBU_PASSWD}" -H "User-Agent: ${GHBU_UNAME}" "${GHBU_API}${GHBU_ORG_URI}/repos?per_page=100&page=$PAGENUM&type=owner" -q)"
check [ "$?" = 0 ]
JSON="$(echo "$JSON" | check jq)"
echo "$JSON" | grep "API rate limit" && check false
REPOLIST_PAGE="$(echo "$JSON" | filter_user_org)"
;;
xgist*)
JSON="$(curl --silent -u "${GHBU_UNAME}:${GHBU_PASSWD}" -H "User-Agent: ${GHBU_UNAME}" "${GHBU_API}${GHBU_ORG_URI}?per_page=100&page=$PAGENUM" -q)"
check [ "$?" = 0 ]
JSON="$(echo "$JSON" | check jq)"
echo "$JSON" | grep "API rate limit" && check false
REPOLIST_PAGE="$(echo "$JSON" | filter_gist)"
GIST_COMMENTLIST_PAGE="$(echo "$JSON" | filter_gist_comments)"
;;
esac
# Not exactly a valid JSON file (gets lists just concatenated), but
# so far good enough to grep it, and some parsers treat it well too
echo "$JSON" >> "${GHBU_BACKUP_DIR}/${GHBU_ORG}-metadata.json.__WRITING__"
if [ -z "$REPOLIST" ] ; then
REPOLIST="$REPOLIST_PAGE"
else
REPOLIST="$REPOLIST
$REPOLIST_PAGE"
fi
if [ -z "$GIST_COMMENTLIST" ] ; then
GIST_COMMENTLIST="$GIST_COMMENTLIST_PAGE"
else
GIST_COMMENTLIST="$GIST_COMMENTLIST
$GIST_COMMENTLIST_PAGE"
fi
if [ 100 -ne `echo $REPOLIST_PAGE | wc -w` ] ; then
break
fi
PAGENUM=$(($PAGENUM+1))
$GHBU_SILENT || echo -n " Fetching next page of repos: $PAGENUM..."
done
REPOS_COUNT=0
REPOS_TOTAL="`echo $REPOLIST | wc -w`"
$GHBU_SILENT || echo " found $REPOS_TOTAL repositories."
mv -f "${GHBU_BACKUP_DIR}/${GHBU_ORG}-metadata.json.__WRITING__" "${GHBU_BACKUP_DIR}/${GHBU_ORG}-metadata.json"
tgz_nonrepo "${GHBU_BACKUP_DIR}/${GHBU_ORG}-metadata.json"
$GHBU_SILENT || (echo "" && echo "=== BACKING UP ===" && echo "")
for REPO in $REPOLIST; do
REPOS_COUNT=$(($REPOS_COUNT+1))
$GHBU_SILENT || echo "Backing up ${GHBU_ORG}/${REPO} ($REPOS_COUNT of $REPOS_TOTAL)"
DIRNAME="`getdir "$REPO"`"
{ getgit "${REPO}" "${DIRNAME}" || {
echo "FAILED to getgit '${REPO}' '${DIRNAME}': will sleep in case it is about the usage quota and try again"
sleep 120
if [ x"$GHBU_FAILFAST_GETGIT" = xtrue ]; then
echo "RETRY getgit '${REPO}' '${DIRNAME}': failure now WILL BE FATAL"
check getgit "${REPO}" "${DIRNAME}"
else
echo "RETRY getgit '${REPO}' '${DIRNAME}': failure now will NOT be fatal"
getgit "${REPO}" "${DIRNAME}"
fi
} ; } && tgz "${DIRNAME}"
# No wikis nor issues for gists; but there are comments (see another loop)
case x"$GHBU_ORGMODE" in
xorg*|xuser*)
$GHBU_SILENT || echo "Backing up ${GHBU_ORG}/${REPO}.wiki (if any)"
DIRNAME="`getdir "$REPO.wiki"`"
# Failure is an option for wikis:
getgit "${REPO}.wiki" "${DIRNAME}" 2>/dev/null && tgz "${DIRNAME}"
# NOTE: While internally issues and PRs seem to be the same,
# at least using the same numbering and responding with them
# to either URL (can discern by "html_url"), they are both
# listed via both numbers: all entities are in "issues" but
# only PRs (with partially different metadata) are in "pulls".
DIRNAME="`getdir "$REPO.issues-and-pulls"`"
$GHBU_SILENT || echo "Preparing local git repo in '${DIRNAME}' to receive issue and PR data"
if [ -d "${DIRNAME}/.git" ]; then
( cd "${DIRNAME}" && git checkout -f
case $? in
0) git clean -fffdddxxx ;;
128) GITOUT="`git log --oneline -1 2>&1`"
# `ls` => empty etags.cache list-issues.json list-pulls.json
if [ $? = 128 ] && ( echo "${GITOUT}" | grep -E 'fatal: (your current branch .* does not have any commits yet|You are on a branch yet to be born)' ) && [ `ls -1 | wc -l` = 3 ] ; then
$GHBU_SILENT || echo "Removing botched earlier preparation of a local git repo in '${DIRNAME}' to receive issue and PR data"
# rm -rf .git
BASE_DIRNAME="`basename "${DIRNAME}"`"
cd .. && rm -rf "${BASE_DIRNAME}"
else
exit 128
fi
;;
esac
) || check [ "$?" = 0 ]
fi
if [ ! -d "${DIRNAME}/.git" ]; then
check mkdir -p "${DIRNAME}" \
&& ( cd "${DIRNAME}" && git init \
&& { git config gc.autodetach false ; git config commit.gpgsign false ; true ; } \
&& touch list-issues.json list-pulls.json etags.cache \
&& git add list-issues.json list-pulls.json etags.cache \
&& git commit -m 'Initial commit' \
) || check [ "$?" = 0 ]
fi
# List of issues:
$GHBU_SILENT || echo "Backing up ${GHBU_ORG}/${REPO} issues"
ISSUES_FILENAME="${DIRNAME}/list-issues.json"
FILENAME="${ISSUES_FILENAME}" \
APIURL="${GHBU_API}/repos/${GHBU_ORG}/${REPO}/issues" \
APIQUERY_SUFFIX="&state=all" \
ENTRYID_REGEX='"url": *"'"${GHBU_API}/repos/${GHBU_ORG}/${REPO}/issues/[0123456789]+"'"' \
get_multipage_file
$GHBU_SILENT || echo "Collected ${ENTRY_COUNT} issues in ${MULTIPAGE_NUM} pages for ${GHBU_ORG}/${REPO}; overall success: ${MULTIPAGE_OK}"
$MULTIPAGE_OK && ( cd "${DIRNAME}" && git add "`basename "$ISSUES_FILENAME"`" ) || MULTIPAGE_OK=false
ISSUES_OK="$MULTIPAGE_OK"
ISSUES_NUM="${ENTRY_COUNT}"
# List of PRs:
$GHBU_SILENT || echo "Backing up ${GHBU_ORG}/${REPO} pull requests"
PULLS_FILENAME="${DIRNAME}/list-pulls.json"
FILENAME="${PULLS_FILENAME}" \
APIURL="${GHBU_API}/repos/${GHBU_ORG}/${REPO}/pulls" \
APIQUERY_SUFFIX="&state=all" \
ENTRYID_REGEX='"url": *"'"${GHBU_API}/repos/${GHBU_ORG}/${REPO}/pulls?/[0123456789]+"'"' \
get_multipage_file
$GHBU_SILENT || echo "Collected ${ENTRY_COUNT} pull requests in ${MULTIPAGE_NUM} pages for ${GHBU_ORG}/${REPO}; overall success: ${MULTIPAGE_OK}"
$MULTIPAGE_OK && ( cd "${DIRNAME}" && git add "`basename "$PULLS_FILENAME"`" ) || MULTIPAGE_OK=false
PULLS_OK="$MULTIPAGE_OK"
PULLS_NUM="${ENTRY_COUNT}"
# Contents
( $ISSUES_OK && grep '"comments_url"' "$ISSUES_FILENAME" || true
$PULLS_OK && grep -E '"(comments_url|review_comments_url|commits_url)"' "$PULLS_FILENAME" || true
) | awk '{print $NF}' | sed -e 's,^",,' -e 's/",*$//' | sort -n | uniq | \
while IFS= read SUB_URL \
; do
SUB_FILENAME="`echo "$SUB_URL" | sed -e "s,^${GHBU_API}/repos/${GHBU_ORG}/${REPO}/,," -e 's,[:/],-,g'`.json"
# Skip user metadata
case "$SUB_FILENAME" in
http---*|https---*|*"{-"*"}"*) continue ;;
esac
$GHBU_SILENT || echo "Backing up ${GHBU_ORG}/${REPO} issue or pull request details from: ${SUB_URL}"
FILENAME="${DIRNAME}/${SUB_FILENAME}" APIURL="${SUB_URL}" \
ENTRYID_REGEX='("sha": *"[0-9a-f]{40}"|"url": *"'"${GHBU_API}/repos/${GHBU_ORG}/${REPO}/(issues|pulls)/comments/[0-9]+"'")' \
get_multipage_file \
&& ( cd "${DIRNAME}" && git add "${SUB_FILENAME}" )
done
$PULLS_OK && $ISSUES_OK \
&& (
GITMSG="Update due to backup at `LANG=C LC_ALL=C TZ=UTC date -u`"
cd "${DIRNAME}" && \
git add etags.cache && \
{ git commit -m "${GITMSG}" || { echo "Retry git commit without GPG" >&2 ; git commit --no-gpg-sign -m "${GITMSG}" ; } ; }
) && tgz "${DIRNAME}"
;;
esac
done
# Assumes GHBU_ORGMODE=gist, but no reason to constrain:
COMMENT_COUNT=0
COMMENT_TOTAL="`echo $GIST_COMMENTLIST | wc -w`"
for COMMENT_URL in $GIST_COMMENTLIST; do
COMMENT_COUNT=$(($COMMENT_COUNT+1))
$GHBU_SILENT || echo "Backing up ${GHBU_ORG}/${COMMENT_URL} comments ($COMMENT_COUNT of $COMMENT_TOTAL)"
FILENAME="`getdir "${COMMENT_URL}.comments" | sed 's,.git$,,'`"
check curl --silent -u "${GHBU_UNAME}:${GHBU_PASSWD}" \
-H "User-Agent: ${GHBU_UNAME}" \
"${COMMENT_URL}" -q \
> "${FILENAME}.__WRITING__" \
&& mv -f "${FILENAME}.__WRITING__" "${FILENAME}" \
&& tgz_nonrepo "${FILENAME}"
done
# NOTE: the "latest" and optional "prev" handling below allows us to leave at
# least one (better two) backup tarballs for each timestamped item sequence.
# GitHub going AWOL and us deleting all backups after 3 days would be folly!
# (Less of a problem if we do keep the repos, but comments/issues/medatata
# are still at risk - maybe GIT their evolution locally?)
# NOTE: according to `man find` (GNU, comments to `-atime` et al handling),
# the fractional parts of "n*24 hours" are ignored, "so to match -atime +1,
# a file has to have been accessed at least two days ago".
# This way, GHBU_PRUNE_AFTER_N_DAYS=0 only chops files older than 24 hours.
if $GHBU_PRUNE_OLD && [ "${GHBU_PRUNE_AFTER_N_DAYS}" -ge 0 ]; then
$GHBU_SILENT || (echo "" && echo "=== PRUNING ===" && echo "")
$GHBU_SILENT || echo "Pruning backup files ${GHBU_PRUNE_AFTER_N_DAYS} days old or older."
$GHBU_SILENT || echo "Found `find $GHBU_BACKUP_DIR -maxdepth 1 -name '*.tar.gz' -a \! -name '*.prev.tar.gz' -a \! -name '*.latest.tar.gz' -mtime +${GHBU_PRUNE_AFTER_N_DAYS} | wc -l` files to prune."
find $GHBU_BACKUP_DIR -maxdepth 1 -name '*.tar.gz' -a \! -name '*.prev.tar.gz' -a \! -name '*.latest.tar.gz' -mtime "+${GHBU_PRUNE_AFTER_N_DAYS}" -exec rm -fv {} > /dev/null \;
$GHBU_SILENT || (echo "" && echo "=== PRUNING FINISHED ===" && echo "")
fi
prune_incomplete
$GHBU_SILENT || (echo "" && echo "=== DONE ===" && echo "")
$GHBU_SILENT || (echo "GitHub backup for ${GHBU_ORG} (${GHBU_ORGMODE}) completed." && echo "")