diff --git a/CI/travis/deploy b/CI/travis/deploy index 63a3f60cd..aa4d086ba 100755 --- a/CI/travis/deploy +++ b/CI/travis/deploy @@ -7,4 +7,4 @@ upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_DEB} ${TARGET_DEB} .deb upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_RPM} ${TARGET_RPM} .rpm upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_TGZ} ${TARGET_TGZ} .tar.gz upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_PKG} ${TARGET_PKG} .pkg - +remove_old_pkgs libiio diff --git a/CI/travis/lib.sh b/CI/travis/lib.sh index 53c8cd13e..450177372 100644 --- a/CI/travis/lib.sh +++ b/CI/travis/lib.sh @@ -225,11 +225,27 @@ sftp_cmd_pipe() { sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" } +sftp_run_cmds() { + local x=5 + while [ "${x}" -gt "0" ] ; do + sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" 0< "$1" && break; + echo_red "failed to ssh, trying again" + x=$((x - 1)) + sleep 10 + done + if [ "${x}" -eq "0" ] ; then + echo_red "failed to upload files" + return 1; + fi + return 0 +} + sftp_rm_artifact() { local artifact="$1" sftp_cmd_pipe <<-EOF cd ${DEPLOY_TO} rm ${artifact} + bye EOF } @@ -297,33 +313,106 @@ upload_file_to_swdownloads() { echo and "${branch}_${LIBNAME}${LDIST}${EXT}" ssh -V - for rmf in ${TO} ${LATE} ; do - sftp_rm_artifact "${rmf}" || \ - echo_blue "Could not delete ${rmf}" - done + local tmpfl=$(mktemp) + echo "cd ${DEPLOY_TO}" > "${tmpfl}" + echo "rm ${TO}" >> "${tmpfl}" + echo "rm ${LATE}" >> "${tmpfl}" + echo "put ${FROM} ${TO}" >> "${tmpfl}" + echo "symlink ${TO} ${LATE}" >> "${tmpfl}" + echo "ls -l ${TO}" >> "${tmpfl}" + echo "ls -l ${LATE}" >> "${tmpfl}" + echo "bye" >> "${tmpfl}" - sftp_upload "${FROM}" "${TO}" "${LATE}" || { - echo_red "Failed to upload artifact from '${FROM}', to '${TO}', symlink '${LATE}'" - return 1 - } + sftp_run_cmds "${tmpfl}" + rm "${tmpfl}" + + return 0 +} +remove_old_pkgs() { # limit things to a few files, so things don't grow forever # we only do this on one build so simultaneous builds don't clobber each other - if [ -n "${GH_DOC_TOKEN}" ] ; then - for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ + if [ -z "${GH_DOC_TOKEN}" ] ; then + return 0 + fi + + if [ -z "${TRAVIS_BUILD_DIR}" ] ; then + echo "TRAVIS_BUILD_DIR not set" + return 0 + fi + + if [ ! -d "${TRAVIS_BUILD_DIR}/.git" ] ; then + echo "No ${TRAVIS_BUILD_DIR}/.git to operate git on" + return 0 + fi + + local LIBNAME=$1 + local old= + + echo "Remove old packages from ${LIBNAME}" + + if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ] ; then + local branch="$TRAVIS_PULL_REQUEST_BRANCH" + else + local branch="$TRAVIS_BRANCH" + fi + + local GLOB=${DEPLOY_TO}/${branch}_${LIBNAME}-* + + # putting everything into a file, and connecting once decreases the chances + # for ssh issues, connections happen once, not every single file + local tmpfl=$(mktemp) + echo "cd ${DEPLOY_TO}" > "${tmpfl}" + for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ "ls -lt ${GLOB}" | tail -n +100 | awk '{print $NF}') - do - ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ - "rm ${DEPLOY_TO}/${files}" || \ - return 1 - done - # provide an index so people can find files. - ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ - "ls -lt ${DEPLOY_TO}" | grep ${LIBNAME} > ${LIBNAME}_index.html - sftp_rm_artifact "${LIBNAME}_index.html" || \ - echo_blue "Could not delete ${LIBNAME}_index.html" - sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html" + do + echo "rm ${files}" >> "${tmpfl}" + done + echo "bye" >> "${tmpfl}" + # if it is only cd & bye, skip it + if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then + sftp_run_cmds "${tmpfl}" fi + rm "${tmpfl}" + # provide an index so people can find files. + ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ + "ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html" + echo "ls captured" + + echo "cd ${DEPLOY_TO}" > "${tmpfl}" + # prune old / removed branches, leave things are are tags/branches + for old in $(sed 's/-> .*$//' libiio_index.html | \ + awk '{print $NF}' | grep -v master | sort | \ + sed "s/_libiio-0.[0-9][0-9].g[a-z0-9]*-/ %% /" | \ + grep "%%" | awk '{print $1}' | sort -u) + do + if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --heads origin "${old}" | wc -l)" -ne "0" ] ; then + echo "${old} is a branch" + else + if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --tags origin "${old}" | wc -l)" -ne "0" ] ; then + echo "${old} is a tag" + else + echo "${old} can be removed" + echo "rm ${old}_${LIBNAME}-*" >> "${tmpfl}" + echo "rm ${old}_latest_${LIBNAME}-*" >> "${tmpfl}" + echo "rm ${old}_lastest_${LIBNAME}-*" >> "${tmpfl}" + fi + fi + done + # cap things at 15, so we don't exceed the time + sed -i 16q "${tmpfl}" + echo "bye" >> "${tmpfl}" + # if it is only cd & bye, skip it + if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then + sftp_run_cmds "${tmpfl}" + fi + rm "${tmpfl}" + + #Now that we removed things, do it again + rm "${LIBNAME}_index.html" + ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \ + "ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html" + sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html" return 0 }