Skip to content

Commit

Permalink
Merge pull request #550 from analogdevicesinc/python-setup-fixes
Browse files Browse the repository at this point in the history
Python setup fixes
  • Loading branch information
rgetz authored Jun 16, 2020
2 parents 21ed20f + e7a7656 commit fc5b671
Show file tree
Hide file tree
Showing 8 changed files with 341 additions and 90 deletions.
10 changes: 7 additions & 3 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,18 @@ matrix:
- TRIGGER_NEXT_BUILD=true

addons:
artifacts: true
ssh_known_hosts:
secure: "q0dQ9MrhnOCo031McxSdQIqw4N6tEyAZLq7mdbWkAJcXOh/CX58NjFvcdSfamleDUYRmg7KpCZPPgnbx2JtqVvWJv8aNnr67CE1GIjRP1Fxh2WaKS+VK+I6rro7GwCO2C9d+uffCt63LfZKqddF1T7vMI2IgHcu9irc5LCuw6Wo="

before_install:
- if [[ "$ARCH" == "arm" ]] ; then ./CI/travis/setup_qemu_for_arm.sh ${OS_VERSION} ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
if [[ "$ARCH" == "arm" ]] ; then
./CI/travis/setup_qemu_for_arm.sh ${OS_VERSION} ;
else
./CI/travis/before_install_linux "$OS_TYPE" ;
fi
fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ./CI/travis/before_install_darwin ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./CI/travis/before_install_linux "$OS_TYPE" ; fi
- if [[ -n "$COVERITY_SCAN_PROJECT_NAME" ]] ; then echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca- ; fi
- if [ -n "$COVERITY_SCAN_PROJECT_NAME" -a "$TRAVIS_EVENT_TYPE" == "cron" ] ; then curl -s 'https://scan.coverity.com/scripts/travisci_build_coverity_scan.sh' | bash || true ; fi

Expand Down
79 changes: 62 additions & 17 deletions CI/travis/before_install_linux
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,36 @@
. CI/travis/lib.sh

install_sphinx() {
if ! command_exists pip3 ; then
pip install sphinx
pip install sphinx-rtd-theme
else
pip3 install sphinx
pip3 install sphinx-rtd-theme
if command_exists python ; then
python --version
command -v python
python -m pip install -U pip
python -m pip install -U setuptools
python -m pip install sphinx
python -m pip install sphinx-rtd-theme
fi
}

install_pyenv() {
if ! command_exists pyenv ; then
echo installing pyenv
git clone git://github.com/yyuu/pyenv.git $HOME/.pyenv
export PATH="$HOME/.pyenv/bin:$PATH"
export PYENV_ROOT="$HOME/.pyenv"
fi
}

install_python() {
echo "### installing python"
command -v pyenv
add_python_path
pyenv install --list | grep "^[[:space:]]*[0-9]"
pyenv install 3.6.3
pyenv global 3.6.3
add_python_path
}


handle_centos() {
# needed for man2html and a few other popular tools
yum search epel-release
Expand All @@ -24,25 +44,40 @@ handle_centos() {
yum -y install cmake libxml2-devel libusb1-devel libaio-devel \
bzip2 gzip rpm rpm-build redhat-lsb-core

# CENTOS 6 doesn't include python 3, it's too old.
if is_centos_at_least_ver "7" ; then
yum -y install python3-pip
# needed for building python with pyenv
yum install -y gcc gcc-c++ make git patch openssl-devel zlib-devel readline-devel sqlite-devel bzip2-devel

# CentOS 6 & 7 don't work with doc, or the latest python.
# install_pyenv
# install_python

if [ "$(get_version | head -c 1)" = "7" ] ; then
# install Cmake3, and make it the default
yum -y install cmake3
alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake 10 \
--slave /usr/local/bin/ctest ctest /usr/bin/ctest \
--slave /usr/local/bin/cpack cpack /usr/bin/cpack \
--slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake \
--family cmake
alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 \
--slave /usr/local/bin/ctest ctest /usr/bin/ctest3 \
--slave /usr/local/bin/cpack cpack /usr/bin/cpack3 \
--slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \
--family cmake
fi

if is_centos_at_least_ver "8" ; then
# On CentOS 8, avahi-devel & doxygen are in this repo; enable it
yum -y install yum-utils
yum config-manager --set-enabled PowerTools
# On CentOS 6 & 7, have issues building or packaging doc
yum -y install python3 doxygen man2html
install_sphinx
else
# On CentOS 8, cdk-devel (Curses Development Kit) does not exist yet
yum -y install ncurses-devel cdk-devel
fi

# CENTOS 6 will has issues with sphinx
if is_centos_at_least_ver "7" ; then
yum -y install doxygen man2html
install_sphinx
fi

yum -y install avahi-devel
}

Expand All @@ -56,15 +91,25 @@ handle_ubuntu_docker() {

handle_default() {
sudo apt-get -qq update
sudo apt-get install -y apt-utils
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake graphviz \
libaio-dev libavahi-client-dev \
libavahi-common-dev libusb-1.0-0-dev libxml2-dev rpm tar \
bzip2 gzip flex bison git lsb-release python3-pip libncurses5-dev libcdk5-dev
bzip2 gzip flex bison git lsb-release libncurses5-dev libcdk5-dev

# Most of these should be here, but are needed for building python by pyenv
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential \
libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev \
libffi-dev liblzma-dev python-openssl git

if [ -n "${GH_DOC_TOKEN}" ] ; then
if ! is_arm ; then
install_pyenv
install_python
sudo apt-get install -y doxygen man2html
install_sphinx
fi

if [ `sudo apt-cache search libserialport-dev | wc -l` -gt 0 ] ; then
sudo apt-get install -y libserialport-dev
fi
Expand Down
2 changes: 1 addition & 1 deletion CI/travis/deploy
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_DEB} ${TARGET_DEB} .deb
upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_RPM} ${TARGET_RPM} .rpm
upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_TGZ} ${TARGET_TGZ} .tar.gz
upload_file_to_swdownloads libiio ${RELEASE_PKG_FILE_PKG} ${TARGET_PKG} .pkg

remove_old_pkgs libiio
201 changes: 177 additions & 24 deletions CI/travis/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,48 @@ echo_red() { printf "\033[1;31m$*\033[m\n"; }
echo_green() { printf "\033[1;32m$*\033[m\n"; }
echo_blue() { printf "\033[1;34m$*\033[m\n"; }

backtrace() {
# shell backtraces only work on bash
if [ ! -z "${BASH}" ] ; then
local i=
i=${#FUNCNAME[@]}
((--i))

while (( i >= 0 ))
do
echo "${BASH_SOURCE[$i]}:${BASH_LINENO[$i]}.${FUNCNAME[$i]}()"
i=$((i - 1))
done
fi
}

add_python_path() {
echo "adding Python to the path"
if [ -d "$HOME/.pyenv/bin" -a "$(echo "$PATH" | grep .pyenv/bin | wc -c)" -eq "0" ] ; then
echo "adding $HOME/.pyenv/bin to path"
export PATH="$HOME/.pyenv/bin:$PATH"
fi
if [ -z "${PYENV_SHELL}" ] ; then
echo init pyenv
eval "$(pyenv init -)"
fi
if [ -d /opt/pyenv/versions/3.6.3/bin -a "$(echo "$PATH" | grep opt/pyenv/versions | wc -c)" -eq "0" ] ; then
echo adding python on opt to PATH
export PATH="/opt/pyenv/versions/3.6.3/bin:$PATH"
fi
if [ -d /root/.pyenv/versions/3.6.3/bin -a "$(echo "$PATH" | grep root/.pyenv/versions | wc -c)" -eq "0" ] ; then
echo adding python on root/.pyenv to PATH
export PATH="/root/.pyenv/versions/3.6.3/bin:$PATH"
fi
if ! command_exists python ; then
echo No python on path
echo "$PATH"
else
python --version
command -v python
fi
}

get_script_path() {
local script="$1"

Expand Down Expand Up @@ -180,14 +222,30 @@ brew_install_if_not_exists() {
}

sftp_cmd_pipe() {
sftp "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}"
sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}"
}

sftp_run_cmds() {
local x=5
while [ "${x}" -gt "0" ] ; do
sftp -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" 0< "$1" && break;
echo_red "failed to ssh, trying again"
x=$((x - 1))
sleep 10
done
if [ "${x}" -eq "0" ] ; then
echo_red "failed to upload files"
return 1;
fi
return 0
}

sftp_rm_artifact() {
local artifact="$1"
sftp_cmd_pipe <<-EOF
cd ${DEPLOY_TO}
rm ${artifact}
bye
EOF
}

Expand All @@ -196,16 +254,26 @@ sftp_upload() {
local TO="$2"
local LATE="$3"

sftp_cmd_pipe <<-EOF
cd ${DEPLOY_TO}
if [ -n "${LATE}" ] ; then
sftp_cmd_pipe <<-EOF
cd ${DEPLOY_TO}
put ${FROM} ${TO}
ls -l ${TO}
put ${FROM} ${TO}
ls -l ${TO}
symlink ${TO} ${LATE}
ls -l ${LATE}
bye
EOF
symlink ${TO} ${LATE}
ls -l ${LATE}
bye
EOF
else
sftp_cmd_pipe <<-EOF
cd ${DEPLOY_TO}
put ${FROM} ${TO}
ls -l ${TO}
bye
EOF
fi
}

upload_file_to_swdownloads() {
Expand Down Expand Up @@ -245,26 +313,106 @@ upload_file_to_swdownloads() {
echo and "${branch}_${LIBNAME}${LDIST}${EXT}"
ssh -V

for rmf in ${TO} ${LATE} ; do
sftp_rm_artifact "${rmf}" || \
echo_blue "Could not delete ${rmf}"
done
local tmpfl=$(mktemp)
echo "cd ${DEPLOY_TO}" > "${tmpfl}"
echo "rm ${TO}" >> "${tmpfl}"
echo "rm ${LATE}" >> "${tmpfl}"
echo "put ${FROM} ${TO}" >> "${tmpfl}"
echo "symlink ${TO} ${LATE}" >> "${tmpfl}"
echo "ls -l ${TO}" >> "${tmpfl}"
echo "ls -l ${LATE}" >> "${tmpfl}"
echo "bye" >> "${tmpfl}"

sftp_upload "${FROM}" "${TO}" "${LATE}" || {
echo_red "Failed to upload artifact from '${FROM}', to '${TO}', symlink '${LATE}'"
return 1
}
sftp_run_cmds "${tmpfl}"
rm "${tmpfl}"

return 0
}

remove_old_pkgs() {
# limit things to a few files, so things don't grow forever
if [ "${EXT}" = ".deb" ] ; then
for files in $(ssh "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
# we only do this on one build so simultaneous builds don't clobber each other
if [ -z "${GH_DOC_TOKEN}" ] ; then
return 0
fi

if [ -z "${TRAVIS_BUILD_DIR}" ] ; then
echo "TRAVIS_BUILD_DIR not set"
return 0
fi

if [ ! -d "${TRAVIS_BUILD_DIR}/.git" ] ; then
echo "No ${TRAVIS_BUILD_DIR}/.git to operate git on"
return 0
fi

local LIBNAME=$1
local old=

echo "Remove old packages from ${LIBNAME}"

if [ -n "$TRAVIS_PULL_REQUEST_BRANCH" ] ; then
local branch="$TRAVIS_PULL_REQUEST_BRANCH"
else
local branch="$TRAVIS_BRANCH"
fi

local GLOB=${DEPLOY_TO}/${branch}_${LIBNAME}-*

# putting everything into a file, and connecting once decreases the chances
# for ssh issues, connections happen once, not every single file
local tmpfl=$(mktemp)
echo "cd ${DEPLOY_TO}" > "${tmpfl}"
for files in $(ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
"ls -lt ${GLOB}" | tail -n +100 | awk '{print $NF}')
do
ssh "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
"rm ${DEPLOY_TO}/${files}" || \
return 1
done
do
echo "rm ${files}" >> "${tmpfl}"
done
echo "bye" >> "${tmpfl}"
# if it is only cd & bye, skip it
if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then
sftp_run_cmds "${tmpfl}"
fi
rm "${tmpfl}"
# provide an index so people can find files.
ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
"ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html"
echo "ls captured"

echo "cd ${DEPLOY_TO}" > "${tmpfl}"
# prune old / removed branches, leave things are are tags/branches
for old in $(sed 's/-> .*$//' libiio_index.html | \
awk '{print $NF}' | grep -v master | sort | \
sed "s/_libiio-0.[0-9][0-9].g[a-z0-9]*-/ %% /" | \
grep "%%" | awk '{print $1}' | sort -u)
do
if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --heads origin "${old}" | wc -l)" -ne "0" ] ; then
echo "${old} is a branch"
else
if [ "$(git --git-dir "${TRAVIS_BUILD_DIR}/.git" ls-remote --tags origin "${old}" | wc -l)" -ne "0" ] ; then
echo "${old} is a tag"
else
echo "${old} can be removed"
echo "rm ${old}_${LIBNAME}-*" >> "${tmpfl}"
echo "rm ${old}_latest_${LIBNAME}-*" >> "${tmpfl}"
echo "rm ${old}_lastest_${LIBNAME}-*" >> "${tmpfl}"
fi
fi
done
# cap things at 15, so we don't exceed the time
sed -i 16q "${tmpfl}"
echo "bye" >> "${tmpfl}"
# if it is only cd & bye, skip it
if [ "$(wc -l "${tmpfl}" | awk '{print $1}')" -gt "2" ] ; then
sftp_run_cmds "${tmpfl}"
fi
rm "${tmpfl}"

#Now that we removed things, do it again
rm "${LIBNAME}_index.html"
ssh -o "StrictHostKeyChecking no" "${EXTRA_SSH}" "${SSHUSER}@${SSHHOST}" \
"ls -lt ${DEPLOY_TO}" | grep "${LIBNAME}" > "${LIBNAME}_index.html"
sftp_upload "${LIBNAME}_index.html" "${LIBNAME}_index.html"

return 0
}
Expand Down Expand Up @@ -347,6 +495,11 @@ is_centos_at_least_ver() {
version_ge "$(get_version)" "$1"
}

is_arm() {
[ "$(dpkg --print-architecture)" = "armhf" ] || return 1
test "$(dpkg --print-architecture)" = "armhf"
}

print_github_api_rate_limits() {
# See https://developer.github.com/v3/rate_limit/
# Note: Accessing this endpoint does not count against your REST API rate limit.
Expand Down
Loading

0 comments on commit fc5b671

Please sign in to comment.