diff --git a/.circleci/config.yml b/.circleci/config.yml index 19548654e67..30f93fa1035 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -329,6 +329,55 @@ jobs: - artifacts - store_artifacts: path: artifacts/ + test-linux-packages: + executor: terraform/default + steps: + - attach_workspace: + at: /tmp/workspace + - checkout + - add_ssh_keys: + fingerprints: + - 3a:d1:7a:b7:57:d7:85:0b:76:79:85:51:38:f3:e4:67 + - terraform/validate: + path: scripts/ci/ + - run: + name: Terraform apply + command: | + set -x + export DEBNAME="$(find /tmp/workspace/artifacts/influxdb3*amd64.deb)" + terraform -chdir=scripts/ci init -input=false + AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \ + -chdir=scripts/ci \ + apply \ + -auto-approve \ + -var package_path=${DEBNAME} \ + -var instance_name=circleci-terraform-${CIRCLE_SHA1} \ + -var additional_files_dir=${PWD}/scripts/ci/tests/ + - run: + name: Install deb + command: | + set -x + export ec2_ip=$(terraform -chdir=scripts/ci output -raw test_node_ssh) + ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip \<< EOF + sudo apt-get update && sudo apt-get install -y /home/ubuntu/influxdb.deb + EOF + - run: + name: Run tests + command: | + set -x + export ec2_ip=$(terraform -chdir=scripts/ci output -raw test_node_ssh) + files=$(ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip 'find /home/ubuntu/files/ -maxdepth 1 -mindepth 1 | sort') + for file in $files; do + ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip "sudo $file" + done + - run: + name: Terraform destroy + when: always + command: | + AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \ + -chdir=scripts/ci \ + destroy \ + -auto-approve sign-packages: circleci_ip_ranges: true docker: @@ -479,11 +528,16 @@ workflows: <<: *any_filter requires: - build-release + - test-linux-packages: + <<: *nofork_filter + requires: + - build-packages - sign-packages: # <<: *release_filter <<: *any_filter requires: - build-packages + - test-linux-packages - test - publish-packages: # <<: *release_filter diff --git a/.circleci/scripts/package-validation/debian b/.circleci/scripts/package-validation/debian new file mode 100755 index 00000000000..f3e16edba74 --- /dev/null +++ b/.circleci/scripts/package-validation/debian @@ -0,0 +1,8 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +"${path}/validate" deb "${1}" diff --git a/.circleci/scripts/package-validation/redhat b/.circleci/scripts/package-validation/redhat new file mode 100755 index 00000000000..db899a9bf66 --- /dev/null +++ b/.circleci/scripts/package-validation/redhat @@ -0,0 +1,97 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +# $1 -> architecture +# $2 -> package path +case ${1} in + x86_64) arch=x86_64 ;; + aarch64) arch=arm64 ;; +esac + +package="$(realpath "${2}")" + +path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" + +terraform_init() { + pushd "${path}/tf" &>/dev/null + + # Unfortunately, CircleCI doesn't offer any RPM based machine images. + # This is required to test the functionality of the systemd services. + # (systemd doesn't run within docker containers). This will spawn a + # Amazon Linux instance in AWS. + terraform init + terraform apply \ + -auto-approve \ + -var "architecture=${1}" \ + -var "package_path=${2}" \ + -var "identifier=${CIRCLE_JOB}" + + popd &>/dev/null +} + +terraform_free() { + pushd "${path}/tf" &>/dev/null + + terraform destroy \ + -auto-approve \ + -var "architecture=${1}" \ + -var "package_path=${2}" \ + -var "identifier=${CIRCLE_JOB}" + + popd &>/dev/null +} + +terraform_ip() { + pushd "${path}/tf" &>/dev/null + + terraform output -raw node_ssh + + popd &>/dev/null +} + + +# This ensures that the associated resources within AWS are released +# upon exit or when encountering an error. This is setup before the +# call to "terraform apply" so even partially initialized resources +# are released. +# shellcheck disable=SC2064 +trap "terraform_free \"${arch}\" \"${package}\"" \ + SIGINT \ + SIGTERM \ + ERR \ + EXIT + +function terraform_setup() +{ + # TODO(bnpfeife): remove this once the executor is updated. + # + # Unfortunately, terraform provided by the CircleCI executor is *terribly* + # out of date. Most Linux distributions are disabling "ssh-rsa" public key + # algorithms which this uses to remote into the ec2 instance . This + # installs the latest version of terraform. + # + # Addendum: the "terraform_version" CircleCI option is broken! +sudo tee /etc/apt/sources.list.d/hashicorp.list </dev/null || true +deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main +EOF + + curl -fL https://apt.releases.hashicorp.com/gpg | gpg --dearmor | \ + sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg >/dev/null + + export DEBIAN_FRONTEND=noninteractive + sudo -E apt-get update + sudo -E apt-get install --yes terraform +} + +terraform_setup + +terraform_init "${arch}" "${package}" + +printf 'Setup complete! Testing %s... (this takes several minutes!)' "${1}" + +# Since terraform *just* created this instance, the host key is not +# known. Therefore, we'll disable StrictHostKeyChecking so ssh does +# not wait for user input. +ssh -o 'StrictHostKeyChecking=no' "ec2-user@$(terraform_ip)" 'sudo ./validate rpm ./influxdb3.rpm' diff --git a/.circleci/scripts/package-validation/tf/main.tf b/.circleci/scripts/package-validation/tf/main.tf new file mode 100644 index 00000000000..fa17c14460b --- /dev/null +++ b/.circleci/scripts/package-validation/tf/main.tf @@ -0,0 +1,114 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 2.70" + } + } +} + +variable "architecture" { + type = string +} + +variable "identifier" { + type = string +} + +variable "package_path" { + type = string +} + +provider "aws" { + region = "us-east-1" +} + +data "aws_ami" "test_ami" { + most_recent = true + + filter { + name = "name" + values = ["al20*-ami-20*"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } + filter { + name = "architecture" + values = [var.architecture] + } + + owners = ["137112412989"] +} + +resource "aws_security_group" "influxdb_test_package_sg" { + ingress { + description = "Allow ssh connection" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + description = "Allow all outgoing" + from_port = 0 + to_port = 0 + protocol = "all" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "test_instance" { + count = 1 + ami = data.aws_ami.test_ami.id + instance_type = var.architecture == "x86_64" ? "t2.micro" : "c6g.medium" + key_name = "circleci-oss-test" + vpc_security_group_ids = [aws_security_group.influxdb_test_package_sg.id] + + tags = { + Name = format("circleci_%s_test_%s", var.identifier, var.architecture) + } + + provisioner "file" { + source = var.package_path + destination = "/home/ec2-user/influxdb3.rpm" + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } + + provisioner "file" { + source = "../validate" + destination = "/home/ec2-user/validate" + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } + + provisioner "remote-exec" { + inline = [ + "chmod +x /home/ec2-user/validate", + ] + + connection { + type = "ssh" + user = "ec2-user" + host = self.public_dns + agent = true + } + } +} + +output "node_ssh" { + value = aws_instance.test_instance.0.public_dns +} diff --git a/.circleci/scripts/package-validation/validate b/.circleci/scripts/package-validation/validate new file mode 100755 index 00000000000..01d884cbc3d --- /dev/null +++ b/.circleci/scripts/package-validation/validate @@ -0,0 +1,118 @@ +#!/bin/bash +set -o errexit \ + -o nounset \ + -o pipefail + +usage() { + cat <<'EOF' +usage: validate [type] [path] + +Program: + This application performs sanity checks on the provided InfluxDB + package. InfluxDB should *not* be installed on the system before + running this application. This validates new installations and + performs specific checks relevant only to InfluxDB. + +Options: + type Must be "deb" or "rpm". This option instructs the + application to use the package manager associated + with "type". + path Path to InfluxDB package to validate. +EOF +} + +if [[ ! "${1:-}" ]] || [[ ! "${2:-}" ]] +then + (usage) && exit 1 +fi +PACKAGE_TYPE="${1}" +PACKAGE_PATH="${2}" + +install_deb() { + # When installing the package, ensure that the latest repository listings + # are available. This might be required so that all dependencies resolve. + # Since this needs to be run by CI, we supply "noninteractive" and "-y" + # so no prompts stall the pipeline. + export DEBIAN_FRONTEND=noninteractive + apt-get update + # "apt-get install" should be used instead of "dpkg -i", because "dpkg" + # does not resolve dependencies. "apt-get" requires that the package + # path looks like a path (either fullpath or prefixed with "./"). + apt-get install -y binutils "$(realpath "${PACKAGE_PATH}")" +} + +install_rpm() { + # see "install_deb" for "update" + yum update -y + yum install -y binutils + yum localinstall -y "$(realpath "${PACKAGE_PATH}")" +} + +case ${PACKAGE_TYPE} +in + deb) + (install_deb) + ;; + rpm) + (install_rpm) + ;; +esac + +if ! which influxdb3 &>/dev/null +then + printf 'ERROR: Failed to locate influxdb3 executable!\n' >&2 + exit 2 +fi + +NEEDED="$(readelf -d "$(which influxdb3)" | (grep 'NEEDED' || true ))" + +# shellcheck disable=SC2181 +if [[ ${?} -ne 0 ]] +then + cat <<'EOF' +ERROR: readelf could not analyze the influxdb3 executable! This + might be the consequence of installing a package built + for another platform OR invalid compiler/linker flags. +EOF + exit 2 +fi + +if [[ "${NEEDED:-}" ]] +then + cat <<'EOF' +ERROR: influxdb3 not statically linked! This may prevent all + platforms from running influxdb3 without installing + separate dependencies. +EOF + exit 2 +fi + +# PIE="$(readelf -d "$(which influxdb3)" | (grep 'Flags: PIE' || true))" +# if [[ ! "${PIE:-}" ]] +# then +# printf 'ERROR: influxdb3 not linked with "-fPIE"!\n' +# exit 2 +# fi + +# We don't have a way to activate it via systemctl for now +# if ! systemctl is-active influxdb3 &>/dev/null +# then +# systemctl start influxdb3 +# fi + + +# for i in 0..2 +# do +# if ! systemctl is-active influxdb3 &>/dev/null +# then +# printf 'ERROR: influxdb3 service failed to start!\n' +# exit 2 +# fi +# # Sometimes the service fails several seconds or minutes after +# # starting. This failure may not propagate to the original +# # "systemctl start " command. Therefore, we'll +# # poll the service several times before exiting. +# sleep 30 +# done + +printf 'Finished validating influxdb3!\n'