Building tiflash using centos7 for official release

.gitignore

@@ -240,6 +240,8 @@ website/presentations
 # TiCS built and test files
 build_docker
 docker/builder/tics
+release-centos7/tiflash
+release-centos7/build-release
 tests/docker/data
 tests/docker/log
 tests/fullstack-test/dml/dml_gen/*

release-centos7/Dockerfile-builder

@@ -0,0 +1,41 @@
+FROM hub.pingcap.net/jenkins/centos7_golang-1.12
+
+USER root
+WORKDIR /root/
+
+ENV HOME /root/
+
+COPY prepare-environments /prepare-environments
+
+RUN yum makecache \
+  && yum install -y \
+         ccache \
+         devscripts \
+         fakeroot \
+         debhelper \
+         libtool \
+         ncurses-static \
+         readline-devel \
+         unixODBC-devel \
+         openssl-devel \
+         libicu-devel \
+         libtool-ltdl-devel \
+         python3-devel \
+  && curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain nightly  \
+  && pip3 install \
+          pybind11 \
+          pyinstaller \
+          dnspython \
+          uri \
+          requests \
+          urllib3 \
+          toml \
+          setuptools \
+  && cd /prepare-environments \
+  && ./install-openssl.sh \
+  && ./install-mysql-dev.sh \
+  && ./install-cmake.sh \
+  && ./install-gcc.sh \
+  && ./install-clang.sh \
+  && ./install-grpc.sh \
+  && yum clean all

release-centos7/Dockerfile-tiflash

@@ -0,0 +1,14 @@
+FROM centos:centos7.7.1908
+
+USER root
+WORKDIR /root/
+
+ENV HOME /root/
+ENV TZ Asia/Shanghai
+ENV LD_LIBRARY_PATH /tiflash
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+COPY tiflash /tiflash
+
+ENTRYPOINT ["/tiflash/theflash", "server"]

release-centos7/Makefile

@@ -0,0 +1,14 @@
+build_release:
+	docker run --rm -v $(realpath ..):/tics hub.pingcap.net/tiflash/tiflash-builder /tics/release-centos7/build/build-release.sh
+
+image_builder:
+	docker build -f Dockerfile-builder -t hub.pingcap.net/tiflash/tiflash-builder .
+
+push_image_builder:
+	docker push hub.pingcap.net/tiflash/tiflash-builder
+
+image_tiflash:
+	docker build -f Dockerfile-tiflash -t hub.pingcap.net/tiflash/tiflash-server .
+
+push_image_tiflash:
+	docker push hub.pingcap.net/tiflash/tiflash-server

release-centos7/build/build-cluster-manager.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ueo pipefail
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+SRCPATH=${1:-$(cd $SCRIPTPATH/../..; pwd -P)}
+
+cd $SRCPATH/cluster_manage
+./release.sh
+cp -r dist/flash_cluster_manager/ /flash_cluster_manager

release-centos7/build/build-release.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+$SCRIPTPATH/build-tiflash-proxy.sh
+$SCRIPTPATH/build-cluster-manager.sh
+$SCRIPTPATH/build-tiflash.sh

release-centos7/build/build-tiflash-proxy.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ueo pipefail
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+SRCPATH=${1:-$(cd $SCRIPTPATH/../..; pwd -P)}
+PATH=$PATH:/root/.cargo/bin
+
+cd / && mkdir libtiflash-proxy
+git clone -b tiflash-proxy-lib https://github.com/solotzg/tikv.git tiflash-proxy
+cd /tiflash-proxy && make release
+cp target/release/libtiflash_proxy.so /libtiflash-proxy
+rm -rf /tiflash-proxy
+
+# rustup self uninstall -y

release-centos7/build/build-tiflash.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -ueo pipefail
+
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+SRCPATH=${1:-$(cd $SCRIPTPATH/../..; pwd -P)}
+NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)}
+CMAKE_BUILD_TYPE="RELWITHDEBINFO"
+ENABLE_EMBEDDED_COMPILER="FALSE"
+
+install_dir="$SRCPATH/release-centos7/tiflash"
+if [ -d "$install_dir" ]; then rm -rf "$install_dir"/*; else mkdir -p "$install_dir"; fi
+
+cp -r /flash_cluster_manager "$install_dir"
+
+if [ -d "$SRCPATH/contrib/kvproto" ]; then
+  cd "$SRCPATH/contrib/kvproto"
+  rm -rf cpp/kvproto
+  ./generate_cpp.sh
+  cd -
+fi
+
+if [ -d "$SRCPATH/contrib/tipb" ]; then
+  cd "$SRCPATH/contrib/tipb"
+  rm -rf cpp/tipb
+  ./generate-cpp.sh
+  cd -
+fi
+
+rm -rf ${SRCPATH}/libs/libtiflash-proxy
+ln -s /libtiflash-proxy ${SRCPATH}/libs/libtiflash-proxy
+
+build_dir="$SRCPATH/release-centos7/build-release"
+rm -rf $build_dir && mkdir -p $build_dir && cd $build_dir
+
+cmake "$SRCPATH" \
+      -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
+      -DENABLE_EMBEDDED_COMPILER=$ENABLE_EMBEDDED_COMPILER \
+      -Wno-dev
+
+make -j $NPROC
+
+cp -f "$build_dir/dbms/src/Server/theflash" "$install_dir/theflash"
+
+ldd "$build_dir/dbms/src/Server/theflash" | grep '/' | grep '=>' | \
+  awk -F '=>' '{print $2}' | awk '{print $1}' | while read lib; do
+  cp -f "$lib" "$install_dir"
+done

release-centos7/prepare-environments/install-clang.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+
+BRANCH="tags/RELEASE_500/final"
+THREADS=$(nproc || grep -c ^processor /proc/cpuinfo)
+
+yum install -y subversion
+
+cd ~
+mkdir llvm
+cd llvm
+svn co "http://llvm.org/svn/llvm-project/llvm/${BRANCH}" llvm
+
+cd llvm/tools
+svn co "http://llvm.org/svn/llvm-project/cfe/${BRANCH}" clang
+svn co "http://llvm.org/svn/llvm-project/lld/${BRANCH}" lld
+svn co "http://llvm.org/svn/llvm-project/polly/${BRANCH}" polly
+
+cd clang/tools
+svn co "http://llvm.org/svn/llvm-project/clang-tools-extra/${BRANCH}" extra
+
+cd ../../../..
+cd llvm/projects/
+svn co "http://llvm.org/svn/llvm-project/compiler-rt/${BRANCH}" compiler-rt
+svn co "http://llvm.org/svn/llvm-project/libcxx/${BRANCH}" libcxx
+svn co "http://llvm.org/svn/llvm-project/libcxxabi/${BRANCH}" libcxxabi
+
+cd ../..
+mkdir build
+cd build/
+cmake -D CMAKE_BUILD_TYPE:STRING=Release ../llvm
+make -j $THREADS
+make install
+
+yum remove -y subversion
+rm -rf ~/llvm

release-centos7/prepare-environments/install-cmake.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+
+VERSION="3.10"
+COMPLETE_VERSION="3.10.2"
+
+cd ~
+wget "https://cmake.org/files/v${VERSION}/cmake-${COMPLETE_VERSION}-Linux-x86_64.tar.gz"
+tar zxvf cmake-${COMPLETE_VERSION}-Linux-x86_64.tar.gz
+
+rm cmake-${COMPLETE_VERSION}-Linux-x86_64.tar.gz
+yum remove cmake -y
+
+ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/ccmake /usr/bin/ccmake
+ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/cmake /usr/bin/cmake
+ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/cmake-gui /usr/bin/cmake-gui
+ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/cpack /usr/bin/cpack
+ln -sf ~/cmake-${COMPLETE_VERSION}-Linux-x86_64/bin/ctest /usr/bin/ctest
+
+cmake --version

release-centos7/prepare-environments/install-gcc.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -e
+
+VERSION="gcc-7.4.0"
+THREADS=$(nproc || grep -c ^processor /proc/cpuinfo)
+
+cd ~
+mkdir gcc
+cd gcc
+wget https://mirrors.ustc.edu.cn/gnu/gcc/${VERSION}/${VERSION}.tar.gz
+tar xf "${VERSION}.tar.gz"
+rm "${VERSION}.tar.gz"
+
+cd ${VERSION}
+./contrib/download_prerequisites
+mkdir gccbuild
+cd gccbuild
+../configure --enable-languages=c,c++ --disable-multilib
+make -j $THREADS
+make install
+ln -s /usr/local/bin/gcc /usr/local/bin/cc
+
+yum remove -y gcc
+
+echo "/usr/local/lib64" | tee /etc/ld.so.conf.d/10_local-lib64.conf
+gcc --version
+
+rm -rf ~/gcc

release-centos7/prepare-environments/install-grpc.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -e
+
+VERSION="v1.14.2"
+THREADS=$(nproc || grep -c ^processor /proc/cpuinfo)
+
+cd ~
+git clone https://github.com/grpc/grpc.git
+cd grpc
+git checkout ${VERSION}
+git submodule update --init
+
+cd ~/grpc
+mkdir .build
+cd .build
+cmake .. -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
+make install -j $THREADS
+
+
+cd ~/grpc
+rm -rf .build
+mkdir .build
+cd .build
+cmake .. -DgRPC_INSTALL=ON -DgRPC_BUILD_TESTS=OFF -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DCMAKE_BUILD_TYPE=Release
+make install -j $THREADS
+
+protoc --version
+
+rm -rf ~/grpc

release-centos7/prepare-environments/install-mysql-dev.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd ~
+mkdir mysql
+cd mysql
+
+wget http://dev.mysql.com/get/mysql57-community-release-el7-9.noarch.rpm
+yum -y --nogpgcheck install mysql57-community-release-el7-9.noarch.rpm
+
+yum -y install mysql-community-devel
+ln -s /usr/lib64/mysql/libmysqlclient.a /usr/lib64/libmysqlclient.a
+
+yum clean all
+rm -rf ~/mysql

release-centos7/prepare-environments/install-openssl.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+curl -s https://packagecloud.io/install/repositories/altinity/clickhouse/script.rpm.sh | bash
+yum install -y openssl-altinity-devel openssl-altinity-static
+yum remove -y openssl-static openssl-devel
+
+# openssl-static
+ln -sf /opt/openssl-1.1.0f/lib/libcrypto.a /usr/lib64/libcrypto.a
+ln -sf /opt/openssl-1.1.0f/lib/libssl.a /usr/lib64/libssl.a
+
+# openssl-devel
+ln -sf /opt/openssl-1.1.0f/include/openssl /usr/include/openssl
+ln -sf /opt/openssl-1.1.0f/lib/libcrypto.so /usr/lib64/libcrypto.so
+ln -sf /opt/openssl-1.1.0f/lib/libssl.so /usr/lib64/libssl.so
+ln -sf /opt/openssl-1.1.0f/lib/pkgconfig/libcrypto.pc /usr/lib64/pkgconfig/libcrypto.pc
+ln -sf /opt/openssl-1.1.0f/lib/pkgconfig/libssl.pc /usr/lib64/pkgconfig/libssl.pc
+ln -sf /opt/openssl-1.1.0f/lib/pkgconfig/openssl.pc /usr/lib64/pkgconfig/openssl.pc