diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 000000000..ec5836731 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,15 @@ +task: + name: + only_if: $CIRRUS_BRANCH == 'Test' + container: + image: ubuntu:latest + memory: 10G + cpu: 4 + kvm: true + prepare_environment_script: + - apt update + - apt install -y qemu git curl + fetch_base_libraries_script: + - ./Tests/Pipeline/Prepare.sh --dry-run + build_and_test_script: + - ./Base/Tests/Pipeline/Create.sh diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml new file mode 100644 index 000000000..6c15c3705 --- /dev/null +++ b/.github/workflows/commit.yml @@ -0,0 +1,19 @@ +name: Verify commit with the simple test suite + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + cxx: [g++, clang++] + + steps: + - uses: actions/checkout@v2 + - name: prepare + run: ./tests/pipeline/prepare.sh + - name: build and test + env: + CXX: ${{ matrix.cxx }} + run: ./tests/pipeline/build.sh diff --git a/.github/workflows/regression.yaml b/.github/workflows/regression.yaml new file mode 100644 index 000000000..bec2b62d7 --- /dev/null +++ b/.github/workflows/regression.yaml @@ -0,0 +1,27 @@ +name: Check if the new PR is working well + +on: + pull_request: + branchs: + - master + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + cxx: [g++] + + steps: + - uses: actions/checkout@v2 + - name: update + run: sudo apt update + - name: prepare + run: while ! sudo apt install qemu; do if ! which qemu-system-x86_64; then sleep 1; else break; fi; done + - name: fetch base libraries + run: ./tests/pipeline/prepare.sh --dry-run + - name: build and test + env: + CXX: ${{ matrix.cxx }} + MEGA: ${{ secrets.MEGA }} + run: ./Base/Tests/Pipeline/Create.sh diff --git a/.gitignore b/.gitignore index 240c473e4..6896427b7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,317 @@ po/insert-header.sin po/quot.sed po/remove-potcdate.sin po/POTFILES + +###Python### + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so +###PyCharm### + +# PyCharm +# http://www.jetbrains.com/pycharm/webhelp/project.html +.idea +.iml + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + + +###Java### + +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + + +###C### +# Object files +*.o +*.ko +*.obj +*.elf + +# Linker output +*.ilk +*.map +*.exp + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb + +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf + + +###C++### + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + + +###vim### + +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist +*~ + + +###gcov### + +# gcc coverage testing tool files + +*.gcno +*.gcda +*.gcov + + +###Gcov### + +# gcc coverage testing tool files + +*.gcno +*.gcda +*.gcov + + +###SublimeText### + +# cache files for sublime text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# workspace files are user-specific +*.sublime-workspace + +# project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using SublimeText +*.sublime-project + +# sftp configuration file +sftp-config.json + + +###D### + +# Compiled Object files +*.o +*.obj + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Compiled Static libraries +*.a +*.lib + +# Executables +*.exe + +# DUB +.dub +docs.json +__dummy.html +docs/ + +# Code coverage +*.lst + +# Tools/Builder +output/.vscode/ +Eevee.xcodeproj/ +.tags_sorted_by_file +.tags +.DS_Store +tags + +# vim +*.config +*.vim +.lvimrc + +# Xcode +*.xcworkspace diff --git a/.requirement.d/debian b/.requirement.d/debian new file mode 100644 index 000000000..8d1f98066 --- /dev/null +++ b/.requirement.d/debian @@ -0,0 +1,16 @@ +gdb +lcov +make +pxelinux +syslinux-common +cmake +atftpd +sshpass +megatools +libx11-dev +libibus-1.0-dev +libgtk-3-dev +genisoimage +squashfs-tools +nfs-ganesha +nfs-kernel-server diff --git a/.travis.yml b/.travis.yml index 293bc8f98..529259aa2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,21 @@ +dist: bionic language: cpp +branches: + only: + - Tests + - build-ui-automate-simulator before_install: - - sudo apt-get install -y libibus-1.0-dev libgtk-3-dev + - sudo apt install -y qemu-kvm qemu-utils addons: apt: update: true -script: mkdir build && cd build && cmake .. && make +install: + - echo "is it suppoted kvm? $(egrep -c '(vmx|svm)' /proc/cpuinfo)" + - sudo lsmod | grep kvm + - if [ -f /dev/kvm ]; then echo "/dev/kvm exists"; fi + +script: + - ./tests/pipeline/prepare.sh --dry-run + - ./Base/Tests/Pipeline/Create.sh diff --git a/tests/pipeline/build.sh b/tests/pipeline/build.sh new file mode 100755 index 000000000..c0400efdd --- /dev/null +++ b/tests/pipeline/build.sh @@ -0,0 +1,430 @@ +#!/bin/bash +# - File: build.sh +# - Description: This bash script will be run right after prepare.sh and it will +# be used to build based on current branch you want to Tests + +PIPELINE="$(dirname "$0" )" +source $PIPELINE/libraries/Logcat.sh +source $PIPELINE/libraries/Package.sh + +SCRIPT="$(basename "$0")" + +if [[ $# -gt 0 ]]; then + MODE=$2 +else + MODE=0 +fi + +if [[ $# -gt 2 ]]; then + NODE=$3 +fi + +info "You have run on machine ${machine} script ${SCRIPT}" +info "Your current dir now is $(pwd)" + +if [ $(which git) ]; then + # @NOTE: jump to branch's test suite and perform build + ROOT="$(git rev-parse --show-toplevel)" + BUILDER=$ROOT/Base/Tools/Builder/build + + if ! $BUILDER --root $ROOT --debug 1 --rebuild 0 --mode $MODE; then + exit -1 + fi + + if [[ $MODE -eq 1 ]] && [[ ${#NODE} -gt 0 ]]; then + FORCE=0 + OTYPE="iso" + IDX=0 + + # @NOTE: fetch the latest release of supported distros so we can + # use them to verify our ibus-unikey black build before we deliver + # this to the marketplace. + + mkdir -p $ROOT/pxeboot/{pxelinux.cfg,rootfs} + + if ! cp /usr/lib/PXELINUX/pxelinux.0 $ROOT/pxeboot; then + error "can't copy pxelinux.0 to $ROOT/pxeboot" + elif ! cp /usr/lib/syslinux/modules/bios/{ldlinux.c32,libcom32.c32,libutil.c32,vesamenu.c32} $ROOT/pxeboot; then + error "can't copy syslinux to $ROOT/pxeboot" + fi + + if [ ! -f /.dockerenv ]; then + $SU systemctl stop nfs-kernel-server + $SU systemctl stop nfs-ganesha + $SU systemctl stop dnsmasq + $SU systemctl stop atftpd + fi + + if [ ! -f $ROOT/pxeboot/nfs-ganesha.config ]; then + if [ -f /.dockerenv ] || [[ $FORCE -eq 1 ]]; then + cat > $ROOT/pxeboot/nfs-ganesha.config << EOF +EXPORT_DEFAULTS +{ + SecType = sys,krb5,krb5i,krb5p; + + # Restrict all exports to NFS v4 unless otherwise specified + Protocols = 3; +} + +LOG { + # Default log level for all components + Default_Log_Level = FULL_DEBUG; + + # Where to log + Facility { + name = FILE; + destination = "$ROOT/pxeboot/nfs-ganesha.log"; + enable = active; + } +} +EOF + fi + fi + + for DISTRO in $(ls -1c $ROOT/tests/pipeline/environments); do + IDX=$((IDX+1)) + + if [ $OTYPE = 'iso' ]; then + BOOT="livecd/casper" + INITRD="initrd=rootfs/$DISTRO/$BOOT/initrd" + else + BOOT="boot" + fi + + mkdir -p $ROOT/pxeboot/rootfs/$DISTRO + + if $ROOT/tools/utilities/generate-customized-livecd-image.sh \ + --input-type iso \ + --output-type $OTYPE \ + --no-image \ + --output $ROOT/pxeboot/rootfs/$DISTRO \ + --env $ROOT/tests/pipeline/environments/$DISTRO; then + info "successful generate $DISTRO's ISO image to start a new simulator" + else + error "can't build $DISTRO's ISO image" + fi + + if [ $OTYPE = 'iso' ]; then + LIVECD="$ROOT/pxeboot/rootfs/$DISTRO/livecd" + MODE="ro,sync,no_wdelay,insecure_locks,no_root_squash,insecure" + + if [ -f /.dockerenv ] || [[ $FORCE -eq 1 ]]; then + cat >> $ROOT/pxeboot/nfs-ganesha.config << EOF +EXPORT +{ + Export_Id = $IDX; + Path = "$LIVECD"; + Pseudo = "/"; + Tag = exp$IDX; + + # Override the default set in EXPORT_DEFAULTS + Protocols = 3; + MaxRead = 65536; + MaxWrite = 65536; + PrefRead = 65536; + PrefWrite = 65536; + + Transports = "UDP", "TCP"; + + # All clients for which there is no CLIENT block that specifies a + # different Access_Type will have RW access (this would be an unusual + # specification in the real world since barring a firewall, this + # export is world readable and writeable). + Access_Type = RW; + + # FSAL block + # + # This is required to indicate which Ganesha File System Abstraction + # Layer (FSAL) will be used for this export. + # + # The only option available for all FSALs is: + # + # Name (required) The name of the FSAL + # + # Some FSALs have additional options, see individual FSAL documentation. + + FSAL + { + Name = VFS; + } + + # CLIENT blocks + # + # An export may optionally have one or more CLIENT blocks. These blocks + # specify export options for a restricted set of clients. The export + # permission options specified in the EXPORT block will apply to any + # client for which there is no applicable CLIENT block. + # + # All export permissions options are available, as well as the + # following: + # + # Clients (required) The list of clients these export permissions + # apply to. Clients may be specified by hostname, + # ip address, netgroup, CIDR network address, + # host name wild card, or simply "*" to apply to + # all clients. + + CLIENT + { + Clients = *; + Squash = No_Root_Squash; + Access_Type = RW; + } +} +EOF + elif echo "$LIVECD 192.168.100.0/24($MODE)" | $SU tee -a /etc/exports &> /dev/null; then + if ! $SU systemctl restart nfs-kernel-server; then + error """can't restart nfs-kernel-server, here is nfs-kernel-server's status: +------------------------------------------------------------------------------- + +$(systemctl status nfs-kernel-server -l) +------------------------------------------------------------------------------- + +list of used ports: +------------------------------------------------------------------------------- + +$(netstat -tunlap) +""" + fi + else + error "can't edit file /etc/exports" + fi + else + LIVECD="$ROOT/pxeboot/rootfs/$DISTRO" + fi + + if [ ! -f $ROOT/pxeboot/rootfs/$DISTRO/$BOOT/vmlinuz ]; then + error "can't generate vmlinuz" + elif [ ! -f $ROOT/pxeboot/rootfs/$DISTRO/$BOOT/initrd ]; then + error "can't generate initrd" + fi + + if [ ! -f $ROOT/pxeboot/pxelinux.cfg/default ]; then + source $ROOT/tests/pipeline/environments/$DISTRO + + cat > $ROOT/pxeboot/pxelinux.cfg/default << EOF +default install +prompt 1 +timeout 1 + +label install + kernel rootfs/$DISTRO/$BOOT/vmlinuz + append $INITRD netboot=nfs nfsroot=192.168.100.1:$LIVECD ip=dhcp rw +EOF + else + cat > $ROOT/pxeboot/pxelinux.cfg/default.${IDX} << EOF +default install +prompt 1 +timeout 1 + +label install + kernel rootfs/$DISTRO/$BOOT/vmlinuz + append $INITRD netboot=nfs nfsroot=192.168.100.1:$LIVECD ip=dhcp rw +EOF + fi + done + + # @NOTE: it seems the developer would like to test with a + # virtual machine, so we should generate file .environment + # here to contain approviated variables to control steps to + # build and test Unikey with our LiveCD collection + + cat > $ROOT/.environment << EOF +source $ROOT/Base/Tests/Pipeline/Libraries/Package.sh +source $ROOT/Base/Tests/Pipeline/Libraries/Logcat.sh +source $ROOT/Base/Tests/Pipeline/Libraries/QEmu.sh + +export VNC="rootroot:$NODE" +export RAM="4G" +export KER_FILENAME="" +export RAM_FILENAME="" +export IMG_FILENAME="" +export TIMEOUT="timeout 60" + +if lsmod | grep 'kvm-intel\\|kvm-amd' &> /dev/null; then + export CPU="host" +fi + +function snift() { + # @NOTE: override this function to prevent starting tcppxeboot since we don't + # need this feature + + return 0 +} + +function start_dhcpd() { + IP=\$(get_ip_interface \$1) + MASK=\$(get_netmask_interface \$1) + + if ! which dnsmasq >& /dev/null; then + return 1 + else + info "start dnsmasq to control pxeboot" + fi + + if [ -f /.dockerenv ] || [ $FORCE = 1 ]; then + screen -ls "nfsd.pid" | grep -E '\\s+[0-9]+\\.' | awk -F ' ' '{print \$1}' | while read s; do screen -XS \$s quit; done + + screen -S "nfsd.pid" -dm \ + $SU ganesha.nfsd -F -p $ROOT/pxeboot/nfs-ganesha.pid \ + -f $ROOT/pxeboot/nfs-ganesha.config \ + -N NIV_FULL_DEBUG + fi + + $SU chmod +x $ROOT/pxeboot + + screen -ls "dhcpd.pid" | grep -E '\\s+[0-9]+\\.' | awk -F ' ' '{print \$1}' | while read s; do screen -XS \$s quit; done + screen -ls "atftp.pid" | grep -E '\\s+[0-9]+\\.' | awk -F ' ' '{print \$1}' | while read s; do screen -XS \$s quit; done + + screen -S "dhcpd.pid" -dm \ + $SU dnsmasq --port 0 \ + --dhcp-boot=pxelinux.0 \ + --dhcp-range=192.168.100.2,192.168.100.2 \ + --dhcp-option=3,0.0.0.0 \ + --dhcp-option=6,0.0.0.0 --dhcp-script \$2 + + echo """ +logfile $ROOT/pxeboot/atftp-console.log +logfile flush 1 +log on +logtstamp after 1 +logtstamp string "[ %t: %Y-%m-%d %c:%s ]\\012" +logtstamp on +"""> $ROOT/vms/atftp.conf + screen -S "atftp.pid" -c $ROOT/vms/atftp.conf -dmL \ + $SU atftpd --daemon --no-fork $ROOT/pxeboot \ + --user root --group root \ + --trace --verbose 7 \ + --logfile $ROOT/pxeboot/atftp-syslog.log + + $SU iptables -A INPUT -p tcp -m tcp --dport 111 -j ACCEPT + $SU iptables -A INPUT -p tcp -m tcp --dport 2049 -j ACCEPT + $SU iptables -A INPUT -p tcp -m tcp --dport 20048 -j ACCEPT + $SU iptables -A INPUT -p udp -m udp --dport 111 -j ACCEPT + $SU iptables -A INPUT -p udp -m udp --dport 2049 -j ACCEPT + $SU iptables -A INPUT -p udp -m udp --dport 20048 -j ACCEPT +} + +function stop_dhcpd() { + info "stop dnsmasq" + + screen -ls "nfsd.pid" | grep -E '\\s+[0-9]+\\.' | awk -F ' ' '{print \$1}' | while read s; do screen -XS \$s quit; done + screen -ls "dhcpd.pid" | grep -E '\\s+[0-9]+\\.' | awk -F ' ' '{print \$1}' | while read s; do screen -XS \$s quit; done + screen -ls "atftp.pid" | grep -E '\\s+[0-9]+\\.' | awk -F ' ' '{print \$1}' | while read s; do screen -XS \$s quit; done +} +EOF + + # @NOTE: well now we define how to test with our virtual + # machines. According the instruction, we should define script + # Test.sh to control how to test automatically + + cat > $ROOT/tests/pipeline/test.sh << EOF +#!/bin/bash +IDX=0 + +function clean() { + if [ -f $ROOT/pxeboot/nfs-ganesha.log ]; then + info """here is the log from nfs-ganesha +------------------------------------------------------------------------------- + +\$(cat $ROOT/pxeboot/nfs-ganesha.log) +""" + fi + + if [ -f $ROOT/pxeboot/atftp-syslog.log ]; then + info """here is the log from atftpd +------------------------------------------------------------------------------- + +\$(cat $ROOT/pxeboot/atftp-syslog.log) +""" + fi + + if [ -f $ROOT/pxeboot/atftp-console.log ]; then + info """here is the log from atftpd console +------------------------------------------------------------------------------- + +\$(cat $ROOT/pxeboot/atftp-console.log) +""" + fi +} + +trap clean EXIT +source $ROOT/Base/Tests/Pipeline/Libraries/Logcat.sh + +PASSED=1 + +if ! ps -aux | grep dnsmasq | grep -v grep &> /dev/null; then + error "there are problems during starting dnsmasq" +fi + +if ! ps -aux | grep atftpd | grep -v grep &> /dev/null; then + error "there are problems during starting atftpd" +fi + +if ! ps -aux | grep qemu | grep -v grep &> /dev/null; then + error "there are problems during starting qemu" +fi + +if [ -f /.dockerenv ] || [[ $FORCE -eq 1 ]]; then + if ! ps -aux | grep nfs | grep -v grep &> /dev/null; then + error "there are problems during starting nfs-ganesha" + fi +fi + +if which qemu-system-x86_64 &> /dev/null; then + if [[ ${#NGROK} -gt 0 ]]; then + info "we're opening a tunnel \$($ROOT/Base/Tools/Utilities/ngrok.sh ngrok --token $NGROK --port 5901), you can use vnc-client to connect to it" + fi +fi + +info "Begin to test Ibus-Unikey" + +for DISTRO in \$(ls -1c $ROOT/tests/pipeline/environments); do + source $ROOT/tests/pipeline/environments/\$DISTRO + + SSHOPTs="-o StrictHostKeyChecking=no" + URL="\$(username)@192.168.100.2" + + IDX=\$((IDX+1)) + AVAILABLE=0 + + for I in {0..50}; do + if ! sshpass -p "\$(password)" ssh \$SSHOPTs \$URL -tt exit 0 &> /dev/null; then + sleep 10 + else + AVAILABLE=1 + + info "machine \$DISTRO is available now, going to test our test suites" + break + fi + done + + if [[ \$PASSED -eq 0 ]]; then + warning "there some issue with distro \$DISTRO" + fi + + if [[ \$AVAILABLE -eq 0 ]]; then + error "it seems the VM take so much time to become available" + elif [ -f $ROOT/pxeboot/pxelinux.cfg/default.\${IDX} ]; then + mv $ROOT/pxeboot/pxelinux.cfg/default.\${IDX} $ROOT/pxeboot/pxelinux.cfg/default + else + break + fi +done + +if [[ \$PASSED -eq 1 ]]; then + info "Finish testing Ibus-Unikey" +else + exit -1 +fi +EOF + $SU chmod +x $ROOT/Tests/Pipeline/Test.sh + $SU chmod -R 755 $ROOT/pxeboot + + info "going to test with virtual machine" + fi +else + error "Please install git first" +fi + +info "Congratulation, you have passed ${SCRIPT}" diff --git a/tests/pipeline/environments/ubuntu b/tests/pipeline/environments/ubuntu new file mode 100644 index 000000000..9f90a3d4e --- /dev/null +++ b/tests/pipeline/environments/ubuntu @@ -0,0 +1,146 @@ +#!/bin/bash + +if [[ $(ls -id / | awk '{ print $1 }') -eq 2 ]]; then + ROOT="$(git rev-parse --show-toplevel)" + BASE="$ROOT/Base" + + source "$BASE/Tests/Pipeline/Libraries/Logcat.sh" + source "$BASE/Tests/Pipeline/Libraries/Package.sh" +fi + +function interface() { + echo "ens4" +} + +function username() { + echo "root" +} + +function password() { + echo "rootroot" +} + +function version() { + URL=$(curl -ksS https://ubuntu.com/download/desktop | grep "/download/desktop/thank-you" | awk '{ split($0,a,"href=\""); split(a[2],b,"\" onclick="); print b[1] }') + + echo "$(echo $URL | awk '{ split($0,a,"version="); split(a[2],b,"&"); print b[1] }')" +} + +function fetch() { + if [ "$1" = 'iso' ]; then + URL=$(curl -ksS https://ubuntu.com/download/desktop | grep "/download/desktop/thank-you" | awk '{ split($0,a,"href=\""); split(a[2],b,"\" onclick="); print b[1] }') + VERSION=$(echo $URL | awk '{ split($0,a,"version="); split(a[2],b,"&"); print b[1] }') + ARCH=$(echo $URL | awk '{ split($0,a,"architecture="); print a[2] }') + + if [[ ${#URL} -gt 0 ]]; then + curl -ksSo ${2}/${3} https://releases.ubuntu.com/$VERSION/ubuntu-${VERSION}-desktop-${ARCH}.iso + else + return -1 + fi + fi +} + +function extract() { + CHECK=$6 + OLDCD=$2/livecd + MIRROR=$5 + ISQUASHFS=$2/squashfs + WORKSPACE=$3 + + if [ "$1" = 'iso' ]; then + info "synchronize $OLDCD to $WORKSPACE" + + if [ -f $MIRROR/ubuntu.squashfs ]; then + if ! mv $MIRROR/ubuntu.squashfs $MIRROR/filesystem.squashfs; then + error "can't rename ${NAME}.squashfs" + fi + elif [[ $CHECK -eq 1 ]]; then + error "can't find $MIRROT/Unbuntu.squashfs as expected" + fi + + if ! rsync --exclude=/livecd/casper/filesystem.squashfs -a $OLDCD $WORKSPACE &> /dev/null; then + error "can't sync $OLDCD to $WORKSPACE" + fi + + return 0 + fi +} + +function configure() { + # @NOTE: this function should be run inside the rootfs when we start chroot + + if [ "$1" = 'iso' ]; then + # @NOTE: change password root to rootroot as the default password + # so we can ssh to this machine and perform testing on it + if ! echo 'root:rootroot' | chpasswd; then + return + fi + + if ! sudo apt-get update &> /dev/null; then + return -1 + elif ! sudo apt-get install -y openssh-server &> /dev/null; then + return -1 + elif ! sudo apt-get clean &> /dev/null; then + return -1 + elif ! sudo systemctl enable ssh &> /dev/null; then + return -1 + fi + + # @NOTE: config nameserver to make nslookup, dig work + echo """ +nameserver 8.8.8.8 +nameserver 8.8.4.4 +""" >> /etc/resolv.conf + + # @NOTE: allow root access + echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + fi +} + +function finish() { + OSQUASHFS="$4" + WORKSPACE="$3" + LIVECD="$2" + + if [ "$1" = 'iso' ]; then + mkdir -p $LIVECD/casper + + info "generate $LIVECD/casper/filesystem.manifest and $LIVECD/casper/filesystem.manifest-desktop" + if [ -f $LIVECD/casper/filesystem.manifest ]; then + $SU chmod +w $LIVECD/casper/filesystem.manifest + else + touch $LIVECD/casper/filesystem.manifest + fi + + if $SU chroot $WORKSPACE/squashfs dpkg-query -W --showformat='${Package} ${Version}\n' | $SU tee $LIVECD/casper/filesystem.manifest &> /dev/null; then + $SU cp $LIVECD/casper/filesystem.manifest $LIVECD/casper/filesystem.manifest-desktop + else + error "can't generate $LIVECD/casper/filesystem.manifest and $LIVECD/casper/filesystem.manifest-desktop" + fi + + info "compress $LIVECD/casper/filesystem.squashfs" + if $SU mksquashfs $WORKSPACE/squashfs $LIVECD/casper/filesystem.squashfs; then + $SU bash -c "cd $LIVECD && find . -type f -exec md5sum {} +" | $SU tee $LIVECD/md5sum.txt &> /dev/null + else + error "can't fully compress filesystem.squashfs" + fi + elif [ -d $OSQUASHFS/casper ]; then + info "copy vmlinuz and initrd from $LIVECD/casper to $OSQUASHFS/casper" + + if ! $SU rsync -zh $LIVECD/casper/vmlinuz $OSQUASHFS/casper; then + error "can't copy $LIVECD/casper/vmlinuz to $OSQUASHFS/casper" + elif ! $SU rsync -zh $LIVECD/casper/initrd $OSQUASHFS/casper; then + error "can't copy $LIVECD/casper/initrd to $OSQUASHFS/casper" + fi + else + mkdir -p $OSQUASHFS/boot + + info "copy vmlinuz and initrd from $LIVECD/casper to $OSQUASHFS/boot" + + if ! $SU rsync -zh $LIVECD/casper/vmlinuz $OSQUASHFS/boot; then + error "can't copy $LIVECD/casper/vmlinuz to $OSQUASHFS/boot" + elif ! $SU rsync -zh $LIVECD/casper/initrd $OSQUASHFS/boot; then + error "can't copy $LIVECD/casper/initrd to $OSQUASHFS/boot" + fi + fi +} diff --git a/tests/pipeline/prepare.sh b/tests/pipeline/prepare.sh new file mode 100755 index 000000000..bc23899c0 --- /dev/null +++ b/tests/pipeline/prepare.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# File: Prepare.sh +# Description: this file should be run first and it will fetch the latest +# LibBase before use this library to build a new Pipeline + +ROOT="$(git rev-parse --show-toplevel)" +BASE="$ROOT/Base" +CMAKED="$ROOT/CMakeD" + +function clean() { + source "$BASE/Tests/Pipeline/Libraries/Logcat.sh" + source "$BASE/Tests/Pipeline/Libraries/Package.sh" + + $SU rm -fr $BASE + $SU rm -fr $ROOT/build + $SU git clean -fdx + exit 0 +} + +DRYRUN=0 + +while [[ $# -gt 0 ]]; do + case $1 in + --verbose) VERBOSE=1;; + --dry-run) DRYRUN=1;; + --clean) clean;; + (--) shift; break;; + (*) if [[ ${#SUBPROJECT} -eq 0 ]]; then SUBPROJECT=$1; else exit -1; fi;; + esac + shift +done + +if [ ! -d $BASE ]; then + if ! git clone https://github.com/hung0913208/Base $BASE; then + exit -1 + fi + + if [[ $DRYRUN -eq 0 ]]; then + if ! git clone https://github.com/dcarp/cmake-d.git $CMAKED; then + exit -1 + else + cp -a $CMAKED/cmake-d/* $BASE/CMakeModules/ + fi + + $BASE/Tests/Pipeline/Prepare.sh Unikey + exit $? + else + exit 0 + fi +else + source "$BASE/Tests/Pipeline/Libraries/Logcat.sh" + source "$BASE/Tests/Pipeline/Libraries/Package.sh" + + PIPELINE="$(dirname "$0")" + CURRENT="$(pwd)" + SCRIPT="$(basename "$0")" +fi diff --git a/tools/utilities/generate-customized-livecd-image.sh b/tools/utilities/generate-customized-livecd-image.sh new file mode 100755 index 000000000..c4d391913 --- /dev/null +++ b/tools/utilities/generate-customized-livecd-image.sh @@ -0,0 +1,319 @@ +#!/bin/bash +# @NOTE: how to use this script? +# ------------------------------ +# +# This script is used to customize a bootable ISO image so we can +# use it to deploy VMs or we can modify LiveCD ISO images and use +# these images to deploy directly to VMs and use these machines for +# testing directly and quickly since almost things are runing directly +# on ramdisk. +# +# For example, you can config the ISO image to run with sshd service +# inside, or you can config to deploy webservice or any kind of services +# you would like to do. +# +# To help anyone to use this script, here is the explanation of how +# to define the environment scripts. The input of this script is +# another script. This script will use keyword `source` to load your +# script and use 4 functions: fetch, extract, configure and finish. +# +# As the names are suggested: +# - function `fecth` is used to download the origin ISO image, +# - function `extract` is used to convert downloaded file to dir `root` +# - function `configure` is used to apply additional features to the new +# image +# - function `finish` use to fill necessary files which is used to during +# boot or verify the image. +# + +ROOT="$(git rev-parse --show-toplevel)" +BASE="$ROOT/Base" + +source "$BASE/Tests/Pipeline/Libraries/Logcat.sh" +source "$BASE/Tests/Pipeline/Libraries/Package.sh" + +function clean() { + rm -fr $ROOT/${NAME}-latest.iso + + if [[ ${#ISQUASHFS} -gt 0 ]] && [ -d $ISQUASHFS ]; then + $SU umount $ISQUASHFS + fi + + if [[ ${#OLDCD} -gt 0 ]] && [ -d $OLDCD ]; then + $SU umount $OLDCD + fi + + if [[ ${#TEMP} -gt 0 ]]; then + $SU rm -fr $TEMP + fi +} + +function usage() { + echo """ +Usage: $(basename $0) [ --input-type TYPE ] [ --env SCRIPT ] [ --output-type ] || + [ --help ] + + --input-type TYPE Which input we would like to use, we could + use \`iso\` to indicate that we will use + iso image as the input of this script or + we can use \`repo\` to indicate that we + will build a new iso with a bundle of packages + + --output-type TYPE Which output we would like to use, we could + use \`iso\` to indicate that the output should + be ISO image + + --env SCRIPT The environment script which is used to help + to provide stub to build ISO images + + --nfs + + --no-image + + --help Show the script's usage + """ +} + +ITYPE="iso" +OTYPE="nfs" +TEMP="$(mktemp -d --tmpdir=$HOME)" +SCRIPT="$(basename "$0")" +ADDRESS="\*" +CURRENT="$(pwd)" +OLDCD=$TEMP/livecd +GENIMAGE=1 +ISQUASHFS=$TEMP/squashfs +WORKSPACE=$TEMP/workspace + +while [ $# -gt 0 ]; do + case $1 in + --input-type) ITYPE="$2"; shift;; + --output-type) OTYPE="$2"; shift;; + --no-image) GENIMAGE=0;; + --output) OUTPUT="$2"; shift;; + --env) ENVIRONMENT="$2"; shift;; + --nfs) ADDRESS="$2"; shift;; + --help) usage;; + (--) shift; break;; + (-*) error "unrecognized option $1";; + (*) break;; + esac + shift +done + +if [[ $GENIMAGE -eq 0 ]]; then + WORKSPACE=$OUTPUT +fi + +NEWCD=$WORKSPACE/livecd +OSQUASHFS=$WORKSPACE/squashfs + +trap clean EXIT + +if [[ ${#ENVIRONMENT} -gt 0 ]]; then + . "$ENVIRONMENT" +else + error "please provide the environment script" +fi + +NAME=$(basename $ENVIRONMENT) +VERSION="$(version)" + +if [ "$ITYPE" = 'iso' ]; then + if ! $SU modinfo squashfs &> /dev/null; then + if ! $SU modprobe squashfs &> /dev/null; then + error "your system don't support squashfs" + fi + fi +fi + +# @TODO: check if our global storage contains the customized livecd. if yes, +# download it and finish. Otherwide, we should generate new livecd and upload +# it to global storage before apply it to our pipeline + +info "download the latest release of $NAME" + +if [ $OTYPE != 'iso' ] && [[ ${#OUTPUT} -gt 0 ]]; then + OSQUASHFS=$OUTPUT +fi + +mkdir -p $WORKSPACE/livecd/casper +mkdir -p $OSQUASHFS + +if [ "$ITYPE" = 'iso' ]; then + if [[ ${#MEGA} -gt 0 ]]; then + USER=$(echo $MEGA | awk '{ split($0,a,":"); print a[1] }') + PASSWORD=$(echo $MEGA | awk '{ split($0,a,":"); print a[2] }') + fi + + if [[ ${#PROJECT} -eq 0 ]]; then + PROJECT=$(basename $ROOT) + fi + + if [[ $GENIMAGE -eq 0 ]]; then + CACHE=${NAME}.squashfs + MIRROR=$WORKSPACE/livecd/casper + else + CACHE=${NAME}.iso + MIRROR=$OUTPUT + fi + + if [[ ${#MEGA} -gt 0 ]] && megaget --path $(pwd)/${NAME}.ver --username $USER --password $PASSWORD /Root/$PROJECT/${NAME}.ver; then + if [ "$VERSION" != "$(cat $(pwd)/${NAME}.ver)" ]; then + MEGA="" + + megarm --username $USER --password $PASSWORD /Root/$PROJECT/${NAME}.ver + megarm --username $USER --password $PASSWORD /Root/$PROJECT/${CACHE} + fi + else + MEGA="" + fi + + # @NOTE: this script supports expecially with ISO image input so i + # will provide features to help to build new ISO image easier + + if [[ ${#MEGA} -eq 0 ]] || ! megaget --path $MIRROR --username $USER --password $PASSWORD /Root/$PROJECT/$CACHE; then + DONE=0 + + if ! fetch $ITYPE $TEMP ${NAME}-latest.iso; then + error "can't fetch $NAME's release" + fi + + mkdir -p $OLDCD + mkdir -p $ISQUASHFS + + # @NOTE: these steps are used to clone a new workspace which base + # on the inpu ISO image + + info "copy $ISQUASHFS to $OSQUASHFS" + + if ! $SU mount -o loop $TEMP/${NAME}-latest.iso $OLDCD &> /dev/null; then + error "can't mount $TEMP/${NAME}-latest.iso" + elif ! $SU mount -t squashfs -o loop $OLDCD/casper/filesystem.squashfs $ISQUASHFS &> /dev/null; then + error "can't mount $OLDCD/casper/filesystem.squashfs" + elif ! $SU cp -a $ISQUASHFS/* $OSQUASHFS &> /dev/null; then + error "can't copy file in $ISQUASHFS to $OSQUASHFS" + fi + elif [[ $GENIMAGE -eq 0 ]]; then + DONE=1 + + if ! fetch $ITYPE $TEMP ${NAME}-latest.iso; then + error "can't fetch $NAME's release" + fi + + mkdir -p $OLDCD + mkdir -p $ISQUASHFS + + if ! $SU mount -o loop $TEMP/${NAME}-latest.iso $OLDCD &> /dev/null; then + error "can't mount $TEMP/${NAME}-latest.iso" + fi + else + exit 0 + fi +elif ! fetch $ITYPE $TEMP $WORKSPACE; then + error "can't fetch files to build a new ISO image" +fi + +if ! extract $ITYPE $TEMP $WORKSPACE $OSQUASHFS $MIRROR $DONE; then + error "extract packages to rootfs" +fi + +if [[ $DONE -eq 1 ]]; then + exit 0 +fi + +$SU cp /etc/resolv.conf $OSQUASHFS/etc/resolv.conf +$SU cp $ENVIRONMENT $OSQUASHFS/$NAME + +info "reconfigure our new livecd" + +if ! $SU chroot $OSQUASHFS /bin/bash -c """ +#!/bin/bash +mount -t proc none /proc/ +mount -t sysfs none /sys/ + +if [[ \$(ls -id / | awk '{ print \$1 }') -ne 2 ]]; then + source /$NAME + + if configure $ITYPE; then + CODE=\$? + fi + + rm -rf /$NAME + umount /proc + umount /sys + + exit \$CODE +else + rm -rf /$NAME + umount /proc + umount /sys + exit -1 +fi +"""; then + error "can't configure a custom livecd" +else + if [ "$OTYPE" = 'iso' ]; then + if ! finish $OTYPE $NEWCD $WORKSPACE $OSQUASHFS; then + error "can't finish preparing stub to build a new ISO image" + fi + else + if ! finish $OTYPE $OLDCD $WORKSPACE $OSQUASHFS; then + error "can't finish preparing stub to build a new ISO image" + fi + fi + + info "generate $ROOT/${NAME}.iso base on the latest release $NAME's livecd" + + if [ "$OTYPE" = 'iso' ]; then + if [[ $GENIMAGE -ne 0 ]]; then + cd $NEWCD || error "can't cd to $NEWCD" + + $SU mkisofs -r -V "Ubuntu-Live" \ + -b isolinux/isolinux.bin \ + -c isolinux/boot.cat \ + -cache-inodes -J -l \ + -no-emul-boot \ + -boot-load-size 4 \ + -boot-info-table \ + -o $OUTPUT/${NAME}.iso . + + if [[ $? -ne 0 ]] || [ ! -f $ROOT/${NAME}.iso ]; then + exit -1 + else + MIRROR=$OUTPUT/${NAME}.iso + fi + else + MIRROR=$(find $NEWCD -name *.squashfs) + fi + + if [[ ${#USER} -gt 0 ]] && [[ ${#PASSWORD} -gt 0 ]]; then + info "upload $MIRROR to global storage" + + echo "$VERSION" > $(pwd)/${NAME}.ver + + if ! megaput --path /Root/$PROJECT/$CACHE --username $USER --password $PASSWORD $MIRROR; then + warning "can't upload $MIRROR to global storage" + fi + + if ! megaput --path /Root/$PROJECT/${NAME}.ver --username $USER --password $PASSWORD $(pwd)/${NAME}.ver; then + warning "can't upload $(pwd)/${NAME}.ver to global storage" + fi + fi + elif [ $OTYPE = 'nfs' ]; then + if echo "$OUTPUT $ADDRESS(ro,sync,no_wdelay,insecure_locks,no_root_squash,insecure)" | $SU tee -a /etc/exports &> /dev/null; then + if ! $SU systemctl restart nfs-kernel-server; then + error "can't restart nfs-kernel-server" + fi + else + error "can't edit file /etc/exports" + fi + fi + + info "clean our workspace" + $SU umount $ISQUASHFS + $SU umount $OLDCD + $SU rm -fr $TEMP +fi + +cd $CURRENT || error "can't cd to $CURRENT"