diff --git a/docker/dist/mixed/Makefile b/docker/dist/mixed/Makefile new file mode 100644 index 000000000..c05fb489c --- /dev/null +++ b/docker/dist/mixed/Makefile @@ -0,0 +1,10 @@ +TERM_INHERIT = $${TERM+--env TERM='${TERM}'} + +login: + docker exec -u ${USER} $(TERM_INHERIT) \ + -w /home/${USER}/gfarm/docker/dist/mixed -it gfarm-c1 /bin/bash + +down: + (cd ../jwt-server; docker compose down) + (cd ../minica; docker compose down) + docker compose down diff --git a/docker/dist/mixed/README.md b/docker/dist/mixed/README.md new file mode 100644 index 000000000..3fecf53cb --- /dev/null +++ b/docker/dist/mixed/README.md @@ -0,0 +1,24 @@ +# Docker containers for Gfarm developers in a heterogeneous environment + +## Explore on virtual clusters + +Install docker compose ([Ubuntu](https://docs.docker.com/engine/install/ubuntu/) | [CentOS](https://docs.docker.com/engine/install/centos/)) and make. + +To allow docker compose to run with user privileges, add $USER to the docker group by `sudo usermod -aG docker $USER` + + % cd gfarm/docker/dist/mixed + % docker compose up -d + % make # login to a container + + (in a container) + % sh ./all.sh + This will install and setup Gfarm. Enjoy! + % Ctrl-D + + % docker compose down + +## Batch tests + +Build, install and setup tests. + + % sh ./batchtest.sh diff --git a/docker/dist/mixed/all.sh b/docker/dist/mixed/all.sh new file mode 100644 index 000000000..e70a119cb --- /dev/null +++ b/docker/dist/mixed/all.sh @@ -0,0 +1,115 @@ +#!/bin/sh +set -xeu +status=1 +PROG=$(basename $0) +trap '[ $status = 0 ] && echo All set || echo NG: $PROG; exit $status' 0 1 2 15 + +REGRESS=false + +# sanity +DISTDIR=$PWD/.. +[ -f $DISTDIR/install.sh ] +[ -f $DISTDIR/config.sh ] + +# for GitHub Actions: +# change the owner of /home/runner/local from root to runner +sudo chown `id -un` ~/local + +# set up .nodelist +sh $DISTDIR/setup.sh + +# install Gfarm +(cd ~/gfarm && sh $DISTDIR/install.sh single) +gfarm-prun -v "(cd ~/gfarm && sh $DISTDIR/install.sh single)" +gfarm-pcp -p ~/.nodelist . +[ -f ~/.gfarm2rc ] && gfarm-prun -a -p "mv ~/.gfarm2rc ~/.gfarm2rc.bak + > /dev/null 2>&1" + +# install Gfarm2fs +PKG=gfarm2fs; export PKG +[ -d ~/gfarm/$PKG ] || git clone https://github.com/oss-tsukuba/$PKG.git +gfarm-prun -a -v "(export PKG=$PKG; cd ~/gfarm/$PKG && sh $DISTDIR/install.sh single)" + +# install jwt-logon +PKG=jwt-logon; export PKG +[ -d ~/gfarm/$PKG ] || git clone https://github.com/oss-tsukuba/$PKG.git +gfarm-prun -a -p "(cd gfarm/$PKG && sudo make PREFIX=/usr/local install + > /dev/null)" + +# install jwt-agent +PKG=jwt-agent; export PKG +[ -d ~/gfarm/$PKG ] || git clone https://github.com/oss-tsukuba/$PKG.git +gfarm-prun -a -v "(cd ~/gfarm/$PKG && make clean > /dev/null && make > /dev/null + && sudo make PREFIX=/usr/local install > /dev/null)" + +# install cyrus-sasl-xoauth2-idp +PKG=cyrus-sasl-xoauth2-idp; export PKG +sasl_libdir=$(pkg-config --variable=libdir libsasl2) +[ -d ~/gfarm/$PKG ] || git clone https://github.com/oss-tsukuba/$PKG.git +gfarm-prun -a -v "(cd ~/gfarm/$PKG && ./autogen.sh && + ./configure --libdir=\$(pkg-config --variable=libdir libsasl2) && + make > /dev/null && sudo make install > /dev/null)" + +cat < /dev/null +log_level: 7 +mech_list: XOAUTH2 ANONYMOUS +xoauth2_scope: hpci +xoauth2_aud: hpci +xoauth2_user_claim: hpci.id +EOF +cat < /dev/null +xoauth2_user_claim: hpci.id +EOF + +cp $sasl_libdir/sasl2/gfarm*.conf ~/local +gfarm-prun -p sudo cp local/gfarm*.conf \$\(pkg-config --variable=libdir libsasl2\)/sasl2 +rm ~/local/gfarm*.conf + +# create empty sasldb2 database, +# because the gdbm backend of Cyrus SAL (e.g. on RHEL9) needs this +# although the berkeley DB backend does NOT +gfarm-prun -p -a "sudo saslpasswd2 -d -u NOT-EXIST NOT-EXIST" + +# XXX - SASL XOAUTH2 fails in gfsd on ubuntu due to the error +# "unable to open Berkeley db /etc/sasldb2: Permission denied" +gfarm-prun -p -a \ + "sudo chown _gfarmfs /etc/sasldb2 /etc/sasl2/sasldb2 > /dev/null 2>&1" + +# set up certificates +sh $DISTDIR/key.sh +sh $DISTDIR/userkey.sh +sh $DISTDIR/cert.sh +sh $DISTDIR/usercert.sh +sh $DISTDIR/tlscert.sh + +# set up Gfarm-1 with 5 nodes +echo c1 c2 c3 c4 c5 | sh $DISTDIR/config.sh - & + +# set up Gfarm-2 to Gfarm-4 with 1 node +for h in c6 c7 c8; do + echo $h | ssh $h sh $DISTDIR/config.sh - & +done +wait + +# Check installation +AUTH= +for a in $(gfstatus -S | grep 'client auth' | grep -v not | awk '{ print $3 }') +do + [ $a = gsi ] && AUTH="$AUTH gsi gsi_auth" + [ $a = tls ] && AUTH="$AUTH tls_sharedsecret tls_client_certificate" + [ $a = sasl ] && AUTH="$AUTH anonymous" +done +AUTH="$AUTH sharedsecret" +for a in $AUTH +do + echo "*** $a ***" + sh $DISTDIR/edconf.sh $a > /dev/null + sh $DISTDIR/check.sh + for h in c6 c7 c8; do + ssh $h sh $DISTDIR/edconf.sh $a > /dev/null + ssh $h sh $DISTDIR/check.sh + done + $REGRESS && sh $DISTDIR/regress.sh +done + +status=0 diff --git a/docker/dist/mixed/batchtest.sh b/docker/dist/mixed/batchtest.sh new file mode 100644 index 000000000..d2da72afc --- /dev/null +++ b/docker/dist/mixed/batchtest.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -xeu +status=1 +trap '[ $status = 0 ] && echo Done || echo NG; exit $status' 0 1 2 15 + +DOCKEREXEC="docker exec -u $USER -w /home/$USER/gfarm/docker/dist/mixed gfarm-c1" + +# clean up +make down + +JWT=true + +docker compose up -d + +# JWT-Server +$JWT && (cd ../jwt-server && docker compose up -d && make setup) + +# execute a script +$DOCKEREXEC sh all.sh + +# SASL XOAUTH2 test +$JWT && $DOCKEREXEC sh ../check-oauth.sh + +# clean up +make down + +status=0 diff --git a/docker/dist/mixed/cert-sign.sh b/docker/dist/mixed/cert-sign.sh new file mode 120000 index 000000000..e5f971733 --- /dev/null +++ b/docker/dist/mixed/cert-sign.sh @@ -0,0 +1 @@ +../cert-sign.sh \ No newline at end of file diff --git a/docker/dist/mixed/check.sh b/docker/dist/mixed/check.sh new file mode 120000 index 000000000..c0555e5a6 --- /dev/null +++ b/docker/dist/mixed/check.sh @@ -0,0 +1 @@ +../check.sh \ No newline at end of file diff --git a/docker/dist/mixed/docker-compose.yaml b/docker/dist/mixed/docker-compose.yaml new file mode 100644 index 000000000..17ec91554 --- /dev/null +++ b/docker/dist/mixed/docker-compose.yaml @@ -0,0 +1,87 @@ +x-ubuntu-build: &x-ubuntu-build + build: + context: ../../.. + dockerfile: docker/dist/ubuntu/Dockerfile + args: + USERNAME: $USER + +x-almalinux8-build: &x-almalinux8-build + build: + context: ../../.. + dockerfile: docker/dist/almalinux8/Dockerfile + args: + USERNAME: $USER + +x-rockylinux9-build: &x-rockylinux9-build + build: + context: ../../.. + dockerfile: docker/dist/rockylinux9/Dockerfile + args: + USERNAME: $USER + +x-common: &x-common + volumes: + - ../../..:/home/$USER/gfarm:cached + - local:/home/$USER/local:cached + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - ../minica/minica.pem:/minica/minica.crt + networks: + - gfarm_net + cap_add: + - SYS_ADMIN + - SYS_PTRACE + devices: + - /dev/fuse + security_opt: + - seccomp:unconfined + - apparmor:unconfined + privileged: false + +services: + c1: + <<: [*x-ubuntu-build, *x-common] + image: gfarm-ubuntu + hostname: c1 + container_name: gfarm-c1 + c2: + <<: [*x-almalinux8-build, *x-common] + image: gfarm-almalinux8 + hostname: c2 + container_name: gfarm-c2 + c3: + <<: [*x-rockylinux9-build, *x-common] + image: gfarm-rockylinux9 + hostname: c3 + container_name: gfarm-c3 + c4: + <<: *x-common + image: gfarm-ubuntu + hostname: c4 + container_name: gfarm-c4 + c5: + <<: *x-common + image: gfarm-almalinux8 + hostname: c5 + container_name: gfarm-c5 + c6: + <<: *x-common + image: gfarm-rockylinux9 + hostname: c6 + container_name: gfarm-c6 + c7: + <<: *x-common + image: gfarm-ubuntu + hostname: c7 + container_name: gfarm-c7 + c8: + <<: *x-common + image: gfarm-almalinux8 + hostname: c8 + container_name: gfarm-c8 + +networks: + gfarm_net: + name: gfarm_net + +volumes: + local: diff --git a/docker/dist/mixed/edconf.sh b/docker/dist/mixed/edconf.sh new file mode 120000 index 000000000..cc138c39d --- /dev/null +++ b/docker/dist/mixed/edconf.sh @@ -0,0 +1 @@ +../edconf.sh \ No newline at end of file diff --git a/docker/dist/mixed/init-jwt.sh b/docker/dist/mixed/init-jwt.sh new file mode 120000 index 000000000..adeed9763 --- /dev/null +++ b/docker/dist/mixed/init-jwt.sh @@ -0,0 +1 @@ +../init-jwt.sh \ No newline at end of file