From 7912aaeaf6086dc72d4bc324dc2b11995abd307b Mon Sep 17 00:00:00 2001 From: Holger Friedrich Date: Thu, 18 Jun 2020 22:04:43 +0200 Subject: [PATCH] Caching essential packages for offline install, #984. * initial functionality for cachin packages added to build.bash * use apt of target system to fetch deb packages * directory prefix for java_zulu_fetch; extraction moved to java_zulu_install * first-boot to install cached packages * verify signature of download images using key from raspberrypi.org website * silence install_zram_code warnings * add openhab2 stable and cache package Signed-off-by: Holger Friedrich --- build-image/apt.conf | 286 ++++++++++++++++++++++++++++++++++++ build-image/first-boot.bash | 8 + build.bash | 87 ++++++++++- functions/java-jre.bash | 16 +- functions/zram.bash | 2 +- 5 files changed, 387 insertions(+), 12 deletions(-) create mode 100644 build-image/apt.conf diff --git a/build-image/apt.conf b/build-image/apt.conf new file mode 100644 index 000000000..eaea82ee6 --- /dev/null +++ b/build-image/apt.conf @@ -0,0 +1,286 @@ +APT ""; +APT::Architecture "armhf"; +APT::Build-Essential ""; +APT::Build-Essential:: "build-essential"; +APT::Install-Recommends "1"; +APT::Install-Suggests "0"; +APT::Sandbox ""; +APT::Sandbox::User "_apt"; +APT::NeverAutoRemove ""; +APT::NeverAutoRemove:: "^firmware-linux.*"; +APT::NeverAutoRemove:: "^linux-firmware$"; +APT::NeverAutoRemove:: "^linux-image-[a-z0-9]*$"; +APT::NeverAutoRemove:: "^linux-image-[a-z0-9]*-[a-z0-9]*$"; +APT::NeverAutoRemove:: "^linux-image-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-image-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-headers-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-headers-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-image-extra-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-image-extra-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-modules-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-modules-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-modules-extra-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-modules-extra-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-signed-image-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-signed-image-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-image-unsigned-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-image-unsigned-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^kfreebsd-image-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^kfreebsd-image-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^kfreebsd-headers-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^kfreebsd-headers-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^gnumach-image-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^gnumach-image-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^.*-modules-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^.*-modules-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^.*-kernel-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^.*-kernel-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-backports-modules-.*-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-backports-modules-.*-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-modules-.*-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-modules-.*-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-tools-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-tools-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-cloud-tools-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-cloud-tools-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-buildinfo-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-buildinfo-4\.19\.97-v7l\+$"; +APT::NeverAutoRemove:: "^linux-source-4\.19\.118-v8\+$"; +APT::NeverAutoRemove:: "^linux-source-4\.19\.97-v7l\+$"; +APT::VersionedKernelPackages ""; +APT::VersionedKernelPackages:: "linux-image"; +APT::VersionedKernelPackages:: "linux-headers"; +APT::VersionedKernelPackages:: "linux-image-extra"; +APT::VersionedKernelPackages:: "linux-modules"; +APT::VersionedKernelPackages:: "linux-modules-extra"; +APT::VersionedKernelPackages:: "linux-signed-image"; +APT::VersionedKernelPackages:: "linux-image-unsigned"; +APT::VersionedKernelPackages:: "kfreebsd-image"; +APT::VersionedKernelPackages:: "kfreebsd-headers"; +APT::VersionedKernelPackages:: "gnumach-image"; +APT::VersionedKernelPackages:: ".*-modules"; +APT::VersionedKernelPackages:: ".*-kernel"; +APT::VersionedKernelPackages:: "linux-backports-modules-.*"; +APT::VersionedKernelPackages:: "linux-modules-.*"; +APT::VersionedKernelPackages:: "linux-tools"; +APT::VersionedKernelPackages:: "linux-cloud-tools"; +APT::VersionedKernelPackages:: "linux-buildinfo"; +APT::VersionedKernelPackages:: "linux-source"; +APT::Never-MarkAuto-Sections ""; +APT::Never-MarkAuto-Sections:: "metapackages"; +APT::Never-MarkAuto-Sections:: "contrib/metapackages"; +APT::Never-MarkAuto-Sections:: "non-free/metapackages"; +APT::Never-MarkAuto-Sections:: "restricted/metapackages"; +APT::Never-MarkAuto-Sections:: "universe/metapackages"; +APT::Never-MarkAuto-Sections:: "multiverse/metapackages"; +APT::Move-Autobit-Sections ""; +APT::Move-Autobit-Sections:: "oldlibs"; +APT::Move-Autobit-Sections:: "contrib/oldlibs"; +APT::Move-Autobit-Sections:: "non-free/oldlibs"; +APT::Move-Autobit-Sections:: "restricted/oldlibs"; +APT::Move-Autobit-Sections:: "universe/oldlibs"; +APT::Move-Autobit-Sections:: "multiverse/oldlibs"; +APT::Periodic ""; +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; +APT::Update ""; +APT::Update::Post-Invoke-Success ""; +#APT::Update::Post-Invoke-Success:: "/usr/bin/test -e /usr/share/dbus-1/system-services/org.freedesktop.PackageKit.service && /usr/bin/test -S /var/run/dbus/system_bus_socket && /usr/bin/gdbus call --system --dest org.freedesktop.PackageKit --object-path /org/freedesktop/PackageKit --timeout 4 --method org.freedesktop.PackageKit.StateHasChanged cache-update > /dev/null; /bin/echo > /dev/null"; +APT::Architectures ""; +APT::Architectures:: "armhf"; +APT::Compressor ""; +APT::Compressor::. ""; +APT::Compressor::.::Name "."; +APT::Compressor::.::Extension ""; +APT::Compressor::.::Binary ""; +APT::Compressor::.::Cost "0"; +APT::Compressor::zstd ""; +APT::Compressor::zstd::Name "zstd"; +APT::Compressor::zstd::Extension ".zst"; +APT::Compressor::zstd::Binary "false"; +APT::Compressor::zstd::Cost "60"; +APT::Compressor::lz4 ""; +APT::Compressor::lz4::Name "lz4"; +APT::Compressor::lz4::Extension ".lz4"; +APT::Compressor::lz4::Binary "false"; +APT::Compressor::lz4::Cost "50"; +APT::Compressor::gzip ""; +APT::Compressor::gzip::Name "gzip"; +APT::Compressor::gzip::Extension ".gz"; +APT::Compressor::gzip::Binary "gzip"; +APT::Compressor::gzip::Cost "100"; +APT::Compressor::gzip::CompressArg ""; +APT::Compressor::gzip::CompressArg:: "-6n"; +APT::Compressor::gzip::UncompressArg ""; +APT::Compressor::gzip::UncompressArg:: "-d"; +APT::Compressor::xz ""; +APT::Compressor::xz::Name "xz"; +APT::Compressor::xz::Extension ".xz"; +APT::Compressor::xz::Binary "xz"; +APT::Compressor::xz::Cost "200"; +APT::Compressor::xz::CompressArg ""; +APT::Compressor::xz::CompressArg:: "-6"; +APT::Compressor::xz::UncompressArg ""; +APT::Compressor::xz::UncompressArg:: "-d"; +APT::Compressor::bzip2 ""; +APT::Compressor::bzip2::Name "bzip2"; +APT::Compressor::bzip2::Extension ".bz2"; +APT::Compressor::bzip2::Binary "bzip2"; +APT::Compressor::bzip2::Cost "300"; +APT::Compressor::bzip2::CompressArg ""; +APT::Compressor::bzip2::CompressArg:: "-6"; +APT::Compressor::bzip2::UncompressArg ""; +APT::Compressor::bzip2::UncompressArg:: "-d"; +APT::Compressor::lzma ""; +APT::Compressor::lzma::Name "lzma"; +APT::Compressor::lzma::Extension ".lzma"; +APT::Compressor::lzma::Binary "xz"; +APT::Compressor::lzma::Cost "400"; +APT::Compressor::lzma::CompressArg ""; +APT::Compressor::lzma::CompressArg:: "--format=lzma"; +APT::Compressor::lzma::CompressArg:: "-6"; +APT::Compressor::lzma::UncompressArg ""; +APT::Compressor::lzma::UncompressArg:: "--format=lzma"; +APT::Compressor::lzma::UncompressArg:: "-d"; +Dir "###BUILDFOLDER###"; +Dir::State "var/lib/apt"; +Dir::State::lists "lists/"; +Dir::State::cdroms "cdroms.list"; +Dir::State::extended_states "extended_states"; +Dir::State::status "/var/lib/dpkg/status"; +Dir::Cache "var/cache/apt"; +Dir::Cache::archives "archives/"; +Dir::Cache::srcpkgcache "srcpkgcache.bin"; +Dir::Cache::pkgcache "pkgcache.bin"; +Dir::Etc "etc/apt"; +Dir::Etc::sourcelist "sources.list"; +Dir::Etc::sourceparts "sources.list.d"; +Dir::Etc::main "apt.conf"; +Dir::Etc::netrc "auth.conf"; +Dir::Etc::netrcparts "auth.conf.d"; +Dir::Etc::parts "apt.conf.d"; +Dir::Etc::preferences "preferences"; +Dir::Etc::preferencesparts "preferences.d"; +Dir::Etc::trusted "trusted.gpg"; +Dir::Etc::trustedparts "trusted.gpg.d"; +Dir::Bin ""; +Dir::Bin::methods "/usr/lib/apt/methods"; +Dir::Bin::solvers ""; +Dir::Bin::solvers:: "/usr/lib/apt/solvers"; +Dir::Bin::planners ""; +Dir::Bin::planners:: "/usr/lib/apt/planners"; +Dir::Bin::dpkg "/usr/bin/dpkg"; +Dir::Bin::gzip "/bin/gzip"; +Dir::Bin::bzip2 "/bin/bzip2"; +Dir::Bin::xz "/usr/bin/xz"; +Dir::Bin::lz4 "/usr/bin/lz4"; +Dir::Bin::zstd "/usr/bin/zstd"; +Dir::Bin::lzma "/usr/bin/xz"; +Dir::Media ""; +Dir::Media::MountPath "/media/apt"; +Dir::Log "var/log/apt"; +Dir::Log::Terminal "term.log"; +Dir::Log::History "history.log"; +Dir::Log::Planner "eipp.log.xz"; +Dir::Ignore-Files-Silently ""; +Dir::Ignore-Files-Silently:: "~$"; +Dir::Ignore-Files-Silently:: "\.disabled$"; +Dir::Ignore-Files-Silently:: "\.bak$"; +Dir::Ignore-Files-Silently:: "\.dpkg-[a-z]+$"; +Dir::Ignore-Files-Silently:: "\.ucf-[a-z]+$"; +Dir::Ignore-Files-Silently:: "\.save$"; +Dir::Ignore-Files-Silently:: "\.orig$"; +Dir::Ignore-Files-Silently:: "\.distUpgrade$"; +Acquire ""; +Acquire::AllowInsecureRepositories "0"; +Acquire::AllowWeakRepositories "0"; +Acquire::AllowDowngradeToInsecureRepositories "0"; +Acquire::cdrom ""; +Acquire::cdrom::mount "/media/cdrom/"; +Acquire::IndexTargets ""; +Acquire::IndexTargets::deb ""; +Acquire::IndexTargets::deb::Packages ""; +Acquire::IndexTargets::deb::Packages::MetaKey "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages"; +Acquire::IndexTargets::deb::Packages::flatMetaKey "Packages"; +Acquire::IndexTargets::deb::Packages::ShortDescription "Packages"; +Acquire::IndexTargets::deb::Packages::Description "$(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages"; +Acquire::IndexTargets::deb::Packages::flatDescription "$(RELEASE) Packages"; +Acquire::IndexTargets::deb::Packages::Optional "0"; +Acquire::IndexTargets::deb::Translations ""; +Acquire::IndexTargets::deb::Translations::MetaKey "$(COMPONENT)/i18n/Translation-$(LANGUAGE)"; +Acquire::IndexTargets::deb::Translations::flatMetaKey "$(LANGUAGE)"; +Acquire::IndexTargets::deb::Translations::ShortDescription "Translation-$(LANGUAGE)"; +Acquire::IndexTargets::deb::Translations::Description "$(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)"; +Acquire::IndexTargets::deb::Translations::flatDescription "$(RELEASE) Translation-$(LANGUAGE)"; +Acquire::IndexTargets::deb-src ""; +Acquire::IndexTargets::deb-src::Sources ""; +Acquire::IndexTargets::deb-src::Sources::MetaKey "$(COMPONENT)/source/Sources"; +Acquire::IndexTargets::deb-src::Sources::flatMetaKey "Sources"; +Acquire::IndexTargets::deb-src::Sources::ShortDescription "Sources"; +Acquire::IndexTargets::deb-src::Sources::Description "$(RELEASE)/$(COMPONENT) Sources"; +Acquire::IndexTargets::deb-src::Sources::flatDescription "$(RELEASE) Sources"; +Acquire::IndexTargets::deb-src::Sources::Optional "0"; +Acquire::Changelogs ""; +Acquire::Changelogs::URI ""; +Acquire::Changelogs::URI::Origin ""; +Acquire::Changelogs::URI::Origin::Debian "https://metadata.ftp-master.debian.org/changelogs/@CHANGEPATH@_changelog"; +Acquire::Changelogs::URI::Origin::Ubuntu "https://changelogs.ubuntu.com/changelogs/pool/@CHANGEPATH@/changelog"; +Acquire::Changelogs::AlwaysOnline ""; +Acquire::Changelogs::AlwaysOnline::Origin ""; +Acquire::Changelogs::AlwaysOnline::Origin::Ubuntu "1"; +Acquire::http ""; +Acquire::http::User-Agent "Debian APT-HTTP/1.3 openHABian"; +Acquire::PDiffs "0"; +Acquire::Languages ""; +Acquire::Languages:: "en_US"; +Acquire::Languages:: "en"; +Acquire::Languages:: "en_US.UTF-8"; +Acquire::Languages:: "none"; +Acquire::CompressionTypes ""; +Acquire::CompressionTypes::xz "xz"; +Acquire::CompressionTypes::bz2 "bzip2"; +Acquire::CompressionTypes::lzma "lzma"; +Acquire::CompressionTypes::gz "gzip"; +Acquire::CompressionTypes::lz4 "lz4"; +Acquire::CompressionTypes::zst "zstd"; +DPkg ""; +DPkg::Path "/usr/sbin:/usr/bin:/sbin:/bin"; +DPkg::Post-Invoke ""; +#DPkg::Post-Invoke:: "if [ -x /usr/local/bin/FireMotD ]; then echo -n 'Updating FireMotD available updates count ... '; /bin/bash /usr/local/bin/FireMotD --skiprepoupdate -S; echo ''; fi"; +#DPkg::Post-Invoke:: "/usr/bin/test -e /usr/share/dbus-1/system-services/org.freedesktop.PackageKit.service && /usr/bin/test -S /var/run/dbus/system_bus_socket && /usr/bin/gdbus call --system --dest org.freedesktop.PackageKit --object-path /org/freedesktop/PackageKit --timeout 4 --method org.freedesktop.PackageKit.StateHasChanged cache-update > /dev/null; /bin/echo > /dev/null"; +DPkg::Pre-Install-Pkgs ""; +#DPkg::Pre-Install-Pkgs:: "/usr/bin/apt-listchanges --apt || test $? -lt 10"; +#DPkg::Pre-Install-Pkgs:: "/usr/sbin/dpkg-preconfigure --apt || true"; +DPkg::Tools ""; +DPkg::Tools::Options ""; +DPkg::Tools::Options::/usr/bin/apt-listchanges ""; +DPkg::Tools::Options::/usr/bin/apt-listchanges::Version "2"; +DPkg::Tools::Options::/usr/bin/apt-listchanges::InfoFD "20"; +Unattended-Upgrade ""; +Unattended-Upgrade::Origins-Pattern ""; +Unattended-Upgrade::Origins-Pattern:: "origin=Debian,codename=${distro_codename},label=Debian"; +Unattended-Upgrade::Origins-Pattern:: "origin=Debian,codename=${distro_codename},label=Debian-Security"; +Binary "apt-config"; +Binary::apt ""; +Binary::apt::APT ""; +Binary::apt::APT::Color "1"; +Binary::apt::APT::Cache ""; +Binary::apt::APT::Cache::Show ""; +Binary::apt::APT::Cache::Show::Version "2"; +Binary::apt::APT::Cache::AllVersions "0"; +Binary::apt::APT::Cache::ShowVirtuals "1"; +Binary::apt::APT::Cache::Search ""; +Binary::apt::APT::Cache::Search::Version "2"; +Binary::apt::APT::Cache::ShowDependencyType "1"; +Binary::apt::APT::Cache::ShowVersion "1"; +Binary::apt::APT::Get ""; +Binary::apt::APT::Get::Upgrade-Allow-New "1"; +Binary::apt::APT::Get::Update ""; +Binary::apt::APT::Get::Update::InteractiveReleaseInfoChanges "1"; +Binary::apt::APT::Cmd ""; +Binary::apt::APT::Cmd::Show-Update-Stats "1"; +Binary::apt::APT::Keep-Downloaded-Packages "0"; +Binary::apt::DPkg ""; +Binary::apt::DPkg::Progress-Fancy "1"; +CommandLine ""; +CommandLine::AsString "apt-config dump"; diff --git a/build-image/first-boot.bash b/build-image/first-boot.bash index 09820e552..135640407 100755 --- a/build-image/first-boot.bash +++ b/build-image/first-boot.bash @@ -2,6 +2,7 @@ # shellcheck source=/etc/openhabian.conf disable=SC1091 CONFIGFILE=/etc/openhabian.conf +PKGCACHEDIR=/opt/openhabian-package-cache # apt/dpkg commands will not try interactive dialogs export DEBIAN_FRONTEND=noninteractive @@ -184,6 +185,13 @@ type openhabian_update &> /dev/null && if ! openhabian_update &> /dev/null; then echo "FAILED" echo "$(timestamp) [openHABian] The git repository on the public internet is not reachable." echo "$(timestamp) [openHABian] We will continue trying to get your system installed, but this is not guaranteed to work." + + if ls "$PKGCACHEDIR" >/dev/null 2>&1; then + echo "$(timestamp) [openHABian] Trying to install bootstrap packages from cache..." + ( + cd "$PKGCACHEDIR" && dpkg -i -- *.deb + ) + fi else echo "OK" fi diff --git a/build.bash b/build.bash index 3b629af04..4335ba71a 100755 --- a/build.bash +++ b/build.bash @@ -26,6 +26,10 @@ cleanup_build() { ########################## # shellcheck source=functions/helpers.bash source "$(dirname "$0")"/functions/helpers.bash +# shellcheck source=functions/java-jre.bash +source "$(dirname "$0")"/functions/java-jre.bash +# shellcheck source=functions/zram.bash +source "$(dirname "$0")"/functions/zram.bash ## This function formats log messages ## @@ -270,9 +274,10 @@ if [[ $hw_platform == "pi-raspios32" ]] || [[ $hw_platform == "pi-raspios64beta" zipfile="$(basename "$zipurl")" # Prerequisites + # TODO: transition to curl, remove wget once all helper functions are using curl echo_process "Checking prerequisites... " - REQ_COMMANDS="git curl unzip crc32 dos2unix xz" - REQ_PACKAGES="git curl unzip libarchive-zip-perl dos2unix xz-utils" + REQ_COMMANDS="git curl wget unzip crc32 dos2unix xz" + REQ_PACKAGES="git curl wget unzip libarchive-zip-perl dos2unix xz-utils" if running_in_docker || running_on_github || is_pi; then # in docker guestfstools are not used; do not install it and all of its prerequisites # -> must be run as root @@ -294,7 +299,7 @@ if [[ $hw_platform == "pi-raspios32" ]] || [[ $hw_platform == "pi-raspios64beta" echo_process "Using local copy of Raspberry Pi OS ($bits-bit) image... " cp "$zipfile" "$buildfolder/$zipfile" else - echo_process "Downloading latest Raspberry Pi OS ($bits-bit) image (no local copy found)... " + echo_process "Downloading latest Raspberry Pi OS ($bits-bit) image (no local copy of $zipfile found)... " curl -L "$baseurl" -o "$zipfile" cp "$zipfile" "$buildfolder/$zipfile" fi @@ -322,6 +327,7 @@ if [[ $hw_platform == "pi-raspios32" ]] || [[ $hw_platform == "pi-raspios64beta" mkdir -p "$buildfolder"/boot "$buildfolder"/root mount_image_file_root "$imagefile" "$buildfolder" + df -h "$buildfolder"/root echo_process "Setting hostname... " # shellcheck disable=SC2154 sed -i "s/127.0.1.1.*/127.0.1.1 $hostname/" "$buildfolder"/root/etc/hosts @@ -341,17 +347,87 @@ if [[ $hw_platform == "pi-raspios32" ]] || [[ $hw_platform == "pi-raspios64beta" ) echo_process "Cloning myself from ${repositoryurl:-https://github.com/openhab/openhabian.git}, ${clonebranch:-stable} branch... " - if ! [[ -d $buildfolder/root/opt/openhabian ]]; then + if ! [[ -d "$buildfolder"/root/opt/openhabian ]]; then git clone "${repositoryurl:-https://github.com/openhab/openhabian.git}" "$buildfolder"/root/opt/openhabian &> /dev/null git -C "$buildfolder"/root/opt/openhabian checkout "${clonebranch:-stable}" &> /dev/null fi touch "$buildfolder"/root/opt/openHABian-install-inprogress + + # caching essential components for install + # 1) ZRAM + echo_process "Downloading ZRAM codebase..." + install_zram_code "$buildfolder"/root/opt/zram + + df -h "$buildfolder"/root + + # 2) caching essential deb packages for install w/o internet connection + mkdir -p "$buildfolder"/root/opt/openhabian-package-cache + chmod 777 "$buildfolder"/root/opt/openhabian-package-cache + ( + set -e -x + # use apt tool of target system to fetch packages for offline install + # apt.conf has been extracted from working install using "apt-config dump" + echo_process "Downloading essential packages..." + # APT_CONFIG needs to be patched to use settings from mounted tartget system + cp "$sourcefolder"/apt.conf "$buildfolder" + sed -i "s|###BUILDFOLDER###|$buildfolder/root|" "$buildfolder"/apt.conf + if [ "$bits" == "64" ]; then sed -i "s|armhf|arm64|" "$buildfolder"/apt.conf; fi + export APT_CONFIG="$buildfolder"/apt.conf + # debug + #cat "$APT_CONFIG" + + # using own folder since first-boot will install _all_ packages from there when internet is down, + # so apt cache folder might not be a good solution in case a few restarts are needed to complete + cd "$buildfolder"/root/opt/openhabian-package-cache + + # add openhab repo and key + repo="deb https://dl.bintray.com/openhab/apt-repo2 stable main" + echo "$repo" > "$buildfolder"/root/etc/apt/sources.list.d/openhab2.list + if ! add_keys "https://bintray.com/user/downloadSubjectPublicKey?username=openhab"; then return 1; fi + + #ls -l "$buildfolder"/root/var/lib/apt/lists/ + apt-get update + ls -l "$buildfolder"/root/var/lib/apt/lists/ + # package list for raspbian lite is provided in .info file, e.g. + # https://downloads.raspberrypi.org/raspbian_lite/images/raspbian_lite-2020-02-14/2020-02-13-raspbian-buster-lite.info + PACKAGES="git libattr1-dev zlib1g libc6 libstdc++6 openhab2" + for i in $PACKAGES; do + # aptitude not installed per default on github actions + #aptitude versions "$i" + #aptitude download "$i" + apt-cache policy "$i" + apt-get download "$i" + done + apt-get clean + ) + chmod 755 "$buildfolder"/root/opt/openhabian-package-cache + + df -h "$buildfolder"/root + + # - 3) JAVA + ( + # source config to set java correctly, not it will break when someone uses != Zulu + # shellcheck source=build-image/openhabian.pi-raspios32.conf + . $sourcefolder/openhabian.$hw_platform.conf + + echo_process "Downloading Java..." + # Using variable hwarch intended for CI only to force use of arm packages. + # java_zulu_fetch takes version/bits as parameter, but internally uses is_arm + # to decide if i686 or arm packages are downloaded. Parameter works for 32 and 64bit. + hwarch="armv6l" java_zulu_fetch "${java_opt:-Zulu8-32}" "$buildfolder"/root + ) + + df -h "$buildfolder"/root umount_image_file_root "$imagefile" "$buildfolder" echo_process "Reactivating SSH... " mount_image_file_boot "$imagefile" "$buildfolder" + df -h "$buildfolder"/boot touch "$buildfolder"/boot/ssh - cp "$sourcefolder"/first-boot.bash "$buildfolder"/boot/first-boot.bash + cp "$sourcefolder"/first-boot.bash "$buildfolder"/boot/ + cp functions/helpers.bash "$buildfolder"/boot/ + cp functions/openhabian.bash "$buildfolder"/boot/ + #sed -i -e '1r functions/helpers.bash' $buildfolder/boot/first-boot.bash # Add platform identification touch "$buildfolder"/boot/first-boot.log unix2dos -q -n "$sourcefolder"/openhabian.${hw_platform}.conf "$buildfolder"/boot/openhabian.conf cp "$sourcefolder"/webserver.bash "$buildfolder"/boot/webserver.bash @@ -363,6 +439,7 @@ if [[ $hw_platform == "pi-raspios32" ]] || [[ $hw_platform == "pi-raspios64beta" echo_process "Closing up image file... " sync + df -h "$buildfolder"/boot umount_image_file_boot "$imagefile" "$buildfolder" fi diff --git a/functions/java-jre.bash b/functions/java-jre.bash index 3d22eb492..9691bc36b 100644 --- a/functions/java-jre.bash +++ b/functions/java-jre.bash @@ -172,6 +172,12 @@ java_zulu_install() { jdkBin="$(find /opt/jdk/*/bin ... -print -quit)" jdkLib="$(find /opt/jdk/*/lib ... -print -quit)" + jdkInstallLocation="/opt/jdk" + + echo -n "$(timestamp) [openHABian] Extracting Java to $jdkInstallLocation" + if ! cond_redirect tar -xpzf "$jdkInstallLocation"/zulu.tar.gz -C "$jdkInstallLocation"; then echo "FAILED (extract)"; rm -rf "${jdkInstallLocation:?}"/*; return 1; fi + if cond_redirect rm -rf "$jdkInstallLocation"/zulu.tar.gz; then echo "OK"; else echo "FAILED (cleanup)"; return 1; fi + if [[ $1 == "Zulu8-64" ]] || [[ $1 == "Zulu11-64" ]]; then if is_aarch64 && [[ $(getconf LONG_BIT) == 64 ]]; then jdkArch="aarch64" @@ -233,14 +239,14 @@ java_zulu_install() { ## Fetch Java Zulu 8 directly from Azul API v1 ## Valid arguments: "Zulu8-32", "Zulu8-64", "Zulu11-32", or "Zulu11-64" ## -## java_zulu_fetch(String type) +## java_zulu_fetch(String arch, String folderPrefix) ## java_zulu_fetch() { local downloadLink local jdkInstallLocation local link - jdkInstallLocation="/opt/jdk" + jdkInstallLocation="$2/opt/jdk" link="https://api.azul.com/zulu/download/community/v1.0/bundles/latest/binary/?os=linux&ext=tar.gz&javafx=false" if [[ $1 == "Zulu8-32" ]]; then @@ -274,11 +280,9 @@ java_zulu_fetch() { fi if [[ -z $downloadLink ]]; then echo "FAILED (download link)"; return 1; fi - if ! mkdir -p $jdkInstallLocation; then echo "FAILED (create directory)"; return 1; fi + if ! mkdir -p "${jdkInstallLocation:?}"; then echo "FAILED (create directory)"; return 1; fi if ! rm -rf "${jdkInstallLocation:?}"/*; then echo "FAILED (clean directory)"; return 1; fi - if ! cond_redirect wget -O "$jdkInstallLocation"/zulu.tar.gz "$downloadLink"; then echo "FAILED (download)"; rm -rf "${jdkInstallLocation:?}"/*; return 1; fi - if ! cond_redirect tar -xpzf "$jdkInstallLocation"/zulu.tar.gz -C "$jdkInstallLocation"; then echo "FAILED (extract)"; rm -rf "${jdkInstallLocation:?}"/*; return 1; fi - if cond_redirect rm -rf "$jdkInstallLocation"/zulu.tar.gz; then echo "OK"; else echo "FAILED (cleanup)"; return 1; fi + if ! cond_redirect wget -qO "$jdkInstallLocation"/zulu.tar.gz "$downloadLink"; then echo "FAILED (download)"; rm -rf "${jdkInstallLocation:?}"/*; return 1; fi } ## Check if a newer version of Java Zulu 8 is available. diff --git a/functions/zram.bash b/functions/zram.bash index c22a463a6..3775d4560 100644 --- a/functions/zram.bash +++ b/functions/zram.bash @@ -13,7 +13,7 @@ install_zram_code() { overlayfsGit="https://github.com/kmxz/overlayfs-tools" zramGit="https://github.com/mstormi/openhabian-zram" - echo -n "$(timestamp) [openHABian] Installing ZRAM code... " + echo -n "$(timestamp) [openHABian] Installing ZRAM code to ${1}... " if ! cond_redirect mkdir -p "$1"; then echo "FAILED (create directory)"; return 1; fi if [[ -d "${1}/overlayfs-tools" ]]; then