cooker

#!/bin/sh
#
# SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
# building. Please read the Cookbook documentation for more information
# and discuss with the AUTHORS before adding anything here. PS: no translations
# here since it's not an end user tool and it's not useful. All devs should
# at least understand basic English.
#

. /usr/lib/slitaz/libcook.sh

# Set pkg name and use same wok as cook.
pkg="$2"
wok="$WOK"

# PID file.
pidfile='/var/run/cooker.pid'

#
# Functions
#

usage() {
	cat <<EOT

Usage: cooker [<command>] [<options>]

Commands with <options>:
  -u  | usage                  Display this short usage.
  -s  | setup                  Setup the Cooker environment.
        setup-cron [<hours>]   Setup a cron job for the Cooker.
        check-cron             Check Cooker cron job.
        arch-db                Create host arch packages DB.
  -n  | note    <note_text>    Add a note to the cooknotes.
  -ns | notes                  Display all the cooknotes.
  -b  | block   <package>      Block a package so cook will skip it.
  -ub | unblock <package>      Unblock a blocked package.
  -R  | reverse <package>      Cook all reverse dependencies for a package.
  -p  | pkg     <package>      Same as 'cook pkg' but with cooker log.
  -f  | flavor  <flavor_name>  Cook all packages of a flavor.
  -l  | list    <list_file>    Cook all packages in the given list.
  -c  | cat     <category>     Cook all packages of a category.
  -r  | rev     <rev_number>   Cook packages of a specific revision.
  -a  | all                    Find and cook all unbuilt packages.
  -T  | tasks                  List existing cooker tasks.
  -t  | task    <task>         Executing specified task.
  -o  | outgoing               Find changes in wok that we can move to wok-hg.
        autodeps               Find dependencies for all packages in wok.

EOT
	exit 0
}


# Some messages occur in activity but log verbose output when checking for commits
# into a log file.

log_commits() {
	sed '/^.\//d' | sed '/^.hg/d' | tee -a $LOGS/commits.log
}


# Clean up before exit when check and cook commits finish.

clean_exit() {
	rm -f $command; touch $command
	[ "$previous_command" ] && ps | grep -q "${previous_command/:/ }" &&
	echo -n "$previous_command" > $command
	rm -f $pidfile
}


# Summary for commits.

commits_summary() {
	msg="from revision $cur to $new"
	[ "$new" == "$cur" ] && msg="revision $new"
	echo "Will cook $msg"
	separator
	title "Summary for commits"
	echo "Hg wok revision  : $cur"
	echo "Pulled revision  : $new"
	echo "Check date       : $(date '+%F %T')"
}


# Return all the names of packages bundled in this receipt

all_names() {
	local split=" $SPLIT "
	unset SPLIT
	. $wok/$pkg/receipt

	if ! head -n1 $WOK/$pkg/receipt | fgrep -q 'v2'; then
		# For receipts v1: $SPLIT may present in the $WANTED package,
		# but split packages have their own receipts
		echo $PACKAGE
	elif [ "${split/ $PACKAGE /}" != "$split" ]; then
		echo $SPLIT
	else
		echo $PACKAGE $SPLIT
	fi
}


# Scan packages build deps and fill up cookorder list.

cook_order_scan() {
	rm -f $cookorder $cookorder.split
	touch $cookorder $cookorder.split

	# Make combined split table: beginning from actual information with fresh
	# commits. Example:
	# freetype	freetype freetype-dev
	# harfbuzz	harfbuzz harfbuzz-apps harfbuzz-dev
	while read pkg; do
		echo "$pkg	$(all_names)" >> $cookorder.split
	done < $cooklist
	cat $cache/split.db >> $cookorder.split

	maxlen=$(wc -L < $cooklist)

	while read pkg; do
		unset WANTED BUILD_DEPENDS
		. $wok/$pkg/receipt
		bdeps=$(
			# Substitite each package of BUILD_DEPENDS list by the "main"
			# receipt which builds this package. Example:
			# BUILD_DEPENDS="freetype-dev harfbuzz-dev" -> bdeps="freetype harfbuzz"
			for i in $BUILD_DEPENDS; do
				main="$(awk -F$'\t' -vi="$i" '{
					if (index(" " $2 " ", i)) {print $1; exit}
				}' $cookorder.split)"
				echo ${main:-$i}
			done
		)
		# The :: is for web interface color.
		bdeps=$(echo $WANTED $bdeps | tr '\n' ' ')
		printf "%-${maxlen}s :: %s\n" "$pkg" "$bdeps"
		for dep in $bdeps; do
			if grep -q "^$dep$" $cooklist; then
				if ! grep -q "^$dep$" $cookorder; then
					echo "$dep" >> $cookorder
				fi
			fi
		done
	done < $cooklist

	# Append unordered packages to cookorder.
	while read pkg; do
		if ! grep -q "^$pkg$" $cookorder; then
			echo "$pkg" >> $cookorder
		fi
	done < $cooklist
}


# Scan and rescan until the cooklist is ordered then handle WANTED.

cook_order() {
	time=$(date +%s)
	scan=0
	rm   -rf $cache/cookorder.d
	mkdir -p $cache/cookorder.d

	# Keep an original cooklist so we do a diff when ordering is finished.
	cp -f $cooklist $cooklist.0
	echo 'cookorder' > $command
	title 'Initial Cooker order scan'
	cook_order_scan

	# Diff between the cooklist and new ordered list ? So copy the last
	# cookorder to cooklist and rescan it.
	while /bin/true; do
		if ! cmp -s $cooklist $cookorder; then
			scan=$(($scan + 1))
			title "Diff scan: $scan"

			md5stamp=$(md5sum $cookorder | cut -d' ' -f1)
			if [ -e "$cache/cookorder.d/$md5stamp" ]; then
				newline
				echo 'A dependency loop was detected. Interrupting the cookorder.'
				break
			fi
			touch $cache/cookorder.d/$md5stamp

			mv -f $cookorder $cooklist
			cook_order_scan

		else
			break
		fi
	done
	# Clean
	rm -rf $cache/cookorder.d; rm $cookorder.split

	# Keep a diff between submitted cooklist and the ordered.
	diff $cooklist.0 $cooklist > $cooklist.diff
	rm -f $cookorder $cooklist.0

	# Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
	# TODO: grep the line number to get pkg position and keep it higher.
	title 'Handle WANTED package'
	while read pkg; do
		unset WANTED
		. $wok/$pkg/receipt
		for wanted in $WANTED; do
			echo "$pkg :: $wanted"
			if grep -q ^${wanted}$ $cooklist; then
				sed -i -e "/^$pkg$/d" \
					-e "/^$wanted$/ a $pkg" $cooklist
			fi
		done
	done < $cooklist

	# Show ordered cooklist
	title 'Cooklist order'
	cat $cooklist
	separator

	time=$(($(date +%s) - $time))
	pkgs=$(wc -l < $cooklist)
	title 'Summary for cookorder'
	cat <<EOT
Ordered packages : $pkgs
Scans executed   : $scan
Scan duration    : ${time}s
EOT
	separator

	rm -f $command
}


# Remove blocked (faster this way than grepping before).

strip_blocked() {
	while read pkg; do
		sed -i "/^${pkg}$/d" $cooklist
	done < $blocked
	sed -i '/^$/d' $cooklist
}


# Use in default mode and with all cmd.

cook_commits() {
	if [ -s "$commits" ]; then
		while read pkg; do
			ps | grep -q "cook $pkg$" && continue
			echo "cook:$pkg" > $command
			cook $pkg || broken
			sed -i "/^${pkg}$/d" $commits
		done < $commits
	fi
}


# Cook all packages in a cooklist.

cook_list() {
	while read pkg; do
		ps | grep -q "cook $pkg$" && continue
		cook $pkg || broken
		sed -i "/^${pkg}$/d" $cooklist
	done < $cooklist
}


# Create a arch.$ARCH file for each package cooked for the target host
# architecture
#
# The deal: we don't want all packages handled by cooker commands and stats,
# since we don't cross compile all packages for each arch but only a set of
# packages to provide one full featured desktop, servers and goodies useful
# for the host system.
#

arch_db() {
	count=0
	echo "Cleaning packages DB : arch.$ARCH"
	rm -f $wok/*/arch.$ARCH && cd $wok
	echo "Creating $ARCH packages DB..."
	for pkg in *; do
		[ -e $wok/$pkg/.hidden ] && continue
		[ -s $wok/$pkg/receipt ] || continue
		HOST_ARCH=
		. $wok/$pkg/receipt
		if [ -n "$HOST_ARCH" ]; then
			if echo "$HOST_ARCH" | egrep -q "$ARCH|any"; then
				count=$(($count + 1))
				echo "Adding: $pkg"
				touch $pkg/arch.$ARCH
			fi
			unset HOST_ARCH
		else
			# HOST_ARCH not set --> package is suitable for current ARCH
			count=$(($count + 1))
			echo "Adding: $pkg"
			touch $pkg/arch.$ARCH
		fi
	done
	echo "Packages for $ARCH : $count"
}


# Compare wok and wok-hg file $1, display signs:
# '+' file added, '-' file removed, '~' file changed, '=' file not changed

compare_wok_file() {
	local f1='n' f2='n' # n: not exists, e: exists
	[ -e "$wok/$1"    ] && f1='e'
	[ -e "$wok-hg/$1" ] && f2='e'
	case "$f1$f2" in
		en) echo "+ $1";;
		ne) [ -n "$del" ] && echo "- $1";;
		ee)
			if cmp -s "$wok/$1" "$wok-hg/$1"; then
				[ -n "$eq" ] && echo "= $1"
			else
				echo "~ $1"
			fi
			;;
	esac
}


# Compare wok and wok-hg folder $1; process only:
# receipt, description.*txt, all files in the stuff folder

compare_wok_folder() {
	IFS=$'\n'
	{
		for i in $wok $wok-hg; do
			ls $i/$1/receipt 2>/dev/null
			ls $i/$1/description.*txt 2>/dev/null
			[ -d $i/$1/stuff ] && find $i/$1/stuff -type f
		done
	} | sed "s|$wok/$1/||; s|$wok-hg/$1/||" | sort -u | \
	while read file; do
		compare_wok_file "$1/$file"
	done
}


# Compare entire wok

compare_wok() {
	{
		cd $wok;    ls
		cd $wok-hg; ls
	} | sort -u | \
	while read folder; do
		result="$(compare_wok_folder $folder)"
		[ -n "$result" ] && echo -e "$result\n"
	done
}


#
# Commands
#

previous_command="$(cat $command 2>/dev/null)"
case "$1" in
	usage|help|-u|-h)
		usage ;;

	setup|-s)
		# Setup the Cooker environment.
		title 'Setting up the Cooker'
		mkdir -p $CACHE
		echo "Cooker setup using: $SLITAZ" | log
		for pkg in $SETUP_PKGS mercurial rsync tazlito; do
			[ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
		done
		mkdir -p $SLITAZ && cd $SLITAZ
		if [ -d "${wok}-hg" ]; then
			echo -e 'Hg wok already exists.\n'
			exit 1
		fi
		if [ -d "$wok" ]; then
			echo -e 'Build wok already exists.\n'
			exit 1
		fi

		# Directories and files
		echo "mkdir's and touch files in: $SLITAZ"
		mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
		for f in $activity $blocked $broken $commits $cooklist $command; do
			touch $f
		done
		hg clone $WOK_URL ${wok}-hg || exit 1
		[ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
		cp -a ${wok}-hg $wok
		footer ;;

	arch-db)
		# Manually create arch packages DB.
		arch_db ;;

	setup-cron)
		# Create cron job for the cooker.
		[ "$2" ] || hours=2
		if [ ! -f "$crontabs" ]; then
			mkdir -p /var/spool/cron/crontabs
		fi
		if ! fgrep -q /usr/bin/cooker $crontabs; then
			cat > $crontabs <<EOT
# Run SliTaz Cooker every $hours hours
59 */$hours * * *  touch $CACHE/cooker-request
*/5 * * * *  [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html
*/5 * * * *  [ -z "$(pidof cooker)" ] && [ -s $CACHE/recook-packages ] && /usr/bin/cooker list $CACHE/recook-packages
EOT
			touch $CACHE/cooker-request $CACHE/recook-packages
			chmod 666 $CACHE/cooker-request $CACHE/recook-packages
			killall crond 2>/dev/null && /etc/init.d/crond start
		fi ;;

	check-cron)
		if [ ! -f "$crontabs" ]; then
			echo "There is no $crontabs here. Use setup-cron option."
			exit 1
		fi
		fgrep /usr/bin/cooker $crontabs ;;

	note|-n)
		# Blocked a pkg and want others to know why? Post a note!
		[ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;

	notes|-ns)
		# View cooknotes.
		title 'Cooknotes'
		cat $cooknotes
		footer ;;

	block|-b)
		# Block a package.
		[ "$pkg" ] && cook $pkg --block ;;

	unblock|-ub)
		# Unblock a package.
		[ "$pkg" ] && cook $pkg --unblock ;;

	reverse|-r)
		# Cook all reverse dependencies for a package. This command lets us
		# control the Cooker manually for commits that will cook a lot of packages.
		#
		# Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"
		#
		if [ ! -d "$wok/$pkg" ]; then
			echo -e "\nNo package $2 found.\n"
			exit 0
		fi
		rm -f $cooklist; touch $cooklist
		title "Reverse cooklist for: $pkg"

		cd $wok
		for rev in *; do
			[ -s $wok/$rev/receipt ] || continue
			unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt
			if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
				echo "$rev" | tee -a $cooklist
			fi
		done
		footer "Reverse dependencies found: $(wc -l < $cooklist)"
		strip_blocked
		cook_order | tee $LOGS/cookorder.log
		cook_list ;;

	pkg|-p)
		# Same as 'cook pkg'.
		ps | grep -q "cook $pkg$" && echo 'Already running' && continue
		cook $pkg || broken
		clean_exit ;;

	cat|-c)
		# Cook all packages of a category.
		cat="$2"
		rm -f $cooklist; touch $cooklist

		cd $wok
		for pkg in *; do
			[ -s $pkg/receipt ] || continue
			unset CATEGORY; . $pkg/receipt
			[ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
		done
		strip_blocked
		cook_order | tee $LOGS/cookorder.log
		cook_list ;;

	flavor|-f)
		# Cook all packages of a flavor.
		name="$2"
		if [ ! -d "$flavors/$name" ]; then
			echo -e "\nSpecified flavor does not exist: $name\n"
			exit 1
		fi
		if [ -d "$flavors/.hg" ]; then
			cd $flavors; hg pull -u
		fi
		list="$flavors/$name/packages.list"
		cp -a $list $cooklist
		strip_blocked
		cook_order | tee $LOGS/cookorder.log
		cook_list ;;

	list|-l)
		# Cook a list of packages given in argument.
		list="$2"
		if [ ! -f "$list" ]; then
			echo -e "\nSpecified list does not exist: $list\n"
			exit 1
		fi
		cat $list >> $cooklist
		echo -n > $list
		strip_blocked
		cook_order | tee $LOGS/cookorder.log
		cook_list ;;

	rev)
		# Cook or recook a specific Hg revision.
		rev="$2"
		[ "$rev" ] || exit 0
		rm -f $cooklist; touch $cooklist

		cd $wok
		for pkg in $(hg log --rev=$rev --template "{files}"); do
			echo "$pkg" | cut -d/ -f1 >> $cooklist
		done
		strip_blocked
		cook_order | tee $LOGS/cookorder.log
		cook_list ;;

	all|-a)
		# Try to build all unbuilt packages except blocked's.
		echo 'cooker:all' > $command
		rm -f $cooklist; touch $cooklist
		title 'Cooker cooklist'

		# Find all unbuilt packages. Get EXTRAVERSION from packed receipt
		# if it exists since extra version is added when packing the package.
		echo 'Searching for all unbuilt packages' | log

		cd $wok
		for pkg in *; do
			[ -s $pkg/receipt ] || continue
			unset EXTRAVERSION
			. $pkg/receipt
			[ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
				. $pkg/taz/$PACKAGE-$VERSION/receipt
			if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then
				echo $pkg; echo $pkg >> $cooklist
			fi
		done
		strip_blocked
		cook_order | tee $LOGS/cookorder.log
		echo "Receipts or stuff changed: $(wc -l < $cooklist)" | log
		cook_list ;;

	tasks|-T)
		# List existing cooker tasks
		[ ! -d "$tasks" ] && echo 'There are no tasks.' && exit 0
		title 'Cooker tasks list'
		last=$(ls $tasks | tail -n1)
		for task in $(ls $tasks); do
			. $tasks/$task
			echo "Task name   : $task"
			echo "Description : $DESC"
			separator $([ $task != $last ] && echo '-')
		done
		newline ;;

	task|-t)
		# Executing specified task
		task="$2"
		title "Executing cooker task: $task"
		. $tasks/$task; task
		footer "Task $task finished" ;;

	outgoing|-o)
		# Find changes in wok that we can move to wok-hg
		compare_wok
		;;

	autodeps)
		# Find dependencies for all packages in wok
		cd $WOK
		for pkg in *; do
			cook $pkg --deps --quiet
		done | tee $cache/autodeps
		;;

	*)
		# Default is to cook all commits if not yet running.
		[ -n "$1" ] && usage
		cooklist=$commits
		if [ -f "$pidfile" ]; then
			pid=$(cat $pidfile)
			if [ -s /proc/$pid/status ]; then
				echo -e "\nStill cooking latest commits with pid:"
				echo -e " $pid\n"
				exit 0
			fi
			rm -f "$pidfile"
		fi

		# Start and get a PID file.
		rm -f $LOGS/commits.log
		newline
		echo 'Checking for commits' | log_commits
		separator | tee -a $LOGS/commits.log

		echo $$ > $pidfile
		trap 'echo -e "\nCooker stopped: PID $$\n" && \
			rm -f $pidfile $command && exit 1' INT TERM

		echo "Cooker PID   : $$" | log_commits
		echo "Cooker date  : $(date '+%F %T')" | log_commits

		# Get revisions. Here we have 2 echoes since we want a msg on screen,
		# in commits log and activity DB without a space before.
		cd $wok || exit 1
		cur=$(hg head --template '{rev}\n')
		echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
		echo "Updating wok: ${wok}-hg" | log
		echo 'hg:pull' > $command
		cd $wok-hg; hg pull -u | log_commits
		new=$(hg head --template '{rev}\n')
		# Store last rev to be used by CGI so it doesn't need to call hg head
		# on each load.
		echo "$new" > $wokrev

		# Sync build wok with rsync so we don't take care about removing old
		# files as before.
		if [ "$new" -gt "$cur" ]; then
			echo "Changes found from: $cur to $new" | log
			echo 'Syncing build wok with Hg wok...' | log_commits
			rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \
				sed '/^$/d' | log_commits
		else
			echo "No revision changes: $cur vs $new" | log
			separator | log_commits
			clean_exit; newline
			exit 0
		fi

		# Get and display modifications.
		cd $wok-hg
		commits_summary | log_commits
		cur=$(($cur + 1))
		rm -f $commits.tmp; touch $commits.tmp
		for rev in $(seq $cur $new); do
			for file in $(hg log --rev=$rev --template "{files}"); do
				pkg=$(echo $file | cut -d/ -f1)
				desc=$(hg log --rev=$rev --template "{desc}" $file)
				echo "Committed package : $pkg - $desc" | log_commits
				echo $pkg >> $commits.tmp
			done
		done

		# We may have deleted packages and files in stuff/. Remove it and
		# clean DB as well as log file.
		cd $wok
		for pkg in *; do
			if [ ! -d "$wok-hg/$pkg" -o -e "$wok-hg/$pkg/.hidden" ]; then
				echo "Removing package: $pkg" | log_commits
				if [ -s $wok/$pkg/receipt ]; then
					. $wok/$pkg/receipt
					rm -f $PKGS/$PACKAGE-$VERSION*
				fi
				rm -rf $wok/$pkg $LOGS/$pkg.log
				sed -i "/^${pkg}$/d" $blocked $broken $commits.tmp
				sed -i "/^$pkg\t/d" $PKGS/packages-$ARCH.info
				sed -i "/^$pkg:/d" $cache/files.list
				sed -i "/^$pkg\t/d" $cache/badges
			fi
			if [ -d "$wok/$pkg/stuff" ]; then
				if [ ! -d "$wok-hg/$pkg/stuff" ]; then
					echo "Removing stuff: $pkg/stuff" | log_commits
					rm -rf $wok/$pkg/stuff
				else
					for stuff_file in $(cd $wok/$pkg/stuff; find \( -type f -o -type l \) | sed 's|^\./||'); do
						if [ ! -f "$wok-hg/$pkg/stuff/$stuff_file" -a \
							 ! -h "$wok-hg/$pkg/stuff/$stuff_file" ]; then
							echo "Removing file from stuff: $wok/$pkg/stuff/$stuff_file" | log_commits
							rm -f $wok/$pkg/stuff/$stuff_file
							rmdir --parents --ignore-fail-on-non-empty $(dirname "$wok/$pkg/stuff/$stuff_file")
						fi
					done
				fi
			fi
		done

		# Keep previous commit and discard duplicate lines
		cat $commits $commits.tmp | sed '/^$/d' > $commits.new
		uniq $commits.new > $commits; rm $commits.*

		# Handle cross compilation. Create arch packages DB and remove pkgs
		# not cooked for this arch from the commits list.
		arch_db
		while read pkg; do
			if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
				echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
					log_commits
				sed -i "/^${pkg}$/d" $commits
			else
				echo "Cooker arch : $ARCH" | log_commits
			fi
		done < $commits

		# Re-create split database
		cook splitdb

		# Stats
		pkgs=$(wc -l < $commits)
		echo "Packages to cook: $pkgs" | log
		echo "Packages to cook : $pkgs" | log_commits
		separator | log_commits
		newline
		# Just update the wok on --update, don't cook any package
		if [ -z "$update" ]; then
			strip_blocked
			cook_order | tee $LOGS/cookorder.log
			cook_commits
		fi
		clean_exit ;;
esac

exit 0