cookutils view cooker @ rev 1032
cook.conf: add some explanations; cook.site: do not force ${datadir} value; cooker: many receipts not include $HOST_ARCH definition, let them build on x86_64 host without obstacles.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Thu Feb 22 15:44:11 2018 +0200 (2018-02-22) |
| parents | 03374082dafa |
| children | 0ff8d93e8855 |
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful. All devs should
7 # at least understand basic English.
8 #
10 . /usr/lib/slitaz/libcook.sh
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14 wok="$WOK"
16 # PID file.
17 pidfile='/var/run/cooker.pid'
19 #
20 # Functions
21 #
23 usage() {
24 cat <<EOT
26 Usage: cooker [<command>] [<options>]
28 Commands with <options>:
29 -u | usage Display this short usage.
30 -s | setup Setup the Cooker environment.
31 setup-cron [<hours>] Setup a cron job for the Cooker.
32 check-cron Check Cooker cron job.
33 arch-db Create host arch packages DB.
34 -n | note <note_text> Add a note to the cooknotes.
35 -ns | notes Display all the cooknotes.
36 -b | block <package> Block a package so cook will skip it.
37 -ub | unblock <package> Unblock a blocked package.
38 -R | reverse <package> Cook all reverse dependencies for a package.
39 -p | pkg <package> Same as 'cook pkg' but with cooker log.
40 -f | flavor <flavor_name> Cook all packages of a flavor.
41 -l | list <list_file> Cook all packages in the given list.
42 -c | cat <category> Cook all packages of a category.
43 -r | rev <rev_number> Cook packages of a specific revision.
44 -a | all Find and cook all unbuilt packages.
45 -T | tasks List existing cooker tasks.
46 -t | task <task> Executing specified task.
47 -o | outgoing Find changes in wok that we can move to wok-hg.
49 EOT
50 exit 0
51 }
54 # Some messages occur in activity but log verbose output when checking for commits
55 # into a log file.
57 log_commits() {
58 sed '/^.\//d' | sed '/^.hg/d' | tee -a $LOGS/commits.log
59 }
62 # Clean up before exit when check and cook commits finish.
64 clean_exit() {
65 rm -f $command; touch $command
66 [ "$previous_command" ] && ps | grep -q "${previous_command/:/ }" &&
67 echo -n "$previous_command" > $command
68 rm -f $pidfile
69 }
72 # Summary for commits.
74 commits_summary() {
75 msg="from revision $cur to $new"
76 [ "$new" == "$cur" ] && msg="revision $new"
77 echo "Will cook $msg"
78 separator
79 title "Summary for commits"
80 echo "Hg wok revision : $cur"
81 echo "Pulled revision : $new"
82 echo "Check date : $(date '+%F %T')"
83 }
86 # Return all the names of packages bundled in this receipt
88 all_names() {
89 local split=" $SPLIT "
90 unset SPLIT
91 . $wok/$pkg/receipt
93 if ! head -n1 $WOK/$pkg/receipt | fgrep -q 'v2'; then
94 # For receipts v1: $SPLIT may present in the $WANTED package,
95 # but split packages have their own receipts
96 echo $PACKAGE
97 elif [ "${split/ $PACKAGE /}" != "$split" ]; then
98 echo $SPLIT
99 else
100 echo $PACKAGE $SPLIT
101 fi
102 }
105 # Scan packages build deps and fill up cookorder list.
107 cook_order_scan() {
108 rm -f $cookorder $cookorder.split
109 touch $cookorder $cookorder.split
111 # Make combined split table: beginning from actual information with fresh
112 # commits. Example:
113 # freetype freetype freetype-dev
114 # harfbuzz harfbuzz harfbuzz-apps harfbuzz-dev
115 while read pkg; do
116 echo "$pkg $(all_names)" >> $cookorder.split
117 done < $cooklist
118 cat $cache/split.db >> $cookorder.split
120 maxlen=$(wc -L < $cooklist)
122 while read pkg; do
123 unset WANTED BUILD_DEPENDS
124 . $wok/$pkg/receipt
125 bdeps=$(
126 # Substitite each package of BUILD_DEPENDS list by the "main"
127 # receipt which builds this package. Example:
128 # BUILD_DEPENDS="freetype-dev harfbuzz-dev" -> bdeps="freetype harfbuzz"
129 for i in $BUILD_DEPENDS; do
130 main="$(awk -F$'\t' -vi="$i" '{
131 if (index(" " $2 " ", i)) {print $1; exit}
132 }' $cookorder.split)"
133 echo ${main:-$i}
134 done
135 )
136 # The :: is for web interface color.
137 bdeps=$(echo $WANTED $bdeps | tr '\n' ' ')
138 printf "%-${maxlen}s :: %s\n" "$pkg" "$bdeps"
139 for dep in $bdeps; do
140 if grep -q "^$dep$" $cooklist; then
141 if ! grep -q "^$dep$" $cookorder; then
142 echo "$dep" >> $cookorder
143 fi
144 fi
145 done
146 done < $cooklist
148 # Append unordered packages to cookorder.
149 while read pkg; do
150 if ! grep -q "^$pkg$" $cookorder; then
151 echo "$pkg" >> $cookorder
152 fi
153 done < $cooklist
154 }
157 # Scan and rescan until the cooklist is ordered then handle WANTED.
159 cook_order() {
160 time=$(date +%s)
161 scan=0
162 rm -rf $cache/cookorder.d
163 mkdir -p $cache/cookorder.d
165 # Keep an original cooklist so we do a diff when ordering is finished.
166 cp -f $cooklist $cooklist.0
167 echo 'cookorder' > $command
168 title 'Initial Cooker order scan'
169 cook_order_scan
171 # Diff between the cooklist and new ordered list ? So copy the last
172 # cookorder to cooklist and rescan it.
173 while /bin/true; do
174 if ! cmp -s $cooklist $cookorder; then
175 scan=$(($scan + 1))
176 title "Diff scan: $scan"
178 md5stamp=$(md5sum $cookorder | cut -d' ' -f1)
179 if [ -e "$cache/cookorder.d/$md5stamp" ]; then
180 newline
181 echo 'A dependency loop was detected. Interrupting the cookorder.'
182 break
183 fi
184 touch $cache/cookorder.d/$md5stamp
186 mv -f $cookorder $cooklist
187 cook_order_scan
189 else
190 break
191 fi
192 done
193 # Clean
194 rm -rf $cache/cookorder.d; rm $cookorder.split
196 # Keep a diff between submitted cooklist and the ordered.
197 diff $cooklist.0 $cooklist > $cooklist.diff
198 rm -f $cookorder $cooklist.0
200 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
201 # TODO: grep the line number to get pkg position and keep it higher.
202 title 'Handle WANTED package'
203 while read pkg; do
204 unset WANTED
205 . $wok/$pkg/receipt
206 for wanted in $WANTED; do
207 echo "$pkg :: $wanted"
208 if grep -q ^${wanted}$ $cooklist; then
209 sed -i -e "/^$pkg$/d" \
210 -e "/^$wanted$/ a $pkg" $cooklist
211 fi
212 done
213 done < $cooklist
215 # Show ordered cooklist
216 title 'Cooklist order'
217 cat $cooklist
218 separator
220 time=$(($(date +%s) - $time))
221 pkgs=$(wc -l < $cooklist)
222 title 'Summary for cookorder'
223 cat <<EOT
224 Ordered packages : $pkgs
225 Scans executed : $scan
226 Scan duration : ${time}s
227 EOT
228 separator
230 rm -f $command
231 }
234 # Remove blocked (faster this way than grepping before).
236 strip_blocked() {
237 while read pkg; do
238 sed -i "/^${pkg}$/d" $cooklist
239 done < $blocked
240 sed -i '/^$/d' $cooklist
241 }
244 # Use in default mode and with all cmd.
246 cook_commits() {
247 if [ -s "$commits" ]; then
248 while read pkg; do
249 ps | grep -q "cook $pkg$" && continue
250 echo "cook:$pkg" > $command
251 cook $pkg || broken
252 sed -i "/^${pkg}$/d" $commits
253 done < $commits
254 fi
255 }
258 # Cook all packages in a cooklist.
260 cook_list() {
261 while read pkg; do
262 ps | grep -q "cook $pkg$" && continue
263 cook $pkg || broken
264 sed -i "/^${pkg}$/d" $cooklist
265 done < $cooklist
266 }
269 # Create a arch.$ARCH file for each package cooked for the target host
270 # architecture
271 #
272 # The deal: we don't want all packages handled by cooker commands and stats,
273 # since we don't cross compile all packages for each arch but only a set of
274 # packages to provide one full featured desktop, servers and goodies useful
275 # for the host system.
276 #
278 arch_db() {
279 count=0
280 echo "Cleaning packages DB : arch.$ARCH"
281 rm -f $wok/*/arch.$ARCH && cd $wok
282 echo "Creating $ARCH packages DB..."
283 for pkg in *; do
284 [ -s $wok/$pkg/receipt ] || continue
285 HOST_ARCH=
286 . $wok/$pkg/receipt
287 if [ -n "$HOST_ARCH" ]; then
288 if $(echo "$HOST_ARCH" | egrep -q "$ARCH|any"); then
289 count=$(($count + 1))
290 echo "Adding: $pkg"
291 touch $pkg/arch.$ARCH
292 fi
293 unset HOST_ARCH
294 else
295 # HOST_ARCH not set --> package is suitable for current ARCH (equivalent to "any")
296 count=$(($count + 1))
297 echo "Adding: $pkg"
298 touch $pkg/arch.$ARCH
299 fi
300 done
301 echo "Packages for $ARCH : $count"
302 }
305 # Compare wok and wok-hg file $1, display signs:
306 # '+' file added, '-' file removed, '~' file changed, '=' file not changed
308 compare_wok_file() {
309 local f1='n' f2='n' # n: not exists, e: exists
310 [ -e "$wok/$1" ] && f1='e'
311 [ -e "$wok-hg/$1" ] && f2='e'
312 case "$f1$f2" in
313 en) echo "+ $1";;
314 ne) [ -n "$del" ] && echo "- $1";;
315 ee)
316 if cmp -s "$wok/$1" "$wok-hg/$1"; then
317 [ -n "$eq" ] && echo "= $1"
318 else
319 echo "~ $1"
320 fi
321 ;;
322 esac
323 }
326 # Compare wok and wok-hg folder $1; process only:
327 # receipt, description.*txt, all files in the stuff folder
329 compare_wok_folder() {
330 IFS=$'\n'
331 {
332 for i in $wok $wok-hg; do
333 ls $i/$1/receipt 2>/dev/null
334 ls $i/$1/description.*txt 2>/dev/null
335 [ -d $i/$1/stuff ] && find $i/$1/stuff -type f
336 done
337 } | sed "s|$wok/$1/||; s|$wok-hg/$1/||" | sort -u | \
338 while read file; do
339 compare_wok_file "$1/$file"
340 done
341 }
344 # Compare entire wok
346 compare_wok() {
347 {
348 cd $wok; ls
349 cd $wok-hg; ls
350 } | sort -u | \
351 while read folder; do
352 result="$(compare_wok_folder $folder)"
353 [ -n "$result" ] && echo -e "$result\n"
354 done
355 }
358 #
359 # Commands
360 #
362 previous_command="$(cat $command 2>/dev/null)"
363 case "$1" in
364 usage|help|-u|-h)
365 usage ;;
367 setup|-s)
368 # Setup the Cooker environment.
369 title 'Setting up the Cooker'
370 mkdir -p $CACHE
371 echo "Cooker setup using: $SLITAZ" | log
372 for pkg in $SETUP_PKGS mercurial rsync tazlito; do
373 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
374 done
375 mkdir -p $SLITAZ && cd $SLITAZ
376 if [ -d "${wok}-hg" ]; then
377 echo -e 'Hg wok already exists.\n'
378 exit 1
379 fi
380 if [ -d "$wok" ]; then
381 echo -e 'Build wok already exists.\n'
382 exit 1
383 fi
385 # Directories and files
386 echo "mkdir's and touch files in: $SLITAZ"
387 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
388 for f in $activity $blocked $broken $commits $cooklist $command; do
389 touch $f
390 done
391 hg clone $WOK_URL ${wok}-hg || exit 1
392 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
393 cp -a ${wok}-hg $wok
394 footer ;;
396 arch-db)
397 # Manually create arch packages DB.
398 arch_db ;;
400 setup-cron)
401 # Create cron job for the cooker.
402 [ "$2" ] || hours=2
403 if [ ! -f "$crontabs" ]; then
404 mkdir -p /var/spool/cron/crontabs
405 fi
406 if ! fgrep -q /usr/bin/cooker $crontabs; then
407 cat > $crontabs <<EOT
408 # Run SliTaz Cooker every $hours hours
409 59 */$hours * * * touch $CACHE/cooker-request
410 */5 * * * * [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html
411 */5 * * * * [ -z "$(pidof cooker)" ] && [ -s $CACHE/recook-packages ] && /usr/bin/cooker list $CACHE/recook-packages
412 EOT
413 touch $CACHE/cooker-request $CACHE/recook-packages
414 chmod 666 $CACHE/cooker-request $CACHE/recook-packages
415 killall crond 2>/dev/null && /etc/init.d/crond start
416 fi ;;
418 check-cron)
419 if [ ! -f "$crontabs" ]; then
420 echo "There is no $crontabs here. Use setup-cron option."
421 exit 1
422 fi
423 fgrep /usr/bin/cooker $crontabs ;;
425 note|-n)
426 # Blocked a pkg and want others to know why? Post a note!
427 [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;
429 notes|-ns)
430 # View cooknotes.
431 title 'Cooknotes'
432 cat $cooknotes
433 footer ;;
435 block|-b)
436 # Block a package.
437 [ "$pkg" ] && cook $pkg --block ;;
439 unblock|-ub)
440 # Unblock a package.
441 [ "$pkg" ] && cook $pkg --unblock ;;
443 reverse|-r)
444 # Cook all reverse dependencies for a package. This command lets us
445 # control the Cooker manually for commits that will cook a lot of packages.
446 #
447 # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"
448 #
449 if [ ! -d "$wok/$pkg" ]; then
450 echo -e "\nNo package $2 found.\n"
451 exit 0
452 fi
453 rm -f $cooklist; touch $cooklist
454 title "Reverse cooklist for: $pkg"
456 cd $wok
457 for rev in *; do
458 [ -s $wok/$rev/receipt ] || continue
459 unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt
460 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
461 echo "$rev" | tee -a $cooklist
462 fi
463 done
464 footer "Reverse dependencies found: $(wc -l < $cooklist)"
465 strip_blocked
466 cook_order | tee $LOGS/cookorder.log
467 cook_list ;;
469 pkg|-p)
470 # Same as 'cook pkg' but with log for web interface.
471 ps | grep -q "cook $pkg$" && echo 'Already running' && continue
472 cook $pkg || broken
473 clean_exit ;;
475 cat|-c)
476 # Cook all packages of a category.
477 cat="$2"
478 rm -f $cooklist; touch $cooklist
480 cd $wok
481 for pkg in *; do
482 [ -s $pkg/receipt ] || continue
483 unset CATEGORY; . $pkg/receipt
484 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
485 done
486 strip_blocked
487 cook_order | tee $LOGS/cookorder.log
488 cook_list ;;
490 flavor|-f)
491 # Cook all packages of a flavor.
492 name="$2"
493 if [ ! -d "$flavors/$name" ]; then
494 echo -e "\nSpecified flavor does not exist: $name\n"
495 exit 1
496 fi
497 if [ -d "$flavors/.hg" ]; then
498 cd $flavors; hg pull -u
499 fi
500 list="$flavors/$name/packages.list"
501 cp -a $list $cooklist
502 strip_blocked
503 cook_order | tee $LOGS/cookorder.log
504 cook_list ;;
506 list|-l)
507 # Cook a list of packages given in argument.
508 list="$2"
509 if [ ! -f "$list" ]; then
510 echo -e "\nSpecified list does not exist: $list\n"
511 exit 1
512 fi
513 cat $list >> $cooklist
514 echo -n > $list
515 strip_blocked
516 cook_order | tee $LOGS/cookorder.log
517 cook_list ;;
519 rev|-r)
520 # Cook or recook a specific Hg revision.
521 rev="$2"
522 [ "$rev" ] || exit 0
523 rm -f $cooklist; touch $cooklist
525 cd $wok
526 for pkg in $(hg log --rev=$rev --template "{files}"); do
527 echo "$pkg" | cut -d/ -f1 >> $cooklist
528 done
529 strip_blocked
530 cook_order | tee $LOGS/cookorder.log
531 cook_list ;;
533 all|-a)
534 # Try to build all unbuilt packages except blocked's.
535 echo 'cooker:all' > $command
536 rm -f $cooklist; touch $cooklist
537 title 'Cooker cooklist'
539 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt
540 # if it exists since extra version is added when packing the package.
541 echo 'Searching for all unbuilt packages' | log
543 cd $wok
544 for pkg in *; do
545 [ -s $pkg/receipt ] || continue
546 unset EXTRAVERSION
547 . $pkg/receipt
548 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
549 . $pkg/taz/$PACKAGE-$VERSION/receipt
550 if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then
551 echo $pkg; echo $pkg >> $cooklist
552 fi
553 done
554 strip_blocked
555 cook_order | tee $LOGS/cookorder.log
556 echo "Packages to cook: $(wc -l < $cooklist)" | log
557 cook_list ;;
559 tasks|-T)
560 # List existing cooker tasks
561 [ ! -d "$tasks" ] && echo 'There are no tasks.' && exit 0
562 title 'Cooker tasks list'
563 last=$(ls $tasks | tail -n1)
564 for task in $(ls $tasks); do
565 . $tasks/$task
566 echo "Task name : $task"
567 echo "Description : $DESC"
568 separator $([ $task != $last ] && echo '-')
569 done
570 newline ;;
572 task|-t)
573 # Executing specified task
574 task="$2"
575 title "Executing cooker task: $task"
576 . $tasks/$task; task
577 footer "Task $task finished" ;;
579 outgoing|-o)
580 # Find changes in wok that we can move to wok-hg
581 compare_wok
582 ;;
584 *)
585 # Default is to cook all commits if not yet running.
586 [ -n "$1" ] && usage
587 cooklist=$commits
588 if [ -f "$pidfile" ]; then
589 pid=$(cat $pidfile)
590 if [ -s /proc/$pid/status ]; then
591 echo -e "\nStill cooking latest commits with pid:"
592 echo -e " $pid\n"
593 exit 0
594 fi
595 rm -f "$pidfile"
596 fi
598 # Start and get a PID file.
599 rm -f $LOGS/commits.log
600 newline
601 echo 'Checking for commits' | log_commits
602 separator | tee -a $LOGS/commits.log
604 echo $$ > $pidfile
605 trap 'echo -e "\nCooker stopped: PID $$\n" && \
606 rm -f $pidfile $command && exit 1' INT TERM
608 echo "Cooker PID : $$" | log_commits
609 echo "Cooker date : $(date '+%F %T')" | log_commits
611 # Get revisions. Here we have 2 echoes since we want a msg on screen,
612 # in commits log and activity DB without a space before.
613 cd $wok || exit 1
614 cur=$(hg head --template '{rev}\n')
615 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
616 echo "Updating wok: ${wok}-hg" | log
617 echo 'hg:pull' > $command
618 cd $wok-hg; hg pull -u | log_commits
619 new=$(hg head --template '{rev}\n')
620 # Store last rev to be used by CGI so it doesn't need to call hg head
621 # on each load.
622 echo "$new" > $wokrev
624 # Sync build wok with rsync so we don't take care about removing old
625 # files as before.
626 if [ "$new" -gt "$cur" ]; then
627 echo "Changes found from: $cur to $new" | log
628 echo 'Syncing build wok with Hg wok...' | log_commits
629 rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \
630 sed '/^$/d' | log_commits
631 else
632 echo "No revision changes: $cur vs $new" | log
633 separator | log_commits
634 clean_exit; newline
635 exit 0
636 fi
638 # Get and display modifications.
639 cd $wok-hg
640 commits_summary | log_commits
641 cur=$(($cur + 1))
642 rm -f $commits.tmp; touch $commits.tmp
643 for rev in $(seq $cur $new); do
644 for file in $(hg log --rev=$rev --template "{files}"); do
645 pkg=$(echo $file | cut -d/ -f1)
646 desc=$(hg log --rev=$rev --template "{desc}" $file)
647 echo "Committed package : $pkg - $desc" | log_commits
648 echo $pkg >> $commits.tmp
649 done
650 done
652 # We may have deleted packages and files in stuff/. Remove it and
653 # clean DB as well as log file.
654 cd $wok
655 for pkg in *; do
656 if [ ! -d "$wok-hg/$pkg" ]; then
657 echo "Removing package: $pkg" | log_commits
658 if [ -s $wok/$pkg/receipt ]; then
659 . $wok/$pkg/receipt
660 rm -f $PKGS/$PACKAGE-$VERSION*
661 fi
662 rm -rf $wok/$pkg $LOGS/$pkg.log
663 sed -i "/^${pkg}$/d" $blocked $broken $commits.tmp
664 sed -i "/^$pkg\t/d" $PKGS/packages.info
665 sed -i "/^$pkg:/d" $cache/files.list
666 fi
667 if [ -d "$wok/$pkg/stuff" ]; then
668 if [ ! -d "$wok-hg/$pkg/stuff" ]; then
669 echo "Removing stuff: $pkg/stuff" | log_commits
670 rm -rf $wok/$pkg/stuff
671 else
672 for stuff_file in $(cd $wok/$pkg/stuff; find \( -type f -o -type l \) | sed 's|^\./||'); do
673 if [ ! -f "$wok-hg/$pkg/stuff/$stuff_file" -a \
674 ! -h "$wok-hg/$pkg/stuff/$stuff_file" ]; then
675 echo "Removing file from stuff: $wok/$pkg/stuff/$stuff_file" | log_commits
676 rm -f $wok/$pkg/stuff/$stuff_file
677 rmdir --parents --ignore-fail-on-non-empty $(dirname "$wok/$pkg/stuff/$stuff_file")
678 fi
679 done
680 fi
681 fi
682 done
684 # Keep previous commit and discard duplicate lines
685 cat $commits $commits.tmp | sed '/^$/d' > $commits.new
686 uniq $commits.new > $commits; rm $commits.*
688 # Handle cross compilation. Create arch packages DB and remove pkgs
689 # not cooked for this arch from the commits list.
690 arch_db
691 while read pkg; do
692 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
693 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
694 log_commits
695 sed -i "/^${pkg}$/d" $commits
696 else
697 echo "Cooker arch : $ARCH" | log_commits
698 fi
699 done < $commits
701 # Re-create split database
702 cook splitdb
704 # Stats
705 pkgs=$(wc -l < $commits)
706 echo "Packages to cook: $pkgs" | log
707 echo "Packages to cook : $pkgs" | log_commits
708 separator | log_commits
709 newline
710 strip_blocked
711 cook_order | tee $LOGS/cookorder.log
712 cook_commits
713 clean_exit ;;
714 esac
716 exit 0