cookutils view cooker @ rev 949
cook: specify we use busybox fold (don't fail when remove coreutils-file-format or coreutils-multicall containing fold); cooker: remove package from packages.info DB too.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Thu Jul 20 05:07:33 2017 +0300 (2017-07-20) |
| parents | dc6b6e10da97 |
| children | 11e28a8b9e5a |
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful. All devs should
7 # at least understand basic English.
8 #
10 . /usr/lib/slitaz/libcook.sh
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14 wok="$WOK"
16 # PID file.
17 pidfile='/var/run/cooker.pid'
19 #
20 # Functions
21 #
23 usage() {
24 cat <<EOT
26 Usage: cooker [<command>] [<options>]
28 Commands with <options>:
29 -u | usage Display this short usage.
30 -s | setup Setup the Cooker environment.
31 setup-cron [<hours>] Setup a cron job for the Cooker.
32 check-cron Check Cooker cron job.
33 arch-db Create host arch packages DB.
34 -n | note <note_text> Add a note to the cooknotes.
35 -ns | notes Display all the cooknotes.
36 -b | block <package> Block a package so cook will skip it.
37 -ub | unblock <package> Unblock a blocked package.
38 -R | reverse <package> Cook all reverse dependencies for a package.
39 -p | pkg <package> Same as 'cook pkg' but with cooker log.
40 -f | flavor <flavor_name> Cook all packages of a flavor.
41 -l | list <list_file> Cook all packages in the given list.
42 -c | cat <category> Cook all packages of a category.
43 -r | rev <rev_number> Cook packages of a specific revision.
44 -a | all Find and cook all unbuilt packages.
45 -T | tasks List existing cooker tasks.
46 -t | task <task> Executing specified task.
48 EOT
49 exit 0
50 }
53 # Some messages occur in activity but log verbose output when checking for commits
54 # into a log file.
56 log_commits() {
57 sed '/^.\//d' | sed '/^.hg/d' | tee -a $LOGS/commits.log
58 }
61 # Clean up before exit when check and cook commits finish.
63 clean_exit() {
64 rm -f $command; touch $command
65 [ "$previous_command" ] && ps | grep -q "${previous_command/:/ }" &&
66 echo -n "$previous_command" > $command
67 rm -f $pidfile
68 }
71 # Summary for commits.
73 commits_summary() {
74 msg="from revision $cur to $new"
75 [ "$new" == "$cur" ] && msg="revision $new"
76 echo "Will cook $msg"
77 separator
78 title "Summary for commits"
79 echo "Hg wok revision : $cur"
80 echo "Pulled revision : $new"
81 echo "Check date : $(date '+%F %T')"
82 }
85 # Return all the names of packages bundled in this receipt
87 all_names() {
88 local split=" $SPLIT "
89 unset SPLIT
90 . $wok/$pkg/receipt
92 if ! head -n1 $WOK/$pkg/receipt | fgrep -q 'v2'; then
93 # For receipts v1: $SPLIT may present in the $WANTED package,
94 # but split packages have their own receipts
95 echo $PACKAGE
96 elif [ "${split/ $PACKAGE /}" != "$split" ]; then
97 echo $SPLIT
98 else
99 echo $PACKAGE $SPLIT
100 fi
101 }
104 # Scan packages build deps and fill up cookorder list.
106 cook_order_scan() {
107 rm -f $cookorder $cookorder.split
108 touch $cookorder $cookorder.split
110 # Make combined split table: beginning from actual information with fresh
111 # commits. Example:
112 # freetype freetype freetype-dev
113 # harfbuzz harfbuzz harfbuzz-apps harfbuzz-dev
114 while read pkg; do
115 echo "$pkg $(all_names)" >> $cookorder.split
116 done < $cooklist
117 cat $cache/split.db >> $cookorder.split
119 maxlen=$(wc -L < $cooklist)
121 while read pkg; do
122 unset WANTED BUILD_DEPENDS
123 . $wok/$pkg/receipt
124 bdeps=$(
125 # Substitite each package of BUILD_DEPENDS list by the "main"
126 # receipt which builds this package. Example:
127 # BUILD_DEPENDS="freetype-dev harfbuzz-dev" -> bdeps="freetype harfbuzz"
128 for i in $BUILD_DEPENDS; do
129 main="$(awk -F$'\t' -vi="$i" '{
130 if (index(" " $2 " ", i)) {print $1; exit}
131 }' $cookorder.split)"
132 echo ${main:-$i}
133 done
134 )
135 # The :: is for web interface color.
136 bdeps=$(echo $WANTED $bdeps | tr '\n' ' ')
137 printf "%-${maxlen}s :: %s\n" "$pkg" "$bdeps"
138 for dep in $bdeps; do
139 if grep -q "^$dep$" $cooklist; then
140 if ! grep -q "^$dep$" $cookorder; then
141 echo "$dep" >> $cookorder
142 fi
143 fi
144 done
145 done < $cooklist
147 # Append unordered packages to cookorder.
148 while read pkg; do
149 if ! grep -q "^$pkg$" $cookorder; then
150 echo "$pkg" >> $cookorder
151 fi
152 done < $cooklist
153 }
156 # Scan and rescan until the cooklist is ordered then handle WANTED.
158 cook_order() {
159 time=$(date +%s)
160 scan=0
161 rm -rf $cache/cookorder.d
162 mkdir -p $cache/cookorder.d
164 # Keep an original cooklist so we do a diff when ordering is finished.
165 cp -f $cooklist $cooklist.0
166 echo 'cookorder' > $command
167 title 'Initial Cooker order scan'
168 cook_order_scan
170 # Diff between the cooklist and new ordered list ? So copy the last
171 # cookorder to cooklist and rescan it.
172 while /bin/true; do
173 if ! cmp -s $cooklist $cookorder; then
174 scan=$(($scan + 1))
175 title "Diff scan: $scan"
177 md5stamp=$(md5sum $cookorder | cut -d' ' -f1)
178 if [ -e "$cache/cookorder.d/$md5stamp" ]; then
179 newline
180 echo 'A dependency loop was detected. Interrupting the cookorder.'
181 break
182 fi
183 touch $cache/cookorder.d/$md5stamp
185 mv -f $cookorder $cooklist
186 cook_order_scan
188 else
189 break
190 fi
191 done
192 # Clean
193 rm -rf $cache/cookorder.d; rm $cookorder.split
195 # Keep a diff between submitted cooklist and the ordered.
196 diff $cooklist.0 $cooklist > $cooklist.diff
197 rm -f $cookorder $cooklist.0
199 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
200 # TODO: grep the line number to get pkg position and keep it higher.
201 title 'Handle WANTED package'
202 while read pkg; do
203 unset WANTED
204 . $wok/$pkg/receipt
205 for wanted in $WANTED; do
206 echo "$pkg :: $wanted"
207 if grep -q ^${wanted}$ $cooklist; then
208 sed -i -e "/^$pkg$/d" \
209 -e "/^$wanted$/ a $pkg" $cooklist
210 fi
211 done
212 done < $cooklist
214 # Show ordered cooklist
215 title 'Cooklist order'
216 cat $cooklist
217 separator
219 time=$(($(date +%s) - $time))
220 pkgs=$(wc -l < $cooklist)
221 title 'Summary for cookorder'
222 cat <<EOT
223 Ordered packages : $pkgs
224 Scans executed : $scan
225 Scan duration : ${time}s
226 EOT
227 separator
229 rm -f $command
230 }
233 # Remove blocked (faster this way than grepping before).
235 strip_blocked() {
236 while read pkg; do
237 sed -i "/^${pkg}$/d" $cooklist
238 done < $blocked
239 sed -i '/^$/d' $cooklist
240 }
243 # Use in default mode and with all cmd.
245 cook_commits() {
246 if [ -s "$commits" ]; then
247 while read pkg; do
248 ps | grep -q "cook $pkg$" && continue
249 echo "cook:$pkg" > $command
250 cook $pkg || broken
251 sed -i "/^${pkg}$/d" $commits
252 done < $commits
253 fi
254 }
257 # Cook all packages in a cooklist.
259 cook_list() {
260 while read pkg; do
261 ps | grep -q "cook $pkg$" && continue
262 cook $pkg || broken
263 sed -i "/^${pkg}$/d" $cooklist
264 done < $cooklist
265 }
268 # Create a arch.$ARCH file for each package cooked for the target host
269 # architecture
270 #
271 # The deal: we don't want all packages handled by cooker commands and stats,
272 # since we don't cross compile all packages for each arch but only a set of
273 # packages to provide one full featured desktop, servers and goodies useful
274 # for the host system.
275 #
277 arch_db() {
278 count=0
279 echo "Cleaning packages DB : arch.$ARCH"
280 rm -f $wok/*/arch.$ARCH && cd $wok
281 echo "Creating $ARCH packages DB..."
282 for pkg in *; do
283 [ -s $wok/$pkg/receipt ] || continue
284 HOST_ARCH=
285 . $wok/$pkg/receipt
286 if [ -n "$HOST_ARCH" ]; then
287 if $(echo "$HOST_ARCH" | egrep -q "$ARCH|any"); then
288 count=$(($count + 1))
289 echo "Adding: $pkg"
290 touch $pkg/arch.$ARCH
291 fi
292 unset HOST_ARCH
293 else
294 # HOST_ARCH not set --> in i486
295 if [ "$ARCH" == 'i486' ]; then
296 count=$(($count + 1))
297 echo "Adding: $pkg"
298 touch $pkg/arch.$ARCH
299 fi
300 fi
301 done
302 echo "Packages for $ARCH : $count"
303 }
306 #
307 # Commands
308 #
310 previous_command="$(cat $command 2>/dev/null)"
311 case "$1" in
312 usage|help|-u|-h)
313 usage ;;
315 setup|-s)
316 # Setup the Cooker environment.
317 title 'Setting up the Cooker'
318 mkdir -p $CACHE
319 echo "Cooker setup using: $SLITAZ" | log
320 for pkg in $SETUP_PKGS mercurial rsync tazlito; do
321 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
322 done
323 mkdir -p $SLITAZ && cd $SLITAZ
324 if [ -d "${wok}-hg" ]; then
325 echo -e 'Hg wok already exists.\n'
326 exit 1
327 fi
328 if [ -d "$wok" ]; then
329 echo -e 'Build wok already exists.\n'
330 exit 1
331 fi
333 # Directories and files
334 echo "mkdir's and touch files in: $SLITAZ"
335 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
336 for f in $activity $blocked $broken $commits $cooklist $command; do
337 touch $f
338 done
339 hg clone $WOK_URL ${wok}-hg || exit 1
340 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
341 cp -a ${wok}-hg $wok
342 footer ;;
344 arch-db)
345 # Manually create arch packages DB.
346 arch_db ;;
348 setup-cron)
349 # Create cron job for the cooker.
350 [ "$2" ] || hours=2
351 if [ ! -f "$crontabs" ]; then
352 mkdir -p /var/spool/cron/crontabs
353 fi
354 if ! fgrep -q /usr/bin/cooker $crontabs; then
355 cat > $crontabs <<EOT
356 # Run SliTaz Cooker every $hours hours
357 59 */$hours * * * touch $CACHE/cooker-request
358 */5 * * * * [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html
359 */5 * * * * [ -z "$(pidof cooker)" ] && [ -s $CACHE/recook-packages ] && /usr/bin/cooker list $CACHE/recook-packages
360 EOT
361 touch $CACHE/cooker-request $CACHE/recook-packages
362 chmod 666 $CACHE/cooker-request $CACHE/recook-packages
363 killall crond 2>/dev/null && /etc/init.d/crond start
364 fi ;;
366 check-cron)
367 if [ ! -f "$crontabs" ]; then
368 echo "There is no $crontabs here. Use setup-cron option."
369 exit 1
370 fi
371 fgrep /usr/bin/cooker $crontabs ;;
373 note|-n)
374 # Blocked a pkg and want others to know why? Post a note!
375 [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;
377 notes|-ns)
378 # View cooknotes.
379 title 'Cooknotes'
380 cat $cooknotes
381 footer ;;
383 block|-b)
384 # Block a package.
385 [ "$pkg" ] && cook $pkg --block ;;
387 unblock|-ub)
388 # Unblock a package.
389 [ "$pkg" ] && cook $pkg --unblock ;;
391 reverse|-r)
392 # Cook all reverse dependencies for a package. This command lets us
393 # control the Cooker manually for commits that will cook a lot of packages.
394 #
395 # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"
396 #
397 if [ ! -d "$wok/$pkg" ]; then
398 echo -e "\nNo package $2 found.\n"
399 exit 0
400 fi
401 rm -f $cooklist; touch $cooklist
402 title "Reverse cooklist for: $pkg"
404 cd $wok
405 for rev in *; do
406 [ -s $wok/$rev/receipt ] || continue
407 unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt
408 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
409 echo "$rev" | tee -a $cooklist
410 fi
411 done
412 footer "Reverse dependencies found: $(wc -l < $cooklist)"
413 strip_blocked
414 cook_order | tee $LOGS/cookorder.log
415 cook_list ;;
417 pkg|-p)
418 # Same as 'cook pkg' but with log for web interface.
419 ps | grep -q "cook $pkg$" && echo 'Already running' && continue
420 cook $pkg || broken
421 clean_exit ;;
423 cat|-c)
424 # Cook all packages of a category.
425 cat="$2"
426 rm -f $cooklist; touch $cooklist
428 cd $wok
429 for pkg in *; do
430 [ -s $pkg/receipt ] || continue
431 unset CATEGORY; . $pkg/receipt
432 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
433 done
434 strip_blocked
435 cook_order | tee $LOGS/cookorder.log
436 cook_list ;;
438 flavor|-f)
439 # Cook all packages of a flavor.
440 name="$2"
441 if [ ! -d "$flavors/$name" ]; then
442 echo -e "\nSpecified flavor does not exist: $name\n"
443 exit 1
444 fi
445 if [ -d "$flavors/.hg" ]; then
446 cd $flavors; hg pull -u
447 fi
448 list="$flavors/$name/packages.list"
449 cp -a $list $cooklist
450 strip_blocked
451 cook_order | tee $LOGS/cookorder.log
452 cook_list ;;
454 list|-l)
455 # Cook a list of packages given in argument.
456 list="$2"
457 if [ ! -f "$list" ]; then
458 echo -e "\nSpecified list does not exist: $list\n"
459 exit 1
460 fi
461 cat $list >> $cooklist
462 echo -n > $list
463 strip_blocked
464 cook_order | tee $LOGS/cookorder.log
465 cook_list ;;
467 rev|-r)
468 # Cook or recook a specific Hg revision.
469 rev="$2"
470 [ "$rev" ] || exit 0
471 rm -f $cooklist; touch $cooklist
473 cd $wok
474 for pkg in $(hg log --rev=$rev --template "{files}"); do
475 echo "$pkg" | cut -d/ -f1 >> $cooklist
476 done
477 strip_blocked
478 cook_order | tee $LOGS/cookorder.log
479 cook_list ;;
481 all|-a)
482 # Try to build all unbuilt packages except blocked's.
483 echo 'cooker:all' > $command
484 rm -f $cooklist; touch $cooklist
485 title 'Cooker cooklist'
487 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt
488 # if it exists since extra version is added when packing the package.
489 echo 'Searching for all unbuilt packages' | log
491 cd $wok
492 for pkg in *; do
493 [ -s $pkg/receipt ] || continue
494 unset EXTRAVERSION
495 . $pkg/receipt
496 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
497 . $pkg/taz/$PACKAGE-$VERSION/receipt
498 if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then
499 echo $pkg; echo $pkg >> $cooklist
500 fi
501 done
502 strip_blocked
503 cook_order | tee $LOGS/cookorder.log
504 echo "Packages to cook: $(wc -l < $cooklist)" | log
505 cook_list ;;
507 tasks|-T)
508 # List existing cooker tasks
509 [ ! -d "$tasks" ] && echo 'There are no tasks.' && exit 0
510 title 'Cooker tasks list'
511 last=$(ls $tasks | tail -n1)
512 for task in $(ls $tasks); do
513 . $tasks/$task
514 echo "Task name : $task"
515 echo "Description : $DESC"
516 separator $([ $task != $last ] && echo '-')
517 done
518 newline ;;
520 task|-t)
521 # Executing specified task
522 task="$2"
523 title "Executing cooker task: $task"
524 . $tasks/$task; task
525 footer "Task $task finished" ;;
527 *)
528 # Default is to cook all commits if not yet running.
529 [ -n "$1" ] && usage
530 cooklist=$commits
531 if [ -f "$pidfile" ]; then
532 pid=$(cat $pidfile)
533 if [ -s /proc/$pid/status ]; then
534 echo -e "\nStill cooking latest commits with pid:"
535 echo -e " $pid\n"
536 exit 0
537 fi
538 rm -f "$pidfile"
539 fi
541 # Start and get a PID file.
542 rm -f $LOGS/commits.log
543 newline
544 echo 'Checking for commits' | log_commits
545 separator | tee -a $LOGS/commits.log
547 echo $$ > $pidfile
548 trap 'echo -e "\nCooker stopped: PID $$\n" && \
549 rm -f $pidfile $command && exit 1' INT TERM
551 echo "Cooker PID : $$" | log_commits
552 echo "Cooker date : $(date '+%F %T')" | log_commits
554 # Get revisions. Here we have 2 echoes since we want a msg on screen,
555 # in commits log and activity DB without a space before.
556 cd $wok || exit 1
557 cur=$(hg head --template '{rev}\n')
558 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
559 echo "Updating wok: ${wok}-hg" | log
560 echo 'hg:pull' > $command
561 cd $wok-hg; hg pull -u | log_commits
562 new=$(hg head --template '{rev}\n')
563 # Store last rev to be used by CGI so it doesn't need to call hg head
564 # on each load.
565 echo "$new" > $wokrev
567 # Sync build wok with rsync so we don't take care about removing old
568 # files as before.
569 if [ "$new" -gt "$cur" ]; then
570 echo "Changes found from: $cur to $new" | log
571 echo 'Syncing build wok with Hg wok...' | log_commits
572 rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \
573 sed '/^$/d' | log_commits
574 else
575 echo "No revision changes: $cur vs $new" | log
576 separator | log_commits
577 clean_exit; newline
578 exit 0
579 fi
581 # Get and display modifications.
582 cd $wok-hg
583 commits_summary | log_commits
584 cur=$(($cur + 1))
585 rm -f $commits.tmp; touch $commits.tmp
586 for rev in $(seq $cur $new); do
587 for file in $(hg log --rev=$rev --template "{files}"); do
588 pkg=$(echo $file | cut -d/ -f1)
589 desc=$(hg log --rev=$rev --template "{desc}" $file)
590 echo "Committed package : $pkg - $desc" | log_commits
591 echo $pkg >> $commits.tmp
592 done
593 done
595 # We may have deleted packages and files in stuff/. Remove it and
596 # clean DB as well as log file.
597 cd $wok
598 for pkg in *; do
599 if [ ! -d "$wok-hg/$pkg" ]; then
600 echo "Removing package: $pkg" | log_commits
601 if [ -s $wok/$pkg/receipt ]; then
602 . $wok/$pkg/receipt
603 rm -f $PKGS/$PACKAGE-$VERSION*
604 fi
605 rm -rf $wok/$pkg $LOGS/$pkg.log
606 sed -i "/^${pkg}$/d" $blocked $broken $commits.tmp
607 sed -i "/^$pkg\t/d" $PKGS/packages.info
608 fi
609 if [ -d "$wok/$pkg/stuff" ]; then
610 if [ ! -d "$wok-hg/$pkg/stuff" ]; then
611 echo "Removing stuff: $pkg/stuff" | log_commits
612 rm -rf $wok/$pkg/stuff
613 else
614 for stuff_file in $(cd $wok/$pkg/stuff; find \( -type f -o -type l \) | sed 's|^\./||'); do
615 if [ ! -f "$wok-hg/$pkg/stuff/$stuff_file" -a \
616 ! -h "$wok-hg/$pkg/stuff/$stuff_file" ]; then
617 echo "Removing file from stuff: $wok/$pkg/stuff/$stuff_file" | log_commits
618 rm -f $wok/$pkg/stuff/$stuff_file
619 rmdir --parents --ignore-fail-on-non-empty $(dirname "$wok/$pkg/stuff/$stuff_file")
620 fi
621 done
622 fi
623 fi
624 done
626 # Keep previous commit and discard duplicate lines
627 cat $commits $commits.tmp | sed '/^$/d' > $commits.new
628 uniq $commits.new > $commits; rm $commits.*
630 # Handle cross compilation. Create arch packages DB and remove pkgs
631 # not cooked for this arch from the commits list.
632 arch_db
633 while read pkg; do
634 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
635 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
636 log_commits
637 sed -i "/^${pkg}$/d" $commits
638 else
639 echo "Cooker arch : $ARCH" | log_commits
640 fi
641 done < $commits
643 # Stats
644 pkgs=$(wc -l < $commits)
645 echo "Packages to cook: $pkgs" | log
646 echo "Packages to cook : $pkgs" | log_commits
647 separator | log_commits
648 newline
649 strip_blocked
650 cook_order | tee $LOGS/cookorder.log
651 cook_commits
652 clean_exit ;;
653 esac
655 exit 0