cookutils view cooker @ rev 481

cooker: Updated fully to what slitaz-tank needs.
author Christopher Rogers <slaxemulator@gmail.com>
date Mon Jun 11 15:49:54 2012 +0000 (2012-06-11)
parents d9b8c75bb5bb
children 4b1f207c4058
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful, all devs should
7 # at least understand basic English.
8 #
9 . /usr/lib/slitaz/libcook.sh
10 . /usr/lib/slitaz/libcookorder.sh
11
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14
15 # PID file.
16 pidfile='/var/run/cooker.pid'
17 COMMAND="$1"
18
19 #
20 # Functions
21 #
22
23 usage() {
24 cat << EOT
25
26 Usage: cooker [command] [pkg|list|note|hours]
27
28 Options:
29 usage|-u $(gettext "Display this short usage.")
30 setup|-s $(gettext "Setup the Cooker environment.")
31 setup-cron $(gettext "Setup a cron job for the Cooker.")
32 arch-db $(gettext "Create host arch packages DB.")
33 note|-n $(gettext "Add a note to the cooknotes.")
34 notes|-ns $(gettext "Display all the cooknotes.")
35 block|-b $(gettext "Block a package so cook will skip it.")
36 unblock|-ub $(gettext "Unblock a blocked package.")
37 pkg|-p $(gettext "Same as 'cook pkg' but with cooker log.")
38 flavor|-f $(gettext "Cook all packages of a flavor.")
39 list|-l $(gettext "Cook all packages in the given list.")
40 cat|-c $(gettext "Cook all packages of a category.")
41 rev|-r $(gettext "Cook packages of a specific revision.")
42 all|-a $(gettext "Find and cook all unbuilt packages.")
43
44 EOT
45 exit 0
46 }
47
48 # Some messages occur in activity but log verbose output when checking for commits
49 # into a log file.
50 log_commits() {
51 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log
52 }
53
54 # Clean up before exit when check and cook commits finish.
55 clean_exit() {
56 rm -f $command && touch $command
57 rm -f $pidfile
58 }
59
60 # Summary for commits.
61 commits_summary() {
62 msg="from revision $cur to $new"
63 [ "$new" == "$cur" ] && msg="revision $new"
64 echo "Will cook $msg"
65 separator
66 echo -e "\nSummary for commits"
67 separator
68 echo "Hg wok revision : $cur"
69 echo "Pulled revision : $new"
70 echo "Check date : $(date '+%Y-%m-%d %H:%M:%S')"
71 }
72
73 # Scan packages build deps and fill up cookorder list.
74 cook_order_scan() {
75 rm -f $cookorder
76 touch $cookorder
77 for pkg in $(cat $cooklist)
78 do
79 unset WANTED BUILD_DEPENDS BDEPS
80 . $WOK/$pkg/receipt
81 # The :: is for web interface color.
82 if [ "$AUTO_DEP" -a ! "$WANTED" ]; then
83 BDEPS=$(scan $PACKAGE --look_for=bdep --with_dev | \
84 grep -v $(for i in $(look_for_rwanted) $PACKAGE; do echo " -e ^$i$"; done))
85 else
86 BDEPS="$BUILD_DEPENDS"
87 fi
88 [ "$WANTED$BDEPS" ] && echo "$pkg :: $WANTED $BDEPS"
89 for dep in $BDEPS
90 do
91 if grep -q "^$dep$" $cooklist; then
92 if ! grep -q "^$dep$" $cookorder; then
93 echo "$dep" >> $cookorder
94 fi
95 fi
96 done
97 done
98
99 # Append unordered packages to cookorder.
100 for pkg in $(cat $cooklist)
101 do
102 if ! grep -q "^$pkg$" $cookorder; then
103 echo "$pkg" >> $cookorder
104 fi
105 done
106 }
107
108 # Scan and rescan until the cooklist is ordered then handle WANTED.
109 cook_order() {
110 time=$(date +%s)
111 scan=0
112
113 # Keep an original cooklist so we do a diff when ordering is finished.
114 cp -f $cooklist $cooklist.0
115 echo "cookorder" > $command
116 echo -e "\nInitial Cooker order scan"
117 separator
118
119 if [ "$COMMAND" = "pkg" ]; then
120 if ! grep -q "^$pkg$" $cooklist; then
121 if [ -f "$WOK/$pkg/receipt" ]; then
122 echo "$pkg" > $cooklist
123 fi
124 fi
125 fi
126
127 cook_order_scan
128
129 # Diff between the cooklist and new ordered list ? So copy the last
130 # cookorder to cooklist and rescan it.
131 while /bin/true
132 do
133 diff $cooklist $cookorder > $cookorder.diff
134 if [ -s "$cookorder.diff" ]; then
135 scan=$(($scan + 1))
136 echo -e "\nDiff scan: $scan"
137 separator
138 mv -f $cookorder $cooklist
139 cook_order_scan
140 else
141 break
142 fi
143 done
144
145 if [ "$COOK_WANTED" ]; then
146 PACKAGE=$pkg
147 for i in $PACKAGE $(look_for_rwanted); do
148 if ! grep -q "^${i}$" $cooklist; then
149 echo $i >> $cooklist
150 fi
151 done
152 fi
153
154 # Keep a diff between submited cooklist and the ordered.
155 diff $cooklist.0 $cooklist > $cooklist.diff
156 rm -f $cookorder $cookorder.diff $cooklist.0
157
158 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
159 # TODO: grep the line number to get pkg position and keep it higher.
160 echo -e "\nHandle WANTED package"
161 separator
162 for pkg in $(cat $cooklist)
163 do
164 unset WANTED
165 . $WOK/$pkg/receipt
166 if [ "$WANTED" ]; then
167 echo "$pkg :: $WANTED"
168 if grep -q ^${WANTED}$ $cooklist; then
169 sed -i -e "/^$pkg$/"d \
170 -e "/^$WANTED$/ a $pkg" $cooklist
171 fi
172 fi
173 done
174
175 # Show ordered cooklist
176 echo -e "\nCooklist order"
177 separator
178 cat $cooklist
179 separator
180 time=$(($(date +%s) - $time))
181 pkgs=$(cat $cooklist | wc -l)
182 echo -e "\nSummary for cookorder"
183 separator
184 cat << EOT
185 Ordered packages : $pkgs
186 Scans executed : $scan
187 Scan duration : ${time}s
188 EOT
189 separator && rm -f $command
190 }
191
192 # Use in default mode and with all cmd.
193 cook_commits() {
194 if [ -s "$commits" ]; then
195 for pkg in $(cat $commits)
196 do
197 echo "cook:$pkg" > $command
198 cook $pkg || broken
199 sed -i /^${pkg}$/d $commits
200 if [ "$COOK_WANTED" ]; then
201 PACKAGE=$pkg
202 for pkg in $(look_for_rwanted); do
203 if ! grep -q "^$pkg$" $commits; then
204 [ -f "$WOK/$pkg/receipt" ] || continue
205 echo "cook:$pkg" > $command
206 cook $pkg
207 fi
208 done
209 fi
210 done
211 fi
212 }
213
214 # Cook all packages in a cooklist.
215 cook_list() {
216 for pkg in $(cat $cooklist)
217 do
218 cook $pkg || broken
219 sed -i /^${pkg}$/d $cooklist
220 if [ "$COOK_WANTED" ]; then
221 PACKAGE=$pkg
222 for wpkg in $(look_for_rwanted); do
223 if ! grep -q "^$wpkg$" $cooklist; then
224 if [ -f "$WOK/$wpkg/receipt" ]; then
225 cook $wpkg
226 sed -i /^${wpkg}$/d $cooklist
227 fi
228 fi
229 done
230 fi
231 if [ "$1" = "--list" ]; then
232 pkgdb incoming
233 fi
234 done
235 }
236
237 # reverse pkg build when doing a big update to
238 # rebuild all packages that depends on it
239 reverse_cook_list() {
240 local pkg="$1"
241 [ ! -f "$WOK/$pkg/receipt" ] && echo -e "\nNo package $pkg found.\n" && exit 0
242 cp -a $cooklist $cooklist.bak
243 rm -f $cooklist && touch $cooklist
244 echo -e "\nReverse cooklist for: $pkg"
245 separator && cd $WOK
246 for rev in *
247 do
248 [ -f $WOK/$rev/receipt ] || continue
249 unset DEPENDS BUILD_DEPENDS && . $WOK/$rev/receipt
250 if echo "$DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
251 echo "$rev" | tee -a $cooklist
252 fi
253 done && separator
254 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
255 strip_blocked
256 cook_order | tee $LOGS/cookorder.log
257 cook_list
258 }
259
260 # Create a arch.$ARCH file for each package cooked for the target host
261 # architecture
262 #
263 # The deal: we dont want all packages handled by cooker commands and stats,
264 # since we dont cross compile all packages for each arch but only a set of
265 # packages to provide one full featured desktop, servers and goodies useful
266 # for the host system.
267 #
268 arch_db() {
269 count=0
270 echo "Cleaning packages DB : arch.$ARCH"
271 rm -f $wok/*/arch.$ARCH && cd $wok
272 echo "Creating $ARCH packages DB..."
273 for pkg in *
274 do
275 unset HOST_ARCH
276 . $wok/$pkg/receipt
277 #: ${HOST_ARCH=i486}
278 if $(echo "$HOST_ARCH" | fgrep -q $ARCH); then
279 count=$(($count + 1))
280 echo "Adding: $pkg"
281 touch $pkg/arch.$ARCH
282 fi
283 done
284 echo "Packages for $ARCH : $count"
285 }
286
287
288 #
289 # Commands
290 #
291 case "$COMMAND" in
292 usage|help|-u|-h)
293 usage ;;
294 setup|-s)
295 # Setup the Cooker environment.
296 echo -e "\nSetting up the Cooker"
297 echo "Cooker setup using: $SLITAZ" | log
298 separator
299 echo "$PKGS" > $LOCALSTATE/mirror
300 mkdir -p $LOCALSTATE/undigest/incoming
301 echo "$INCOMING" > $LOCALSTATE/undigest/incoming/mirror
302 echo "incoming" > $LOCALSTATE/priority
303 tazpkg recharge
304 for pkg in $INSTALL_PKGS
305 do
306 [ ! -f "$INSTALLED/$pkg/receipt" ] && tazpkg get-install $pkg
307 done
308 ls -1 "$INSTALLED" > $CACHE/chroot-pkgs
309 mkdir -p $SLITAZ && cd $SLITAZ
310 [ -d "$WOKHG" ] || hg clone $WOK_URL $WOKHG
311 rsync_wok
312 init_db_files
313 cook gen-wok-db
314 echo "Make unbuild list"
315 cook unbuild --full
316 [ "$AUTO_PURGE_SRC" ] && cook gen-src $SRC
317 #[ -d "$FLAVORS" ] || hg clone $FLAVORS_URL flavors
318 separator && newline ;;
319 arch-db)
320 # Manually create arch packages DB.
321 arch_db ;;
322 setup-cron)
323 # Create cron job for the cooker.
324 [ "$2" ] || hours=2
325 if [ ! -f "$crontabs" ]; then
326 mkdir -p /var/spool/cron/crontabs
327 fi
328 if ! fgrep -q /usr/bin/cooker $crontabs; then
329 echo "# Run SliTaz Cooker every $hours hours" > $crontabs
330 echo "0 */$hours * * * /usr/bin/cooker --output=html" >> $crontabs
331 killall crond 2>/dev/null && /etc/init.d/crond start
332 fi ;;
333 check-cron)
334 [ -f "$crontabs" ] || \
335 echo "There is no $crontabs here. Use setup-cron option." && exit 1
336 fgrep /usr/bin/cooker $crontabs ;;
337 note|-n)
338 # Blocked a pkg and want others to know why ? Post a note!
339 note="$2"
340 date=$(date "+%Y-%m-%d %H:%M")
341 [ "$note" ] && echo "$date : $note" >> $cooknotes ;;
342 notes|-ns)
343 # View cooknotes.
344 echo -e "\nCooknotes"
345 separator
346 cat $cooknotes
347 separator && newline ;;
348 block|-b)
349 # Block a package.
350 [ "$pkg" ] && cook $pkg --block ;;
351 unblock|-ub)
352 # Unblock a package.
353 [ "$pkg" ] && cook $pkg --unblock ;;
354 reverse|-r)
355 # Cook all reverse dependencies for a package. This command lets us
356 # control the Cooker manually for commits that will cook a lot of packages.
357 #
358 # Use hg commit ? Ex: hg commit -m "Message bla bla | cooker:reverse"
359 #
360 reverse_cook_list $pkg ;;
361 chroot)
362 [ -d $SLITAZ/chroot ] || tazdev gen-chroot $2
363 [ -d $SLITAZ/chroot ] && tazdev chroot $2
364 ;;
365 toolchain)
366 toolchain=$CACHE/toolchain
367 [ -f $PKGS/fullco.txt ] || cook gen-wok-db
368 cook gen-cooklist toolchain > $toolchain
369 newline > $cooklist
370 echo "cooker:toolchain" > $command
371 echo "First build slitaz-toolchain"
372 [ -f $blocked ] && rm -f $blocked
373 touch $blocked
374 cook slitaz-toolchain
375 SKIP="glibc gcc binutils linux-api-headers slitaz-toolchain cookutils"
376 for i in $SKIP; do
377 echo "$i" >> $blocked
378 done
379 for pkg in $(cat $blocked)
380 do
381 sed -i /^${pkg}$/d $toolchain
382 done && sed -i /^$/d $toolchain
383 for pkg in $(cat $toolchain)
384 do
385 [ -f "$WOK/$pkg/receipt" ] || continue
386 cook $pkg
387 sed -i /^${pkg}$/d $toolchain
388 if [ "$COOK_WANTED" ]; then
389 PACKAGE=$pkg
390 for pkg in $(look_for_rwanted); do
391 if ! grep -q "^$pkg$" $toolchain; then
392 [ -f "$WOK/$pkg/receipt" ] || continue
393 cook $pkg
394 fi
395 done
396 fi
397 cook pkgdb incoming
398 done
399 [ -f $toolchain ] && rm -f $toolchain
400 clean_exit ;;
401 pkg|-p)
402 # Same as 'cook pkg' but with log for web interface.
403 [ ! -f "$WOK/$pkg/receipt" ] && echo "$pkg doesn't exist in wok" && exit 1
404 if [ "$COOK_WANTED" ]; then
405 PACKAGE=$pkg
406 strip_blocked
407 cook_order | tee $LOGS/cookorder.log
408 cook_list
409 if [ "$AUTO_UPDATE_DB" ]; then
410 cook pkgdb $INCOMING
411 fi
412 else
413 cook $pkg
414 fi
415 clean_exit ;;
416 cat|-c)
417 # Cook all packages of a category.
418 cat="$2"
419 cp -a $cooklist $cooklist.bak
420 rm -f $cooklist && touch $cooklist && cd $WOK
421 for pkg in *
422 do
423 unset CATEGORY && . $pkg/receipt
424 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
425 done
426 strip_blocked
427 cook_order | tee $LOGS/cookorder.log
428 cook_list --list ;;
429 flavor|-f)
430 # Cook all packages of a flavor.
431 name="$2"
432 [ ! -d "$FLAVORS/$name" ] && \
433 echo -e "\nSpecified flavor does not exist: $name\n" && exit 1
434 [ -d "$FLAVORS/.hg" ] && cd $FLAVORS && hg pull -u
435 list=$FLAVORS/$name/packages.list
436 cp -a $list $cooklist
437 strip_blocked
438 cook_order | tee $LOGS/cookorder.log
439 cook_list ;;
440 list|-l)
441 # Cook a list of packages given in argument.
442 list="$cooklist"
443 [ "$2" ] && list="$2"
444 [ -f "$list" ] || $(gettext -e "$list doesn't exist." && exit 1)
445 if [ "$list" != "$cooklist" ]; then
446 cp -a $list $cooklist
447 fi
448 unset pkg
449 strip_blocked
450 if [ "$UPCOOKLIST" ]; then
451 gen_cook_list
452 else
453 cook_order | tee $LOGS/cookorder.log
454 fi
455 cook_list --list
456 cook pkgdb $INCOMING ;;
457 rev|-r)
458 # Cook or recook a specific Hg revision.
459 rev="$2"
460 [ "$rev" ] || exit 0
461 cd $WOK
462 cp -a $cooklist $cooklist.bak
463 rm -f $cooklist && touch $cooklist
464 for pkg in $(hg log --rev=$rev --template "{files}")
465 do
466 echo "$pkg" | cut -d "/" -f 1 >> $cooklist
467 done
468 strip_blocked
469 cook_order | tee $LOGS/cookorder.log
470 cook_list ;;
471 all|-a)
472 # Try to build all unbuilt packages except blocked's.
473 echo "cooker:all" > $command
474 cp -a $cooklist $cooklist.bak
475 rm -f $cooklist && touch $cooklist
476 newline && cd $WOK
477 echo "Cooker cooklist"
478 separator
479
480 cook unbuild
481 cp -a $unbuild $cooklist
482 strip_blocked
483 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
484 cook_list ;;
485 new|pkgdb|get-src|check-incoming|gen-wok-db|upwok)
486 cook "$COMMAND" "$2" "$3" "$4" "$5" ;;
487 *)
488 # Default is to cook all commits if not yet running.
489 [ "$COMMAND" ] && usage
490 cooklist=$commits
491 if [ -f "$pidfile" ]; then
492 pid=$(cat $pidfile)
493 if [ -s /proc/$pid/status ]; then
494 gettext -e "\nStill cooking latest commits with pid:"
495 echo -e " $pid\n" && exit 0
496 fi
497 rm -f "$pidfile"
498 fi
499
500 # Start and get a PID file.
501 rm -f $LOGS/commits.log
502 newline
503 echo "Checking for commits" | log_commits
504 separator | tee -a $LOGS/commits.log
505
506 echo $$ > $pidfile
507 trap 'echo -e "\nCooker stopped: PID $$\n" && \
508 rm -f $pidfile $command && exit 1' INT TERM
509
510 echo "Cooker PID : $$" | log_commits
511 echo "Cooker date : $(date '+%Y-%m-%d %H:%M:%S')" | log_commits
512
513 # Get revisions. Here we have 2 echoes since we want a msg on screen,
514 # in commits log and activity DB without a space before.
515 cd $WOK || exit 1
516 branch=$(cat /etc/slitaz-release)
517 cur=$(hg head $branch --template '{rev}\n')
518 echo "Updating wok : $WOKHG (rev $cur)" | log_commits
519 echo "Updating wok: $WOKHG" | log
520 echo "hg:pull" > $command
521 cd $WOKHG && hg pull -u | log_commits
522 new=$(hg head $branch --template '{rev}\n')
523 # Store last rev to be used by CGI so it doesn't need to call hg head
524 # on each load.
525 echo "$new" > $wokrev
526
527 # Sync build wok with rsync so we don't take care about removing old
528 # files as before.
529 if [ "$new" -gt "$cur" ]; then
530 echo "Changes found from: $cur to $new" | log
531 echo "Syncing build wok with Hg wok..." | log_commits
532 rsync -r -t -c -l -u -v -D -E $WOKHG/ $WOK/ | \
533 sed '/^$/'d | log_commits
534 else
535 echo "No revision changes: $cur vs $new" | log
536 separator | log_commits
537 clean_exit && newline && exit 0
538 fi
539
540 # Get and display modifications.
541 cd $WOKHG
542 commits_summary | log_commits
543 cur=$(($cur + 1))
544 rm -f $commits.tmp && touch $commits.tmp
545 for rev in $(seq $cur $new)
546 do
547 for file in $(hg log --rev=$rev --template "{files}")
548 do
549 pkg=$(echo $file | cut -d "/" -f 1)
550 desc=$(hg log --rev=$rev --template "{desc}" $file)
551 echo "Commited package : $pkg - $desc" | log_commits
552 echo $pkg >> $commits.tmp
553 done
554 done
555
556 # We may have deleted packages and files in stuff/. Remove it and
557 # clean DB as well as log file.
558 cd $WOK
559 for pkg in *
560 do
561 if [ ! -d "$WOKHG/$pkg" ]; then
562 echo "Removing package: $pkg" | log_commits
563 . $WOK/$pkg/receipt
564 if [ -f "$INCOMING/$PACKAGE-$VERSION*" ]; then
565 rm -rf $INCOMING/$PACKAGE-$VERSION*
566 fi
567 if [ -f "$PKGS/$PACKAGE-$VERSION*" ]; then
568 rm -rf $PKGS/$PACKAGE-$VERSION*
569 fi
570 rm -rf $WOK/$pkg $LOGS/$pkg.log
571 sed -i "/^${pkg}$/"d $blocked $broken $commits.tmp
572 fi
573 done
574
575 # Keep previous commit and discard duplicate lines
576 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
577 uniq $commits.new > $commits && rm $commits.*
578
579 # Handle cross compilation. Create arch packages DB and remove pkgs
580 # not cooked for this arch from the commits list.
581 case "$ARCH" in
582 arm)
583 arch_db | log_commits
584 for pkg in $(cat $commits)
585 do
586 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
587 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
588 log_commits
589 sed -i "/^${pkg}$/"d $commits
590 fi
591 done ;;
592 i486)
593 echo "Cooker arch : $ARCH (default)" | log_commits ;;
594 *)
595 echo "Cooker arch : $ARCH" | log_commits ;;
596 esac
597
598 # Stats
599 pkgs=$(cat $commits | wc -l)
600 echo "Packages to cook: $pkgs" | log
601 echo "Packages to cook : $pkgs" | log_commits
602 separator | log_commits
603 newline
604 strip_blocked
605 cook_order | tee $LOGS/cookorder.log
606 cook_commits
607 pkgdb $INCOMING
608 clean_exit ;;
609 esac
610
611 exit 0