wok view tazbb/stuff/tazbb @ rev 4394
tazbb: allow packages with symlinks only
| author | Pascal Bellard <pascal.bellard@slitaz.org> |
|---|---|
| date | Fri Oct 16 15:30:12 2009 +0200 (2009-10-16) |
| parents | 64353f1da619 |
| children | e408b254cf08 |
line source
1 #!/bin/sh
2 # Tazbb - SliTaz Build Bot.
3 # System wide config file: /etc/slitaz/tazbb.conf
4 #
5 # Tazbb is a tool to automate package building, it can be run manually
6 # or via a cron job. On SliTaz build host, tazbb is run in a chroot env.
7 #
8 # (c) 2009 SliTaz GNU/Linux project - GNU gpl v3
9 #
11 # Include config file or exit if no file found.
12 if [ -f "./tazbb.conf" ]; then
13 . ./tazbb.conf
14 elif [ -f "/etc/slitaz/tazbb.conf" ]; then
15 . /etc/slitaz/tazbb.conf
16 else
17 echo -e "\nNo config file found: tazbb.conf...\n" && exit 0
18 fi
20 # Tazbb is only for root.
21 if test $(id -u) != 0 ; then
22 echo -e "\nYou must be root to run: `basename $0`.\n" && exit 0
23 fi
25 # Let tazbb finish is work and make sure needed files exist.
26 if [ -f $LOCK_FILE ]; then
27 case $1 in
28 usage|list-*|*block)
29 continue ;;
30 *)
31 echo -e "\nTazbb is already running and locked...\n"
32 exit 0 ;;
33 esac
34 else
35 mkdir -p $DB_DIR $LOG_DIR
36 touch $LOCK_FILE $DB_DIR/blocked
37 fi
39 # Set KERNEL variable
40 if [ -s $BUILD_WOK/linux/receipt ]; then
41 . $BUILD_WOK/linux/receipt
42 KERNEL=$VERSION
43 fi
45 usage()
46 {
47 echo -e "\nSliTaz developers and build host tool\n
48 \033[1mUsage: \033[0m `basename $0` [command] [--option]
49 \033[1mCommands: \033[0m\n
50 usage Print this short usage and command list.
51 list-pkgs List last cooked packages with date.
52 report Run in report mode and dont cook anything [--verbose].
53 cook Cook, install and log a single package build.
54 cook-all Cook all missing, modified or unbuilt packages.
55 cook-commit Cook all packages affected by a commit in the last update.
56 test-pkgs Execute a test suite on all packages [--verbose].
57 [un]block Block or unblock a package to skip or enable building.
58 mail Send mail to package maintainer with tazbbmail.
59 clean-up Remove old packages [--verbose|--dry-run].
60 clean-log Remove all generated build log files.\n"
61 }
63 status()
64 {
65 local CHECK=$?
66 echo -en "\033[70G"
67 if [ $CHECK = 0 ]; then
68 echo "Done"
69 else
70 echo "Failed"
71 fi
72 return $CHECK
73 }
75 top_summary()
76 {
77 cat > $DB_DIR/summary << _EOT_
78 Update : `date`
79 Revision : $NEW_REV (<a href="$HG_URL/log/$NEW_REV">changelog</a>)
80 _EOT_
81 }
83 packages_summary()
84 {
85 if ! grep -q "^Packages" $DB_DIR/summary; then
86 cat >> $DB_DIR/summary << _EOT_
87 Packages : `ls $BUILD_WOK | wc -l` in the wok, `cat $DB_DIR/cooklist | wc -l` to cook, \
88 `cat $DB_DIR/blocked | wc -l` blocked, `cat $DB_DIR/corrupted | wc -l` corrupted
89 _EOT_
90 fi
91 }
93 packages_summary_update()
94 {
95 sed -i s/"[0-9]* in the wok"/"`ls $BUILD_WOK | wc -l` in the wok"/ \
96 $DB_DIR/summary
97 sed -i s/"[0-9]* to cook"/"`cat $DB_DIR/cooklist | wc -l` to cook"/ \
98 $DB_DIR/summary
99 sed -i s/"[0-9]* blocked"/"`cat $DB_DIR/blocked | wc -l` blocked"/ \
100 $DB_DIR/summary
101 sed -i s/"[0-9]* corrupted"/"`cat $DB_DIR/corrupted | wc -l` corrupted"/ \
102 $DB_DIR/summary
103 }
105 list_packages()
106 {
107 cd $PACKAGES_REPOSITORY
108 ls -1t *.tazpkg | head -20 | \
109 while read file
110 do
111 echo -n $(stat -c '%y' $PACKAGES_REPOSITORY/$file | cut -d. -f1)
112 echo " $file"
113 done
114 }
116 show_report()
117 {
118 echo "Cooklist"
119 echo "================================================================================"
120 cat $DB_DIR/cooklist && echo ""
121 echo "Blocked"
122 echo "================================================================================"
123 cat $DB_DIR/blocked && echo ""
124 echo ""
125 }
127 # URL encoding
128 escape()
129 {
130 echo $1 | sed -e 's/+/%2B/g' -e 's|/|%2F|g' -e 's/:/%3A/g'
131 }
133 update_wok()
134 {
135 echo ""
136 echo "(updating wok)" > $DB_DIR/running
137 cd $HG_WOK
138 LAST_REV=`hg head --template '{rev}\n'`
139 hg pull && hg update
140 NEW_REV=`hg head --template '{rev}\n'`
141 # Gen a new summary and link last revision for the web interface.
142 echo -e "\nHg wok : $HG_WOK ($NEW_REV)"
143 echo -e "Build wok : $BUILD_WOK ($LAST_REV)\n"
144 top_summary
145 # Copy Hg wok if new revision or exit to stop process since nothing
146 # have change (--forced can be used).
147 if [ "$NEW_REV" != "$LAST_REV" ]; then
148 size=`du -sh $HG_WOK | awk '{ print $1 }'`
149 echo -n "Copying Hg wok to the build wok ($size)... "
150 #rsync -r -n -t $HG_WOK/ $BUILD_WOK/
151 cp -a $HG_WOK/* $BUILD_WOK
152 cp -a $HG_WOK/.hg $BUILD_WOK
153 echo -e "Done\n"
154 else
155 if [ "$1" = "cook-all" ] || [ "$1" = "cook-commit" ]; then
156 if [ "$2" != "--forced" ]; then
157 echo -e "Nothing to cook...\n"
158 packages_summary
159 rm -f $LOCK_FILE && exit 0
160 fi
161 fi
162 fi
163 }
165 # Running 'tazbb report' should not cook anything and --verbose option
166 # can be used to display more messages.
167 check_wok()
168 {
169 # Clean up last results.
170 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
171 rm -f $DB_DIR/report && touch $DB_DIR/report
172 rm -f $DB_DIR/unbuilt && touch $DB_DIR/unbuilt
173 echo "Checking all files in: $HG_WOK"
174 echo "================================================================================"
175 echo "(checking wok)" > $DB_DIR/running
176 for pkg in $HG_WOK/*
177 do
178 EXTRAVERSION=""
179 WANTED=""
180 . $pkg/receipt
181 [ "$2" = "--verbose" ] && echo "Package : $PACKAGE"
182 # Skip blocked packages.
183 if grep -qs "^$PACKAGE$" $DB_DIR/blocked; then
184 echo "Blocked : $PACKAGE ($VERSION)" && continue
185 fi
187 # Bristuff hack until the receipt are improved...
188 #[ "$VERSION" = "bristuff" ] && VERSION=`get_version`
189 if [ "$VERSION" = "bristuff" ]; then
190 . $BUILD_WOK/$PACKAGE/taz/*/receipt
191 fi
193 # First check if package exit. Package naming _must_ be in the form of:
194 # $PACKAGE-$VERSION or $PACKAGE-${VERSION}$EXTRAVERSION (Kernel string).
195 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-$VERSION.tazpkg ]; then
196 [ -z "$EXTRAVERSION" ] && EXTRAVERSION="_$KERNEL"
197 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg ]; then
198 [ "$1" = "report" ] && echo "Missing : $PACKAGE ($VERSION)"
199 echo "Missing : $PACKAGE ($VERSION)" >> $DB_DIR/report
200 echo "$PACKAGE" >> $DB_DIR/cooklist
201 fi
202 else
203 # Check if package is up-to-date.
204 PKG_DATE=`date -u -r $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg '+%m%d%H%M%Y'`
205 for file in `find $pkg -type f`
206 do
207 FILE_DATE=`date -u -r $file '+%m%d%H%M%Y'`
208 [ "$2" = "--verbose" ] && echo " -> Checking: $file"
209 if [ "$FILE_DATE" -gt "$PKG_DATE" ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
210 [ "$1" = "report" ] && echo "Refresh : $PACKAGE ($VERSION)"
211 echo "Refresh : $PACKAGE ($VERSION)" >> $DB_DIR/report
212 echo "$PACKAGE" >> $DB_DIR/cooklist
213 fi
214 done
215 fi
216 # Now check if package is built and not already in the list.
217 if [ ! -d $BUILD_WOK/$PACKAGE/taz ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
218 [ "$1" = "report" ] && echo "Unbuilt : $PACKAGE ($VERSION)"
219 echo "Unbuilt : $PACKAGE ($VERSION)" >> $DB_DIR/report
220 echo "$PACKAGE" >> $DB_DIR/cooklist
221 fi
222 # Rebuild unbuilt packages list with link to log file. This list
223 # is also generated by cook_inslall to have real time stats.
224 if [ ! -d $BUILD_WOK/$PACKAGE/taz ]; then
225 echo "<a href=\"log.php?package=$(escape $PACKAGE)\">$PACKAGE</a>" \
226 >> $DB_DIR/unbuilt
227 fi
228 done
229 packages_summary
230 }
232 # Create a new cooklist and summary (dont modify report) so 'tazbb cook-commit'
233 # can cook last changes.
234 check_commit()
235 {
236 echo "(checking commit)" > $DB_DIR/running
237 cd $HG_WOK
238 # Clean up last results.
239 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
240 # Get the name of modified packages by the revision range. +1 last
241 # commit was build by the previous build.
242 LAST_REV=$(($LAST_REV+1))
243 echo -e "Will cook from revision $LAST_REV to $NEW_REV\n"
244 for file in `hg log --rev=$LAST_REV:$NEW_REV --template '{files}\n'`
245 do
246 pkg=`echo $file | cut -d "/" -f 1`
247 if ! grep -q ^$pkg$ $DB_DIR/cooklist; then
248 . $pkg/receipt
249 echo "Commit : $PACKAGE ($VERSION)" >> $DB_DIR/report
250 echo "$PACKAGE" >> $DB_DIR/cooklist
251 fi
252 done
253 packages_summary
254 }
256 # Cook one package
257 cook_package()
258 {
259 EXTRAVERSION=""
260 DEPENDS=""
261 BUILD_DEPENDS=""
262 SOURCE=""
263 WANTED=""
264 echo "(cooking <a href=\"log.php?package=$(escape $pkg)\">$pkg</a>)" > $DB_DIR/running
265 tazwok clean $pkg
266 script -c "echo 'install' | tazwok cook $pkg" $LOG_DIR/$pkg.log
267 # Install new package (important for new shared libs). Note
268 # that tests are done separatly with 'test_packages' and should
269 # be done by tazwok.
270 if [ -f $BUILD_WOK/$pkg/taz/*/receipt ]; then
271 . $BUILD_WOK/$pkg/taz/*/receipt
272 echo "(installing $PACKAGE-${VERSION}$EXTRAVERSION.tazpkg)" \
273 > $DB_DIR/running
274 yes | tazpkg install \
275 $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg \
276 --forced
277 return 0
278 fi
279 return 1
280 }
282 # Sort list according WANTED and BUILD_DEPENDS
283 sort_cook_list()
284 {
285 sort | while read pkg; do
286 echo -n "$pkg"
287 WANTED=""
288 BUILD_DEPENDS=""
289 . $BUILD_WOK/$pkg/receipt
290 MISSING=""
291 for i in $WANTED $BUILD_DEPENDS ; do
292 if [ ! -f $BUILD_WOK/$i/taz/*/receipt ]; then
293 case " $MISSING " in
294 *\ $i\ *);;
295 *) echo -n " $i";;
296 esac
297 MISSING="$MISSING $i"
298 fi
299 done
300 echo ""
301 done | awk '
302 function show(name)
303 {
304 print name;
305 got[name]++;
306 if (revdepcnt[name] > 0)
307 for (i = split(revdep[name], pkg, " "); i > 0; i--)
308 if (--depcnt[pkg[i]] == 0) show(pkg[i]);
309 }
311 {
312 if ($2 == "") show($1);
313 else {
314 depcnt[$1] = NF - 1;
315 unres = unres " " $1;
316 for (i = 2; i <= NF; i++) {
317 if (got[$i] > 0) continue;
318 revdepcnt[$i]++;
319 revdep[$i] = revdep[$i] " " $1;
320 }
321 }
322 }
323 END {
324 for (i = split(unres, pkg, " "); i > 0; i--)
325 if (depcnt[pkg[i]] > 0) print pkg[i];
326 }
327 '
328 }
330 # Here we cook all packages found in the cooklist.
331 cook_install()
332 {
333 echo "" > $DB_DIR/unbuilt
334 for pkg in `cat $DB_DIR/cooklist | sort_cook_list`
335 do
336 if ! cook_package $pkg; then
337 # Link to build log.
338 echo "<a href=\"log.php?package=$(escape $pkg)\">$pkg</a>" >> \
339 $DB_DIR/unbuilt
340 fi
341 # Remove package from the cooklist and empty lines for HTML <pre>.
342 sed -i /"^$pkg$"/d $DB_DIR/cooklist
343 sed -i '/^$/d' $DB_DIR/cooklist
344 packages_summary_update
345 done
346 }
348 # Remove old packages in the build wok and clean pkgs repository. The
349 # Hg wok is copied into the build wok so packages removed by hg must be
350 # removed. To remove old packages in the repository we look into the
351 # build wok and dont remove unbuilt packages. Clean-up will also remove
352 # all corrupted packages.
353 clean_up()
354 {
355 touch $DB_DIR/removed
356 echo -e "\nCleaning the build wok, old and corrupted packages...\n"
357 echo "(cleaning)" > $DB_DIR/running
358 for pkg in `ls $BUILD_WOK`
359 do
360 if [ ! -d $HG_WOK/$pkg ]; then
361 case $2 in
362 --dry-run)
363 echo "Removing directory : $pkg" ;;
364 --verbose)
365 echo "Removing directory : $pkg"
366 rm -rf $BUILD_WOK/$pkg ;;
367 *)
368 rm -rf $BUILD_WOK/$pkg ;;
369 esac
370 fi
371 done
372 # Build a packages list with EXTRAVERSION so we can grep into it.
373 rm -f $DB_DIR/packaged && touch $DB_DIR/packaged
374 for receipt in $BUILD_WOK/*/taz/*/receipt
375 do
376 EXTRAVERSION=""
377 . $receipt
378 echo "$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg" >> $DB_DIR/packaged
379 done
380 for pkg in `cd $PACKAGES_REPOSITORY && ls *.tazpkg`
381 do
382 if ! grep -q "^$pkg$" $DB_DIR/packaged; then
383 case $2 in
384 --dry-run)
385 echo "Removing package : $pkg" ;;
386 --verbose)
387 echo "Removing package : $pkg"
388 echo "$pkg" >> $DB_DIR/removed
389 rm -f $PACKAGES_REPOSITORY/$pkg ;;
390 *)
391 echo "$pkg" >> $DB_DIR/removed
392 rm -f $PACKAGES_REPOSITORY/$pkg ;;
393 esac
394 fi
395 done
396 # Remove all corrupted packages
397 for pkg in `cat $DB_DIR/corrupted | awk '{ print $3 }'`
398 do
399 case $2 in
400 --dry-run)
401 echo "Removing corrupted: $pkg" ;;
402 --verbose)
403 echo "Removing corrupted: $pkg"
404 echo "$pkg" >> $DB_DIR/removed
405 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
406 *)
407 echo "$pkg" >> $DB_DIR/removed
408 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
409 esac
410 done
411 echo ""
412 # Keep the 20 last removed packages list.
413 cat $DB_DIR/removed | tail -n 20 > /tmp/removed.tail
414 mv -f /tmp/removed.tail $DB_DIR/removed
415 }
417 blocked_urls()
418 {
419 rm -f $DB_DIR/blocked.urls
420 for pkg in `cat $DB_DIR/blocked`
421 do
422 if [ -f $LOG_DIR/$pkg.log ]; then
423 echo "<a href=\"log.php?package=$(escape $pkg)\">$pkg</a>" >> \
424 $DB_DIR/blocked.urls
425 else
426 echo "$pkg" >> $DB_DIR/blocked.urls
427 fi
428 done
429 }
431 # 4k, not a meta or a get-* package and no files = buggy package
432 test_packages()
433 {
434 echo -e "\nTesting all packages in: $PACKAGES_REPOSITORY"
435 echo "================================================================================"
436 echo "(testing packages)" > $DB_DIR/running
437 rm -f $DB_DIR/corrupted && touch $DB_DIR/corrupted
438 for pkg in $PACKAGES_REPOSITORY/*.tazpkg
439 do
440 tmp=/tmp/bb-test.$$
441 CATEGORY=""
442 if du $pkg | grep -qw '^4' && ! echo `basename $pkg` | grep -q '^get-'; then
443 mkdir -p $tmp && cd $tmp
444 cpio -i receipt 2>/dev/null < $pkg
445 . ./receipt
446 if [ "$CATEGORY" != "meta" ]; then
447 [ "$2" = "--verbose" ] && echo "Testing: $PACKAGE"
448 cpio -i fs.cpio.gz 2>/dev/null < $pkg
449 if [ ! -f fs.cpio.gz ]; then
450 echo "Missing filesystem `basename $pkg`"
451 if [ -f $LOG_DIR/$PACKAGE.log ];then
452 echo "Missing filesystem `basename $pkg` <a href=\"log.php?package=$(escape $PACKAGE)\">Log</a>" \
453 >> $DB_DIR/corrupted
454 else
455 echo "Missing filesystem `basename $pkg`" \
456 >> $DB_DIR/corrupted
457 fi
458 else
459 zcat fs.cpio.gz | cpio -id 2>/dev/null
460 files=`find fs -type f -o type l`
461 if [ -z "$files" ]; then
462 echo "Empty filesystem `basename $pkg`"
463 if [ -f $LOG_DIR/$PACKAGE.log ]; then
464 echo "Empty filesystem `basename $pkg` <a href=\"log.php?package=$(escape $PACKAGE)\">Log</a>" \
465 >> $DB_DIR/corrupted
466 else
467 echo "Empty filesystem `basename $pkg`" \
468 >> $DB_DIR/corrupted
469 fi
470 fi
471 fi
472 fi
473 cd .. && rm -rf $tmp
474 fi
475 done
476 packages_summary_update
477 echo ""
478 }
480 case "$1" in
481 list-pkgs)
482 # List last cooked packages.
483 list_packages ;;
484 report)
485 # Run in report mode. If an update is done we must cook-all to
486 # rebuild all updated packages.
487 [ "$2" == "--update" ] && update_wok $@ || echo ""
488 check_wok $@
489 test_packages $@
490 show_report ;;
491 cook)
492 # Cook, install and log a single package build.
493 if [ -z $2 ]; then
494 echo "Please specify a package on the command line."
495 rm -f $LOCK_FILE && exit 0
496 fi
497 pkg=$2
498 echo "Starting to cook and install: $pkg"
499 if ! cook_package $pkg; then
500 echo "Unable to install: $pkg"
501 fi ;;
502 cook-all)
503 # Update wok, gen report (with cooklist), cook all packages, test,
504 # clean, gen new report and lists.
505 update_wok $@
506 check_wok $@
507 cook_install
508 test_packages $@
509 clean_up $@
510 check_wok $@
511 echo "(generating lists)" > $DB_DIR/running
512 tazwok gen-list --text
513 echo "" ;;
514 cook-commit)
515 # Cook all packages affected by the last commits in the wok.
516 # Clean up is done only by cook-all to avoid rebuild of corrupted
517 # packages on each commit.
518 update_wok $@
519 check_commit
520 cook_install
521 test_packages $@
522 check_wok $@
523 echo "(generating lists)" > $DB_DIR/running
524 tazwok gen-list --text
525 echo "" ;;
526 block)
527 # Add a pkg name to the list of blocked packages.
528 echo ""
529 if grep -qs "^$2$" $DB_DIR/blocked; then
530 echo -e "$2 is already in the blocked packages list."
531 else
532 echo -n "Adding $2 to : $DB_DIR/blocked... "
533 echo "$2" >> $DB_DIR/blocked && echo "Done"
534 if grep -q "^$2$" $DB_DIR/cooklist; then
535 echo -n "Removing $2 from : $DB_DIR/cooklist... "
536 sed -i /"^$2$"/d $DB_DIR/cooklist && echo "Done"
537 packages_summary_update
538 fi
539 fi
540 blocked_urls
541 echo "" ;;
542 unblock)
543 # Remove a pkg name from the list of blocked packages.
544 echo ""
545 if grep -qs "^$2$" $DB_DIR/blocked; then
546 echo -n "Removing $2 from : $DB_DIR/blocked... "
547 sed -i /"^$2$"/d $DB_DIR/blocked
548 sed -i '/^$/d' $DB_DIR/blocked && echo "Done"
549 echo -n "Adding $2 to : $DB_DIR/cooklist... "
550 echo "$2" >> $DB_DIR/cooklist && echo "Done"
551 packages_summary_update
552 else
553 echo -e "$2 is not in the blocked packages list."
554 fi
555 blocked_urls
556 echo "" ;;
557 test-pkgs)
558 # Start a test suite on all builded packages.
559 test_packages $@ ;;
560 test-suite)
561 # Start a test suite on all builded package and the wok using
562 # the great 'tazwok check'.
563 #
564 # test_packages > $LOG_DIR/test-suite.log
565 # tazwok check >> $LOG_DIR/test-suite.log
566 #
567 test_packages $@
568 script -c "tazwok check" $LOG_DIR/test-suite.log ;;
569 mail)
570 # Tazbbmail Pythom script wrapper.
571 PACKAGE=$2
572 tazbbmail $PACKAGE ;;
573 clean-up)
574 # Remove old packages and generate new packages lists.
575 update_wok $@
576 clean_up $@
577 packages_summary_update
578 [ "$2" != "--dry-run" ] && tazwok gen-list --text ;;
579 clean-log)
580 logs=`ls $LOG_DIR | wc -l`
581 echo -n "Cleaning: $LOG_DIR... "
582 rm -rf $LOG_DIR/*
583 echo "$logs log removed" ;;
584 *)
585 usage ;;
586 esac
588 echo "" > $DB_DIR/running
589 rm -f $LOCK_FILE
591 exit 0