wok view tazbb/stuff/tazbb @ rev 4117

Up asterisk-sound, asterisk-sound-fr (1.4.16)
author Pascal Bellard <pascal.bellard@slitaz.org>
date Sat Sep 19 23:08:58 2009 +0200 (2009-09-19)
parents 0c813a4a2d8b
children 9df47d7de942
line source
1 #!/bin/sh
2 # Tazbb - SliTaz Build Bot.
3 # System wide config file: /etc/slitaz/tazbb.conf
4 #
5 # Tazbb is a tool to automate package building, it can be run manually
6 # or via a cron job. On SliTaz build host, tazbb is run in a chroot env.
7 #
8 # (c) 2009 SliTaz GNU/Linux project - GNU gpl v3
9 #
10
11 # Include config file or exit if no file found.
12 if [ -f "./tazbb.conf" ]; then
13 . ./tazbb.conf
14 elif [ -f "/etc/slitaz/tazbb.conf" ]; then
15 . /etc/slitaz/tazbb.conf
16 else
17 echo -e "\nNo config file found: tazbb.conf...\n" && exit 0
18 fi
19
20 # Tazbb is only for root.
21 if test $(id -u) != 0 ; then
22 echo -e "\nYou must be root to run: `basename $0`.\n" && exit 0
23 fi
24
25 # Let tazbb finish is work and make sure needed files exist.
26 if [ -f $LOCK_FILE ]; then
27 case $1 in
28 usage|list-*|*block)
29 continue ;;
30 *)
31 echo -e "\nTazbb is already running and locked...\n"
32 exit 0 ;;
33 esac
34 else
35 mkdir -p $DB_DIR $LOG_DIR
36 touch $LOCK_FILE $DB_DIR/blocked
37 fi
38
39 usage()
40 {
41 echo -e "\nSliTaz developers and build host tool\n
42 \033[1mUsage: \033[0m `basename $0` [command] [--option]
43 \033[1mCommands: \033[0m\n
44 usage Print this short usage and command list.
45 list-pkgs List last cooked packages with date.
46 report Run in report mode and dont cook anything [--verbose].
47 cook Cook, install and log a single package build.
48 cook-all Cook all missing, modified or unbuilt packages.
49 cook-commit Cook all packages affected by a commit in the last update.
50 test-pkgs Execute a test suite on all packages [--verbose].
51 [un]block Block or unblock a package to skip or enable building.
52 mail Send mail to package maintainer with tazbbmail.
53 clean-up Remove old packages [--verbose|--dry-run].
54 clean-log Remove all generated build log files.\n"
55 }
56
57 status()
58 {
59 local CHECK=$?
60 echo -en "\033[70G"
61 if [ $CHECK = 0 ]; then
62 echo "Done"
63 else
64 echo "Failed"
65 fi
66 return $CHECK
67 }
68
69 top_summary()
70 {
71 cat > $DB_DIR/summary << _EOT_
72 Update : `date`
73 Revision : $NEW_REV (<a href="$HG_URL/log/$NEW_REV">changelog</a>)
74 _EOT_
75 }
76
77 packages_summary()
78 {
79 if ! grep -q "^Packages" $DB_DIR/summary; then
80 cat >> $DB_DIR/summary << _EOT_
81 Packages : `ls $BUILD_WOK | wc -l` in the wok, `cat $DB_DIR/cooklist | wc -l` to cook, \
82 `cat $DB_DIR/blocked | wc -l` blocked, `cat $DB_DIR/corrupted | wc -l` corrupted
83 _EOT_
84 fi
85 }
86
87 packages_summary_update()
88 {
89 sed -i s/"[0-9]* in the wok"/"`ls $BUILD_WOK | wc -l` in the wok"/ \
90 $DB_DIR/summary
91 sed -i s/"[0-9]* to cook"/"`cat $DB_DIR/cooklist | wc -l` to cook"/ \
92 $DB_DIR/summary
93 sed -i s/"[0-9]* blocked"/"`cat $DB_DIR/blocked | wc -l` blocked"/ \
94 $DB_DIR/summary
95 sed -i s/"[0-9]* corrupted"/"`cat $DB_DIR/corrupted | wc -l` corrupted"/ \
96 $DB_DIR/summary
97 }
98
99 list_packages()
100 {
101 cd $PACKAGES_REPOSITORY
102 ls -1t *.tazpkg | head -20 | \
103 while read file
104 do
105 echo -n $(stat -c '%y' $PACKAGES_REPOSITORY/$file | cut -d. -f1)
106 echo " $file"
107 done
108 }
109
110 show_report()
111 {
112 echo "Cooklist"
113 echo "================================================================================"
114 cat $DB_DIR/cooklist && echo ""
115 echo "Blocked"
116 echo "================================================================================"
117 cat $DB_DIR/blocked && echo ""
118 echo ""
119 }
120
121 update_wok()
122 {
123 echo ""
124 echo "(updating wok)" > $DB_DIR/running
125 cd $HG_WOK
126 LAST_REV=`hg head --template '{rev}\n'`
127 hg pull && hg update
128 NEW_REV=`hg head --template '{rev}\n'`
129 # Gen a new summary and link last revision for the web interface.
130 echo -e "\nHg wok : $HG_WOK ($NEW_REV)"
131 echo -e "Build wok : $BUILD_WOK ($LAST_REV)\n"
132 top_summary
133 # Copy Hg wok if new revision or exit to stop process since nothing
134 # have change (--forced can be used).
135 if [ "$NEW_REV" != "$LAST_REV" ]; then
136 size=`du -sh $HG_WOK | awk '{ print $1 }'`
137 echo -n "Copying Hg wok to the build wok ($size)... "
138 #rsync -r -n -t $HG_WOK/ $BUILD_WOK/
139 cp -a $HG_WOK/* $BUILD_WOK
140 cp -a $HG_WOK/.hg $BUILD_WOK
141 echo -e "Done\n"
142 else
143 if [ "$1" = "cook-all" ] || [ "$1" = "cook-commit" ]; then
144 if [ "$2" != "--forced" ]; then
145 echo -e "Nothing to cook...\n"
146 packages_summary
147 rm -f $LOCK_FILE && exit 0
148 fi
149 fi
150 fi
151 }
152
153 # Running 'tazbb report' should not cook anything and --verbose option
154 # can be used to display more messages.
155 check_wok()
156 {
157 # Clean up last results.
158 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
159 rm -f $DB_DIR/report && touch $DB_DIR/report
160 rm -f $DB_DIR/unbuilt && touch $DB_DIR/unbuilt
161 echo "Checking all files in: $HG_WOK"
162 echo "================================================================================"
163 echo "(checking wok)" > $DB_DIR/running
164 for pkg in $HG_WOK/*
165 do
166 EXTRAVERSION=""
167 WANTED=""
168 . $pkg/receipt
169 [ "$2" = "--verbose" ] && echo "Package : $PACKAGE"
170 # Skip blocked packages.
171 if grep -qs "^$PACKAGE$" $DB_DIR/blocked; then
172 echo "Blocked : $PACKAGE ($VERSION)" && continue
173 fi
174
175 # Bristuff hack until the receipt are improved...
176 #[ "$VERSION" = "bristuff" ] && VERSION=`get_version`
177 if [ "$VERSION" = "bristuff" ]; then
178 . $BUILD_WOK/$PACKAGE/taz/*/receipt
179 fi
180
181 # First check if package exit. Package naming _must_ be in the form of:
182 # $PACKAGE-$VERSION or $PACKAGE-${VERSION}$EXTRAVERSION (Kernel string).
183 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-$VERSION.tazpkg ]; then
184 [ -z "$EXTRAVERSION" ] && EXTRAVERSION="_$KERNEL"
185 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg ]; then
186 [ "$1" = "report" ] && echo "Missing : $PACKAGE ($VERSION)"
187 echo "Missing : $PACKAGE ($VERSION)" >> $DB_DIR/report
188 echo "$PACKAGE" >> $DB_DIR/cooklist
189 fi
190 else
191 # Check if package is up-to-date.
192 PKG_DATE=`date -u -r $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg '+%m%d%H%M%Y'`
193 for file in `find $pkg -type f`
194 do
195 FILE_DATE=`date -u -r $file '+%m%d%H%M%Y'`
196 [ "$2" = "--verbose" ] && echo " -> Checking: $file"
197 if [ "$FILE_DATE" -gt "$PKG_DATE" ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
198 [ "$1" = "report" ] && echo "Refresh : $PACKAGE ($VERSION)"
199 echo "Refresh : $PACKAGE ($VERSION)" >> $DB_DIR/report
200 echo "$PACKAGE" >> $DB_DIR/cooklist
201 fi
202 done
203 fi
204 # Now check if package is built and not already in the list.
205 if [ ! -d $BUILD_WOK/$PACKAGE/taz ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
206 [ "$1" = "report" ] && echo "Unbuilt : $PACKAGE ($VERSION)"
207 echo "Unbuilt : $PACKAGE ($VERSION)" >> $DB_DIR/report
208 echo "$PACKAGE" >> $DB_DIR/cooklist
209 fi
210 # Rebuild unbuilt packages list with link to log file. This list
211 # is also generated by cook_inslall to have real time stats.
212 if [ ! -d $BUILD_WOK/$PACKAGE/taz ]; then
213 echo "<a href=\"log.php?package=$PACKAGE\">$PACKAGE</a>" \
214 >> $DB_DIR/unbuilt
215 fi
216 done
217 packages_summary
218 }
219
220 # Create a new cooklist and summary (dont modify report) so 'tazbb cook-commit'
221 # can cook last changes.
222 check_commit()
223 {
224 echo "(checking commit)" > $DB_DIR/running
225 cd $HG_WOK
226 # Clean up last results.
227 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
228 # Get the name of modified packages by the revision range. +1 last
229 # commit was build by the previous build.
230 LAST_REV=$(($LAST_REV+1))
231 echo -e "Will cook from revision $LAST_REV to $NEW_REV\n"
232 for file in `hg log --rev=$LAST_REV:$NEW_REV --template '{files}\n'`
233 do
234 pkg=`echo $file | cut -d "/" -f 1`
235 if ! grep -q ^$pkg$ $DB_DIR/cooklist; then
236 . $pkg/receipt
237 echo "Commit : $PACKAGE ($VERSION)" >> $DB_DIR/report
238 echo "$PACKAGE" >> $DB_DIR/cooklist
239 fi
240 done
241 packages_summary
242 }
243
244 # Here we cook all packages found in the cooklist.
245 cook_install()
246 {
247 echo "" > $DB_DIR/unbuilt
248 for pkg in `cat $DB_DIR/cooklist | sort`
249 do
250 EXTRAVERSION=""
251 DEPENDS=""
252 BUILD_DEPENDS=""
253 SOURCE=""
254 WANTED=""
255 echo "(cooking <a href=\"log.php?package=$pkg\">$pkg</a>)" > $DB_DIR/running
256 tazwok clean $pkg
257 script -c "echo 'install' | tazwok cook $pkg" $LOG_DIR/$pkg.log
258 # Install new package (important for new shared libs). Note
259 # that tests are done separatly with 'test_packages' and should
260 # be done by tazwok.
261 if [ -f $BUILD_WOK/$pkg/taz/*/receipt ]; then
262 . $BUILD_WOK/$pkg/taz/*/receipt
263 echo "(installing $PACKAGE-${VERSION}$EXTRAVERSION.tazpkg)" \
264 > $DB_DIR/running
265 yes | tazpkg install \
266 $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg \
267 --forced
268 else
269 # Link to build log.
270 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
271 $DB_DIR/unbuilt
272 fi
273 # Remove package from the cooklist and empty lines for HTML <pre>.
274 sed -i /"^$pkg$"/d $DB_DIR/cooklist
275 sed -i '/^$/d' $DB_DIR/cooklist
276 packages_summary_update
277 done
278 }
279
280 # Remove old packages in the build wok and clean pkgs repository. The
281 # Hg wok is copied into the build wok so packages removed by hg must be
282 # removed. To remove old packages in the repository we look into the
283 # build wok and dont remove unbuilt packages. Clean-up will also remove
284 # all corrupted packages.
285 clean_up()
286 {
287 touch $DB_DIR/removed
288 echo -e "\nCleaning the build wok, old and corrupted packages...\n"
289 echo "(cleaning)" > $DB_DIR/running
290 for pkg in `ls $HG_WOK`
291 do
292 if [ ! -d $BUILD_WOK/$pkg ]; then
293 case $2 in
294 --dry-run)
295 echo "Removing directory : $pkg" ;;
296 --verbose)
297 echo "Removing directory : $pkg"
298 rm -rf $BUILD_WOK/$pkg ;;
299 *)
300 rm -rf $BUILD_WOK/$pkg ;;
301 esac
302 fi
303 done
304 # Build a packages list with EXTRAVERSION so we can grep into it.
305 rm -f $DB_DIR/packaged && touch $DB_DIR/packaged
306 for receipt in $BUILD_WOK/*/taz/*/receipt
307 do
308 EXTRAVERSION=""
309 . $receipt
310 echo "$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg" >> $DB_DIR/packaged
311 done
312 for pkg in `cd $PACKAGES_REPOSITORY && ls *.tazpkg`
313 do
314 if ! grep -q "^$pkg$" $DB_DIR/packaged; then
315 case $2 in
316 --dry-run)
317 echo "Removing package : $pkg" ;;
318 --verbose)
319 echo "Removing package : $pkg"
320 echo "$pkg" >> $DB_DIR/removed
321 rm -f $PACKAGES_REPOSITORY/$pkg ;;
322 *)
323 echo "$pkg" >> $DB_DIR/removed
324 rm -f $PACKAGES_REPOSITORY/$pkg ;;
325 esac
326 fi
327 done
328 # Remove all corrupted packages
329 for pkg in `cat $DB_DIR/corrupted | awk '{ print $3 }'`
330 do
331 case $2 in
332 --dry-run)
333 echo "Removing corrupted: $pkg" ;;
334 --verbose)
335 echo "Removing corrupted: $pkg"
336 echo "$pkg" >> $DB_DIR/removed
337 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
338 *)
339 echo "$pkg" >> $DB_DIR/removed
340 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
341 esac
342 done
343 echo ""
344 # Keep the 20 last removed packages list.
345 cat $DB_DIR/removed | tail -n 20 > /tmp/removed.tail
346 mv -f /tmp/removed.tail $DB_DIR/removed
347 }
348
349 blocked_urls()
350 {
351 rm -f $DB_DIR/blocked.urls
352 for pkg in `cat $DB_DIR/blocked`
353 do
354 if [ -f $LOG_DIR/$pkg.log ]; then
355 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
356 $DB_DIR/blocked.urls
357 else
358 echo "$pkg" >> $DB_DIR/blocked.urls
359 fi
360 done
361 }
362
363 # 4k, not a meta or a get-* package and no files = buggy package
364 test_packages()
365 {
366 echo -e "\nTesting all packages in: $PACKAGES_REPOSITORY"
367 echo "================================================================================"
368 echo "(testing packages)" > $DB_DIR/running
369 rm -f $DB_DIR/corrupted && touch $DB_DIR/corrupted
370 for pkg in $PACKAGES_REPOSITORY/*.tazpkg
371 do
372 tmp=/tmp/bb-test.$$
373 CATEGORY=""
374 if du $pkg | grep -qw '^4' && ! echo `basename $pkg` | grep -q '^get-'; then
375 mkdir -p $tmp && cd $tmp
376 cpio -i receipt 2>/dev/null < $pkg
377 . ./receipt
378 if [ "$CATEGORY" != "meta" ]; then
379 [ "$2" = "--verbose" ] && echo "Testing: $PACKAGE"
380 cpio -i fs.cpio.gz 2>/dev/null < $pkg
381 if [ ! -f fs.cpio.gz ]; then
382 echo "Missing filesystem `basename $pkg`"
383 if [ -f $LOG_DIR/$PACKAGE.log ];then
384 echo "Missing filesystem `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
385 >> $DB_DIR/corrupted
386 else
387 echo "Missing filesystem `basename $pkg`" \
388 >> $DB_DIR/corrupted
389 fi
390 else
391 zcat fs.cpio.gz | cpio -id 2>/dev/null
392 files=`find fs -type f`
393 if [ -z "$files" ]; then
394 echo "Empty filesystem `basename $pkg`"
395 if [ -f $LOG_DIR/$PACKAGE.log ]; then
396 echo "Empty filesystem `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
397 >> $DB_DIR/corrupted
398 else
399 echo "Empty filesystem `basename $pkg`" \
400 >> $DB_DIR/corrupted
401 fi
402 fi
403 fi
404 fi
405 cd .. && rm -rf $tmp
406 fi
407 done
408 packages_summary_update
409 echo ""
410 }
411
412 case "$1" in
413 list-pkgs)
414 # List last cooked packages.
415 list_packages ;;
416 report)
417 # Run in report mode. If an update is done we must cook-all to
418 # rebuild all updated packages.
419 [ "$2" == "--update" ] && update_wok $@ || echo ""
420 check_wok $@
421 test_packages $@
422 show_report ;;
423 cook)
424 # Cook, install and log a single package build.
425 if [ -z $2 ]; then
426 echo "Please specify a package on the command line."
427 rm -f $LOCK_FILE && exit 0
428 fi
429 pkg=$2
430 echo "Starting to cook and install: $pkg"
431 echo "(cooking <a href=\"log.php?package=$pkg\">$pkg</a>)" > $DB_DIR/running
432 tazwok clean $pkg
433 script -c "echo 'install' | tazwok cook $pkg" $LOG_DIR/$pkg.log
434 if [ -f $BUILD_WOK/$pkg/taz/*/receipt ]; then
435 . $BUILD_WOK/$pkg/taz/*/receipt
436 echo "(installing $PACKAGE-${VERSION}$EXTRAVERSION.tazpkg)" \
437 > $DB_DIR/running
438 yes | tazpkg install \
439 $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg \
440 --forced
441 else
442 echo "Unable to install: $pkg"
443 fi ;;
444 cook-all)
445 # Update wok, gen report (with cooklist), cook all packages, test,
446 # clean, gen new report and lists.
447 update_wok $@
448 check_wok $@
449 cook_install
450 test_packages $@
451 clean_up $@
452 check_wok $@
453 echo "(generating lists)" > $DB_DIR/running
454 tazwok gen-list --text
455 echo "" ;;
456 cook-commit)
457 # Cook all packages affected by the last commits in the wok.
458 # Clean up is done only by cook-all to avoid rebuild of corrupted
459 # packages on each commit.
460 update_wok $@
461 check_commit
462 cook_install
463 test_packages $@
464 check_wok $@
465 echo "(generating lists)" > $DB_DIR/running
466 tazwok gen-list --text
467 echo "" ;;
468 block)
469 # Add a pkg name to the list of blocked packages.
470 echo ""
471 if grep -qs "^$2$" $DB_DIR/blocked; then
472 echo -e "$2 is already in the blocked packages list."
473 else
474 echo -n "Adding $2 to : $DB_DIR/blocked... "
475 echo "$2" >> $DB_DIR/blocked && echo "Done"
476 if grep -q "^$2$" $DB_DIR/cooklist; then
477 echo -n "Removing $2 from : $DB_DIR/cooklist... "
478 sed -i /"^$2$"/d $DB_DIR/cooklist && echo "Done"
479 packages_summary_update
480 fi
481 fi
482 blocked_urls
483 echo "" ;;
484 unblock)
485 # Remove a pkg name from the list of blocked packages.
486 echo ""
487 if grep -qs "^$2$" $DB_DIR/blocked; then
488 echo -n "Removing $2 from : $DB_DIR/blocked... "
489 sed -i /"^$2$"/d $DB_DIR/blocked
490 sed -i '/^$/d' $DB_DIR/blocked && echo "Done"
491 echo -n "Adding $2 to : $DB_DIR/cooklist... "
492 echo "$2" >> $DB_DIR/cooklist && echo "Done"
493 packages_summary_update
494 else
495 echo -e "$2 is not in the blocked packages list."
496 fi
497 blocked_urls
498 echo "" ;;
499 test-pkgs)
500 # Start a test suite on all builded packages.
501 test_packages $@ ;;
502 test-suite)
503 # Start a test suite on all builded package and the wok using
504 # the great 'tazwok check'.
505 #
506 # test_packages > $LOG_DIR/test-suite.log
507 # tazwok check >> $LOG_DIR/test-suite.log
508 #
509 test_packages $@
510 script -c "tazwok check" $LOG_DIR/test-suite.log ;;
511 mail)
512 # Tazbbmail Pythom script wrapper.
513 PACKAGE=$2
514 tazbbmail $PACKAGE ;;
515 clean-up)
516 # Remove old packages and generate new packages lists.
517 update_wok $@
518 clean_up $@
519 packages_summary_update
520 [ "$2" != "--dry-run" ] && tazwok gen-list --text ;;
521 clean-log)
522 logs=`ls $LOG_DIR | wc -l`
523 echo -n "Cleaning: $LOG_DIR... "
524 rm -rf $LOG_DIR/*
525 echo "$logs log removed" ;;
526 *)
527 usage ;;
528 esac
529
530 echo "" > $DB_DIR/running
531 rm -f $LOCK_FILE
532
533 exit 0