wok view tazbb/stuff/tazbb @ rev 3973
merge
| author | Christophe Lincoln <pankso@slitaz.org> |
|---|---|
| date | Thu Aug 27 11:36:57 2009 +0200 (2009-08-27) |
| parents | 28ce60b11e25 |
| children | d5010a514900 |
line source
1 #!/bin/sh
2 # Tazbb - SliTaz Build Bot.
3 # System wide config file: /etc/slitaz/tazbb.conf
4 #
5 # Tazbb is a tool to automate package building, it can be run manually
6 # or via a cron job. On SliTaz build host, tazbb is run in a chroot env.
7 #
8 # (c) 2009 SliTaz GNU/Linux project - GNU gpl v3
9 #
11 # Include config file or exit if no file found.
12 if [ -f "./tazbb.conf" ]; then
13 . ./tazbb.conf
14 elif [ -f "/etc/slitaz/tazbb.conf" ]; then
15 . /etc/slitaz/tazbb.conf
16 else
17 echo -e "\nNo config file found: tazbb.conf...\n" && exit 0
18 fi
20 # Tazbb is only for root.
21 if test $(id -u) != 0 ; then
22 echo -e "\nYou must be root to run: `basename $0`.\n" && exit 0
23 fi
25 # Let tazbb finish is work and make sure needed files exist.
26 if [ -f $LOCK_FILE ]; then
27 case $1 in
28 usage|list-*|*block)
29 continue ;;
30 *)
31 echo -e "\nTazbb is already running and locked...\n"
32 exit 0 ;;
33 esac
34 else
35 mkdir -p $DB_DIR $LOG_DIR
36 touch $LOCK_FILE $DB_DIR/blocked
37 fi
39 usage()
40 {
41 echo -e "\nSliTaz developers and build host tool\n
42 \033[1mUsage: \033[0m `basename $0` [command] [--option]
43 \033[1mCommands: \033[0m\n
44 usage Print this short usage and command list.
45 list-pkgs List last cooked packages with date.
46 report Run in report mode and dont cook anything [--verbose].
47 cook-all Cook all missing, modified or unbuilt packages.
48 cook-commit Cook all packages affected by a commit in the last update.
49 test-pkgs Execute a test suite on all packages [--verbose].
50 [un]block Block or unblock a package to skip or enable building.
51 mail Send mail to package maintainer with tazbbmail.
52 clean-up Remove old packages [--verbose|--dry-run].
53 clean-log Remove all generated build log files.\n"
54 }
56 status()
57 {
58 local CHECK=$?
59 echo -en "\033[70G"
60 if [ $CHECK = 0 ]; then
61 echo "Done"
62 else
63 echo "Failed"
64 fi
65 return $CHECK
66 }
68 top_summary()
69 {
70 cat > $DB_DIR/summary << _EOT_
71 Update : `date`
72 Revision : $NEW_REV (<a href="$HG_URL/log/$NEW_REV">changelog</a>)
73 _EOT_
74 }
76 packages_summary()
77 {
78 if ! grep -q "^Packages" $DB_DIR/summary; then
79 cat >> $DB_DIR/summary << _EOT_
80 Packages : `ls $BUILD_WOK | wc -l` in the wok, `cat $DB_DIR/cooklist | wc -l` to cook, \
81 `cat $DB_DIR/blocked | wc -l` blocked, `cat $DB_DIR/corrupted | wc -l` corrupted
82 _EOT_
83 fi
84 }
86 packages_summary_update()
87 {
88 sed -i s/"[0-9]* in the wok"/"`ls $BUILD_WOK | wc -l` in the wok"/ \
89 $DB_DIR/summary
90 sed -i s/"[0-9]* to cook"/"`cat $DB_DIR/cooklist | wc -l` to cook"/ \
91 $DB_DIR/summary
92 sed -i s/"[0-9]* blocked"/"`cat $DB_DIR/blocked | wc -l` blocked"/ \
93 $DB_DIR/summary
94 sed -i s/"[0-9]* corrupted"/"`cat $DB_DIR/corrupted | wc -l` corrupted"/ \
95 $DB_DIR/summary
96 }
98 list_packages()
99 {
100 cd $PACKAGES_REPOSITORY
101 ls -1t *.tazpkg | head -20 | \
102 while read file
103 do
104 echo -n $(stat -c '%y' $PACKAGES_REPOSITORY/$file | cut -d. -f1)
105 echo " $file"
106 done
107 }
109 show_report()
110 {
111 echo "Cooklist"
112 echo "================================================================================"
113 cat $DB_DIR/cooklist && echo ""
114 echo "Blocked"
115 echo "================================================================================"
116 cat $DB_DIR/blocked && echo ""
117 echo ""
118 }
120 update_wok()
121 {
122 echo ""
123 echo "(updating wok)" > $DB_DIR/running
124 cd $HG_WOK
125 LAST_REV=`hg head --template '{rev}\n'`
126 hg pull && hg update
127 NEW_REV=`hg head --template '{rev}\n'`
128 # Gen a new summary and link last revision for the web interface.
129 echo -e "\nHg wok : $HG_WOK ($NEW_REV)"
130 echo -e "Build wok : $BUILD_WOK ($LAST_REV)\n"
131 top_summary
132 # Copy Hg wok if new revision or exit to stop process since nothing
133 # have change (--forced can be used).
134 if [ "$NEW_REV" != "$LAST_REV" ]; then
135 size=`du -sh $HG_WOK | awk '{ print $1 }'`
136 echo -n "Copying Hg wok to the build wok ($size)... "
137 #rsync -r -n -t $HG_WOK/ $BUILD_WOK/
138 cp -a $HG_WOK/* $BUILD_WOK
139 cp -a $HG_WOK/.hg $BUILD_WOK
140 echo -e "Done\n"
141 else
142 if [ "$1" = "cook-all" ] || [ "$1" = "cook-commit" ]; then
143 if [ "$2" != "--forced" ]; then
144 echo -e "Nothing to cook...\n"
145 packages_summary
146 rm -f $LOCK_FILE && exit 0
147 fi
148 fi
149 fi
150 }
152 # Running 'tazbb report' should not cook anything and --verbose option
153 # can be used to display more messages.
154 check_wok()
155 {
156 # Clean up last results.
157 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
158 rm -f $DB_DIR/report && touch $DB_DIR/report
159 rm -f $DB_DIR/unbuilt && touch $DB_DIR/unbuilt
160 echo "Checking all files in: $HG_WOK"
161 echo "================================================================================"
162 echo "(checking wok)" > $DB_DIR/running
163 for pkg in $HG_WOK/*
164 do
165 EXTRAVERSION=""
166 WANTED=""
167 . $pkg/receipt
168 [ "$2" = "--verbose" ] && echo "Package : $PACKAGE"
169 # Skip blocked packages.
170 if grep -qs "^$PACKAGE$" $DB_DIR/blocked; then
171 echo "Blocked : $PACKAGE ($VERSION)" && continue
172 fi
174 # Bristuff hack until the receipt are improved...
175 #[ "$VERSION" = "bristuff" ] && VERSION=`get_version`
176 if [ "$VERSION" = "bristuff" ]; then
177 . $BUILD_WOK/$PACKAGE/taz/*/receipt
178 fi
180 # First check if package exit. Package naming _must_ be in the form of:
181 # $PACKAGE-$VERSION or $PACKAGE-${VERSION}$EXTRAVERSION (Kernel string).
182 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-$VERSION.tazpkg ]; then
183 [ -z "$EXTRAVERSION" ] && EXTRAVERSION="_$KERNEL"
184 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg ]; then
185 [ "$1" = "report" ] && echo "Missing : $PACKAGE ($VERSION)"
186 echo "Missing : $PACKAGE ($VERSION)" >> $DB_DIR/report
187 echo "$PACKAGE" >> $DB_DIR/cooklist
188 fi
189 else
190 # Check if package is up-to-date.
191 PKG_DATE=`date -u -r $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg '+%m%d%H%M%Y'`
192 for file in `find $pkg -type f`
193 do
194 FILE_DATE=`date -u -r $file '+%m%d%H%M%Y'`
195 [ "$2" = "--verbose" ] && echo " -> Checking: $file"
196 if [ "$FILE_DATE" -gt "$PKG_DATE" ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
197 [ "$1" = "report" ] && echo "Refresh : $PACKAGE ($VERSION)"
198 echo "Refresh : $PACKAGE ($VERSION)" >> $DB_DIR/report
199 echo "$PACKAGE" >> $DB_DIR/cooklist
200 fi
201 done
202 fi
203 # Now check if package is built and not already in the list.
204 if [ ! -d $BUILD_WOK/$PACKAGE/taz ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
205 [ "$1" = "report" ] && echo "Unbuilt : $PACKAGE ($VERSION)"
206 echo "Unbuilt : $PACKAGE ($VERSION)" >> $DB_DIR/report
207 echo "$PACKAGE" >> $DB_DIR/cooklist
208 fi
209 # Rebuild unbuilt packages list with link to log file. This list
210 # is also generated by cook_inslall to have real time stats.
211 if [ ! -d $BUILD_WOK/$PACKAGE/taz ]; then
212 echo "<a href=\"log.php?package=$PACKAGE\">$PACKAGE</a>" \
213 >> $DB_DIR/unbuilt
214 fi
215 done
216 packages_summary
217 }
219 # Create a new cooklist and summary (dont modify report) so 'tazbb cook-commit'
220 # can cook last changes.
221 check_commit()
222 {
223 echo "(checking commit)" > $DB_DIR/running
224 cd $HG_WOK
225 # Clean up last results.
226 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
227 # Get the name of modified packages by the revision range. +1 last
228 # commit was build by the previous build.
229 LAST_REV=$(($LAST_REV+1))
230 echo -e "Will cook from revision $LAST_REV to $NEW_REV\n"
231 for file in `hg log --rev=$LAST_REV:$NEW_REV --template '{files}\n'`
232 do
233 pkg=`echo $file | cut -d "/" -f 1`
234 if ! grep -q ^$pkg$ $DB_DIR/cooklist; then
235 . $pkg/receipt
236 echo "Commit : $PACKAGE ($VERSION)" >> $DB_DIR/report
237 echo "$PACKAGE" >> $DB_DIR/cooklist
238 fi
239 done
240 packages_summary
241 }
243 # Here we cook all packages found in the cooklist.
244 cook_install()
245 {
246 echo "" > $DB_DIR/unbuilt
247 for pkg in `cat $DB_DIR/cooklist | sort`
248 do
249 EXTRAVERSION=""
250 DEPENDS=""
251 BUILD_DEPENDS=""
252 SOURCE=""
253 WANTED=""
254 echo "(cooking <a href=\"log.php?package=$pkg\">$pkg</a>)" > $DB_DIR/running
255 tazwok clean $pkg
256 script -c "echo 'install' | tazwok cook $pkg" $LOG_DIR/$pkg.log
257 # Install new package (important for new shared libs). Note
258 # that tests are done separatly with 'test_packages' and should
259 # be done by tazwok.
260 if [ -f $BUILD_WOK/$pkg/taz/*/receipt ]; then
261 . $BUILD_WOK/$pkg/taz/*/receipt
262 echo "(installing $PACKAGE-${VERSION}$EXTRAVERSION.tazpkg)" \
263 > $DB_DIR/running
264 yes | tazpkg install \
265 $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg \
266 --forced
267 else
268 # Link to build log.
269 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
270 $DB_DIR/unbuilt
271 fi
272 # Remove package from the cooklist and empty lines for HTML <pre>.
273 sed -i /"^$pkg$"/d $DB_DIR/cooklist
274 sed -i '/^$/d' $DB_DIR/cooklist
275 packages_summary_update
276 done
277 }
279 # Remove old packages in the build wok and clean pkgs repository. The
280 # Hg wok is copied into the build wok so packages removed by hg must be
281 # removed. To remove old packages in the repository we look into the
282 # build wok and dont remove unbuilt packages. Clean-up will also remove
283 # all corrupted packages.
284 clean_up()
285 {
286 touch $DB_DIR/removed
287 echo -e "\nCleaning the build wok, old and corrupted packages...\n"
288 echo "(cleaning)" > $DB_DIR/running
289 for pkg in `ls $HG_WOK`
290 do
291 if [ ! -d $BUILD_WOK/$pkg ]; then
292 case $2 in
293 --dry-run)
294 echo "Removing directory : $pkg" ;;
295 --verbose)
296 echo "Removing directory : $pkg"
297 rm -rf $BUILD_WOK/$pkg ;;
298 *)
299 rm -rf $BUILD_WOK/$pkg ;;
300 esac
301 fi
302 done
303 # Build a packages list with EXTRAVERSION so we can grep into it.
304 rm -f $DB_DIR/packaged && touch $DB_DIR/packaged
305 for receipt in $BUILD_WOK/*/taz/*/receipt
306 do
307 EXTRAVERSION=""
308 . $receipt
309 echo "$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg" >> $DB_DIR/packaged
310 done
311 for pkg in `cd $PACKAGES_REPOSITORY && ls *.tazpkg`
312 do
313 if ! grep -q "^$pkg$" $DB_DIR/packaged; then
314 case $2 in
315 --dry-run)
316 echo "Removing package : $pkg" ;;
317 --verbose)
318 echo "Removing package : $pkg"
319 echo "$pkg" >> $DB_DIR/removed
320 rm -f $PACKAGES_REPOSITORY/$pkg ;;
321 *)
322 echo "$pkg" >> $DB_DIR/removed
323 rm -f $PACKAGES_REPOSITORY/$pkg ;;
324 esac
325 fi
326 done
327 # Remove all corrupted packages
328 for pkg in `cat $DB_DIR/corrupted | awk '{ print $3 }'`
329 do
330 case $2 in
331 --dry-run)
332 echo "Removing corrupted: $pkg" ;;
333 --verbose)
334 echo "Removing corrupted: $pkg"
335 echo "$pkg" >> $DB_DIR/removed
336 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
337 *)
338 echo "$pkg" >> $DB_DIR/removed
339 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
340 esac
341 done
342 echo ""
343 # Keep the 20 last removed packages list.
344 cat $DB_DIR/removed | tail -n 20 > /tmp/removed.tail
345 mv -f /tmp/removed.tail $DB_DIR/removed
346 }
348 blocked_urls()
349 {
350 rm -f $DB_DIR/blocked.urls
351 for pkg in `cat $DB_DIR/blocked`
352 do
353 if [ -f $LOG_DIR/$pkg.log ]; then
354 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
355 $DB_DIR/blocked.urls
356 else
357 echo "$pkg" >> $DB_DIR/blocked.urls
358 fi
359 done
360 }
362 # 4k, not a meta or a get-* package and no files = buggy package
363 test_packages()
364 {
365 echo -e "\nTesting all packages in: $PACKAGES_REPOSITORY"
366 echo "================================================================================"
367 echo "(testing packages)" > $DB_DIR/running
368 rm -f $DB_DIR/corrupted && touch $DB_DIR/corrupted
369 for pkg in $PACKAGES_REPOSITORY/*.tazpkg
370 do
371 tmp=/tmp/bb-test.$$
372 CATEGORY=""
373 if du $pkg | grep -qw '^4' && ! echo `basename $pkg` | grep -q '^get-'; then
374 mkdir -p $tmp && cd $tmp
375 cpio -i receipt 2>/dev/null < $pkg
376 . ./receipt
377 if [ "$CATEGORY" != "meta" ]; then
378 [ "$2" = "--verbose" ] && echo "Testing: $PACKAGE"
379 cpio -i fs.cpio.gz 2>/dev/null < $pkg
380 if [ ! -f fs.cpio.gz ]; then
381 echo "Missing filesystem `basename $pkg`"
382 if [ -f $LOG_DIR/$PACKAGE.log ];then
383 echo "Missing filesystem `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
384 >> $DB_DIR/corrupted
385 else
386 echo "Missing filesystem `basename $pkg`" \
387 >> $DB_DIR/corrupted
388 fi
389 else
390 zcat fs.cpio.gz | cpio -id 2>/dev/null
391 files=`find fs -type f`
392 if [ -z "$files" ]; then
393 echo "Empty filesystem `basename $pkg`"
394 if [ -f $LOG_DIR/$PACKAGE.log ]; then
395 echo "Empty filesystem `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
396 >> $DB_DIR/corrupted
397 else
398 echo "Empty filesystem `basename $pkg`" \
399 >> $DB_DIR/corrupted
400 fi
401 fi
402 fi
403 fi
404 cd .. && rm -rf $tmp
405 fi
406 done
407 packages_summary_update
408 echo ""
409 }
411 case "$1" in
412 list-pkgs)
413 # List last cooked packages.
414 list_packages ;;
415 report)
416 # Run in report mode. If an update is done we must cook-all to
417 # rebuild all updated packages.
418 [ "$2" == "--update" ] && update_wok $@ || echo ""
419 check_wok $@
420 test_packages $@
421 show_report ;;
422 cook-all)
423 # Update wok, gen report (with cooklist), cook all packages, test,
424 # clean, gen new report and lists.
425 update_wok $@
426 check_wok $@
427 cook_install
428 test_packages $@
429 clean_up $@
430 check_wok $@
431 echo "(generating lists)" > $DB_DIR/running
432 tazwok gen-list --text
433 echo "" ;;
434 cook-commit)
435 # Cook all packages affected by the last commits in the wok.
436 # Clean up is done only by cook-all to avoid rebuild of corrupted
437 # packages on each commit.
438 update_wok $@
439 check_commit
440 cook_install
441 test_packages $@
442 check_wok $@
443 echo "(generating lists)" > $DB_DIR/running
444 tazwok gen-list --text
445 echo "" ;;
446 block)
447 # Add a pkg name to the list of blocked packages.
448 echo ""
449 if grep -qs "^$2$" $DB_DIR/blocked; then
450 echo -e "$2 is already in the blocked packages list."
451 else
452 echo -n "Adding $2 to : $DB_DIR/blocked... "
453 echo "$2" >> $DB_DIR/blocked && echo "Done"
454 if grep -q "^$2$" $DB_DIR/cooklist; then
455 echo -n "Removing $2 from : $DB_DIR/cooklist... "
456 sed -i /"^$2$"/d $DB_DIR/cooklist && echo "Done"
457 packages_summary_update
458 fi
459 fi
460 blocked_urls
461 echo "" ;;
462 unblock)
463 # Remove a pkg name from the list of blocked packages.
464 echo ""
465 if grep -qs "^$2$" $DB_DIR/blocked; then
466 echo -n "Removing $2 from : $DB_DIR/blocked... "
467 sed -i /"^$2$"/d $DB_DIR/blocked
468 sed -i '/^$/d' $DB_DIR/blocked && echo "Done"
469 echo -n "Adding $2 to : $DB_DIR/cooklist... "
470 echo "$2" >> $DB_DIR/cooklist && echo "Done"
471 packages_summary_update
472 else
473 echo -e "$2 is not in the blocked packages list."
474 fi
475 blocked_urls
476 echo "" ;;
477 test-pkgs)
478 # Start a test suite on all builded packages.
479 test_packages $@ ;;
480 test-suite)
481 # Start a test suite on all builded package and the wok using
482 # the great 'tazwok check'.
483 #
484 # test_packages > $LOG_DIR/test-suite.log
485 # tazwok check >> $LOG_DIR/test-suite.log
486 #
487 test_packages $@
488 script -c "tazwok check" $LOG_DIR/test-suite.log ;;
489 mail)
490 # Tazbbmail Pythom script wrapper.
491 PACKAGE=$2
492 tazbbmail $PACKAGE ;;
493 clean-up)
494 # Remove old packages and generate new packages lists.
495 update_wok $@
496 clean_up $@
497 packages_summary_update
498 [ "$2" != "--dry-run" ] && tazwok gen-list --text ;;
499 clean-log)
500 logs=`ls $LOG_DIR | wc -l`
501 echo -n "Cleaning: $LOG_DIR... "
502 rm -rf $LOG_DIR/*
503 echo "$logs log removed" ;;
504 *)
505 usage ;;
506 esac
508 echo "" > $DB_DIR/running
509 rm -f $LOCK_FILE
511 exit 0