wok view tazbb/stuff/tazbb @ rev 3521
Add Tazpkg-web - SliTaz Packages Web interface
| author | Christophe Lincoln <pankso@slitaz.org> |
|---|---|
| date | Sun Jun 21 03:26:15 2009 +0200 (2009-06-21) |
| parents | 60003751479d |
| children | 87cd9284651c |
line source
1 #!/bin/sh
2 # Tazbb - SliTaz Build Bot.
3 # System wide config file: /etc/slitaz/tazbb.conf
4 #
5 # Tazbb is a tool to automate package building, it can be run manually
6 # or via a cron job. On SliTaz build host, tazbb is run in a chroot env.
7 #
8 # (c) 2009 SliTaz GNU/Linux project - GNU gpl v3
9 #
11 # Include config file or exit if no file found.
12 if [ -f "./tazbb.conf" ]; then
13 . ./tazbb.conf
14 elif [ -f "/etc/slitaz/tazbb.conf" ]; then
15 . /etc/slitaz/tazbb.conf
16 else
17 echo -e "\nNo config file found: tazbb.conf...\n" && exit 0
18 fi
20 # Tazbb is only for root.
21 if test $(id -u) != 0 ; then
22 echo -e "\nYou must be root to run: `basename $0`.\n" && exit 0
23 fi
25 # Let tazbb finish is work and make sure needed files exist.
26 if [ -f $LOCK_FILE ]; then
27 case $1 in
28 usage|list-*|*block)
29 continue ;;
30 *)
31 echo -e "\nTazbb is already running and locked...\n"
32 exit 0 ;;
33 esac
34 else
35 mkdir -p $DB_DIR $LOG_DIR
36 touch $LOCK_FILE $DB_DIR/blocked
37 fi
39 usage()
40 {
41 echo -e "\nSliTaz developers and build host tool\n
42 \033[1mUsage: \033[0m `basename $0` [command] [--option]
43 \033[1mCommands: \033[0m\n
44 usage Print this short usage and command list.
45 list-pkgs List last cooked packages with date.
46 report Run in report mode and dont cook anything [--verbose].
47 cook-all Cook all missing, modified or unbuilt packages.
48 cook-commit Cook all packages affected by a commit in the last update.
49 test-pkgs Execute a test suite on all packages [--verbose].
50 [un]block Block or unblock a package to skip or enable building.
51 clean-up Remove old packages [--verbose|--dry-run].
52 clean-log Remove all generated build log files.\n"
53 }
55 status()
56 {
57 local CHECK=$?
58 echo -en "\033[70G"
59 if [ $CHECK = 0 ]; then
60 echo "Done"
61 else
62 echo "Failed"
63 fi
64 return $CHECK
65 }
67 top_summary()
68 {
69 cat > $DB_DIR/summary << _EOT_
70 Update : `date`
71 Revision : $NEW_REV (<a href="$HG_URL/log/$NEW_REV">changelog</a>)
72 _EOT_
73 }
75 packages_summary()
76 {
77 if ! grep -q "^Packages" $DB_DIR/summary; then
78 cat >> $DB_DIR/summary << _EOT_
79 Packages : `ls $BUILD_WOK | wc -l` in the wok, `cat $DB_DIR/cooklist | wc -l` to cook, \
80 `cat $DB_DIR/blocked | wc -l` blocked, `cat $DB_DIR/corrupted | wc -l` corrupted
81 _EOT_
82 fi
83 }
85 packages_summary_update()
86 {
87 sed -i s/"[0-9]* in the wok"/"`ls $BUILD_WOK | wc -l` in the wok"/ \
88 $DB_DIR/summary
89 sed -i s/"[0-9]* to cook"/"`cat $DB_DIR/cooklist | wc -l` to cook"/ \
90 $DB_DIR/summary
91 sed -i s/"[0-9]* blocked"/"`cat $DB_DIR/blocked | wc -l` blocked"/ \
92 $DB_DIR/summary
93 sed -i s/"[0-9]* corrupted"/"`cat $DB_DIR/corrupted | wc -l` corrupted"/ \
94 $DB_DIR/summary
95 }
97 list_packages()
98 {
99 cd $PACKAGES_REPOSITORY
100 ls -1t *.tazpkg | head -20 | \
101 while read file
102 do
103 echo -n $(stat -c '%y' $PACKAGES_REPOSITORY/$file | cut -d. -f1)
104 echo " $file"
105 done
106 }
108 show_report()
109 {
110 echo "Cooklist"
111 echo "================================================================================"
112 cat $DB_DIR/cooklist && echo ""
113 echo "Blocked"
114 echo "================================================================================"
115 cat $DB_DIR/blocked && echo ""
116 echo ""
117 }
119 update_wok()
120 {
121 echo ""
122 echo "(updating wok)" > $DB_DIR/running
123 cd $HG_WOK
124 LAST_REV=`hg head --template '{rev}\n'`
125 hg pull && hg update
126 NEW_REV=`hg head --template '{rev}\n'`
127 # Gen a new summary and link last revision for the web interface.
128 echo -e "\nHg wok : $HG_WOK ($NEW_REV)"
129 echo -e "Build wok : $BUILD_WOK ($LAST_REV)\n"
130 top_summary
131 # Copy Hg wok if new revision or exit to stop process since nothing
132 # have change (--forced can be used).
133 if [ "$NEW_REV" != "$LAST_REV" ]; then
134 size=`du -sh $HG_WOK | awk '{ print $1 }'`
135 echo -n "Copying Hg wok to the build wok ($size)... "
136 cp -a $HG_WOK/* $BUILD_WOK
137 cp -a $HG_WOK/.hg $BUILD_WOK
138 echo -e "Done\n"
139 else
140 if [ "$1" = "cook-all" ] || [ "$1" = "cook-commit" ]; then
141 if [ "$2" != "--forced" ]; then
142 echo -e "Nothing to cook...\n"
143 packages_summary
144 rm -f $LOCK_FILE && exit 0
145 fi
146 fi
147 fi
148 }
150 # Running 'tazbb report' should not cook anything and --verbose option
151 # can be used to display more messages.
152 check_wok()
153 {
154 # Clean up last results.
155 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
156 rm -f $DB_DIR/report && touch $DB_DIR/report
157 rm -f $DB_DIR/unbuilt && touch $DB_DIR/unbuilt
158 echo "Checking all files in: $HG_WOK"
159 echo "================================================================================"
160 echo "(checking wok)" > $DB_DIR/running
161 for pkg in $HG_WOK/*
162 do
163 EXTRAVERSION=""
164 WANTED=""
165 . $pkg/receipt
166 [ "$2" = "--verbose" ] && echo "Package : $PACKAGE"
167 # Skip blocked packages.
168 if grep -qs "^$PACKAGE$" $DB_DIR/blocked; then
169 echo "Blocked : $PACKAGE ($VERSION)" && continue
170 fi
172 # Bristuff hack until the receipt are improved...
173 #[ "$VERSION" = "bristuff" ] && VERSION=`get_version`
174 if [ "$VERSION" = "bristuff" ]; then
175 . $BUILD_WOK/$PACKAGE/taz/*/receipt
176 fi
178 # First check if package exit. Package naming _must_ be in the form of:
179 # $PACKAGE-$VERSION or $PACKAGE-${VERSION}$EXTRAVERSION (Kernel string).
180 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-$VERSION.tazpkg ]; then
181 [ -z "$EXTRAVERSION" ] && EXTRAVERSION="_$KERNEL"
182 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg ]; then
183 [ "$1" = "report" ] && echo "Missing : $PACKAGE ($VERSION)"
184 echo "Missing : $PACKAGE ($VERSION)" >> $DB_DIR/report
185 echo "$PACKAGE" >> $DB_DIR/cooklist
186 fi
187 else
188 # Check if package is up-to-date.
189 PKG_DATE=`date -u -r $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg '+%m%d%H%M%Y'`
190 for file in `find $pkg -type f`
191 do
192 FILE_DATE=`date -u -r $file '+%m%d%H%M%Y'`
193 [ "$2" = "--verbose" ] && echo " -> Checking: $file"
194 if [ "$FILE_DATE" -gt "$PKG_DATE" ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
195 [ "$1" = "report" ] && echo "Refresh : $PACKAGE ($VERSION)"
196 echo "Refresh : $PACKAGE ($VERSION)" >> $DB_DIR/report
197 echo "$PACKAGE" >> $DB_DIR/cooklist
198 fi
199 done
200 fi
201 # Now check if package is built and not already in the list.
202 if [ ! -d $BUILD_WOK/$PACKAGE/taz ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
203 [ "$1" = "report" ] && echo "Unbuilt : $PACKAGE ($VERSION)"
204 echo "Unbuilt : $PACKAGE ($VERSION)" >> $DB_DIR/report
205 echo "$PACKAGE" >> $DB_DIR/cooklist
206 fi
207 # Rebuild unbuilt packages list with link to log file. This list
208 # is also generated by cook_inslall to have real time stats.
209 if [ ! -d $BUILD_WOK/$PACKAGE/taz ]; then
210 echo "<a href=\"log.php?package=$PACKAGE\">$PACKAGE</a>" \
211 >> $DB_DIR/unbuilt
212 fi
213 done
214 packages_summary
215 }
217 # Create a new cooklist and summary (dont modify report) so 'tazbb cook-commit'
218 # can cook last changes.
219 check_commit()
220 {
221 echo "(checking commit)" > $DB_DIR/running
222 cd $HG_WOK
223 # Clean up last results.
224 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
225 # Get the name of modified packages by the revision range. +1 last
226 # commit was build by the previous build.
227 LAST_REV=$(($LAST_REV+1))
228 echo -e "Will cook from revision $LAST_REV to $NEW_REV\n"
229 for file in `hg log --rev=$LAST_REV:$NEW_REV --template '{files}\n'`
230 do
231 pkg=`echo $file | cut -d "/" -f 1`
232 if ! grep -q ^$pkg$ $DB_DIR/cooklist; then
233 . $pkg/receipt
234 echo "Commit : $PACKAGE ($VERSION)" >> $DB_DIR/report
235 echo "$PACKAGE" >> $DB_DIR/cooklist
236 fi
237 done
238 packages_summary
239 }
241 # Here we cook all packages found in the cooklist.
242 cook_install()
243 {
244 echo "" > $DB_DIR/unbuilt
245 for pkg in `cat $DB_DIR/cooklist | sort`
246 do
247 EXTRAVERSION=""
248 DEPENDS=""
249 BUILD_DEPENDS=""
250 SOURCE=""
251 WANTED=""
252 echo "(cooking <a href=\"log.php?package=$pkg\">$pkg</a>)" > $DB_DIR/running
253 tazwok clean $pkg
254 script -c "echo 'install' | tazwok cook $pkg" $LOG_DIR/$pkg.log
255 # Install new package (important for new shared libs). Note
256 # that tests are done separatly with 'test_packages' and should
257 # be done by tazwok.
258 if [ -f $BUILD_WOK/$pkg/taz/*/receipt ]; then
259 . $BUILD_WOK/$pkg/taz/*/receipt
260 echo "(installing $PACKAGE-${VERSION}$EXTRAVERSION.tazpkg)" \
261 > $DB_DIR/running
262 yes | tazpkg install \
263 $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg \
264 --forced
265 else
266 # Link to build log.
267 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
268 $DB_DIR/unbuilt
269 fi
270 # Remove package from the cooklist and empty lines for HTML <pre>.
271 sed -i /"^$pkg$"/d $DB_DIR/cooklist
272 sed -i '/^$/d' $DB_DIR/cooklist
273 done
274 }
276 # Remove old packages in the build wok and clean pkgs repository. The
277 # Hg wok is copied into the build wok so packages removed by hg must be
278 # removed. To remove old packages in the repository we look into the
279 # build wok and dont remove unbuilt packages. Clean-up will also remove
280 # all corrupted packages.
281 clean_up()
282 {
283 touch $DB_DIR/removed
284 echo -e "\nCleaning the build wok, old and corrupted packages...\n"
285 echo "(cleaning)" > $DB_DIR/running
286 for pkg in `ls $HG_WOK`
287 do
288 if [ ! -d $BUILD_WOK/$pkg ]; then
289 case $2 in
290 --dry-run)
291 echo "Removing directory : $pkg" ;;
292 --verbose)
293 echo "Removing directory : $pkg"
294 rm -rf $BUILD_WOK/$pkg ;;
295 *)
296 rm -rf $BUILD_WOK/$pkg ;;
297 esac
298 fi
299 done
300 # Build a packages list with EXTRAVERSION so we can grep into it.
301 rm -f $DB_DIR/packaged && touch $DB_DIR/packaged
302 for receipt in $BUILD_WOK/*/taz/*/receipt
303 do
304 EXTRAVERSION=""
305 . $receipt
306 echo "$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg" >> $DB_DIR/packaged
307 done
308 for pkg in `cd $PACKAGES_REPOSITORY && ls *.tazpkg`
309 do
310 if ! grep -q "^$pkg$" $DB_DIR/packaged; then
311 case $2 in
312 --dry-run)
313 echo "Removing package : $pkg" ;;
314 --verbose)
315 echo "Removing package : $pkg"
316 echo "$pkg" >> $DB_DIR/removed
317 rm -f $PACKAGES_REPOSITORY/$pkg ;;
318 *)
319 echo "$pkg" >> $DB_DIR/removed
320 rm -f $PACKAGES_REPOSITORY/$pkg ;;
321 esac
322 fi
323 done
324 # Remove all corrupted packages
325 for pkg in `cat $DB_DIR/corrupted | awk '{ print $3 }'`
326 do
327 case $2 in
328 --dry-run)
329 echo "Removing corrupted: $pkg" ;;
330 --verbose)
331 echo "Removing corrupted: $pkg"
332 echo "$pkg" >> $DB_DIR/removed
333 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
334 *)
335 echo "$pkg" >> $DB_DIR/removed
336 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
337 esac
338 done
339 echo ""
340 # Keep the 20 last removed packages list.
341 cat $DB_DIR/removed | tail -n 20 > /tmp/removed.tail
342 mv -f /tmp/removed.tail $DB_DIR/removed
343 }
345 blocked_urls()
346 {
347 rm -f $DB_DIR/blocked.urls
348 for pkg in `cat $DB_DIR/blocked`
349 do
350 if [ -f $LOG_DIR/$pkg.log ]; then
351 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
352 $DB_DIR/blocked.urls
353 else
354 echo "$pkg" >> $DB_DIR/blocked.urls
355 fi
356 done
357 }
359 # 4k, not a meta or a get-* package and no files = buggy package
360 test_packages()
361 {
362 echo -e "\nTesting all packages in: $PACKAGES_REPOSITORY"
363 echo "================================================================================"
364 echo "(testing packages)" > $DB_DIR/running
365 rm -f $DB_DIR/corrupted && touch $DB_DIR/corrupted
366 for pkg in $PACKAGES_REPOSITORY/*.tazpkg
367 do
368 tmp=/tmp/bb-test.$$
369 CATEGORY=""
370 if du $pkg | grep -qw '^4' && ! echo `basename $pkg` | grep -q '^get-'; then
371 mkdir -p $tmp && cd $tmp
372 cpio -i receipt 2>/dev/null < $pkg
373 . ./receipt
374 if [ "$CATEGORY" != "meta" ]; then
375 [ "$2" = "--verbose" ] && echo "Testing: $PACKAGE"
376 cpio -i fs.cpio.gz 2>/dev/null < $pkg
377 if [ ! -f fs.cpio.gz ]; then
378 echo "Missing filesystem `basename $pkg`"
379 if [ -f $LOG_DIR/$PACKAGE.log ];then
380 echo "Missing filesystem `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
381 >> $DB_DIR/corrupted
382 else
383 echo "Missing filesystem `basename $pkg`" \
384 >> $DB_DIR/corrupted
385 fi
386 else
387 zcat fs.cpio.gz | cpio -id 2>/dev/null
388 files=`find fs -type f`
389 if [ -z "$files" ]; then
390 echo "Empty filesystem `basename $pkg`"
391 if [ -f $LOG_DIR/$PACKAGE.log ]; then
392 echo "Empty filesystem `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
393 >> $DB_DIR/corrupted
394 else
395 echo "Empty filesystem `basename $pkg`" \
396 >> $DB_DIR/corrupted
397 fi
398 fi
399 fi
400 fi
401 cd .. && rm -rf $tmp
402 fi
403 done
404 packages_summary_update
405 echo ""
406 }
408 case "$1" in
409 list-pkgs)
410 # List last cooked packages.
411 list_packages ;;
412 report)
413 # Run in report mode. If an update is done we must cook-all to
414 # rebuild all updated packages.
415 [ "$2" == "--update" ] && update_wok $@ || echo ""
416 check_wok $@
417 test_packages $@
418 show_report ;;
419 cook-all)
420 # Update wok, gen report (with cooklist), cook all packages, test,
421 # clean, gen new report and lists.
422 update_wok $@
423 check_wok $@
424 cook_install
425 test_packages $@
426 clean_up $@
427 check_wok $@
428 echo "(generating lists)" > $DB_DIR/running
429 tazwok gen-list --text
430 echo "" ;;
431 cook-commit)
432 # Cook all packages affected by the last commits in the wok.
433 update_wok $@
434 check_commit
435 cook_install
436 test_packages $@
437 clean_up $@
438 check_wok $@
439 echo "(generating lists)" > $DB_DIR/running
440 tazwok gen-list --text
441 echo "" ;;
442 block)
443 # Add a pkg name to the list of blocked packages.
444 echo ""
445 if grep -qs "^$2$" $DB_DIR/blocked; then
446 echo -e "$2 is already in the blocked packages list."
447 else
448 echo -n "Adding $2 to : $DB_DIR/blocked... "
449 echo "$2" >> $DB_DIR/blocked && echo "Done"
450 if grep -q "^$2$" $DB_DIR/cooklist; then
451 echo -n "Removing $2 from : $DB_DIR/cooklist... "
452 sed -i /"^$2$"/d $DB_DIR/cooklist && echo "Done"
453 packages_summary_update
454 fi
455 fi
456 blocked_urls
457 echo "" ;;
458 unblock)
459 # Remove a pkg name from the list of blocked packages.
460 echo ""
461 if grep -qs "^$2$" $DB_DIR/blocked; then
462 echo -n "Removing $2 from : $DB_DIR/blocked... "
463 sed -i /"^$2$"/d $DB_DIR/blocked
464 sed -i '/^$/d' $DB_DIR/blocked && echo "Done"
465 echo -n "Adding $2 to : $DB_DIR/cooklist... "
466 echo "$2" >> $DB_DIR/cooklist && echo "Done"
467 packages_summary_update
468 else
469 echo -e "$2 is not in the blocked packages list."
470 fi
471 blocked_urls
472 echo "" ;;
473 test-pkgs)
474 # Start a test suite on all builded packages.
475 test_packages $@ ;;
476 test-suite)
477 # Start a test suite on all builded package and the wok using
478 # the great 'tazwok check'.
479 #
480 # test_packages > $LOG_DIR/test-suite.log
481 # tazwok check >> $LOG_DIR/test-suite.log
482 #
483 test_packages $@
484 script -c "tazwok check" $LOG_DIR/test-suite.log ;;
485 clean-up)
486 # Remove old packages and generate new packages lists.
487 update_wok $@
488 clean_up $@
489 packages_summary_update
490 [ "$2" != "--dry-run" ] && tazwok gen-list --text ;;
491 clean-log)
492 logs=`ls $LOG_DIR | wc -l`
493 echo -n "Cleaning: $LOG_DIR... "
494 rm -rf $LOG_DIR/*
495 echo "$logs log removed" ;;
496 *)
497 usage ;;
498 esac
500 echo "" > $DB_DIR/running
501 rm -f $LOCK_FILE
503 exit 0