cookutils: f92f75dce828 cooker

cookutils view cooker @ rev 1019

cook: add fix() to use '--as-needed' linker flag in compile_rules(); cookit(): make QA fail on empty vars / bad values; remove_already_packed(): fix bug when $PACKAGE not listed in $SPLIT and we use this function for the default set. lighttpd/index.cgi: sort orphans. modules/precheck: separate error message by empty lines.

author	Aleksej Bobylev <al.bobylev@gmail.com>
date	Thu Dec 07 14:31:28 2017 +0200 (2017-12-07)
parents	9c835beabf65
children	54c97f545127

line source

1 #!/bin/sh

3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package

4 # building. Please read the Cookbook documentation for more information

5 # and discuss with the AUTHORS before adding anything here. PS: no translations

6 # here since it's not an end user tool and it's not useful. All devs should

7 # at least understand basic English.

10 . /usr/lib/slitaz/libcook.sh

12 # Set pkg name and use same wok as cook.

13 pkg="$2"

14 wok="$WOK"

16 # PID file.

17 pidfile='/var/run/cooker.pid'

20 # Functions

23 usage() {

24 cat <<EOT

26 Usage: cooker [<command>] [<options>]

28 Commands with <options>:

29 -u | usage Display this short usage.

30 -s | setup Setup the Cooker environment.

31 setup-cron [<hours>] Setup a cron job for the Cooker.

32 check-cron Check Cooker cron job.

33 arch-db Create host arch packages DB.

34 -n | note <note_text> Add a note to the cooknotes.

35 -ns | notes Display all the cooknotes.

36 -b | block <package> Block a package so cook will skip it.

37 -ub | unblock <package> Unblock a blocked package.

38 -R | reverse <package> Cook all reverse dependencies for a package.

39 -p | pkg <package> Same as 'cook pkg' but with cooker log.

40 -f | flavor <flavor_name> Cook all packages of a flavor.

41 -l | list <list_file> Cook all packages in the given list.

42 -c | cat <category> Cook all packages of a category.

43 -r | rev <rev_number> Cook packages of a specific revision.

44 -a | all Find and cook all unbuilt packages.

45 -T | tasks List existing cooker tasks.

46 -t | task <task> Executing specified task.

47 -o | outgoing Find changes in wok that we can move to wok-hg.

50 exit 0

54 # Some messages occur in activity but log verbose output when checking for commits

55 # into a log file.

57 log_commits() {

58 sed '/^.\//d' | sed '/^.hg/d' | tee -a $LOGS/commits.log

62 # Clean up before exit when check and cook commits finish.

64 clean_exit() {

65 rm -f $command; touch $command

66 [ "$previous_command" ] && ps | grep -q "${previous_command/:/ }" &&

67 echo -n "$previous_command" > $command

68 rm -f $pidfile

72 # Summary for commits.

74 commits_summary() {

75 msg="from revision $cur to $new"

76 [ "$new" == "$cur" ] && msg="revision $new"

77 echo "Will cook $msg"

78 separator

79 title "Summary for commits"

80 echo "Hg wok revision : $cur"

81 echo "Pulled revision : $new"

82 echo "Check date : $(date '+%F %T')"

86 # Return all the names of packages bundled in this receipt

88 all_names() {

89 local split=" $SPLIT "

90 unset SPLIT

91 . $wok/$pkg/receipt

93 if ! head -n1 $WOK/$pkg/receipt | fgrep -q 'v2'; then

94 # For receipts v1: $SPLIT may present in the $WANTED package,

95 # but split packages have their own receipts

96 echo $PACKAGE

97 elif [ "${split/ $PACKAGE /}" != "$split" ]; then

98 echo $SPLIT

99 else

100 echo $PACKAGE $SPLIT

105 # Scan packages build deps and fill up cookorder list.

107 cook_order_scan() {

108 rm -f $cookorder $cookorder.split

109 touch $cookorder $cookorder.split

111 # Make combined split table: beginning from actual information with fresh

112 # commits. Example:

113 # freetype freetype freetype-dev

114 # harfbuzz harfbuzz harfbuzz-apps harfbuzz-dev

115 while read pkg; do

116 echo "$pkg $(all_names)" >> $cookorder.split

117 done < $cooklist

118 cat $cache/split.db >> $cookorder.split

120 maxlen=$(wc -L < $cooklist)

122 while read pkg; do

123 unset WANTED BUILD_DEPENDS

124 . $wok/$pkg/receipt

125 bdeps=$(

126 # Substitite each package of BUILD_DEPENDS list by the "main"

127 # receipt which builds this package. Example:

128 # BUILD_DEPENDS="freetype-dev harfbuzz-dev" -> bdeps="freetype harfbuzz"

129 for i in $BUILD_DEPENDS; do

130 main="$(awk -F$'\t' -vi="$i" '{

131 if (index(" " $2 " ", i)) {print $1; exit}

132 }' $cookorder.split)"

133 echo ${main:-$i}

134 done

136 # The :: is for web interface color.

137 bdeps=$(echo $WANTED $bdeps | tr '\n' ' ')

138 printf "%-${maxlen}s :: %s\n" "$pkg" "$bdeps"

139 for dep in $bdeps; do

140 if grep -q "^$dep$" $cooklist; then

141 if ! grep -q "^$dep$" $cookorder; then

142 echo "$dep" >> $cookorder

143 fi

144 fi

145 done

146 done < $cooklist

148 # Append unordered packages to cookorder.

149 while read pkg; do

150 if ! grep -q "^$pkg$" $cookorder; then

151 echo "$pkg" >> $cookorder

152 fi

153 done < $cooklist

157 # Scan and rescan until the cooklist is ordered then handle WANTED.

159 cook_order() {

160 time=$(date +%s)

161 scan=0

162 rm -rf $cache/cookorder.d

163 mkdir -p $cache/cookorder.d

165 # Keep an original cooklist so we do a diff when ordering is finished.

166 cp -f $cooklist $cooklist.0

167 echo 'cookorder' > $command

168 title 'Initial Cooker order scan'

169 cook_order_scan

171 # Diff between the cooklist and new ordered list ? So copy the last

172 # cookorder to cooklist and rescan it.

173 while /bin/true; do

174 if ! cmp -s $cooklist $cookorder; then

175 scan=$(($scan + 1))

176 title "Diff scan: $scan"

178 md5stamp=$(md5sum $cookorder | cut -d' ' -f1)

179 if [ -e "$cache/cookorder.d/$md5stamp" ]; then

180 newline

181 echo 'A dependency loop was detected. Interrupting the cookorder.'

182 break

183 fi

184 touch $cache/cookorder.d/$md5stamp

186 mv -f $cookorder $cooklist

187 cook_order_scan

189 else

190 break

191 fi

192 done

193 # Clean

194 rm -rf $cache/cookorder.d; rm $cookorder.split

196 # Keep a diff between submitted cooklist and the ordered.

197 diff $cooklist.0 $cooklist > $cooklist.diff

198 rm -f $cookorder $cooklist.0

200 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.

201 # TODO: grep the line number to get pkg position and keep it higher.

202 title 'Handle WANTED package'

203 while read pkg; do

204 unset WANTED

205 . $wok/$pkg/receipt

206 for wanted in $WANTED; do

207 echo "$pkg :: $wanted"

208 if grep -q ^${wanted}$ $cooklist; then

209 sed -i -e "/^$pkg$/d" \

210 -e "/^$wanted$/ a $pkg" $cooklist

211 fi

212 done

213 done < $cooklist

215 # Show ordered cooklist

216 title 'Cooklist order'

217 cat $cooklist

218 separator

220 time=$(($(date +%s) - $time))

221 pkgs=$(wc -l < $cooklist)

222 title 'Summary for cookorder'

223 cat <<EOT

224 Ordered packages : $pkgs

225 Scans executed : $scan

226 Scan duration : ${time}s

228 separator

230 rm -f $command

234 # Remove blocked (faster this way than grepping before).

236 strip_blocked() {

237 while read pkg; do

238 sed -i "/^${pkg}$/d" $cooklist

239 done < $blocked

240 sed -i '/^$/d' $cooklist

244 # Use in default mode and with all cmd.

246 cook_commits() {

247 if [ -s "$commits" ]; then

248 while read pkg; do

249 ps | grep -q "cook $pkg$" && continue

250 echo "cook:$pkg" > $command

251 cook $pkg || broken

252 sed -i "/^${pkg}$/d" $commits

253 done < $commits

258 # Cook all packages in a cooklist.

260 cook_list() {

261 while read pkg; do

262 ps | grep -q "cook $pkg$" && continue

263 cook $pkg || broken

264 sed -i "/^${pkg}$/d" $cooklist

265 done < $cooklist

269 # Create a arch.$ARCH file for each package cooked for the target host

270 # architecture

272 # The deal: we don't want all packages handled by cooker commands and stats,

273 # since we don't cross compile all packages for each arch but only a set of

274 # packages to provide one full featured desktop, servers and goodies useful

275 # for the host system.

278 arch_db() {

279 count=0

280 echo "Cleaning packages DB : arch.$ARCH"

281 rm -f $wok/*/arch.$ARCH && cd $wok

282 echo "Creating $ARCH packages DB..."

283 for pkg in *; do

284 [ -s $wok/$pkg/receipt ] || continue

285 HOST_ARCH=

286 . $wok/$pkg/receipt

287 if [ -n "$HOST_ARCH" ]; then

288 if $(echo "$HOST_ARCH" | egrep -q "$ARCH|any"); then

289 count=$(($count + 1))

290 echo "Adding: $pkg"

291 touch $pkg/arch.$ARCH

292 fi

293 unset HOST_ARCH

294 else

295 # HOST_ARCH not set --> in i486

296 if [ "$ARCH" == 'i486' ]; then

297 count=$(($count + 1))

298 echo "Adding: $pkg"

299 touch $pkg/arch.$ARCH

300 fi

301 fi

302 done

303 echo "Packages for $ARCH : $count"

307 # Compare wok and wok-hg file $1, display signs:

308 # '+' file added, '-' file removed, '~' file changed, '=' file not changed

310 compare_wok_file() {

311 local f1='n' f2='n' # n: not exists, e: exists

312 [ -e "$wok/$1" ] && f1='e'

313 [ -e "$wok-hg/$1" ] && f2='e'

314 case "$f1$f2" in

315 en) echo "+ $1";;

316 ne) [ -n "$del" ] && echo "- $1";;

317 ee)

318 if cmp -s "$wok/$1" "$wok-hg/$1"; then

319 [ -n "$eq" ] && echo "= $1"

320 else

321 echo "~ $1"

322 fi

323 ;;

324 esac

328 # Compare wok and wok-hg folder $1; process only:

329 # receipt, description.*txt, all files in the stuff folder

331 compare_wok_folder() {

332 IFS=$'\n'

334 for i in $wok $wok-hg; do

335 ls $i/$1/receipt 2>/dev/null

336 ls $i/$1/description.*txt 2>/dev/null

337 [ -d $i/$1/stuff ] && find $i/$1/stuff -type f

338 done

339 } | sed "s|$wok/$1/||; s|$wok-hg/$1/||" | sort -u | \

340 while read file; do

341 compare_wok_file "$1/$file"

342 done

346 # Compare entire wok

348 compare_wok() {

350 cd $wok; ls

351 cd $wok-hg; ls

352 } | sort -u | \

353 while read folder; do

354 result="$(compare_wok_folder $folder)"

355 [ -n "$result" ] && echo -e "$result\n"

356 done

361 # Commands

364 previous_command="$(cat $command 2>/dev/null)"

365 case "$1" in

366 usage|help|-u|-h)

367 usage ;;

369 setup|-s)

370 # Setup the Cooker environment.

371 title 'Setting up the Cooker'

372 mkdir -p $CACHE

373 echo "Cooker setup using: $SLITAZ" | log

374 for pkg in $SETUP_PKGS mercurial rsync tazlito; do

375 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg

376 done

377 mkdir -p $SLITAZ && cd $SLITAZ

378 if [ -d "${wok}-hg" ]; then

379 echo -e 'Hg wok already exists.\n'

380 exit 1

381 fi

382 if [ -d "$wok" ]; then

383 echo -e 'Build wok already exists.\n'

384 exit 1

385 fi

387 # Directories and files

388 echo "mkdir's and touch files in: $SLITAZ"

389 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC

390 for f in $activity $blocked $broken $commits $cooklist $command; do

391 touch $f

392 done

393 hg clone $WOK_URL ${wok}-hg || exit 1

394 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors

395 cp -a ${wok}-hg $wok

396 footer ;;

398 arch-db)

399 # Manually create arch packages DB.

400 arch_db ;;

402 setup-cron)

403 # Create cron job for the cooker.

404 [ "$2" ] || hours=2

405 if [ ! -f "$crontabs" ]; then

406 mkdir -p /var/spool/cron/crontabs

407 fi

408 if ! fgrep -q /usr/bin/cooker $crontabs; then

409 cat > $crontabs <<EOT

410 # Run SliTaz Cooker every $hours hours

411 59 */$hours * * * touch $CACHE/cooker-request

412 */5 * * * * [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html

413 */5 * * * * [ -z "$(pidof cooker)" ] && [ -s $CACHE/recook-packages ] && /usr/bin/cooker list $CACHE/recook-packages

415 touch $CACHE/cooker-request $CACHE/recook-packages

416 chmod 666 $CACHE/cooker-request $CACHE/recook-packages

417 killall crond 2>/dev/null && /etc/init.d/crond start

418 fi ;;

420 check-cron)

421 if [ ! -f "$crontabs" ]; then

422 echo "There is no $crontabs here. Use setup-cron option."

423 exit 1

424 fi

425 fgrep /usr/bin/cooker $crontabs ;;

427 note|-n)

428 # Blocked a pkg and want others to know why? Post a note!

429 [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;

431 notes|-ns)

432 # View cooknotes.

433 title 'Cooknotes'

434 cat $cooknotes

435 footer ;;

437 block|-b)

438 # Block a package.

439 [ "$pkg" ] && cook $pkg --block ;;

441 unblock|-ub)

442 # Unblock a package.

443 [ "$pkg" ] && cook $pkg --unblock ;;

445 reverse|-r)

446 # Cook all reverse dependencies for a package. This command lets us

447 # control the Cooker manually for commits that will cook a lot of packages.

449 # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"

451 if [ ! -d "$wok/$pkg" ]; then

452 echo -e "\nNo package $2 found.\n"

453 exit 0

454 fi

455 rm -f $cooklist; touch $cooklist

456 title "Reverse cooklist for: $pkg"

458 cd $wok

459 for rev in *; do

460 [ -s $wok/$rev/receipt ] || continue

461 unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt

462 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then

463 echo "$rev" | tee -a $cooklist

464 fi

465 done

466 footer "Reverse dependencies found: $(wc -l < $cooklist)"

467 strip_blocked

468 cook_order | tee $LOGS/cookorder.log

469 cook_list ;;

471 pkg|-p)

472 # Same as 'cook pkg' but with log for web interface.

473 ps | grep -q "cook $pkg$" && echo 'Already running' && continue

474 cook $pkg || broken

475 clean_exit ;;

477 cat|-c)

478 # Cook all packages of a category.

479 cat="$2"

480 rm -f $cooklist; touch $cooklist

482 cd $wok

483 for pkg in *; do

484 [ -s $pkg/receipt ] || continue

485 unset CATEGORY; . $pkg/receipt

486 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist

487 done

488 strip_blocked

489 cook_order | tee $LOGS/cookorder.log

490 cook_list ;;

492 flavor|-f)

493 # Cook all packages of a flavor.

494 name="$2"

495 if [ ! -d "$flavors/$name" ]; then

496 echo -e "\nSpecified flavor does not exist: $name\n"

497 exit 1

498 fi

499 if [ -d "$flavors/.hg" ]; then

500 cd $flavors; hg pull -u

501 fi

502 list="$flavors/$name/packages.list"

503 cp -a $list $cooklist

504 strip_blocked

505 cook_order | tee $LOGS/cookorder.log

506 cook_list ;;

508 list|-l)

509 # Cook a list of packages given in argument.

510 list="$2"

511 if [ ! -f "$list" ]; then

512 echo -e "\nSpecified list does not exist: $list\n"

513 exit 1

514 fi

515 cat $list >> $cooklist

516 echo -n > $list

517 strip_blocked

518 cook_order | tee $LOGS/cookorder.log

519 cook_list ;;

521 rev|-r)

522 # Cook or recook a specific Hg revision.

523 rev="$2"

524 [ "$rev" ] || exit 0

525 rm -f $cooklist; touch $cooklist

527 cd $wok

528 for pkg in $(hg log --rev=$rev --template "{files}"); do

529 echo "$pkg" | cut -d/ -f1 >> $cooklist

530 done

531 strip_blocked

532 cook_order | tee $LOGS/cookorder.log

533 cook_list ;;

535 all|-a)

536 # Try to build all unbuilt packages except blocked's.

537 echo 'cooker:all' > $command

538 rm -f $cooklist; touch $cooklist

539 title 'Cooker cooklist'

541 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt

542 # if it exists since extra version is added when packing the package.

543 echo 'Searching for all unbuilt packages' | log

545 cd $wok

546 for pkg in *; do

547 [ -s $pkg/receipt ] || continue

548 unset EXTRAVERSION

549 . $pkg/receipt

550 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \

551 . $pkg/taz/$PACKAGE-$VERSION/receipt

552 if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then

553 echo $pkg; echo $pkg >> $cooklist

554 fi

555 done

556 strip_blocked

557 cook_order | tee $LOGS/cookorder.log

558 echo "Packages to cook: $(wc -l < $cooklist)" | log

559 cook_list ;;

561 tasks|-T)

562 # List existing cooker tasks

563 [ ! -d "$tasks" ] && echo 'There are no tasks.' && exit 0

564 title 'Cooker tasks list'

565 last=$(ls $tasks | tail -n1)

566 for task in $(ls $tasks); do

567 . $tasks/$task

568 echo "Task name : $task"

569 echo "Description : $DESC"

570 separator $([ $task != $last ] && echo '-')

571 done

572 newline ;;

574 task|-t)

575 # Executing specified task

576 task="$2"

577 title "Executing cooker task: $task"

578 . $tasks/$task; task

579 footer "Task $task finished" ;;

581 outgoing|-o)

582 # Find changes in wok that we can move to wok-hg

583 compare_wok

584 ;;

587 # Default is to cook all commits if not yet running.

588 [ -n "$1" ] && usage

589 cooklist=$commits

590 if [ -f "$pidfile" ]; then

591 pid=$(cat $pidfile)

592 if [ -s /proc/$pid/status ]; then

593 echo -e "\nStill cooking latest commits with pid:"

594 echo -e " $pid\n"

595 exit 0

596 fi

597 rm -f "$pidfile"

598 fi

600 # Start and get a PID file.

601 rm -f $LOGS/commits.log

602 newline

603 echo 'Checking for commits' | log_commits

604 separator | tee -a $LOGS/commits.log

606 echo $$ > $pidfile

607 trap 'echo -e "\nCooker stopped: PID $$\n" && \

608 rm -f $pidfile $command && exit 1' INT TERM

610 echo "Cooker PID : $$" | log_commits

611 echo "Cooker date : $(date '+%F %T')" | log_commits

613 # Get revisions. Here we have 2 echoes since we want a msg on screen,

614 # in commits log and activity DB without a space before.

615 cd $wok || exit 1

616 cur=$(hg head --template '{rev}\n')

617 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits

618 echo "Updating wok: ${wok}-hg" | log

619 echo 'hg:pull' > $command

620 cd $wok-hg; hg pull -u | log_commits

621 new=$(hg head --template '{rev}\n')

622 # Store last rev to be used by CGI so it doesn't need to call hg head

623 # on each load.

624 echo "$new" > $wokrev

626 # Sync build wok with rsync so we don't take care about removing old

627 # files as before.

628 if [ "$new" -gt "$cur" ]; then

629 echo "Changes found from: $cur to $new" | log

630 echo 'Syncing build wok with Hg wok...' | log_commits

631 rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \

632 sed '/^$/d' | log_commits

633 else

634 echo "No revision changes: $cur vs $new" | log

635 separator | log_commits

636 clean_exit; newline

637 exit 0

638 fi

640 # Get and display modifications.

641 cd $wok-hg

642 commits_summary | log_commits

643 cur=$(($cur + 1))

644 rm -f $commits.tmp; touch $commits.tmp

645 for rev in $(seq $cur $new); do

646 for file in $(hg log --rev=$rev --template "{files}"); do

647 pkg=$(echo $file | cut -d/ -f1)

648 desc=$(hg log --rev=$rev --template "{desc}" $file)

649 echo "Committed package : $pkg - $desc" | log_commits

650 echo $pkg >> $commits.tmp

651 done

652 done

654 # We may have deleted packages and files in stuff/. Remove it and

655 # clean DB as well as log file.

656 cd $wok

657 for pkg in *; do

658 if [ ! -d "$wok-hg/$pkg" ]; then

659 echo "Removing package: $pkg" | log_commits

660 if [ -s $wok/$pkg/receipt ]; then

661 . $wok/$pkg/receipt

662 rm -f $PKGS/$PACKAGE-$VERSION*

663 fi

664 rm -rf $wok/$pkg $LOGS/$pkg.log

665 sed -i "/^${pkg}$/d" $blocked $broken $commits.tmp

666 sed -i "/^$pkg\t/d" $PKGS/packages.info

667 sed -i "/^$pkg:/d" $cache/files.list

668 fi

669 if [ -d "$wok/$pkg/stuff" ]; then

670 if [ ! -d "$wok-hg/$pkg/stuff" ]; then

671 echo "Removing stuff: $pkg/stuff" | log_commits

672 rm -rf $wok/$pkg/stuff

673 else

674 for stuff_file in $(cd $wok/$pkg/stuff; find \( -type f -o -type l \) | sed 's|^\./||'); do

675 if [ ! -f "$wok-hg/$pkg/stuff/$stuff_file" -a \

676 ! -h "$wok-hg/$pkg/stuff/$stuff_file" ]; then

677 echo "Removing file from stuff: $wok/$pkg/stuff/$stuff_file" | log_commits

678 rm -f $wok/$pkg/stuff/$stuff_file

679 rmdir --parents --ignore-fail-on-non-empty $(dirname "$wok/$pkg/stuff/$stuff_file")

680 fi

681 done

682 fi

683 fi

684 done

686 # Keep previous commit and discard duplicate lines

687 cat $commits $commits.tmp | sed '/^$/d' > $commits.new

688 uniq $commits.new > $commits; rm $commits.*

690 # Handle cross compilation. Create arch packages DB and remove pkgs

691 # not cooked for this arch from the commits list.

692 arch_db

693 while read pkg; do

694 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then

695 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \

696 log_commits

697 sed -i "/^${pkg}$/d" $commits

698 else

699 echo "Cooker arch : $ARCH" | log_commits

700 fi

701 done < $commits

703 # Re-create split database

704 cook splitdb

706 # Stats

707 pkgs=$(wc -l < $commits)

708 echo "Packages to cook: $pkgs" | log

709 echo "Packages to cook : $pkgs" | log_commits

710 separator | log_commits

711 newline

712 strip_blocked

713 cook_order | tee $LOGS/cookorder.log

714 cook_commits

715 clean_exit ;;

716 esac

718 exit 0