cookutils annotate cooker @ rev 341

cooker: Add setup-cron to enable cron job
author Christophe Lincoln <pankso@slitaz.org>
date Sat Mar 17 23:33:55 2012 +0100 (2012-03-17)
parents 20f9b75ec056
children 4f9d23e13883
rev   line source
pankso@4 1 #!/bin/sh
pankso@4 2 #
pankso@4 3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
pankso@4 4 # building. Please read the Cookbook documentation for more information
paul@54 5 # and discuss with the AUTHORS before adding anything here. PS: no translations
paul@54 6 # here since it's not an end user tool and it's not useful, all devs should
pankso@4 7 # at least understand basic English.
pankso@4 8 #
pankso@4 9
pankso@4 10 [ -f "/etc/slitaz/cook.conf" ] && . /etc/slitaz/cook.conf
pankso@4 11 [ -f "cook.conf" ] && . ./cook.conf
pankso@4 12
pankso@49 13 # Set pkg name and use same wok as cook.
pankso@49 14 pkg="$2"
pankso@4 15 wok="$WOK"
pankso@31 16 flavors="$SLITAZ/flavors"
pankso@4 17
pankso@4 18 # Cooker DB files.
pankso@4 19 activity="$CACHE/activity"
pankso@4 20 commits="$CACHE/commits"
pankso@4 21 cooklist="$CACHE/cooklist"
pankso@4 22 cookorder="$CACHE/cookorder"
pankso@14 23 command="$CACHE/command"
pankso@4 24 blocked="$CACHE/blocked"
pankso@4 25 broken="$CACHE/broken"
pankso@22 26 cooknotes="$CACHE/cooknotes"
pankso@341 27 crontabs="/var/spool/cron/crontabs/root"
pankso@4 28
pankso@81 29 # PID file.
pankso@81 30 pidfile='/var/run/cooker.pid'
pankso@81 31
pankso@4 32 #
pankso@4 33 # Functions
pankso@4 34 #
pankso@4 35
pankso@4 36 usage() {
pankso@4 37 cat << EOT
pankso@4 38
pankso@341 39 Usage: cooker [command] [pkg|list|note|hours]
pankso@4 40
pankso@4 41 Options:
pankso@49 42 usage|-u Display this short usage.
pankso@49 43 setup|-s Setup the Cooker environment.
pankso@341 44 setup-cron Setup a cron job for the Cooker.
pankso@49 45 note|-n Add a note to the cooknotes.
pankso@49 46 notes|-ns Display all the cooknotes.
pankso@49 47 block|-b Block a package so cook will skip it.
pankso@49 48 unblock|-ub Unblock a blocked package.
pankso@49 49 pkg|-p Same as 'cook pkg' but with cooker log.
pankso@49 50 flavor|-f Cook all packages of a flavor.
paul@54 51 list|-l Cook all packages in the given list.
pankso@49 52 cat|-c Cook all packages of a category.
pankso@79 53 rev|-r Cook packages of a specific revision.
pankso@49 54 all|-a Find and cook all unbuilt packages.
pankso@4 55
pankso@4 56 EOT
pankso@4 57 exit 0
pankso@4 58 }
pankso@4 59
pankso@4 60 separator() {
pankso@4 61 echo "================================================================================"
pankso@4 62 }
pankso@4 63
pankso@4 64 # Lograte activity.
pankso@65 65 [ -s "$activity" ] && tail -n 60 $activity > /tmp/tail && \
pankso@4 66 mv -f /tmp/tail $activity
pankso@4 67
pankso@13 68 # Log activities, we want first letter capitalized.
pankso@4 69 log() {
pankso@27 70 grep ^[A-Z] | \
pankso@4 71 sed s"#^[A-Z]\([^']*\)#$(date '+%Y-%m-%d %H:%M') : \0#" >> $activity
pankso@4 72 }
pankso@4 73
paul@54 74 # Some messages occur in activity but log verbose output when checking for commits
pankso@14 75 # into a log file.
pankso@14 76 log_commits() {
pankso@35 77 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log
pankso@14 78 }
pankso@14 79
pankso@74 80 # Log broken packages.
pankso@4 81 broken() {
pankso@74 82 if ! grep -q "^$pkg$" $broken; then
pankso@74 83 echo "$pkg" >> $broken
pankso@74 84 fi
pankso@4 85 }
pankso@4 86
pankso@81 87 # Clean up before exit when check and cook commits finish.
pankso@81 88 clean_exit() {
pankso@14 89 rm -f $command && touch $command
pankso@81 90 rm -f $pidfile
pankso@14 91 }
pankso@14 92
pankso@35 93 # Summary for commits.
pankso@35 94 commits_summary() {
pankso@35 95 msg="from revision $cur to $new"
pankso@35 96 [ "$new" == "$cur" ] && msg="revision $new"
pankso@35 97 echo "Will cook $msg"
pankso@35 98 separator
pankso@35 99 echo -e "\nSummary for commits"
pankso@35 100 separator
pankso@35 101 echo "Hg wok revision : $cur"
pankso@35 102 echo "Pulled revision : $new"
pankso@82 103 echo "Check date : $(date '+%Y-%m-%d %H:%M:%S')"
pankso@35 104 }
pankso@35 105
paul@54 106 # Scan packages build deps and fill up cookorder list.
pankso@4 107 cook_order_scan() {
pankso@124 108 rm -f $cookorder
pankso@4 109 for pkg in $(cat $cooklist)
pankso@4 110 do
pascal@273 111 unset WANTED BUILD_DEPENDS
pankso@14 112 . $wok/$pkg/receipt
pankso@29 113 # The :: is for web interface color.
pascal@273 114 [ "$WANTED$BUILD_DEPENDS" ] && echo "$pkg :: $WANTED $BUILD_DEPENDS"
pascal@273 115 for dep in $WANTED $BUILD_DEPENDS
pankso@4 116 do
pankso@4 117 if grep -q "^$dep$" $cooklist; then
pankso@4 118 if ! grep -q "^$dep$" $cookorder; then
pankso@4 119 echo "$dep" >> $cookorder
pankso@4 120 fi
pankso@4 121 fi
pankso@4 122 done
pankso@4 123 done
pankso@4 124
paul@54 125 # Append unordered packages to cookorder.
pankso@4 126 for pkg in $(cat $cooklist)
pankso@4 127 do
pankso@4 128 if ! grep -q "^$pkg$" $cookorder; then
pankso@4 129 echo "$pkg" >> $cookorder
pankso@4 130 fi
pankso@4 131 done
pankso@4 132 }
pankso@4 133
paul@54 134 # Scan and rescan until the cooklist is ordered then handle WANTED.
pankso@4 135 cook_order() {
pankso@4 136 time=$(date +%s)
pankso@4 137 scan=0
pankso@4 138
pankso@4 139 # Keep an original cooklist so we do a diff when ordering is finished.
pankso@4 140 cp -f $cooklist $cooklist.0
pankso@99 141 echo "cookorder" > $command
pankso@4 142 echo -e "\nInitial Cooker order scan"
pankso@4 143 separator
pankso@4 144 cook_order_scan
pankso@4 145
pankso@4 146 # Diff between the cooklist and new ordered list ? So copy the last
pankso@4 147 # cookorder to cooklist and rescan it.
pankso@4 148 while /bin/true
pankso@4 149 do
pankso@4 150 diff $cooklist $cookorder > $cookorder.diff
pankso@4 151 if [ -s "$cookorder.diff" ]; then
pankso@4 152 scan=$(($scan + 1))
pankso@4 153 echo -e "\nDiff scan: $scan"
pankso@4 154 separator
pankso@4 155 mv -f $cookorder $cooklist
pankso@4 156 cook_order_scan
pankso@4 157 else
pankso@4 158 break
pankso@4 159 fi
pankso@4 160 done
pankso@4 161
pankso@4 162 # Keep a diff between submited cooklist and the ordered.
pankso@4 163 diff $cooklist.0 $cooklist > $cooklist.diff
pankso@4 164 rm -f $cookorder $cookorder.diff $cooklist.0
pankso@4 165
pankso@107 166 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
paul@214 167 # TODO: grep the line number to get pkg position and keep it higher.
pankso@4 168 echo -e "\nHandle WANTED package"
pankso@4 169 separator
pankso@4 170 for pkg in $(cat $cooklist)
pankso@4 171 do
pankso@4 172 unset WANTED
pankso@14 173 . $wok/$pkg/receipt
pankso@299 174 for wanted in $WANTED
pankso@299 175 do
pascal@291 176 echo "$pkg :: $wanted"
pascal@291 177 if grep -q ^${wanted}$ $cooklist; then
pankso@107 178 sed -i -e "/^$pkg$/"d \
pascal@291 179 -e "/^$wanted$/ a $pkg" $cooklist
pankso@107 180 fi
pascal@291 181 done
pankso@4 182 done
pankso@4 183
pankso@4 184 # Show ordered cooklist
pankso@4 185 echo -e "\nCooklist order"
pankso@4 186 separator
pankso@4 187 cat $cooklist
pankso@4 188 separator
pankso@4 189 time=$(($(date +%s) - $time))
pankso@4 190 pkgs=$(cat $cooklist | wc -l)
pankso@35 191 echo -e "\nSummary for cookorder"
pankso@4 192 separator
pankso@4 193 cat << EOT
pankso@4 194 Ordered packages : $pkgs
pankso@4 195 Scans executed : $scan
pankso@4 196 Scan duration : ${time}s
pankso@4 197 EOT
pankso@99 198 separator && rm -f $command
pankso@4 199 }
pankso@4 200
pankso@14 201 # Remove blocked (faster this way than grepping before).
pankso@14 202 strip_blocked() {
pankso@14 203 for pkg in $(cat $blocked)
pankso@14 204 do
pankso@14 205 sed -i /^${pkg}$/d $cooklist
pankso@14 206 done && sed -i /^$/d $cooklist
pankso@14 207 }
pankso@14 208
paul@54 209 # Use in default mode and with all cmd.
pankso@4 210 cook_commits() {
pankso@4 211 if [ -s "$commits" ]; then
pankso@4 212 for pkg in $(cat $commits)
pankso@4 213 do
pankso@14 214 echo "cook:$pkg" > $command
pankso@4 215 cook $pkg || broken
pankso@4 216 sed -i /^${pkg}$/d $commits
pankso@4 217 done
pankso@4 218 fi
pankso@4 219 }
pankso@4 220
paul@54 221 # Cook all packages in a cooklist.
pankso@14 222 cook_list() {
pankso@14 223 for pkg in $(cat $cooklist)
pankso@14 224 do
pankso@79 225 cook $pkg || broken
pankso@79 226 sed -i /^${pkg}$/d $cooklist
pankso@14 227 done
pankso@14 228 }
pankso@14 229
pankso@4 230 #
pankso@4 231 # Commands
pankso@4 232 #
pankso@4 233 case "$1" in
pankso@31 234 usage|help|-u|-h)
pankso@4 235 usage ;;
pankso@31 236 setup|-s)
pankso@4 237 # Setup the Cooker environment.
pankso@4 238 echo -e "\nSetting up the Cooker"
pankso@31 239 echo "Cooker setup using: $SLITAZ" | log
pankso@4 240 separator
pankso@309 241 for pkg in $SETUP_PKGS mercurial rsync tazlito
pankso@4 242 do
pankso@4 243 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
pankso@4 244 done
pankso@4 245 mkdir -p $SLITAZ && cd $SLITAZ
paul@54 246 [ -d "${wok}-hg" ] && echo -e "Hg wok already exists.\n" && exit 1
paul@54 247 [ -d "$wok" ] && echo -e "Build wok already exists.\n" && exit 1
pankso@4 248
pankso@4 249 # Directories and files
pankso@4 250 echo "mkdir's and touch files in: $SLITAZ"
pankso@311 251 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
pankso@14 252 for f in $activity $blocked $broken $commits $cooklist $command
pankso@4 253 do
pankso@4 254 touch $f
pankso@4 255 done
pankso@14 256 hg clone $WOK_URL ${wok}-hg || exit 1
pankso@60 257 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
pankso@4 258 cp -a ${wok}-hg $wok
pankso@4 259 separator && echo "" ;;
pankso@341 260 setup-cron)
pankso@341 261 # Create cron job for the cooker.
pankso@341 262 [ "$2" ] || hours=2
pankso@341 263 if [ ! -f "$crontabs" ]; then
pankso@341 264 mkdir -p /var/spool/cron/crontabs
pankso@341 265 echo "# Run SliTaz Cooker each $hours hours" > $crontabs
pankso@341 266 echo "0 */$hours * * * /usr/bin/cooker" >> $crontabs
pankso@341 267 /etc/init.d/crond start
pankso@341 268 fi
pankso@341 269 if ! fgrep -q /usr/bin/cooker $crontabs; then
pankso@341 270 echo "# Run SliTaz Cooker each $hours hours" > $crontabs
pankso@341 271 echo "0 */$hours * * * /usr/bin/cooker" >> $crontabs
pankso@341 272 killall crond 2>/dev/null && /etc/init.d/crond start
pankso@341 273 fi ;;
pankso@341 274 check-cron)
pankso@341 275 fgrep /usr/bin/cooker $crontabs ;;
pankso@31 276 note|-n)
paul@54 277 # Blocked a pkg and want others to know why ? Post a note!
pankso@31 278 note="$2"
pankso@48 279 date=$(date "+%Y-%m-%d %H:%M")
pankso@48 280 [ "$note" ] && echo "$date : $note" >> $cooknotes ;;
pankso@31 281 notes|-ns)
pankso@29 282 # View cooknotes.
pankso@29 283 echo -e "\nCooknotes"
pankso@29 284 separator
pankso@29 285 cat $cooknotes
pankso@29 286 separator && echo "" ;;
pankso@49 287 block|-b)
pankso@49 288 # Block a package.
pankso@49 289 [ "$pkg" ] && cook $pkg --block ;;
pankso@49 290 unblock|-ub)
pankso@49 291 # Unblock a package.
pankso@49 292 [ "$pkg" ] && cook $pkg --unblock ;;
pankso@31 293 reverse|-r)
paul@54 294 # Cook all reverse dependencies for a package. This command lets us
paul@54 295 # control the Cooker manually for commits that will cook a lot of packages.
pankso@14 296 #
pankso@21 297 # Use hg commit ? Ex: hg commit -m "Message bla bla | cooker:reverse"
pankso@14 298 #
pankso@32 299 [ ! -d "$wok/$pkg" ] && echo -e "\nNo package $2 found.\n" && exit 0
pankso@32 300 rm -f $cooklist && touch $cooklist && cd $wok
pankso@32 301 echo -e "\nReverse cooklist for: $pkg"
pankso@32 302 separator && cd $wok
pankso@4 303 for rev in *
pankso@4 304 do
pascal@273 305 unset WANTED DEPENDS BUILD_DEPENDS && . $wok/$rev/receipt
pascal@273 306 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
pankso@32 307 echo "$rev" | tee -a $cooklist
pankso@4 308 fi
pankso@32 309 done && separator
pankso@32 310 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
pankso@32 311 strip_blocked
pankso@32 312 cook_order | tee $LOGS/cookorder.log
pankso@32 313 cook_list ;;
pankso@31 314 pkg|-p)
pankso@12 315 # Same as 'cook pkg' but with log for web interface.
pankso@32 316 cook $pkg || broken
pankso@81 317 clean_exit ;;
pankso@31 318 cat|-c)
pankso@4 319 # Cook all packages of a category.
pankso@31 320 cat="$2"
pankso@14 321 rm -f $cooklist && touch $cooklist && cd $wok
pankso@4 322 for pkg in *
pankso@4 323 do
pankso@4 324 unset CATEGORY && . $pkg/receipt
pankso@4 325 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
pankso@4 326 done
pankso@14 327 strip_blocked
pankso@14 328 cook_order | tee $LOGS/cookorder.log
pankso@26 329 cook_list ;;
pankso@31 330 flavor|-f)
pankso@21 331 # Cook all packages of a flavor.
pankso@31 332 name="$2"
pankso@31 333 [ ! -d "$flavors/$name" ] && \
pankso@31 334 echo -e "\nSpecified flavor does not exist: $name\n" && exit 1
pankso@60 335 [ -d "$flavors/.hg" ] && cd $flavors && hg pull -u
pankso@31 336 list=$flavors/$name/packages.list
pankso@21 337 cp -a $list $cooklist
pankso@21 338 strip_blocked
pankso@21 339 cook_order | tee $LOGS/cookorder.log
pankso@32 340 cook_list ;;
pankso@31 341 list|-l)
paul@54 342 # Cook a list of packages given in argument.
pankso@31 343 list="$2"
pankso@26 344 [ ! -f "$list" ] && \
pankso@26 345 echo -e "\nSpecified list does not exist: $list\n" && exit 1
pankso@26 346 cp -a $list $cooklist
pankso@26 347 strip_blocked
pankso@98 348 cook_order | tee $LOGS/cookorder.log
pankso@98 349 cook_list ;;
pankso@79 350 rev|-r)
pankso@79 351 # Cook or recook a specific Hg revision.
pankso@79 352 rev="$2"
pankso@79 353 [ "$rev" ] || exit 0
pankso@79 354 cd $wok
pankso@79 355 rm -f $cooklist && touch $cooklist
pankso@80 356 for pkg in $(hg log --rev=$rev --template "{files}")
pankso@79 357 do
pankso@80 358 echo "$pkg" | cut -d "/" -f 1 >> $cooklist
pankso@79 359 done
pankso@79 360 strip_blocked
pankso@26 361 cook_order | tee $LOGS/cookorder.log
pankso@26 362 cook_list ;;
pankso@31 363 all|-a)
pankso@4 364 # Try to build all unbuilt packages except blocked's.
pankso@31 365 echo "cooker:all" > $command
pankso@4 366 rm -f $cooklist && touch $cooklist
pankso@4 367 echo "" && cd $wok
pankso@4 368 echo "Cooker cooklist"
pankso@4 369 separator
pankso@4 370
pankso@118 371 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt
paul@132 372 # if it exists since extra version is added when packing the package.
pankso@4 373 echo "Searching for all unbuilt packages" | log
pankso@4 374 for pkg in *
pankso@4 375 do
pankso@118 376 unset EXTRAVERSION
pankso@4 377 . $pkg/receipt
pankso@118 378 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
pankso@118 379 . $pkg/taz/$PACKAGE-$VERSION/receipt
pankso@118 380 if [ ! -f "$PKGS/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg" ]
pankso@118 381 then
pankso@118 382 echo $pkg && echo $pkg >> $cooklist
pankso@118 383 fi
pankso@4 384 done
pankso@14 385 strip_blocked
pankso@99 386 cook_order | tee $LOGS/cookorder.log
pankso@13 387 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
pankso@79 388 cook_list ;;
pankso@4 389 *)
pankso@81 390 # Default is to cook all commits if not yet running.
pankso@4 391 [ "$1" ] && usage
pankso@4 392 cooklist=$CACHE/commits
pankso@81 393 if [ -f "$pidfile" ]; then
pankso@81 394 pid=$(cat /var/run/cooker.pid)
pascal@274 395 if [ -s /proc/$pid/status ]; then
pascal@274 396 gettext -e "\nStill cooking latest commits with pid:"
pascal@274 397 echo -e " $pid\n" && exit 0
pascal@274 398 fi
pascal@274 399 rm -f "$pidfile"
pankso@81 400 fi
pankso@81 401
pankso@81 402 # Start and get a PID file.
pankso@14 403 rm -f $LOGS/commits.log
pankso@14 404 echo ""
pankso@14 405 echo "Checking for commits" | log_commits
pankso@14 406 separator | tee -a $LOGS/commits.log
pankso@233 407
pankso@82 408 echo $$ > $pidfile
pankso@233 409 trap 'echo -e "\nCooker stopped: PID $$\n" && \
pankso@233 410 rm -f $pidfile $command && exit 1' INT TERM
pankso@233 411
pankso@82 412 echo "Cooker PID : $$" | log_commits
pankso@82 413 echo "Cooker date : $(date '+%Y-%m-%d %H:%M:%S')" | log_commits
pankso@4 414
paul@132 415 # Get revisions. Here we have 2 echoes since we want a msg on screen,
paul@132 416 # in commits log and activity DB without a space before.
pankso@60 417 cd $wok || exit 1
pankso@4 418 cur=$(hg head --template '{rev}\n')
pankso@82 419 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
pankso@82 420 echo "Updating wok: ${wok}-hg" | log
pankso@14 421 echo "hg:pull" > $command
pankso@60 422 cd ${wok}-hg && hg pull -u | log_commits
pankso@4 423 new=$(hg head --template '{rev}\n')
paul@168 424 # Store last rev to be used by CGI so it doesn't need to call hg head
pankso@164 425 # on each load.
pankso@164 426 echo "$new" > $CACHE/wokrev
pankso@14 427
paul@54 428 # Sync build wok with rsync so we don't take care about removing old
pankso@4 429 # files as before.
pankso@4 430 if [ "$new" -gt "$cur" ]; then
pankso@4 431 echo "Changes found from: $cur to $new" | log
pankso@35 432 echo "Syncing build wok with Hg wok..." | log_commits
pankso@78 433 rsync -r -t -c -l -u -v -D -E ${wok}-hg/ $wok/ | \
pankso@35 434 sed '/^$/'d | log_commits
pankso@4 435 else
pankso@13 436 echo "No revision changes: $cur vs $new" | log
pankso@14 437 separator | log_commits
pankso@81 438 clean_exit && echo "" && exit 0
pankso@4 439 fi
pankso@4 440
pankso@35 441 # Get and display modifications.
pankso@4 442 cd ${wok}-hg
pankso@110 443 commits_summary | log_commits
pankso@14 444 cur=$(($cur + 1))
pankso@4 445 rm -f $commits.tmp && touch $commits.tmp
pankso@80 446 for rev in $(seq $cur $new)
pankso@80 447 do
pankso@80 448 for file in $(hg log --rev=$rev --template "{files}")
pankso@80 449 do
pankso@80 450 pkg=$(echo $file | cut -d "/" -f 1)
pankso@36 451 desc=$(hg log --rev=$rev --template "{desc}" $file)
pankso@180 452 echo "Commited package : $pkg - $desc" | log_commits
pankso@80 453 echo $pkg >> $commits.tmp
pankso@4 454 done
pankso@4 455 done
pankso@248 456
pankso@248 457 # We may have deleted packages and files in stuff/. Remove it and
pankso@248 458 # clean DB as well as log file.
pankso@248 459 cd $wok
pankso@248 460 for pkg in *
pankso@248 461 do
pankso@248 462 if [ ! -d "${wok}-hg/$pkg" ]; then
pankso@248 463 echo "Removing package: $pkg" | log_commits
pankso@248 464 . $wok/$pkg/receipt
pankso@248 465 rm -rf $PKGS/$PACKAGE-$VERSION* $wok/$pkg $LOGS/$pkg.log
pankso@248 466 sed -i "/^${pkg}$/"d $CACHE/blocked $CACHE/broken $commits.tmp
pankso@248 467 fi
pankso@248 468 done
pankso@4 469
paul@54 470 # Keep previous commit and discard duplicate lines
pankso@4 471 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
pankso@4 472 uniq $commits.new > $commits && rm $commits.*
pankso@109 473 pkgs=$(cat $commits | wc -l)
pankso@100 474 echo "Packages to cook: $pkgs" | log
pankso@109 475 echo "Packages to cook : $pkgs" | log_commits
pankso@35 476 separator | log_commits
pankso@35 477 echo ""
pankso@14 478 strip_blocked
pankso@14 479 cook_order | tee $LOGS/cookorder.log
pankso@81 480 cook_commits
pankso@81 481 clean_exit ;;
pankso@4 482 esac
pankso@4 483
pankso@4 484 exit 0