cookutils view cooker @ rev 32

cooker: finalize reverse command
author Christophe Lincoln <pankso@slitaz.org>
date Fri May 06 17:36:35 2011 +0200 (2011-05-06)
parents 62fa13d02142
children 9c4eb4fc7273
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and put talk to AUTHORS before adding anything here. PS: no translation
6 # here since it's not a end user tool and it's not usfull, all devs should
7 # at least understand basic English.
8 #
9
10 [ -f "/etc/slitaz/cook.conf" ] && . /etc/slitaz/cook.conf
11 [ -f "cook.conf" ] && . ./cook.conf
12
13 # The same wok as cook.
14 wok="$WOK"
15 flavors="$SLITAZ/flavors"
16
17 # Cooker DB files.
18 activity="$CACHE/activity"
19 commits="$CACHE/commits"
20 cooklist="$CACHE/cooklist"
21 cookorder="$CACHE/cookorder"
22 command="$CACHE/command"
23 blocked="$CACHE/blocked"
24 broken="$CACHE/broken"
25 cooknotes="$CACHE/cooknotes"
26
27 #
28 # Functions
29 #
30
31 usage() {
32 cat << EOT
33
34 Usage: cooker [command] [pkg|list|note]
35
36 Options:
37 usage|-u Display this short usage.
38 setup|-s Setup the Cooker environment.
39 note|-n Add a note to the cooknotes.
40 notes|-ns Display all the cooknotes.
41 pkg|-p Same as 'cook pkg' but with cooker log.
42 flavor|-f Cook all packages of a flavor.
43 list|-l Cook all package in the given list.
44 cat|-c Cook all packages of a category.
45 all|-a Find and cook all unbuilt packages.
46
47 EOT
48 exit 0
49 }
50
51 separator() {
52 echo "================================================================================"
53 }
54
55 # Lograte activity.
56 [ -s "$activity" ] && tail -n 20 $activity > /tmp/tail && \
57 mv -f /tmp/tail $activity
58
59 # Log activities, we want first letter capitalized.
60 log() {
61 grep ^[A-Z] | \
62 sed s"#^[A-Z]\([^']*\)#$(date '+%Y-%m-%d %H:%M') : \0#" >> $activity
63 }
64
65 # Some message goes in activity but log verbose output when checking for commits
66 # into a log file.
67 log_commits() {
68 tee -a $LOGS/commits.log
69 }
70
71 # Log broken packages
72 broken() {
73 echo "$pkg" >> $broken
74 }
75
76 # Clean up after cook sucess.
77 empty_command() {
78 rm -f $command && touch $command
79 }
80
81 # Scan packages build deps an fill up cookorder list.
82 cook_order_scan() {
83 touch $cooklist $cookorder
84 for pkg in $(cat $cooklist)
85 do
86 unset BUILD_DEPENDS
87 . $wok/$pkg/receipt
88 # The :: is for web interface color.
89 [ "$BUILD_DEPENDS" ] && echo "$pkg :: $BUILD_DEPENDS"
90 for dep in $BUILD_DEPENDS
91 do
92 if grep -q "^$dep$" $cooklist; then
93 if ! grep -q "^$dep$" $cookorder; then
94
95 echo "$dep" >> $cookorder
96 fi
97 fi
98 done
99 done
100
101 # Append unordered packages to cookored.
102 for pkg in $(cat $cooklist)
103 do
104 if ! grep -q "^$pkg$" $cookorder; then
105 echo "$pkg" >> $cookorder
106 fi
107 done
108 }
109
110 # Scan and rescan untill the cooklist is ordered then handle WANTED.
111 cook_order() {
112 time=$(date +%s)
113 scan=0
114
115 # Keep an original cooklist so we do a diff when ordering is finished.
116 cp -f $cooklist $cooklist.0
117 echo -e "\nInitial Cooker order scan"
118 separator
119 cook_order_scan
120
121 # Diff between the cooklist and new ordered list ? So copy the last
122 # cookorder to cooklist and rescan it.
123 while /bin/true
124 do
125 diff $cooklist $cookorder > $cookorder.diff
126 if [ -s "$cookorder.diff" ]; then
127 scan=$(($scan + 1))
128 echo -e "\nDiff scan: $scan"
129 separator
130 mv -f $cookorder $cooklist
131 cook_order_scan
132 else
133 break
134 fi
135 done
136
137 # Keep a diff between submited cooklist and the ordered.
138 diff $cooklist.0 $cooklist > $cooklist.diff
139 rm -f $cookorder $cookorder.diff $cooklist.0
140
141 # Scan is finish: append pkg to WANTED
142 echo -e "\nHandle WANTED package"
143 separator
144 for pkg in $(cat $cooklist)
145 do
146 unset WANTED
147 . $wok/$pkg/receipt
148 if [ "$WANTED" ]; then
149 echo "$pkg :: $WANTED"
150 sed -i -e "/^$pkg$/"d \
151 -e "/^$WANTED$/ a $pkg" $cooklist
152 fi
153 done
154
155 # Show ordered cooklist
156 echo -e "\nCooklist order"
157 separator
158 cat $cooklist
159 separator
160 time=$(($(date +%s) - $time))
161 pkgs=$(cat $cooklist | wc -l)
162 echo -e "\nSummary"
163 separator
164 cat << EOT
165 Ordered packages : $pkgs
166 Scans executed : $scan
167 Scan duration : ${time}s
168 EOT
169 separator && echo ""
170 }
171
172 # Remove blocked (faster this way than grepping before).
173 strip_blocked() {
174 for pkg in $(cat $blocked)
175 do
176 sed -i /^${pkg}$/d $cooklist
177 done && sed -i /^$/d $cooklist
178 }
179
180 # Uses in default mode and with all cmd.
181 cook_commits() {
182 if [ -s "$commits" ]; then
183 for pkg in $(cat $commits)
184 do
185 echo "Cook started for: <a href='cooker.cgi?pkg=$pkg'>$pkg</a>" | log
186 echo "cook:$pkg" > $command
187 cook $pkg || broken
188 sed -i /^${pkg}$/d $commits
189 done
190 fi
191 }
192
193 # Cook all package in a cooklist.
194 cook_list() {
195 for pkg in $(cat $cooklist)
196 do
197 if [ ! -d "$wok/$pkg/install" ]; then
198 echo "Cook started for: <a href='cooker.cgi?pkg=$pkg'>$pkg</a>" | log
199 cook $pkg || broken
200 sed -i /^${pkg}$/d $cooklist
201 fi
202 done
203 }
204
205 #
206 # Commands
207 #
208 case "$1" in
209 usage|help|-u|-h)
210 usage ;;
211 setup|-s)
212 # Setup the Cooker environment.
213 echo -e "\nSetting up the Cooker"
214 echo "Cooker setup using: $SLITAZ" | log
215 separator
216 for pkg in $SETUP_PKGS mercurial rsync
217 do
218 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
219 done
220 mkdir -p $SLITAZ && cd $SLITAZ
221 [ -d "${wok}-hg" ] && echo -e "Hg wok already exist.\n" && exit 1
222 [ -d "$wok" ] && echo -e "Build wok already exist.\n" && exit 1
223 [ -d "$flavors" ] && echo -e "Flavors repo already exist.\n" && exit 1
224
225 # Directories and files
226 echo "mkdir's and touch files in: $SLITAZ"
227 mkdir -p $PKGS $LOGS $CACHE $SRC
228 for f in $activity $blocked $broken $commits $cooklist $command
229 do
230 touch $f
231 done
232 hg clone $WOK_URL ${wok}-hg || exit 1
233 hg clone $FLAVORS_URL flavors
234 cp -a ${wok}-hg $wok
235 separator && echo "" ;;
236 note|-n)
237 # Blocked a pkg and want other to know why ? Post a note!
238 note="$2"
239 [ "$note" ] && echo "$note" >> $cooknotes ;;
240 notes|-ns)
241 # View cooknotes.
242 echo -e "\nCooknotes"
243 separator
244 cat $cooknotes
245 separator && echo "" ;;
246 reverse|-r)
247 # Cook all reverse dependencies for a packages. This command let us
248 # control the Cooker manually for commit that will cook a lot of packages.
249 #
250 # Use hg commit ? Ex: hg commit -m "Message bla bla | cooker:reverse"
251 #
252 pkg="$2"
253 [ ! -d "$wok/$pkg" ] && echo -e "\nNo package $2 found.\n" && exit 0
254 rm -f $cooklist && touch $cooklist && cd $wok
255 echo -e "\nReverse cooklist for: $pkg"
256 separator && cd $wok
257 for rev in *
258 do
259 unset DEPENDS BUILD_DEPENDS && . $wok/$rev/receipt
260 if echo "$DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
261 echo "$rev" | tee -a $cooklist
262 fi
263 done && separator
264 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
265 strip_blocked
266 cook_order | tee $LOGS/cookorder.log
267 cook_list ;;
268 pkg|-p)
269 # Same as 'cook pkg' but with log for web interface.
270 pkg="$2"
271 echo "Cook started for: <a href='cooker.cgi?pkg=$pkg'>$pkg</a>" | log
272 cook $pkg || broken
273 empty_command ;;
274 cat|-c)
275 # Cook all packages of a category.
276 cat="$2"
277 rm -f $cooklist && touch $cooklist && cd $wok
278 for pkg in *
279 do
280 unset CATEGORY && . $pkg/receipt
281 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
282 done
283 strip_blocked
284 cook_order | tee $LOGS/cookorder.log
285 cook_list ;;
286 flavor|-f)
287 # Cook all packages of a flavor.
288 name="$2"
289 [ ! -d "$flavors/$name" ] && \
290 echo -e "\nSpecified flavor does not exist: $name\n" && exit 1
291 list=$flavors/$name/packages.list
292 cp -a $list $cooklist
293 strip_blocked
294 cook_order | tee $LOGS/cookorder.log
295 cook_list ;;
296 list|-l)
297 # Cook a list og package given in argument.
298 list="$2"
299 [ ! -f "$list" ] && \
300 echo -e "\nSpecified list does not exist: $list\n" && exit 1
301 cp -a $list $cooklist
302 strip_blocked
303 cook_order | tee $LOGS/cookorder.log
304 cook_list ;;
305 all|-a)
306 # Try to build all unbuilt packages except blocked's.
307 echo "cooker:all" > $command
308 rm -f $cooklist && touch $cooklist
309 echo "" && cd $wok
310 echo "Cooker cooklist"
311 separator
312
313 # Find all unbuilt packages.
314 echo "Searching for all unbuilt packages" | log
315 for pkg in *
316 do
317 . $pkg/receipt
318 [ ! -f "$PKGS/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg" ] && \
319 echo $pkg >> $cooklist
320 done
321
322 strip_blocked
323 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
324 cook_order | tee $LOGS/cookorder.log
325 cook_list && empty_command;;
326 *)
327 # Default is to cook all commits.
328 [ "$1" ] && usage
329 cooklist=$CACHE/commits
330 rm -f $LOGS/commits.log
331 echo ""
332 echo "Checking for commits" | log_commits
333 separator | tee -a $LOGS/commits.log
334
335 # Get revisions.
336 cd $wok || ( echo "No wok found: $wok" && exit 1 )
337 cur=$(hg head --template '{rev}\n')
338 echo "Updating Hg wok: ${wok}-hg" | log
339 echo "hg:pull" > $command
340 cd ${wok}-hg && hg pull -u | log_commits
341 new=$(hg head --template '{rev}\n')
342 echo "Hg wok revision : $cur" | log_commits
343 echo "Pulled revision : $new" | log_commits
344 echo "Check date : $(date '+%Y-%m-%d %H:%M')" | log_commits
345
346 # Sync build wok with rsync so we dont take care about removing old
347 # files as before.
348 if [ "$new" -gt "$cur" ]; then
349 echo "Changes found from: $cur to $new" | log
350 echo "Syncing build wok with Hg wok..."
351 #cp -a ${wok}-hg/* $wok
352 #cp -a ${wok}-hg/.hg $wok
353 rsync -r -t -c -l -u -D -E --delete ${wok}-hg/ $wok/ | log_commits
354 else
355 echo "No revision changes: $cur vs $new" | log
356 separator | log_commits
357 empty_command && echo "" && exit 0
358 fi
359
360 # Get modifications
361 cd ${wok}-hg
362 cur=$(($cur + 1))
363 msg="from revision $cur to $new"
364 [ "$new" == "$cur" ] && msg="revision: $new"
365 echo -e "Will cook $msg\n"
366 rm -f $commits.tmp && touch $commits.tmp
367 for rev in $(seq $cur $new); do
368 pkg=$(hg log --rev=$rev --template "{files}\n" | cut -d "/" -f 1)
369 for file in $log; do
370 echo "Commited file: $file" log_commits
371 echo $file >> $commits.tmp
372 done
373 done
374
375 # Keep previews commit and discard duplicate lines
376 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
377 uniq $commits.new > $commits && rm $commits.*
378 echo "Packages to cook: $(cat $commits | wc -l)" | log
379 separator && echo "" | log_commits
380 strip_blocked
381 cook_order | tee $LOGS/cookorder.log
382 cook_commits && empty_command ;;
383 esac
384
385 exit 0