cookutils view modules/pkgdb @ rev 1078
cook: allow multi-set & no-src receipts (sources may be downloaded inside compile_rules() individually for each set), allow change SHORT_DESC & WEB_SITE in genpkg_rules() (for differents sets);
lighttpd/cooker.css: add style for log line "Switching to the set...";
lighttpd/index.cgi: remove false-positive coloring with word "fatal", mark line "Switching to the set...", don't hide "Logs" button (fresh log initially empty and it may lasts for many seconds due to buffered "lazy write";
modules/compressor: better remove unwanted Perl files;
modules/mk_pkg_receipt: follow changed SHORT_DESC & WEB_SITE, improve condition speed (from 10 sec to 0.01 sec) when SPLIT is relatively long - looks like ${A/ $B /} is slow.
lighttpd/cooker.css: add style for log line "Switching to the set...";
lighttpd/index.cgi: remove false-positive coloring with word "fatal", mark line "Switching to the set...", don't hide "Logs" button (fresh log initially empty and it may lasts for many seconds due to buffered "lazy write";
modules/compressor: better remove unwanted Perl files;
modules/mk_pkg_receipt: follow changed SHORT_DESC & WEB_SITE, improve condition speed (from 10 sec to 0.01 sec) when SPLIT is relatively long - looks like ${A/ $B /} is slow.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Thu Jun 21 12:57:52 2018 +0300 (2018-06-21) |
parents | 3911a2292888 |
children | 0385051c3996 |
line source
1 #!/bin/sh
2 #
3 # pkgdb - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
10 #
11 # Functions
12 #
14 dblog() { tee -a $LOGS/pkgdb.log; }
17 # Return size of file in human readable format
18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
26 # Create suitable packages list for TazPkg and only for built packages
27 # as well as flavors files for TazLiTo.
29 # Find how much time was spent the last time (for web interface)
30 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
31 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
32 while read cmd duration start; do
33 [ $(($start + $duration)) -lt $(date +%s) ] &&
34 echo "sed -i '/^$cmd $duration/d' $cooktime"
35 done < $cooktime | sh
37 rm $LOGS/pkgdb.log 2>/dev/null
39 case "$1" in
40 --flavors|--rmpkg) ;;
41 *)
42 [ -n "$1" ] && PKGS="$1"
43 if [ ! -d "$PKGS" ]; then
44 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
45 exit 1
46 fi ;;
47 esac
49 time=$(date +%s)
50 flavors="$SLITAZ/flavors"
51 live="$SLITAZ/live"
53 arch=''
54 case "$ARCH" in
55 arm*|x86_64) arch="-$ARCH" ;;
56 esac
58 echo 'cook:pkgdb' > $command
59 _ 'Cook pkgdb: Creating all packages lists' | log
60 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
62 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
64 cd $PKGS
66 # Web interface database files should exist on the mirror1
67 >.folderlist; >.filelist
68 chmod 666 .folderlist .filelist
70 # Command `cook pkgdb` may be executed by cron. Creating a packages database
71 # takes some time, during which previously pending packages may continue to
72 # be created. This will result in an error due to the lack of a database. This in
73 # turn can lead to errors in the creating of many subsequent packages and will
74 # require manual intervention.
75 # Solution is an atomic update of the packages database: first we create new files
76 # (in the separate temp dir), and then instantly (well, almost) replace the old
77 # files by new ones.
79 dbs=$(mktemp -d)
81 _n 'Creating file "%s"' 'packages.list' | dblog
82 find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $dbs/packages.list
83 echo " ($(filesize $dbs/packages.list))" | dblog
85 _n 'Creating file "%s"' 'packages.md5' | dblog
86 find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $dbs/packages.md5
87 echo " ($(filesize $dbs/packages.md5))" | dblog
88 cp $dbs/packages.md5 $dbs/packages.toremove # list of unnecessary packages
90 md5sum $dbs/packages.md5 | cut -d' ' -f1 > $dbs/ID
91 ( cat $dbs/ID | tr $'\n' ' '; date -ur $dbs/ID +%s ) > $dbs/IDs # md5 and timestamp
93 _n 'Creating file "%s"' 'descriptions.txt' | dblog
94 for i in $(ls $WOK | sort); do
95 [ -d "$WOK/$i/taz" ] || continue
97 for j in $(ls $WOK/$i/taz | sort); do
98 [ -e "$WOK/$i/taz/$j/description.txt" ] || continue
100 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
101 echo "$pkgname"
102 sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
103 # if description.txt doesn't end with \n then add one
104 [ -z "$(tail -c1 $WOK/$i/taz/$j/description.txt)" ] || echo
105 echo
106 done >> $dbs/descriptions.txt
107 done
108 echo " ($(filesize $dbs/descriptions.txt))" | dblog
110 _n 'Creating file "%s"' 'bdeps.txt' | dblog
111 for i in $(ls $WOK | sort); do
112 [ -s "$WOK/$i/receipt" ] || continue
113 bdeps=$(. $WOK/$i/receipt; echo $BUILD_DEPENDS) # remove newlines from some receipts
114 echo "$i $bdeps" >> $dbs/bdeps.txt
115 done
116 echo " ($(filesize $dbs/bdeps.txt))" | dblog
119 _ 'Creating lists from "%s"' "$WOK" | dblog
120 cd $WOK
121 rsumf=$(mktemp)
122 touch $dbs/packages.desc $dbs/packages.txt $dbs/packages.info $dbs/packages.equiv
123 for i in *; do
124 [ -d "$WOK/$i/taz" ] || continue
126 for j in $(ls $WOK/$i/taz | sort); do
127 pack="$i/taz/$j"
128 [ -f "$WOK/$pack/receipt" ] || continue
129 unset_receipt
130 . ./$pack/receipt
132 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" ]; then
134 # packages.desc lets us search easily in DB
135 cat >> $dbs/packages.desc <<EOT
136 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
137 EOT
139 # packages.txt used by tazpkg-web also to provide
140 # a human readable package list with version and description.
141 cat >> $dbs/packages.txt <<EOT
142 $PACKAGE
143 $VERSION$EXTRAVERSION
144 $SHORT_DESC
145 $PACKED_SIZE ($UNPACKED_SIZE installed)
147 EOT
149 # packages.info combines TazPkg separate files
150 # and will substitute them all
151 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
152 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
154 #MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
155 cp $pack/md5sum $rsumf
156 md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
157 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
158 rsum=$(md5sum $rsumf | awk '{print $1}')
160 cat >> $dbs/packages.info <<EOT
161 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $rsum $PROVIDE
162 EOT
164 # packages.equiv is used by tazpkg install to check depends.
165 for k in $PROVIDE; do
166 DEST=''
167 echo $k | fgrep -q : && DEST="${k#*:}:"
168 if grep -qs ^${k%:*}= $dbs/packages.equiv; then
169 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
170 $dbs/packages.equiv
171 else
172 echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv
173 fi
174 done
176 # files.list provides a list of all packages files.
177 sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $dbs/files.list
179 # list of unnecessary packages
180 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg/d" $dbs/packages.toremove
181 else
182 # if receipt variable HOST_ARCH absent/empty or contains ARCH
183 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
184 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
185 fi
186 fi
187 done
188 done
189 rm $rsumf
192 # Display list size.
193 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $dbs/packages.desc)" | dblog
194 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $dbs/packages.txt)" | dblog
195 _ 'Done: %s (%s)' 'packages.info' "$(filesize $dbs/packages.info)" | dblog
196 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $dbs/packages.equiv)" | dblog
198 cd $PKGS
201 # Check for unnecessary packages
202 if [ -s "$dbs/packages.toremove" ]; then
203 newline | dblog
204 case x$rmpkg in
205 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
206 *) _ 'Removing unnecessary packages:' | dblog;;
207 esac
208 while read pkgsum pkgfile; do
209 echo " - $pkgfile" | dblog
210 sed -i "/${pkgfile%.tazpkg}/d" $dbs/packages.list
211 sed -i "/ $pkgfile/d" $dbs/packages.md5
212 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
213 done < $dbs/packages.toremove
214 newline | dblog
215 fi
216 rm $dbs/packages.toremove
219 # files.list.lzma
220 _n 'Creating file "%s"' 'files.list.lzma' | dblog
221 touch $dbs/files.list
222 # pkgs.slitaz.org strongly depends on list sorted by packages names
223 #lzma e $dbs/files.list $dbs/files.list.lzma
224 /bin/lzma -zeqcT0 $dbs/files.list > $dbs/files.list.lzma
225 echo " ($(filesize $dbs/files.list.lzma))" | dblog
227 # Pre-sorting filenames causes 10% smaller resulting lzma file
228 _n 'Creating file "%s"' 'files-list.lzma' | dblog
229 cat $dbs/files.list | sort -k2 -o $dbs/files.list.sorted
230 #lzma e $dbs/files.list.sorted $dbs/files-list.lzma
231 /bin/lzma -zeqcT0 $dbs/files.list.sorted > $dbs/files-list.lzma
232 rm -f $dbs/files.list $dbs/files.list.sorted
233 echo " ($(filesize $dbs/files-list.lzma))" | dblog
235 md5sum $dbs/files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > $dbs/files-list.md5
237 # Make bundle to fast recharge
238 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
239 # Make sure to get "mirrors" file
240 until [ -e "$dbs/mirrors" ]; do
241 wget -q -O $dbs/mirrors http://mirror1.slitaz.org/mirrors
242 echo -n '.' | dblog; sleep 5
243 done
244 # Make sure to get "extra.list" file
245 until [ -e "$dbs/extra.list" ]; do
246 wget -q -O $dbs/extra.list http://mirror1.slitaz.org/packages/get.list
247 echo -n '.' | dblog; sleep 5
248 done
249 (
250 cd $dbs
251 busybox tar -chf bundle.tar \
252 mirrors extra.list files-list.md5 packages.info descriptions.txt \
253 packages.desc packages.md5 packages.txt packages.list packages.equiv
254 )
255 /bin/lzma -zeqcT0 $dbs/bundle.tar > $dbs/bundle.tar.lzma
256 rm $dbs/bundle.tar $dbs/mirrors
257 echo " ($(filesize $dbs/bundle.tar.lzma))" | dblog
259 # Display some info.
260 separator | dblog
261 nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
262 time=$(($(date +%s) - $time))
263 # L10n: 's' is for seconds (cooking time)
264 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
266 # "Atomic" update now
267 cp -f $dbs/* $PKGS
268 rm -r $dbs
271 # Create all flavors files at once. Do we really need code to monitor
272 # flavors changes? Let's just build them with packages lists before
273 # syncing the mirror.
274 [ "$1" != '--flavors' ] && rm $command && exit 1
276 if [ ! -d "$flavors" ]; then
277 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
278 rm $command
279 exit 1
280 fi
282 [ ! -d "$live" ] && mkdir -p $live
283 _ 'Creating flavors files in "%s"' "$live" | dblog
284 _ 'Cook pkgdb: Creating all flavors' | log
285 separator | dblog
287 _ 'Recharging lists to use latest packages...' | dblog
288 tazpkg recharge >/dev/null 2>/dev/null
290 # We need a custom tazlito config to set working dir to /home/slitaz.
291 if [ ! -f "$live/tazlito.conf" ]; then
292 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
293 cp /etc/tazlito/tazlito.conf $live
294 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
295 $live/tazlito.conf
296 fi
298 # Update Hg flavors repo and pack.
299 if [ -d "$flavors/.hg" ]; then
300 cd $flavors; hg pull -u
301 fi
303 cd $live
304 _ 'Starting to generate flavors...' | dblog
305 rm -f flavors.list *.flavor
306 for i in $flavors/*; do
307 fl=$(basename $i)
308 _ 'Packing flavor "%s"' "$fl" | dblog
309 tazlito pack-flavor $fl >/dev/null || exit 1
310 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
311 done
312 cp -f $live/*.flavor $live/flavors.list $PKGS
313 separator | dblog
314 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
315 separator | dblog
316 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
318 rm $command
321 exit 0