cookutils view modules/pkgdb @ rev 860

cook: changed the algorithm for determining the free memory: Busybox-1.26.1 has different output of the `free` command.
I believe with /proc/meminfo it will be more consistent result.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Sun Jan 08 01:50:38 2017 +0200 (2017-01-08)
parents
children 1520231a65c2
line source
1 #!/bin/sh
2 #
3 # pkgdb - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
6
7 . /usr/lib/slitaz/libcook.sh
8
9 export output=raw
10
11 echo "flavors='$flavors' rmpkg='$rmpkg' 1='$1' 2='$2'"
12
13 # Internationalization.
14
15 export TEXTDOMAIN='cook'
16
17
18 #
19 # Functions
20 #
21
22 dblog() { tee -a $LOGS/pkgdb.log; }
23
24
25 # Return size of file in human readible format
26 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
27
28 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
29
30
31
32
33
34 # Create suitable packages list for TazPkg and only for built packages
35 # as well as flavors files for TazLiTo. We don't need logs since we do it
36 # manually to ensure everything is fine before syncing the mirror.
37
38 rm $LOGS/pkgdb.log 2>/dev/null
39
40 case "$1" in
41 --flavors|--rmpkg) ;;
42 *)
43 [ -n "$1" ] && PKGS="$1"
44 if [ ! -d "$PKGS" ]; then
45 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
46 exit 1
47 fi ;;
48 esac
49
50 time=$(date +%s)
51 flavors="$SLITAZ/flavors"
52 live="$SLITAZ/live"
53
54 echo 'cook:pkgdb' > $command
55 _ 'Cook pkgdb: Creating all packages lists' | log
56 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
57
58 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
59
60 cd $PKGS
61 rm -f packages.* extra.list
62 touch packages.equiv
63
64 _n 'Creating file "%s"' 'packages.list' | dblog
65 ls -1 *.tazpkg | sed s'/.tazpkg//' > $PKGS/packages.list
66 echo " ($(filesize $PKGS/packages.list))" | dblog
67
68 _n 'Creating file "%s"' 'packages.md5' | dblog
69 md5sum *.tazpkg > $PKGS/packages.md5
70 echo " ($(filesize $PKGS/packages.md5))" | dblog
71 cp $PKGS/packages.md5 $PKGS/packages.toremove # list of duplicates
72
73 md5sum packages.md5 | cut -d' ' -f1 > ID
74 ( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp
75
76 _n 'Creating file "%s"' 'descriptions.txt' | dblog
77 rm $PKGS/descriptions.txt 2>/dev/null
78 for i in $(ls $WOK | sort); do
79 if [ -e "$WOK/$i/description.txt" ]; then
80 echo "$i" >> descriptions.txt
81 cat "$WOK/$i/description.txt" | sed 's|^$| |' >> descriptions.txt
82 echo >> descriptions.txt
83 fi
84 done
85 echo " ($(filesize $PKGS/descriptions.txt))" | dblog
86
87
88 _ 'Creating lists from "%s"' "$WOK" | dblog
89 cd $WOK
90 for pkg in *; do
91 unset_receipt
92 . $pkg/receipt
93 # PACKED_SIZE and UNPACKED_SIZE are only in built receipt
94 [ -s $pkg/taz/*/receipt ] && . $pkg/taz/*/receipt
95
96 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ] || \
97 [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-$ARCH.tazpkg" ]; then
98
99 # packages.desc lets us search easily in DB
100 cat >> $PKGS/packages.desc <<EOT
101 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
102 EOT
103
104 # packages.txt used by tazpkg and tazpkg-web also to provide
105 # a human readable package list with version and description.
106 cat >> $PKGS/packages.txt <<EOT
107 $PACKAGE
108 $VERSION$EXTRAVERSION
109 $SHORT_DESC
110 $PACKED_SIZE ($UNPACKED_SIZE installed)
111
112 EOT
113
114 # packages.info combines TazPkg separate files
115 # and will substitute them all
116 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
117 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
118 MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
119 cat >> $PKGS/packages.info <<EOT
120 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $MD5
121 EOT
122
123 # packages.equiv is used by tazpkg install to check depends.
124 for i in $PROVIDE; do
125 DEST=''
126 echo $i | fgrep -q : && DEST="${i#*:}:"
127 if grep -qs ^${i%:*}= $PKGS/packages.equiv; then
128 sed -i "s/^${i%:*}=/${i%:*}=$DEST$PACKAGE /" \
129 $PKGS/packages.equiv
130 else
131 echo "${i%:*}=$DEST$PACKAGE" >> $PKGS/packages.equiv
132 fi
133 done
134
135 # files.list provides a list of all packages files.
136 cat $pkg/taz/*/files.list | sed s/^/"$pkg: \0"/ >> \
137 $PKGS/files.list
138
139 # list of duplicates
140 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION.tazpkg/d" $PKGS/packages.toremove
141 else
142 # if receipt variable HOST_ARCH absent/empty or contains ARCH
143 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
144 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
145 fi
146 fi
147 done
148
149 # Display list size.
150 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $PKGS/packages.desc)" | dblog
151 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $PKGS/packages.txt)" | dblog
152 _ 'Done: %s (%s)' 'packages.info' "$(filesize $PKGS/packages.info)" | dblog
153 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog
154
155 cd $PKGS
156
157
158 # Check package duplicates
159 if [ -s "$PKGS/packages.toremove" ]; then
160 newline | dblog
161 _ 'Removing duplicates:' | dblog
162 while read pkgsum pkgfile; do
163 echo " - $pkgfile" | dblog
164 sed -i "/${pkgfile%.tazpkg}/d" $PKGS/packages.list
165 sed -i "/ $pkgfile/d" $PKGS/packages.md5
166 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
167 done < $PKGS/packages.toremove
168 newline | dblog
169 fi
170 rm $PKGS/packages.toremove
171
172
173 # files.list.lzma
174 _n 'Creating file "%s"' 'files.list.lzma' | dblog
175 touch files.list
176 # pkgs.slitaz.org strongly depends on list sorted by packages names
177 lzma e files.list files.list.lzma
178 echo " ($(filesize $PKGS/files.list.lzma))" | dblog
179
180 # Pre-sorting filenames causes 10% smaller resulting lzma file
181 _n 'Creating file "%s"' 'files-list.lzma' | dblog
182 cat files.list | sort -k2 -o files.list.sorted
183 lzma e files.list.sorted files-list.lzma
184 rm -f files.list files.list.sorted
185 echo " ($(filesize $PKGS/files-list.lzma))" | dblog
186
187 [ -e files.list.md5 ] && rm files.list.md5
188 md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5
189
190 # packages.info.lzma
191 PI=packages.info
192 _n 'Creating file "%s"' 'packages.info.lzma' | dblog
193 touch $PI
194 lzma e $PI $PI.lzma
195 echo " ($(filesize $PKGS/packages.info.lzma))" | dblog
196
197 # Make bundle to fast recharge
198 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
199 [ -f bundle.tar.lzma ] && rm bundle.tar.lzma
200 # Make sure to get "mirrors" file
201 until [ -e 'mirrors' ]; do
202 wget -q http://mirror1.slitaz.org/mirrors
203 echo -n '.' | dblog; sleep 5
204 done
205 # Make sure to get "extra.list" file
206 until [ -e 'extra.list' ]; do
207 wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list
208 echo -n '.' | dblog; sleep 5
209 done
210 busybox tar -chaf bundle.tar.lzma \
211 mirrors extra.list files-list.md5 packages.info descriptions.txt \
212 packages.desc packages.md5 packages.txt packages.list packages.equiv
213 rm ./mirrors
214 echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog
215
216 # Display some info.
217 separator | dblog
218 nb=$(ls $PKGS/*.tazpkg | wc -l)
219 time=$(($(date +%s) - $time))
220 # L10n: 's' is for seconds (cooking time)
221 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
222
223
224 # Create all flavors files at once. Do we really need code to monitor
225 # flavors changes? Let's just build them with packages lists before
226 # syncing the mirror.
227 [ "$1" != '--flavors' ] && exit 1
228
229 if [ ! -d "$flavors" ]; then
230 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
231 exit 1
232 fi
233
234 [ ! -d "$live" ] && mkdir -p $live
235 _ 'Creating flavors files in "%s"' "$live" | dblog
236 _ 'Cook pkgdb: Creating all flavors' | log
237 separator | dblog
238
239 _ 'Recharging lists to use latest packages...' | dblog
240 tazpkg recharge >/dev/null 2>/dev/null
241
242 # We need a custom tazlito config to set working dir to /home/slitaz.
243 if [ ! -f "$live/tazlito.conf" ]; then
244 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
245 cp /etc/tazlito/tazlito.conf $live
246 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
247 $live/tazlito.conf
248 fi
249
250 # Update Hg flavors repo and pack.
251 if [ -d "$flavors/.hg" ]; then
252 cd $flavors; hg pull -u
253 fi
254
255 cd $live
256 _ 'Starting to generate flavors...' | dblog
257 rm -f flavors.list *.flavor
258 for i in $flavors/*; do
259 fl=$(basename $i)
260 _ 'Packing flavor "%s"' "$fl" | dblog
261 tazlito pack-flavor $fl >/dev/null || exit 1
262 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
263 done
264 cp -f $live/*.flavor $live/flavors.list $PKGS
265 separator | dblog
266 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
267 rm -f $command
268 separator | dblog
269 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
270
271
272 exit 0