cookutils annotate modules/pkgdb @ rev 1063

lighttpd/index.cgi: load main page parts asynchronously
author Aleksej Bobylev <al.bobylev@gmail.com>
date Mon Jun 04 11:27:06 2018 +0300 (2018-06-04)
parents 3911a2292888
children 0385051c3996
rev   line source
al@857 1 #!/bin/sh
al@857 2 #
al@857 3 # pkgdb - module of the SliTaz Cook
al@857 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
al@857 5 #
al@857 6
al@857 7 . /usr/lib/slitaz/libcook.sh
al@857 8
al@857 9
al@857 10 #
al@857 11 # Functions
al@857 12 #
al@857 13
al@857 14 dblog() { tee -a $LOGS/pkgdb.log; }
al@857 15
al@857 16
paul@1061 17 # Return size of file in human readable format
al@857 18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
al@857 19
al@857 20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
al@857 21
al@857 22
al@857 23
al@857 24
al@857 25
al@857 26 # Create suitable packages list for TazPkg and only for built packages
al@1059 27 # as well as flavors files for TazLiTo.
al@857 28
al@914 29 # Find how much time was spent the last time (for web interface)
al@899 30 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
al@899 31 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
al@899 32 while read cmd duration start; do
al@899 33 [ $(($start + $duration)) -lt $(date +%s) ] &&
al@899 34 echo "sed -i '/^$cmd $duration/d' $cooktime"
al@899 35 done < $cooktime | sh
al@899 36
al@857 37 rm $LOGS/pkgdb.log 2>/dev/null
al@857 38
al@857 39 case "$1" in
al@857 40 --flavors|--rmpkg) ;;
al@857 41 *)
al@857 42 [ -n "$1" ] && PKGS="$1"
al@857 43 if [ ! -d "$PKGS" ]; then
al@857 44 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
al@857 45 exit 1
al@857 46 fi ;;
al@857 47 esac
al@857 48
al@857 49 time=$(date +%s)
al@857 50 flavors="$SLITAZ/flavors"
al@857 51 live="$SLITAZ/live"
al@857 52
al@1024 53 arch=''
al@1024 54 case "$ARCH" in
al@1024 55 arm*|x86_64) arch="-$ARCH" ;;
al@1024 56 esac
al@1024 57
al@857 58 echo 'cook:pkgdb' > $command
al@857 59 _ 'Cook pkgdb: Creating all packages lists' | log
al@857 60 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
al@857 61
al@857 62 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
al@857 63
al@857 64 cd $PKGS
al@857 65
al@975 66 # Web interface database files should exist on the mirror1
al@975 67 >.folderlist; >.filelist
al@976 68 chmod 666 .folderlist .filelist
al@975 69
al@1059 70 # Command `cook pkgdb` may be executed by cron. Creating a packages database
paul@1061 71 # takes some time, during which previously pending packages may continue to
paul@1061 72 # be created. This will result in an error due to the lack of a database. This in
al@1059 73 # turn can lead to errors in the creating of many subsequent packages and will
al@1059 74 # require manual intervention.
paul@1061 75 # Solution is an atomic update of the packages database: first we create new files
al@1059 76 # (in the separate temp dir), and then instantly (well, almost) replace the old
al@1059 77 # files by new ones.
al@1059 78
al@1059 79 dbs=$(mktemp -d)
al@1059 80
al@857 81 _n 'Creating file "%s"' 'packages.list' | dblog
al@1059 82 find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $dbs/packages.list
al@1059 83 echo " ($(filesize $dbs/packages.list))" | dblog
al@857 84
al@857 85 _n 'Creating file "%s"' 'packages.md5' | dblog
al@1059 86 find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $dbs/packages.md5
al@1059 87 echo " ($(filesize $dbs/packages.md5))" | dblog
al@1059 88 cp $dbs/packages.md5 $dbs/packages.toremove # list of unnecessary packages
al@857 89
al@1059 90 md5sum $dbs/packages.md5 | cut -d' ' -f1 > $dbs/ID
al@1059 91 ( cat $dbs/ID | tr $'\n' ' '; date -ur $dbs/ID +%s ) > $dbs/IDs # md5 and timestamp
al@857 92
al@857 93 _n 'Creating file "%s"' 'descriptions.txt' | dblog
al@857 94 for i in $(ls $WOK | sort); do
al@914 95 [ -d "$WOK/$i/taz" ] || continue
al@914 96
al@914 97 for j in $(ls $WOK/$i/taz | sort); do
al@914 98 [ -e "$WOK/$i/taz/$j/description.txt" ] || continue
al@914 99
al@914 100 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
al@951 101 echo "$pkgname"
al@951 102 sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
paul@1061 103 # if description.txt doesn't end with \n then add one
al@1059 104 [ -z "$(tail -c1 $WOK/$i/taz/$j/description.txt)" ] || echo
al@951 105 echo
al@1059 106 done >> $dbs/descriptions.txt
al@857 107 done
al@1059 108 echo " ($(filesize $dbs/descriptions.txt))" | dblog
al@857 109
al@982 110 _n 'Creating file "%s"' 'bdeps.txt' | dblog
al@982 111 for i in $(ls $WOK | sort); do
al@982 112 [ -s "$WOK/$i/receipt" ] || continue
al@982 113 bdeps=$(. $WOK/$i/receipt; echo $BUILD_DEPENDS) # remove newlines from some receipts
al@1059 114 echo "$i $bdeps" >> $dbs/bdeps.txt
al@982 115 done
al@1060 116 echo " ($(filesize $dbs/bdeps.txt))" | dblog
al@982 117
al@857 118
al@857 119 _ 'Creating lists from "%s"' "$WOK" | dblog
al@857 120 cd $WOK
al@945 121 rsumf=$(mktemp)
al@1059 122 touch $dbs/packages.desc $dbs/packages.txt $dbs/packages.info $dbs/packages.equiv
al@914 123 for i in *; do
al@914 124 [ -d "$WOK/$i/taz" ] || continue
al@857 125
al@914 126 for j in $(ls $WOK/$i/taz | sort); do
al@945 127 pack="$i/taz/$j"
al@945 128 [ -f "$WOK/$pack/receipt" ] || continue
al@914 129 unset_receipt
al@951 130 . ./$pack/receipt
al@857 131
al@1024 132 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" ]; then
al@914 133
al@914 134 # packages.desc lets us search easily in DB
al@1059 135 cat >> $dbs/packages.desc <<EOT
al@857 136 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
al@857 137 EOT
al@857 138
al@1059 139 # packages.txt used by tazpkg-web also to provide
al@914 140 # a human readable package list with version and description.
al@1059 141 cat >> $dbs/packages.txt <<EOT
al@857 142 $PACKAGE
al@857 143 $VERSION$EXTRAVERSION
al@857 144 $SHORT_DESC
al@857 145 $PACKED_SIZE ($UNPACKED_SIZE installed)
al@857 146
al@857 147 EOT
al@857 148
al@914 149 # packages.info combines TazPkg separate files
al@914 150 # and will substitute them all
al@914 151 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
al@914 152 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
al@945 153
al@945 154 #MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
al@945 155 cp $pack/md5sum $rsumf
al@945 156 md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
al@945 157 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
al@945 158 rsum=$(md5sum $rsumf | awk '{print $1}')
al@945 159
al@1059 160 cat >> $dbs/packages.info <<EOT
al@945 161 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $rsum $PROVIDE
al@857 162 EOT
al@857 163
al@914 164 # packages.equiv is used by tazpkg install to check depends.
al@914 165 for k in $PROVIDE; do
al@914 166 DEST=''
al@914 167 echo $k | fgrep -q : && DEST="${k#*:}:"
al@1059 168 if grep -qs ^${k%:*}= $dbs/packages.equiv; then
al@914 169 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
al@1059 170 $dbs/packages.equiv
al@914 171 else
al@1059 172 echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv
al@914 173 fi
al@914 174 done
al@914 175
al@914 176 # files.list provides a list of all packages files.
al@1059 177 sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $dbs/files.list
al@914 178
al@914 179 # list of unnecessary packages
al@1059 180 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg/d" $dbs/packages.toremove
al@914 181 else
al@914 182 # if receipt variable HOST_ARCH absent/empty or contains ARCH
al@914 183 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
al@914 184 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
al@857 185 fi
al@857 186 fi
al@914 187 done
al@857 188 done
al@945 189 rm $rsumf
al@945 190
al@857 191
al@857 192 # Display list size.
al@1059 193 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $dbs/packages.desc)" | dblog
al@1059 194 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $dbs/packages.txt)" | dblog
al@1059 195 _ 'Done: %s (%s)' 'packages.info' "$(filesize $dbs/packages.info)" | dblog
al@1059 196 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $dbs/packages.equiv)" | dblog
al@857 197
al@857 198 cd $PKGS
al@857 199
al@857 200
al@862 201 # Check for unnecessary packages
al@1059 202 if [ -s "$dbs/packages.toremove" ]; then
al@857 203 newline | dblog
al@862 204 case x$rmpkg in
al@862 205 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
al@862 206 *) _ 'Removing unnecessary packages:' | dblog;;
al@862 207 esac
al@857 208 while read pkgsum pkgfile; do
al@857 209 echo " - $pkgfile" | dblog
al@1059 210 sed -i "/${pkgfile%.tazpkg}/d" $dbs/packages.list
al@1059 211 sed -i "/ $pkgfile/d" $dbs/packages.md5
al@857 212 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
al@1059 213 done < $dbs/packages.toremove
al@857 214 newline | dblog
al@857 215 fi
al@1059 216 rm $dbs/packages.toremove
al@857 217
al@857 218
al@857 219 # files.list.lzma
al@857 220 _n 'Creating file "%s"' 'files.list.lzma' | dblog
al@1059 221 touch $dbs/files.list
al@857 222 # pkgs.slitaz.org strongly depends on list sorted by packages names
al@1059 223 #lzma e $dbs/files.list $dbs/files.list.lzma
al@1059 224 /bin/lzma -zeqcT0 $dbs/files.list > $dbs/files.list.lzma
al@1059 225 echo " ($(filesize $dbs/files.list.lzma))" | dblog
al@857 226
al@857 227 # Pre-sorting filenames causes 10% smaller resulting lzma file
al@857 228 _n 'Creating file "%s"' 'files-list.lzma' | dblog
al@1059 229 cat $dbs/files.list | sort -k2 -o $dbs/files.list.sorted
al@1059 230 #lzma e $dbs/files.list.sorted $dbs/files-list.lzma
al@1059 231 /bin/lzma -zeqcT0 $dbs/files.list.sorted > $dbs/files-list.lzma
al@1059 232 rm -f $dbs/files.list $dbs/files.list.sorted
al@1059 233 echo " ($(filesize $dbs/files-list.lzma))" | dblog
al@857 234
al@1059 235 md5sum $dbs/files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > $dbs/files-list.md5
al@857 236
al@857 237 # Make bundle to fast recharge
al@857 238 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
al@857 239 # Make sure to get "mirrors" file
al@1059 240 until [ -e "$dbs/mirrors" ]; do
al@1059 241 wget -q -O $dbs/mirrors http://mirror1.slitaz.org/mirrors
al@857 242 echo -n '.' | dblog; sleep 5
al@857 243 done
al@857 244 # Make sure to get "extra.list" file
al@1059 245 until [ -e "$dbs/extra.list" ]; do
al@1059 246 wget -q -O $dbs/extra.list http://mirror1.slitaz.org/packages/get.list
al@857 247 echo -n '.' | dblog; sleep 5
al@857 248 done
al@1059 249 (
al@1059 250 cd $dbs
al@1059 251 busybox tar -chf bundle.tar \
al@1059 252 mirrors extra.list files-list.md5 packages.info descriptions.txt \
al@1059 253 packages.desc packages.md5 packages.txt packages.list packages.equiv
al@1059 254 )
al@1059 255 /bin/lzma -zeqcT0 $dbs/bundle.tar > $dbs/bundle.tar.lzma
al@1059 256 rm $dbs/bundle.tar $dbs/mirrors
al@1059 257 echo " ($(filesize $dbs/bundle.tar.lzma))" | dblog
al@857 258
al@857 259 # Display some info.
al@857 260 separator | dblog
al@968 261 nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
al@857 262 time=$(($(date +%s) - $time))
al@857 263 # L10n: 's' is for seconds (cooking time)
al@857 264 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
al@857 265
al@1059 266 # "Atomic" update now
al@1059 267 cp -f $dbs/* $PKGS
al@1059 268 rm -r $dbs
al@1059 269
al@857 270
al@857 271 # Create all flavors files at once. Do we really need code to monitor
al@857 272 # flavors changes? Let's just build them with packages lists before
al@857 273 # syncing the mirror.
al@899 274 [ "$1" != '--flavors' ] && rm $command && exit 1
al@857 275
al@857 276 if [ ! -d "$flavors" ]; then
al@857 277 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
al@899 278 rm $command
al@857 279 exit 1
al@857 280 fi
al@857 281
al@857 282 [ ! -d "$live" ] && mkdir -p $live
al@857 283 _ 'Creating flavors files in "%s"' "$live" | dblog
al@857 284 _ 'Cook pkgdb: Creating all flavors' | log
al@857 285 separator | dblog
al@857 286
al@857 287 _ 'Recharging lists to use latest packages...' | dblog
al@857 288 tazpkg recharge >/dev/null 2>/dev/null
al@857 289
al@857 290 # We need a custom tazlito config to set working dir to /home/slitaz.
al@857 291 if [ ! -f "$live/tazlito.conf" ]; then
al@857 292 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
al@857 293 cp /etc/tazlito/tazlito.conf $live
al@857 294 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
al@857 295 $live/tazlito.conf
al@857 296 fi
al@857 297
al@857 298 # Update Hg flavors repo and pack.
al@857 299 if [ -d "$flavors/.hg" ]; then
al@857 300 cd $flavors; hg pull -u
al@857 301 fi
al@857 302
al@857 303 cd $live
al@857 304 _ 'Starting to generate flavors...' | dblog
al@857 305 rm -f flavors.list *.flavor
al@857 306 for i in $flavors/*; do
al@857 307 fl=$(basename $i)
al@857 308 _ 'Packing flavor "%s"' "$fl" | dblog
al@857 309 tazlito pack-flavor $fl >/dev/null || exit 1
al@857 310 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
al@857 311 done
al@857 312 cp -f $live/*.flavor $live/flavors.list $PKGS
al@857 313 separator | dblog
al@857 314 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
al@857 315 separator | dblog
al@857 316 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
al@857 317
al@899 318 rm $command
al@876 319
al@857 320
al@857 321 exit 0