cookutils annotate modules/pkgdb @ rev 1078

cook: allow multi-set & no-src receipts (sources may be downloaded inside compile_rules() individually for each set), allow change SHORT_DESC & WEB_SITE in genpkg_rules() (for differents sets);
lighttpd/cooker.css: add style for log line "Switching to the set...";
lighttpd/index.cgi: remove false-positive coloring with word "fatal", mark line "Switching to the set...", don't hide "Logs" button (fresh log initially empty and it may lasts for many seconds due to buffered "lazy write";
modules/compressor: better remove unwanted Perl files;
modules/mk_pkg_receipt: follow changed SHORT_DESC & WEB_SITE, improve condition speed (from 10 sec to 0.01 sec) when SPLIT is relatively long - looks like ${A/ $B /} is slow.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Thu Jun 21 12:57:52 2018 +0300 (2018-06-21)
parents 3911a2292888
children 0385051c3996
rev   line source
al@857 1 #!/bin/sh
al@857 2 #
al@857 3 # pkgdb - module of the SliTaz Cook
al@857 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
al@857 5 #
al@857 6
al@857 7 . /usr/lib/slitaz/libcook.sh
al@857 8
al@857 9
al@857 10 #
al@857 11 # Functions
al@857 12 #
al@857 13
al@857 14 dblog() { tee -a $LOGS/pkgdb.log; }
al@857 15
al@857 16
paul@1061 17 # Return size of file in human readable format
al@857 18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
al@857 19
al@857 20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
al@857 21
al@857 22
al@857 23
al@857 24
al@857 25
al@857 26 # Create suitable packages list for TazPkg and only for built packages
al@1059 27 # as well as flavors files for TazLiTo.
al@857 28
al@914 29 # Find how much time was spent the last time (for web interface)
al@899 30 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
al@899 31 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
al@899 32 while read cmd duration start; do
al@899 33 [ $(($start + $duration)) -lt $(date +%s) ] &&
al@899 34 echo "sed -i '/^$cmd $duration/d' $cooktime"
al@899 35 done < $cooktime | sh
al@899 36
al@857 37 rm $LOGS/pkgdb.log 2>/dev/null
al@857 38
al@857 39 case "$1" in
al@857 40 --flavors|--rmpkg) ;;
al@857 41 *)
al@857 42 [ -n "$1" ] && PKGS="$1"
al@857 43 if [ ! -d "$PKGS" ]; then
al@857 44 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
al@857 45 exit 1
al@857 46 fi ;;
al@857 47 esac
al@857 48
al@857 49 time=$(date +%s)
al@857 50 flavors="$SLITAZ/flavors"
al@857 51 live="$SLITAZ/live"
al@857 52
al@1024 53 arch=''
al@1024 54 case "$ARCH" in
al@1024 55 arm*|x86_64) arch="-$ARCH" ;;
al@1024 56 esac
al@1024 57
al@857 58 echo 'cook:pkgdb' > $command
al@857 59 _ 'Cook pkgdb: Creating all packages lists' | log
al@857 60 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
al@857 61
al@857 62 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
al@857 63
al@857 64 cd $PKGS
al@857 65
al@975 66 # Web interface database files should exist on the mirror1
al@975 67 >.folderlist; >.filelist
al@976 68 chmod 666 .folderlist .filelist
al@975 69
al@1059 70 # Command `cook pkgdb` may be executed by cron. Creating a packages database
paul@1061 71 # takes some time, during which previously pending packages may continue to
paul@1061 72 # be created. This will result in an error due to the lack of a database. This in
al@1059 73 # turn can lead to errors in the creating of many subsequent packages and will
al@1059 74 # require manual intervention.
paul@1061 75 # Solution is an atomic update of the packages database: first we create new files
al@1059 76 # (in the separate temp dir), and then instantly (well, almost) replace the old
al@1059 77 # files by new ones.
al@1059 78
al@1059 79 dbs=$(mktemp -d)
al@1059 80
al@857 81 _n 'Creating file "%s"' 'packages.list' | dblog
al@1059 82 find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $dbs/packages.list
al@1059 83 echo " ($(filesize $dbs/packages.list))" | dblog
al@857 84
al@857 85 _n 'Creating file "%s"' 'packages.md5' | dblog
al@1059 86 find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $dbs/packages.md5
al@1059 87 echo " ($(filesize $dbs/packages.md5))" | dblog
al@1059 88 cp $dbs/packages.md5 $dbs/packages.toremove # list of unnecessary packages
al@857 89
al@1059 90 md5sum $dbs/packages.md5 | cut -d' ' -f1 > $dbs/ID
al@1059 91 ( cat $dbs/ID | tr $'\n' ' '; date -ur $dbs/ID +%s ) > $dbs/IDs # md5 and timestamp
al@857 92
al@857 93 _n 'Creating file "%s"' 'descriptions.txt' | dblog
al@857 94 for i in $(ls $WOK | sort); do
al@914 95 [ -d "$WOK/$i/taz" ] || continue
al@914 96
al@914 97 for j in $(ls $WOK/$i/taz | sort); do
al@914 98 [ -e "$WOK/$i/taz/$j/description.txt" ] || continue
al@914 99
al@914 100 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
al@951 101 echo "$pkgname"
al@951 102 sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
paul@1061 103 # if description.txt doesn't end with \n then add one
al@1059 104 [ -z "$(tail -c1 $WOK/$i/taz/$j/description.txt)" ] || echo
al@951 105 echo
al@1059 106 done >> $dbs/descriptions.txt
al@857 107 done
al@1059 108 echo " ($(filesize $dbs/descriptions.txt))" | dblog
al@857 109
al@982 110 _n 'Creating file "%s"' 'bdeps.txt' | dblog
al@982 111 for i in $(ls $WOK | sort); do
al@982 112 [ -s "$WOK/$i/receipt" ] || continue
al@982 113 bdeps=$(. $WOK/$i/receipt; echo $BUILD_DEPENDS) # remove newlines from some receipts
al@1059 114 echo "$i $bdeps" >> $dbs/bdeps.txt
al@982 115 done
al@1060 116 echo " ($(filesize $dbs/bdeps.txt))" | dblog
al@982 117
al@857 118
al@857 119 _ 'Creating lists from "%s"' "$WOK" | dblog
al@857 120 cd $WOK
al@945 121 rsumf=$(mktemp)
al@1059 122 touch $dbs/packages.desc $dbs/packages.txt $dbs/packages.info $dbs/packages.equiv
al@914 123 for i in *; do
al@914 124 [ -d "$WOK/$i/taz" ] || continue
al@857 125
al@914 126 for j in $(ls $WOK/$i/taz | sort); do
al@945 127 pack="$i/taz/$j"
al@945 128 [ -f "$WOK/$pack/receipt" ] || continue
al@914 129 unset_receipt
al@951 130 . ./$pack/receipt
al@857 131
al@1024 132 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" ]; then
al@914 133
al@914 134 # packages.desc lets us search easily in DB
al@1059 135 cat >> $dbs/packages.desc <<EOT
al@857 136 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
al@857 137 EOT
al@857 138
al@1059 139 # packages.txt used by tazpkg-web also to provide
al@914 140 # a human readable package list with version and description.
al@1059 141 cat >> $dbs/packages.txt <<EOT
al@857 142 $PACKAGE
al@857 143 $VERSION$EXTRAVERSION
al@857 144 $SHORT_DESC
al@857 145 $PACKED_SIZE ($UNPACKED_SIZE installed)
al@857 146
al@857 147 EOT
al@857 148
al@914 149 # packages.info combines TazPkg separate files
al@914 150 # and will substitute them all
al@914 151 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
al@914 152 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
al@945 153
al@945 154 #MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
al@945 155 cp $pack/md5sum $rsumf
al@945 156 md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
al@945 157 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
al@945 158 rsum=$(md5sum $rsumf | awk '{print $1}')
al@945 159
al@1059 160 cat >> $dbs/packages.info <<EOT
al@945 161 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $rsum $PROVIDE
al@857 162 EOT
al@857 163
al@914 164 # packages.equiv is used by tazpkg install to check depends.
al@914 165 for k in $PROVIDE; do
al@914 166 DEST=''
al@914 167 echo $k | fgrep -q : && DEST="${k#*:}:"
al@1059 168 if grep -qs ^${k%:*}= $dbs/packages.equiv; then
al@914 169 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
al@1059 170 $dbs/packages.equiv
al@914 171 else
al@1059 172 echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv
al@914 173 fi
al@914 174 done
al@914 175
al@914 176 # files.list provides a list of all packages files.
al@1059 177 sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $dbs/files.list
al@914 178
al@914 179 # list of unnecessary packages
al@1059 180 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg/d" $dbs/packages.toremove
al@914 181 else
al@914 182 # if receipt variable HOST_ARCH absent/empty or contains ARCH
al@914 183 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
al@914 184 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
al@857 185 fi
al@857 186 fi
al@914 187 done
al@857 188 done
al@945 189 rm $rsumf
al@945 190
al@857 191
al@857 192 # Display list size.
al@1059 193 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $dbs/packages.desc)" | dblog
al@1059 194 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $dbs/packages.txt)" | dblog
al@1059 195 _ 'Done: %s (%s)' 'packages.info' "$(filesize $dbs/packages.info)" | dblog
al@1059 196 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $dbs/packages.equiv)" | dblog
al@857 197
al@857 198 cd $PKGS
al@857 199
al@857 200
al@862 201 # Check for unnecessary packages
al@1059 202 if [ -s "$dbs/packages.toremove" ]; then
al@857 203 newline | dblog
al@862 204 case x$rmpkg in
al@862 205 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
al@862 206 *) _ 'Removing unnecessary packages:' | dblog;;
al@862 207 esac
al@857 208 while read pkgsum pkgfile; do
al@857 209 echo " - $pkgfile" | dblog
al@1059 210 sed -i "/${pkgfile%.tazpkg}/d" $dbs/packages.list
al@1059 211 sed -i "/ $pkgfile/d" $dbs/packages.md5
al@857 212 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
al@1059 213 done < $dbs/packages.toremove
al@857 214 newline | dblog
al@857 215 fi
al@1059 216 rm $dbs/packages.toremove
al@857 217
al@857 218
al@857 219 # files.list.lzma
al@857 220 _n 'Creating file "%s"' 'files.list.lzma' | dblog
al@1059 221 touch $dbs/files.list
al@857 222 # pkgs.slitaz.org strongly depends on list sorted by packages names
al@1059 223 #lzma e $dbs/files.list $dbs/files.list.lzma
al@1059 224 /bin/lzma -zeqcT0 $dbs/files.list > $dbs/files.list.lzma
al@1059 225 echo " ($(filesize $dbs/files.list.lzma))" | dblog
al@857 226
al@857 227 # Pre-sorting filenames causes 10% smaller resulting lzma file
al@857 228 _n 'Creating file "%s"' 'files-list.lzma' | dblog
al@1059 229 cat $dbs/files.list | sort -k2 -o $dbs/files.list.sorted
al@1059 230 #lzma e $dbs/files.list.sorted $dbs/files-list.lzma
al@1059 231 /bin/lzma -zeqcT0 $dbs/files.list.sorted > $dbs/files-list.lzma
al@1059 232 rm -f $dbs/files.list $dbs/files.list.sorted
al@1059 233 echo " ($(filesize $dbs/files-list.lzma))" | dblog
al@857 234
al@1059 235 md5sum $dbs/files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > $dbs/files-list.md5
al@857 236
al@857 237 # Make bundle to fast recharge
al@857 238 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
al@857 239 # Make sure to get "mirrors" file
al@1059 240 until [ -e "$dbs/mirrors" ]; do
al@1059 241 wget -q -O $dbs/mirrors http://mirror1.slitaz.org/mirrors
al@857 242 echo -n '.' | dblog; sleep 5
al@857 243 done
al@857 244 # Make sure to get "extra.list" file
al@1059 245 until [ -e "$dbs/extra.list" ]; do
al@1059 246 wget -q -O $dbs/extra.list http://mirror1.slitaz.org/packages/get.list
al@857 247 echo -n '.' | dblog; sleep 5
al@857 248 done
al@1059 249 (
al@1059 250 cd $dbs
al@1059 251 busybox tar -chf bundle.tar \
al@1059 252 mirrors extra.list files-list.md5 packages.info descriptions.txt \
al@1059 253 packages.desc packages.md5 packages.txt packages.list packages.equiv
al@1059 254 )
al@1059 255 /bin/lzma -zeqcT0 $dbs/bundle.tar > $dbs/bundle.tar.lzma
al@1059 256 rm $dbs/bundle.tar $dbs/mirrors
al@1059 257 echo " ($(filesize $dbs/bundle.tar.lzma))" | dblog
al@857 258
al@857 259 # Display some info.
al@857 260 separator | dblog
al@968 261 nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
al@857 262 time=$(($(date +%s) - $time))
al@857 263 # L10n: 's' is for seconds (cooking time)
al@857 264 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
al@857 265
al@1059 266 # "Atomic" update now
al@1059 267 cp -f $dbs/* $PKGS
al@1059 268 rm -r $dbs
al@1059 269
al@857 270
al@857 271 # Create all flavors files at once. Do we really need code to monitor
al@857 272 # flavors changes? Let's just build them with packages lists before
al@857 273 # syncing the mirror.
al@899 274 [ "$1" != '--flavors' ] && rm $command && exit 1
al@857 275
al@857 276 if [ ! -d "$flavors" ]; then
al@857 277 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
al@899 278 rm $command
al@857 279 exit 1
al@857 280 fi
al@857 281
al@857 282 [ ! -d "$live" ] && mkdir -p $live
al@857 283 _ 'Creating flavors files in "%s"' "$live" | dblog
al@857 284 _ 'Cook pkgdb: Creating all flavors' | log
al@857 285 separator | dblog
al@857 286
al@857 287 _ 'Recharging lists to use latest packages...' | dblog
al@857 288 tazpkg recharge >/dev/null 2>/dev/null
al@857 289
al@857 290 # We need a custom tazlito config to set working dir to /home/slitaz.
al@857 291 if [ ! -f "$live/tazlito.conf" ]; then
al@857 292 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
al@857 293 cp /etc/tazlito/tazlito.conf $live
al@857 294 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
al@857 295 $live/tazlito.conf
al@857 296 fi
al@857 297
al@857 298 # Update Hg flavors repo and pack.
al@857 299 if [ -d "$flavors/.hg" ]; then
al@857 300 cd $flavors; hg pull -u
al@857 301 fi
al@857 302
al@857 303 cd $live
al@857 304 _ 'Starting to generate flavors...' | dblog
al@857 305 rm -f flavors.list *.flavor
al@857 306 for i in $flavors/*; do
al@857 307 fl=$(basename $i)
al@857 308 _ 'Packing flavor "%s"' "$fl" | dblog
al@857 309 tazlito pack-flavor $fl >/dev/null || exit 1
al@857 310 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
al@857 311 done
al@857 312 cp -f $live/*.flavor $live/flavors.list $PKGS
al@857 313 separator | dblog
al@857 314 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
al@857 315 separator | dblog
al@857 316 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
al@857 317
al@899 318 rm $command
al@876 319
al@857 320
al@857 321 exit 0