tazpkg annotate modules/get @ rev 944

modules/summary: fix $((arithmetic))
Busybox ash vs. Bash problem?
author Aleksej Bobylev <al.bobylev@gmail.com>
date Wed Feb 22 04:05:32 2017 +0200 (2017-02-22)
parents bc02a72abd85
children dbf40347342a
rev   line source
al@844 1 #!/bin/sh
al@844 2 # TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg
al@844 3 # get - TazPkg module
al@844 4 # Get package to the cache directory
al@844 5
al@844 6
al@844 7 # 1. "Plain packages" - compiled and packed on cook.slitaz.org using receipts.
al@844 8 #
al@844 9 # Recently added other type of packages: packages that can't be compiled due to the closed sources
al@844 10 # or due to the giant size of that sources. In this case special script: a) downloads package
al@844 11 # compiled for other Linux from official web site, b) converts this package to the TazPkg format,
al@844 12 # c) installs this package as "plain" package.
al@844 13 #
al@844 14 # 2. "Get-packages" - package contains only one special script for downloading/converting.
al@844 15 # Sometimes get-package can get some specified version of the package, sometimes - different
al@844 16 # (latest) version.
al@844 17 #
al@844 18 # Packages 1. and 2. can be found in the wok repository: http://hg.slitaz.org/wok/
al@844 19 #
al@844 20 # 3. "Extra" get-scripts. Next addition - only script, without packaging.
al@844 21 # Extra get-scripts repository: http://hg.slitaz.org/get-scripts/
al@844 22 # Extra get-scripts mirror: http://mirror.slitaz.org/packages/get/
al@844 23 #
al@844 24 # 4. Converted "extra" packages. Next addition - some packages like LibreOffice / OpenOffice are
al@844 25 # big to compile them like _1. "Plain packages"_ as well as big to convert them on the user side
al@844 26 # (you need a lot of time, CPU, and RAM) and sometimes it is unable on the weak user machines.
al@844 27 # So, some selected and free packages are converted on the SliTaz server side, packaged and
al@844 28 # are ready to download by any user: http://mirror.slitaz.org/packages/extra/
al@844 29 #
al@844 30 #
al@844 31 # Algorithm to get package:
al@844 32 # - search package in question in the mirrored packages list, download if exists;
al@844 33 # - search "get-package" in the mirrored packages list, download if exists;
al@844 34 # - search package in the "extra" get-scripts list, download if exists and run it, get the package.
al@844 35 #
al@844 36 # Note, here doubling. Imagine, you may wanted "palemoon" package.
al@844 37 # 1. No "plain package";
al@844 38 # 2. Existing "get-palemoon" package;
al@844 39 # 3. Existing "palemoon" extra get-script;
al@844 40 # 4. Existing "palemoon" extra converted package.
al@844 41 # User will get 2. He can specify "--extra" option to get 3.
al@844 42 # FIXME: do something with doubling.
al@844 43
al@844 44
al@844 45
al@844 46
al@844 47 # Connect function libraries
al@844 48 . /lib/libtaz.sh
al@844 49 . /usr/lib/slitaz/libpkg.sh
al@844 50
al@844 51 # Get TazPkg working environment
al@844 52 . @@MODULES@@/getenv
al@844 53
al@844 54 . @@MODULES@@/find-depends
al@844 55
al@844 56
al@844 57
al@844 58
al@844 59 # Get cache directory path
al@844 60
al@844 61 get_cache_path() {
al@844 62 # input: $1 = DB folder such as "$PKGS_DB" or "$PKGS_DB/undigest/*"
al@844 63 # $2 (optional): 'extra' for cache for extra-package scripts
al@844 64 # output: path to cache directory
al@844 65
al@844 66 local cache_dir
al@844 67
al@844 68 if [ "$2" == 'extra' ]; then
al@844 69 cache_dir="$SAVE_CACHE_DIR/extra/packages"
al@844 70 elif [ "$1" == "$PKGS_DB" ]; then
al@844 71 # Main repository
al@844 72 cache_dir="$SAVE_CACHE_DIR/$SLITAZ_RELEASE/packages"
al@844 73 else
al@844 74 # Undigest repository
al@844 75 cache_dir="$SAVE_CACHE_DIR/${1##*/}/packages"
al@844 76 fi
al@844 77
al@844 78 # Make cache directory if absent
al@844 79 mkdir -p "$cache_dir"
al@844 80 chmod a+w "$cache_dir"
al@844 81
al@844 82 echo "$cache_dir"
al@844 83 }
al@844 84
al@844 85
al@844 86 # Download a file from mirror
al@844 87
al@844 88 download_from() {
al@844 89 # input: "<mirror_url>" "<package_name>-<version>.tazpkg"
al@844 90
al@844 91 debug "\ndownload_from('$1', '$2')"
al@844 92
al@844 93 case "$1" in
al@844 94 # Mirror URL can have a trailing slash or not.
al@844 95 http://* | https://* | ftp://*)
al@862 96 debug " wget -c -T 30 -U $UA ${1%/}/$2"
al@877 97 q=''; [ -n "$quiet" ] && q='-q'
al@864 98 # Display abridged wget status (skip info about connections)
al@937 99 wget -c $q -T 30 -U $UA ${1%/}/$2 2>&1 | awk '$0~"%"{printf "%s\r",$0}' >&2
al@844 100 ;;
al@844 101 *)
al@863 102 debug " cp ${1%/}/$2 ."
al@863 103 cp ${1%/}/$2 .;;
al@844 104 esac
al@844 105 }
al@844 106
al@844 107
al@844 108 # This function may be called by a get script.
al@844 109
al@844 110 abort_package() {
al@844 111 cd "$CUR_DIR"
al@844 112 rm -rf "$tmp_dir"
pascal@911 113 [ -n "$1" ] ||
pascal@911 114 set -- 'Could not download "%s" from "%s". Exiting.' "${TARBALL:-$PACKAGE}" "${WGET_URL:-$WEB_SITE}"
pascal@911 115 printf "$@"
al@844 116 exit 1
al@844 117 }
al@844 118
al@844 119
al@844 120 # Get extra-package file to the cache
al@844 121
al@844 122 get_pkg_extra() {
al@844 123 # input: $1 extra-package name (like 'dropbox')
al@844 124 # $2 (internal): empty or 'redo' (when recursive calling)
al@844 125 # action: download stuff and make package
al@844 126 # output: full path to created package
al@844 127
al@844 128 debug "\nget_pkg_extra('$1', '$2')"
al@844 129
al@844 130 local mirror extra_cache converted tmp_dir script
al@844 131
al@844 132 # Check extra.list
al@844 133 if [ ! -s "$PKGS_DB/extra.list" ]; then
al@844 134 # Empty extra.list
al@844 135 if [ "$2" != 'redo' ]; then
al@844 136 tazpkg recharge >&2
al@844 137 get_pkg_extra "$1" 'redo'; exit 0
al@844 138 else
al@844 139 # Give up
al@844 140 _ 'File "%s" empty.' 'extra.list' >&2
al@844 141 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
al@844 142 fi
al@844 143 fi
al@844 144
al@844 145 # Extra.list exists and not empty
al@844 146 if ! grep -q "^$1|" "$PKGS_DB/extra.list"; then
al@844 147 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
al@844 148 fi
al@844 149
al@844 150 # Extra-package found in extra.list
al@844 151
al@844 152 if [ -z "$nocache" ]; then
al@844 153 # Go to cache for "get-install" command; don't move for "get" command
al@844 154 extra_cache="$(get_cache_path "$PKGS_DB" 'extra')"
al@844 155 debug "cd '$extra_cache'"
al@844 156 cd "$extra_cache"
al@844 157
al@844 158 # Extra-cache should contain packages DB files (packages.info, etc.)
al@844 159 # to list extra-packages contained in the extra-cache
al@844 160 [ ! -f 'packages.info' ] && tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
al@844 161
al@844 162 if [ -f 'packages.info' ]; then
al@844 163 awk -F$'\t' -vp="$1" '$1==p{exit 1}' packages.info
al@844 164 if [ "$?" -eq 1 ]; then
al@876 165 [ -z "$quiet" ] && _ 'Package "%s" already in the cache' "$1" >&2
al@844 166 echo -n "$(pwd)/"
al@844 167 awk -F$'\t' -vp="$1" '$1==p{printf "%s-%s.tazpkg\n", $1, $2; exit}' packages.info
al@844 168 exit 0
al@844 169 fi
al@844 170 fi
al@844 171 fi
al@844 172
al@844 173 mirror="$(cat "$PKGS_DB/mirror")"
al@844 174 debug "mirror='$mirror'"
al@844 175
al@844 176
al@844 177 # Try converted extra-packages first
al@844 178 # FIXME: Workaround to get packages filenames (even better to have names and versions separate)
al@846 179 converted="$(wget -O - http://mirror1.slitaz.org/packages/extra/ 2>/dev/null | \
al@844 180 tr \'\" $'\n' | grep "$1-.*\.tazpkg$" | sort -u)"
al@844 181 debug "converted='$converted'"
al@844 182 if [ -n "$converted" ]; then
al@844 183 case "$mirror" in
al@844 184 http://*|https://*|ftp://*)
al@844 185 # Default 'http://mirror.slitaz.org/packages/cooking/'
al@844 186 # -> 'http://mirror.slitaz.org/packages/extra/'
al@862 187 debug "wget -T 30 -U '$UA' '${mirror%packages/*}packages/extra/$converted'"
al@877 188 q=''; [ -n "$quiet" ] && q='-q'
al@877 189 wget $q -T 30 -U "$UA" "${mirror%packages/*}packages/extra/$converted" \
al@938 190 2>&1 | awk '$0~"%"{printf "%s\r",$0}' >&2;;
al@844 191 esac
al@844 192 if [ -f "$converted" ]; then
al@844 193 echo "$extra_cache/$converted"; exit 0
al@844 194 fi
al@844 195 fi
al@844 196
al@844 197
al@844 198 # Fails to download converted extra-package; now try to download and execute get-script
al@844 199 cd ..
al@844 200 case "$mirror" in
al@844 201 http://*|https://*|ftp://*)
al@844 202 # Default 'http://mirror.slitaz.org/packages/cooking/'
al@844 203 # -> 'http://mirror.slitaz.org/packages/get/'
al@862 204 debug "wget -T 30 -U '$UA' '${mirror%packages/*}packages/get/$1'"
al@877 205 q=''; [ -n "$quiet" ] && q='-q'
al@877 206 wget $q -T 30 -U "$UA" "${mirror%packages/*}packages/get/$1" \
al@938 207 2>&1 | awk '$0~"%"{printf "%s\r",$0}' >&2;;
al@844 208 esac
al@844 209
al@844 210 if [ ! -f "$1" ]; then
al@844 211 # TODO: extra package or extra package script? :) Too complicated...
al@844 212 die 'Unable to download extra package "%s".' "$(boldify $1)"
al@844 213 fi
al@844 214
al@844 215 # Extra-package script downloaded in the /var/cache/tazpkg/extra/
al@844 216
al@844 217 unset_receipt
al@844 218 PACKAGE="$1"
al@844 219 script="$(realpath $1)"
al@844 220 tmp_dir="$(mktemp -d)"; cd "$tmp_dir"
al@844 221 # Run script
al@844 222 set -e
al@844 223 . "$script"
al@844 224 set +e
al@844 225
pascal@918 226 [ "$PWD" != "$tmp_dir" ] && mv $PACKAGE* $tmp_dir
pascal@918 227 cd $tmp_dir
pascal@918 228 [ -n "$VERSION" ] || VERSION="$(date +%Y%m%d)"
pascal@918 229 [ -d "$PACKAGE-$VERSION" ] || mv $PACKAGE $PACKAGE-$VERSION || abort_package
pascal@918 230 [ -n "$TARBALL" ] || TARBALL="$(basename $WGET_URL)"
al@844 231
al@844 232 if [ ! -s "$PACKAGE-$VERSION/receipt" ]; then
al@844 233 # Create receipt (if script not created it early) using variables from script
al@844 234 cat > "$PACKAGE-$VERSION/receipt" <<EOT
al@844 235 # SliTaz package receipt.
al@844 236
al@844 237 PACKAGE="$PACKAGE"
pascal@918 238 VERSION="$VERSION"
al@844 239 CATEGORY="${CATEGORY:-non-free}"
pascal@919 240 WEB_SITE="${WEB_SITE:-$WEBSITE}"
pascal@918 241 SHORT_DESC="${SHORT_DESC:-The $PACKAGE software package of SliTaz}"
al@844 242 MAINTAINER="${MAINTAINER:-nobody@slitaz.org}"
pascal@912 243 LICENSE="${LICENSE:-unknown}"
pascal@918 244 TARBALL="$TARBALL"
pascal@919 245 WGET_URL="${WGET_URL:-$WGETURL}"
al@844 246 CONFIG_FILES="$CONFIG_FILES"
al@844 247 SUGGESTED="$SUGGESTED"
al@844 248 PROVIDE="$PROVIDE"
al@844 249 DEPENDS="$DEPENDS"
al@844 250 HOST_ARCH="$HOST_ARCH"
al@844 251 TAGS="$TAGS"
al@844 252 EXTRA_SOURCE_FILES="$EXTRA_SOURCE_FILES"
al@844 253 EOT
al@844 254 fi
al@844 255
al@844 256 # Add dependencies which was found to already defined dependencies
al@844 257 DEPENDS="$(unset DEPENDS; . "$PACKAGE-$VERSION/receipt"; echo $DEPENDS)"
al@898 258 TMP_DIR="$tmp_dir"
al@844 259 for i in $(find_depends "$PACKAGE-$VERSION/fs"); do
al@844 260 case " $DEPENDS " in
al@844 261 *\ $i\ *) continue;;
al@844 262 esac
al@844 263 sed -i "s/^DEPENDS=\"/&$i /" "$PACKAGE-$VERSION/receipt"
al@844 264 done
al@844 265
al@844 266 # Remove empty variables
al@844 267 sed -i '/=""$/d' "$PACKAGE-$VERSION/receipt"
al@844 268
al@844 269 tazpkg pack "$PACKAGE-$VERSION" gzip >&2
al@844 270
al@844 271 if [ -z "nocache" ]; then
al@844 272 # move package to the extra-cache for "get-install" command
al@844 273 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$extra_cache"
al@844 274 # Re-make extra packages database
al@844 275 tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
al@844 276 else
al@844 277 # move package to directory where "tazpkg get" command invoked
al@844 278 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$CUR_DIR"
al@844 279 fi
al@844 280
al@844 281 # Clean
al@844 282 rm -rf "$tmp_dir"
al@844 283
al@844 284 # Function output: path to package
al@844 285 echo "$CUR_DIR/$PACKAGE-$VERSION.tazpkg"
al@844 286 }
al@844 287
al@844 288
pascal@873 289 # return possible name for a virtual package name
pascal@873 290
pascal@922 291 virtual_name() {
pascal@873 292 # input: $1 virtual package name
pascal@873 293 # $2 repository db directory
pascal@873 294 local i
al@887 295 unset IFS
pascal@873 296 for i in $(grep -hs "^$1=" "$2/packages.equiv" | sed "s/^$1=//"); do
pascal@873 297 if echo $i | fgrep -q : ; then
pascal@873 298 # format 'alternative:newname'
pascal@873 299 # if alternative is installed then substitute newname
pascal@874 300 if [ -f $INSTALLED/${i%:*}/receipt ]; then
pascal@873 301 # substitute package dependency
pascal@873 302 echo ${i#*:}
pascal@873 303 return
pascal@873 304 fi
al@887 305 elif ! grep -q "^$1 " "$2/packages.info" || [ -f "$INSTALLED/$i/receipt" ]; then
pascal@873 306 # unconditional substitution
pascal@873 307 echo $i
pascal@873 308 return
pascal@873 309 fi
pascal@873 310 done
pascal@879 311 # the real package name
pascal@879 312 echo $1
pascal@873 313 }
pascal@873 314
pascal@922 315 virtual_pkg() {
pascal@922 316 # input: $1 virtual package name
pascal@922 317 # $2 repository db directory
pascal@922 318 # output: display possible package name
pascal@922 319
pascal@922 320 debug "\nvirtual_pkg('$1', '$2')"
pascal@922 321
pascal@922 322 if [ "$tazpkg_command" != 'get-install' ]; then
pascal@922 323 # 'get' command: download any package
pascal@922 324 if [ -z "$(awk -F$'\t' -vp="$1" '{if ($1 == p) print p}' "$2/packages.info")" ]; then
paul@923 325 # This package does not exist in the list, it may be a virtual package
pascal@922 326 virtual_name "$1" "$2"
pascal@922 327 else
pascal@922 328 echo $1
pascal@922 329 fi
pascal@922 330 return
pascal@922 331 fi
pascal@922 332
pascal@922 333 virtual_name "$1" "$2"
pascal@922 334 }
pascal@922 335
al@887 336
al@844 337 # Download package file to the cache
al@844 338
al@844 339 get_pkg() {
al@844 340 # input: $1 package name either "name" or "name-version"
al@844 341 # $2 (internal): empty or 'redo' (when recursive calling)
al@844 342 # action: download package from mirror to the cache directory
al@844 343 # output: full path to the downloaded package
al@844 344
al@844 345 debug "\nget_pkg('$1', '$2')"
al@844 346 local repo line namever pkgsum
al@844 347
al@844 348 IFS=$'\n'
al@844 349 for rep in $PRIORITY; do
al@844 350 [ ! -f "$rep/packages.info" ] && continue
al@844 351 # If found, output string "<name>-<ver>:<sum>"
pascal@879 352 line=$(awk -F$'\t' -vpkg="$(virtual_pkg "$1" "$rep")" \
pascal@879 353 '$1==pkg || $1"-"$2==pkg {printf "%s-%s:%s", $1, $2, $9; exit}' "$rep/packages.info")
al@844 354 if [ -n "$line" ]; then
al@844 355 namever=${line%:*}; pkgsum=${line#*:}
al@844 356 break
al@844 357 fi
al@844 358 done
al@844 359 unset IFS
al@844 360
al@844 361 debug " rep='$rep'\n namever='$namever'\n pkgsum='$pkgsum'"
al@844 362
al@844 363 if [ -z "$line" ]; then
al@889 364 _ 'Unable to find package "%s" in the mirrored packages list.' "$1" >&2
al@844 365 # Retry with "get-package"; prevent looping with 'redo'
al@844 366 if [ "$2" != 'redo' ]; then
al@844 367 get_pkg "get-$1" 'redo'; exit 0
al@844 368 else
al@844 369 # Retry with extra-package
al@844 370 get_pkg_extra "${1#get-}"
al@844 371 exit 0
al@844 372 fi
al@844 373 fi
al@844 374
al@844 375 # We need package "$namever.tazpkg" from "$rep" with "$pkgsum"
al@844 376
al@844 377 if [ -z "$nocache" ]; then
al@844 378 # Go to cache for "get-install" command; don't move for "get" command
al@844 379 debug "cd '$(get_cache_path "$rep")'"
al@844 380 cd "$(get_cache_path "$rep")"
al@844 381 fi
al@844 382
al@844 383 # Check if package already downloaded
al@844 384 if [ -f "$namever.tazpkg" ]; then
al@876 385 [ -z "$nocache" -a -z "$quiet" ] && _ 'Package "%s" already in the cache' "$namever" >&2
al@844 386
paul@943 387 # Check if downloading complete, resume if not complete
al@844 388 if ! tail -c 2k "$namever.tazpkg" | fgrep -q '00000000TRAILER'; then
al@876 389 [ -z "$quiet" ] && _ 'Continuing package "%s" download' "$namever" >&2
al@844 390 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
al@844 391 fi
al@844 392 else
al@844 393 # Package absent in the cache, "cold" downloading
al@844 394 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
al@844 395 fi
al@844 396
al@844 397 # Make sure we downloaded what we want with checksum
al@844 398
al@844 399 if [ "$($CHECKSUM "$namever.tazpkg" | cut -d' ' -f1)" != "$pkgsum" ]; then
al@845 400 _ 'Checksum error for "%s"' "$namever.tazpkg" >&2
al@844 401 rm "$namever.tazpkg"
al@844 402
al@844 403 # Recharge DBs and try again; prevent looping with 'redo'
al@844 404 if [ "$2" != 'redo' ]; then
al@844 405 tazpkg recharge >&2
al@844 406 get_pkg "$1" 'redo'; exit 0
al@844 407 else
al@844 408 # Give up
al@844 409 # TODO: try another mirror?
al@844 410 die 'Please wait until the mirror synchronization is complete and try again.'
al@844 411 fi
al@844 412 fi
al@844 413
al@844 414 # Output: path to downloaded package
al@844 415 printf '%s/%s.tazpkg\n' "$(pwd)" "$namever"
al@844 416 }
al@844 417
al@844 418
al@941 419 # Get local package from /home/slitaz/packages to co-work with the cookutils
al@941 420
al@941 421 get_pkg_cookmode() {
al@941 422 # input: $1 package name (like 'advancecomp')
al@941 423 # action: find package in /home/slitaz/packages (or where defined in the cook.conf)
al@941 424 # output: full path to the found package
al@941 425 # ROOT NOT USED
al@941 426
al@941 427 local PKGS='/home/slitaz/packages' found='0'
al@941 428 [ -e "/etc/slitaz/cook.conf" ] && . "/etc/slitaz/cook.conf"
al@941 429
al@941 430 # Find local package
al@941 431 tempd="$(mktemp -d)"; cd "$tempd"
al@941 432
al@941 433 for pkgfile in $PKGS/$1-*.tazpkg; do
al@941 434 [ -e "$pkgfile" ] || continue
al@941 435 # Extract receipt from each matched package
al@941 436 cpio -F "$pkgfile" -i receipt >/dev/null 2>&1
al@941 437 name=$(. receipt; echo $PACKAGE)
al@941 438 rm receipt
al@941 439 if [ "$name" == "$1" ]; then
al@941 440 # Use the first matched package: normally there is only one package
al@941 441 # with the $PACKAGE matched in the receipt
al@941 442 found='1'
al@941 443 # Output: path to found package and exit loop
al@941 444 echo "$pkgfile"
al@941 445 break
al@941 446 fi
al@941 447 done
al@941 448 rm -r "$tempd"
al@941 449
al@941 450 # Proceed to get package as usual (non-local)
al@941 451 if [ "$found" -eq 0 ]; then
al@941 452 get_pkg "$1"
al@941 453 fi
al@941 454 }
al@941 455
al@941 456
al@844 457
al@844 458
al@864 459 # Command 'get-install' calls 'get', then 'install' modules. Check package presence here, on the
al@864 460 # first stage, if '--forced' option not given
al@866 461 if [ "$tazpkg_command" == 'get-install' ]; then
al@866 462 if grep -qs "^$1$" "$BLOCKED"; then
al@866 463 _ 'Package "%s" blocked.' "$1" >&2
al@864 464 exit 1
al@864 465 fi
al@866 466
al@866 467 if [ -z "$forced" ]; then
al@866 468 awk -F$'\t' -vpv="$1" '$1==pv { exit 1 }' "$PKGS_DB/installed.info"
al@866 469 if [ "$?" -eq 1 ]; then
al@880 470 [ -z "$quiet" ] && (
al@866 471 newline
al@866 472 _ '"%s" package is already installed.' "$(colorize 34 "$1")"
al@866 473 longline "$(_ 'You can use the --forced option to force installation.')"
al@866 474 newline
al@866 475 ) >&2
al@866 476 # Prevent execution 'install' stage:
al@866 477 exit 1
al@866 478 fi
al@866 479 fi
al@864 480 fi
al@864 481
al@941 482 if [ -n "$cookmode" ]; then
al@941 483 # When '--cookmode' option given, try to find package in the local cook repository,
al@941 484 # then, if fails, try to get the package as usual
al@941 485 get_pkg_cookmode "$1"
al@941 486 elif [ -n "$extra" ]; then
al@844 487 # When '--extra' option given, extra-package has priority over 'regular' packages
al@844 488 get_pkg_extra "$1"
al@844 489 else
al@844 490 # Try to get 'package', then 'get-package', then extra 'package'
al@844 491 get_pkg "$1"
al@844 492 fi