tazpkg view modules/get @ rev 863
Module "get": copy source in download_from() for local repo; module "install": use version for depmod
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Mon Nov 16 10:45:25 2015 +0200 (2015-11-16) |
| parents | 8c6346ff1cef |
| children | 67996045442a |
line source
1 #!/bin/sh
2 # TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg
3 # get - TazPkg module
4 # Get package to the cache directory
7 # 1. "Plain packages" - compiled and packed on cook.slitaz.org using receipts.
8 #
9 # Recently added other type of packages: packages that can't be compiled due to the closed sources
10 # or due to the giant size of that sources. In this case special script: a) downloads package
11 # compiled for other Linux from official web site, b) converts this package to the TazPkg format,
12 # c) installs this package as "plain" package.
13 #
14 # 2. "Get-packages" - package contains only one special script for downloading/converting.
15 # Sometimes get-package can get some specified version of the package, sometimes - different
16 # (latest) version.
17 #
18 # Packages 1. and 2. can be found in the wok repository: http://hg.slitaz.org/wok/
19 #
20 # 3. "Extra" get-scripts. Next addition - only script, without packaging.
21 # Extra get-scripts repository: http://hg.slitaz.org/get-scripts/
22 # Extra get-scripts mirror: http://mirror.slitaz.org/packages/get/
23 #
24 # 4. Converted "extra" packages. Next addition - some packages like LibreOffice / OpenOffice are
25 # big to compile them like _1. "Plain packages"_ as well as big to convert them on the user side
26 # (you need a lot of time, CPU, and RAM) and sometimes it is unable on the weak user machines.
27 # So, some selected and free packages are converted on the SliTaz server side, packaged and
28 # are ready to download by any user: http://mirror.slitaz.org/packages/extra/
29 #
30 #
31 # Algorithm to get package:
32 # - search package in question in the mirrored packages list, download if exists;
33 # - search "get-package" in the mirrored packages list, download if exists;
34 # - search package in the "extra" get-scripts list, download if exists and run it, get the package.
35 #
36 # Note, here doubling. Imagine, you may wanted "palemoon" package.
37 # 1. No "plain package";
38 # 2. Existing "get-palemoon" package;
39 # 3. Existing "palemoon" extra get-script;
40 # 4. Existing "palemoon" extra converted package.
41 # User will get 2. He can specify "--extra" option to get 3.
42 # FIXME: do something with doubling.
47 # Connect function libraries
48 . /lib/libtaz.sh
49 . /usr/lib/slitaz/libpkg.sh
51 # Get TazPkg working environment
52 . @@MODULES@@/getenv
54 . @@MODULES@@/find-depends
59 # Get cache directory path
61 get_cache_path() {
62 # input: $1 = DB folder such as "$PKGS_DB" or "$PKGS_DB/undigest/*"
63 # $2 (optional): 'extra' for cache for extra-package scripts
64 # output: path to cache directory
66 local cache_dir
68 if [ "$2" == 'extra' ]; then
69 cache_dir="$SAVE_CACHE_DIR/extra/packages"
70 elif [ "$1" == "$PKGS_DB" ]; then
71 # Main repository
72 cache_dir="$SAVE_CACHE_DIR/$SLITAZ_RELEASE/packages"
73 else
74 # Undigest repository
75 cache_dir="$SAVE_CACHE_DIR/${1##*/}/packages"
76 fi
78 # Make cache directory if absent
79 mkdir -p "$cache_dir"
80 chmod a+w "$cache_dir"
82 echo "$cache_dir"
83 }
86 # Download a file from mirror
88 download_from() {
89 # input: "<mirror_url>" "<package_name>-<version>.tazpkg"
91 debug "\ndownload_from('$1', '$2')"
93 case "$1" in
94 # Mirror URL can have a trailing slash or not.
95 http://* | https://* | ftp://*)
96 debug " wget -c -T 30 -U $UA ${1%/}/$2"
97 # TODO: display abridged wget status
98 wget -c -T 30 -U $UA ${1%/}/$2
99 ;;
100 *)
101 debug " cp ${1%/}/$2 ."
102 cp ${1%/}/$2 .;;
103 esac
104 }
107 # This function may be called by a get script.
109 abort_package() {
110 cd "$CUR_DIR"
111 rm -rf "$tmp_dir"
112 echo "${1:-Abort $PACKAGE.}"
113 exit 1
114 }
117 # Get extra-package file to the cache
119 get_pkg_extra() {
120 # input: $1 extra-package name (like 'dropbox')
121 # $2 (internal): empty or 'redo' (when recursive calling)
122 # action: download stuff and make package
123 # output: full path to created package
125 debug "\nget_pkg_extra('$1', '$2')"
127 local mirror extra_cache converted tmp_dir script
129 # Check extra.list
130 if [ ! -s "$PKGS_DB/extra.list" ]; then
131 # Empty extra.list
132 if [ "$2" != 'redo' ]; then
133 tazpkg recharge >&2
134 get_pkg_extra "$1" 'redo'; exit 0
135 else
136 # Give up
137 _ 'File "%s" empty.' 'extra.list' >&2
138 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
139 fi
140 fi
142 # Extra.list exists and not empty
143 if ! grep -q "^$1|" "$PKGS_DB/extra.list"; then
144 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
145 fi
147 # Extra-package found in extra.list
149 if [ -z "$nocache" ]; then
150 # Go to cache for "get-install" command; don't move for "get" command
151 extra_cache="$(get_cache_path "$PKGS_DB" 'extra')"
152 debug "cd '$extra_cache'"
153 cd "$extra_cache"
155 # Extra-cache should contain packages DB files (packages.info, etc.)
156 # to list extra-packages contained in the extra-cache
157 [ ! -f 'packages.info' ] && tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
159 if [ -f 'packages.info' ]; then
160 awk -F$'\t' -vp="$1" '$1==p{exit 1}' packages.info
161 if [ "$?" -eq 1 ]; then
162 _ 'Package "%s" already in the cache' "$1" >&2
163 echo -n "$(pwd)/"
164 awk -F$'\t' -vp="$1" '$1==p{printf "%s-%s.tazpkg\n", $1, $2; exit}' packages.info
165 exit 0
166 fi
167 fi
168 fi
170 mirror="$(cat "$PKGS_DB/mirror")"
171 debug "mirror='$mirror'"
174 # Try converted extra-packages first
175 # FIXME: Workaround to get packages filenames (even better to have names and versions separate)
176 converted="$(wget -O - http://mirror1.slitaz.org/packages/extra/ 2>/dev/null | \
177 tr \'\" $'\n' | grep "$1-.*\.tazpkg$" | sort -u)"
178 debug "converted='$converted'"
179 if [ -n "$converted" ]; then
180 case "$mirror" in
181 http://*|https://*|ftp://*)
182 # Default 'http://mirror.slitaz.org/packages/cooking/'
183 # -> 'http://mirror.slitaz.org/packages/extra/'
184 debug "wget -T 30 -U '$UA' '${mirror%packages/*}packages/extra/$converted'"
185 wget -T 30 -U "$UA" "${mirror%packages/*}packages/extra/$converted";;
186 esac
187 if [ -f "$converted" ]; then
188 echo "$extra_cache/$converted"; exit 0
189 fi
190 fi
193 # Fails to download converted extra-package; now try to download and execute get-script
194 cd ..
195 case "$mirror" in
196 http://*|https://*|ftp://*)
197 # Default 'http://mirror.slitaz.org/packages/cooking/'
198 # -> 'http://mirror.slitaz.org/packages/get/'
199 debug "wget -T 30 -U '$UA' '${mirror%packages/*}packages/get/$1'"
200 wget -T 30 -U "$UA" "${mirror%packages/*}packages/get/$1";;
201 esac
203 if [ ! -f "$1" ]; then
204 # TODO: extra package or extra package script? :) Too complicated...
205 die 'Unable to download extra package "%s".' "$(boldify $1)"
206 fi
208 # Extra-package script downloaded in the /var/cache/tazpkg/extra/
210 unset_receipt
211 PACKAGE="$1"
212 script="$(realpath $1)"
213 tmp_dir="$(mktemp -d)"; cd "$tmp_dir"
214 # Run script
215 set -e
216 . "$script"
217 set +e
219 if [ ! -d "$PACKAGE-$VERSION" ]; then
220 abort_package 'Could not download "%s" from "%s". Exiting.' "${TARBALL:-$PACKAGE}" "${WGET_URL:-$WEB_SITE}"
221 fi
223 if [ ! -s "$PACKAGE-$VERSION/receipt" ]; then
224 # Create receipt (if script not created it early) using variables from script
225 cat > "$PACKAGE-$VERSION/receipt" <<EOT
226 # SliTaz package receipt.
228 PACKAGE="$PACKAGE"
229 VERSION="${VERSION:-unknown}"
230 CATEGORY="${CATEGORY:-non-free}"
231 WEB_SITE="$WEB_SITE"
232 SHORT_DESC="${SHORT_DESC:-$PACKAGE}"
233 MAINTAINER="${MAINTAINER:-nobody@slitaz.org}"
234 LICENSE="$LICENSE"
235 TARBALL="$TARBALL"
236 WGET_URL="$WGET_URL"
237 CONFIG_FILES="$CONFIG_FILES"
238 SUGGESTED="$SUGGESTED"
239 PROVIDE="$PROVIDE"
240 DEPENDS="$DEPENDS"
241 HOST_ARCH="$HOST_ARCH"
242 TAGS="$TAGS"
243 EXTRA_SOURCE_FILES="$EXTRA_SOURCE_FILES"
244 EOT
245 fi
247 # Add dependencies which was found to already defined dependencies
248 DEPENDS="$(unset DEPENDS; . "$PACKAGE-$VERSION/receipt"; echo $DEPENDS)"
249 for i in $(find_depends "$PACKAGE-$VERSION/fs"); do
250 case " $DEPENDS " in
251 *\ $i\ *) continue;;
252 esac
253 sed -i "s/^DEPENDS=\"/&$i /" "$PACKAGE-$VERSION/receipt"
254 done
256 # Remove empty variables
257 sed -i '/=""$/d' "$PACKAGE-$VERSION/receipt"
259 tazpkg pack "$PACKAGE-$VERSION" gzip >&2
261 if [ -z "nocache" ]; then
262 # move package to the extra-cache for "get-install" command
263 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$extra_cache"
264 # Re-make extra packages database
265 tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
266 else
267 # move package to directory where "tazpkg get" command invoked
268 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$CUR_DIR"
269 fi
271 # Clean
272 rm -rf "$tmp_dir"
274 # Function output: path to package
275 echo "$CUR_DIR/$PACKAGE-$VERSION.tazpkg"
276 }
279 # Download package file to the cache
281 get_pkg() {
282 # input: $1 package name either "name" or "name-version"
283 # $2 (internal): empty or 'redo' (when recursive calling)
284 # action: download package from mirror to the cache directory
285 # output: full path to the downloaded package
287 debug "\nget_pkg('$1', '$2')"
288 local repo line namever pkgsum
290 IFS=$'\n'
291 for rep in $PRIORITY; do
292 [ ! -f "$rep/packages.info" ] && continue
293 # If found, output string "<name>-<ver>:<sum>"
294 line=$(awk -F$'\t' -vpkg="$1" \
295 '$1==pkg || $1"-"$2==pkg {printf "%s-%s:%s", $1, $2, $9; exit}' "$rep/packages.info")
296 if [ -n "$line" ]; then
297 namever=${line%:*}; pkgsum=${line#*:}
298 break
299 fi
300 done
301 unset IFS
303 debug " rep='$rep'\n namever='$namever'\n pkgsum='$pkgsum'"
305 if [ -z "$line" ]; then
306 _ 'Unable to find package "%s" in the mirrored packages list.' "$(boldify $1)" >&2
307 # Retry with "get-package"; prevent looping with 'redo'
308 if [ "$2" != 'redo' ]; then
309 get_pkg "get-$1" 'redo'; exit 0
310 else
311 # Retry with extra-package
312 get_pkg_extra "${1#get-}"
313 exit 0
314 fi
315 fi
317 # We need package "$namever.tazpkg" from "$rep" with "$pkgsum"
319 if [ -z "$nocache" ]; then
320 # Go to cache for "get-install" command; don't move for "get" command
321 debug "cd '$(get_cache_path "$rep")'"
322 cd "$(get_cache_path "$rep")"
323 fi
325 # Check if package already downloaded
326 if [ -f "$namever.tazpkg" ]; then
327 [ -z "$nocache" ] && _ 'Package "%s" already in the cache' "$namever" >&2
329 # Check if downloading complete, resume it not complete
330 if ! tail -c 2k "$namever.tazpkg" | fgrep -q '00000000TRAILER'; then
331 _ 'Continuing package "%s" download' "$namever" >&2
332 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
333 fi
334 else
335 # Package absent in the cache, "cold" downloading
336 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
337 fi
339 # Make sure we downloaded what we want with checksum
341 if [ "$($CHECKSUM "$namever.tazpkg" | cut -d' ' -f1)" != "$pkgsum" ]; then
342 _ 'Checksum error for "%s"' "$namever.tazpkg" >&2
343 rm "$namever.tazpkg"
345 # Recharge DBs and try again; prevent looping with 'redo'
346 if [ "$2" != 'redo' ]; then
347 tazpkg recharge >&2
348 get_pkg "$1" 'redo'; exit 0
349 else
350 # Give up
351 # TODO: try another mirror?
352 die 'Please wait until the mirror synchronization is complete and try again.'
353 fi
354 fi
356 # Output: path to downloaded package
357 printf '%s/%s.tazpkg\n' "$(pwd)" "$namever"
358 }
363 if [ -n "$extra" ]; then
364 # When '--extra' option given, extra-package has priority over 'regular' packages
365 get_pkg_extra "$1"
366 else
367 # Try to get 'package', then 'get-package', then extra 'package'
368 get_pkg "$1"
369 fi