tazpkg view modules/get @ rev 846

Remove "busybox" "prefixes" (thanks llev)
We used "busybox wget", etc. to be sure we called Busybox's "wget", not any other "wget". Workaround already done in "getenv" module.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Fri Oct 09 13:14:01 2015 +0300 (2015-10-09)
parents ce7009ff237b
children 8c6346ff1cef
line source
1 #!/bin/sh
2 # TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg
3 # get - TazPkg module
4 # Get package to the cache directory
5
6
7 # 1. "Plain packages" - compiled and packed on cook.slitaz.org using receipts.
8 #
9 # Recently added other type of packages: packages that can't be compiled due to the closed sources
10 # or due to the giant size of that sources. In this case special script: a) downloads package
11 # compiled for other Linux from official web site, b) converts this package to the TazPkg format,
12 # c) installs this package as "plain" package.
13 #
14 # 2. "Get-packages" - package contains only one special script for downloading/converting.
15 # Sometimes get-package can get some specified version of the package, sometimes - different
16 # (latest) version.
17 #
18 # Packages 1. and 2. can be found in the wok repository: http://hg.slitaz.org/wok/
19 #
20 # 3. "Extra" get-scripts. Next addition - only script, without packaging.
21 # Extra get-scripts repository: http://hg.slitaz.org/get-scripts/
22 # Extra get-scripts mirror: http://mirror.slitaz.org/packages/get/
23 #
24 # 4. Converted "extra" packages. Next addition - some packages like LibreOffice / OpenOffice are
25 # big to compile them like _1. "Plain packages"_ as well as big to convert them on the user side
26 # (you need a lot of time, CPU, and RAM) and sometimes it is unable on the weak user machines.
27 # So, some selected and free packages are converted on the SliTaz server side, packaged and
28 # are ready to download by any user: http://mirror.slitaz.org/packages/extra/
29 #
30 #
31 # Algorithm to get package:
32 # - search package in question in the mirrored packages list, download if exists;
33 # - search "get-package" in the mirrored packages list, download if exists;
34 # - search package in the "extra" get-scripts list, download if exists and run it, get the package.
35 #
36 # Note, here doubling. Imagine, you may wanted "palemoon" package.
37 # 1. No "plain package";
38 # 2. Existing "get-palemoon" package;
39 # 3. Existing "palemoon" extra get-script;
40 # 4. Existing "palemoon" extra converted package.
41 # User will get 2. He can specify "--extra" option to get 3.
42 # FIXME: do something with doubling.
43
44
45
46
47 # Connect function libraries
48 . /lib/libtaz.sh
49 . /usr/lib/slitaz/libpkg.sh
50
51 # Get TazPkg working environment
52 . @@MODULES@@/getenv
53
54 . @@MODULES@@/find-depends
55
56
57
58
59 # Get cache directory path
60
61 get_cache_path() {
62 # input: $1 = DB folder such as "$PKGS_DB" or "$PKGS_DB/undigest/*"
63 # $2 (optional): 'extra' for cache for extra-package scripts
64 # output: path to cache directory
65
66 local cache_dir
67
68 if [ "$2" == 'extra' ]; then
69 cache_dir="$SAVE_CACHE_DIR/extra/packages"
70 elif [ "$1" == "$PKGS_DB" ]; then
71 # Main repository
72 cache_dir="$SAVE_CACHE_DIR/$SLITAZ_RELEASE/packages"
73 else
74 # Undigest repository
75 cache_dir="$SAVE_CACHE_DIR/${1##*/}/packages"
76 fi
77
78 # Make cache directory if absent
79 mkdir -p "$cache_dir"
80 chmod a+w "$cache_dir"
81
82 echo "$cache_dir"
83 }
84
85
86 # Download a file from mirror
87
88 download_from() {
89 # input: "<mirror_url>" "<package_name>-<version>.tazpkg"
90
91 debug "\ndownload_from('$1', '$2')"
92
93 case "$1" in
94 # Mirror URL can have a trailing slash or not.
95 http://* | https://* | ftp://*)
96 debug " wget -c -q -T 30 -U $UA ${1%/}/$2"
97 # TODO: display abridged wget status
98 wget -c -q -T 30 -U $UA ${1%/}/$2
99 ;;
100 *)
101 debug " ln -sf ${1%/}/$2 ."
102 ln -sf ${1%/}/$2 .;;
103 esac
104 }
105
106
107 # This function may be called by a get script.
108
109 abort_package() {
110 cd "$CUR_DIR"
111 rm -rf "$tmp_dir"
112 echo "${1:-Abort $PACKAGE.}"
113 exit 1
114 }
115
116
117 # Get extra-package file to the cache
118
119 get_pkg_extra() {
120 # input: $1 extra-package name (like 'dropbox')
121 # $2 (internal): empty or 'redo' (when recursive calling)
122 # action: download stuff and make package
123 # output: full path to created package
124
125 debug "\nget_pkg_extra('$1', '$2')"
126
127 local mirror extra_cache converted tmp_dir script
128
129 # Check extra.list
130 if [ ! -s "$PKGS_DB/extra.list" ]; then
131 # Empty extra.list
132 if [ "$2" != 'redo' ]; then
133 tazpkg recharge >&2
134 get_pkg_extra "$1" 'redo'; exit 0
135 else
136 # Give up
137 _ 'File "%s" empty.' 'extra.list' >&2
138 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
139 fi
140 fi
141
142 # Extra.list exists and not empty
143 if ! grep -q "^$1|" "$PKGS_DB/extra.list"; then
144 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
145 fi
146
147 # Extra-package found in extra.list
148
149 if [ -z "$nocache" ]; then
150 # Go to cache for "get-install" command; don't move for "get" command
151 extra_cache="$(get_cache_path "$PKGS_DB" 'extra')"
152 debug "cd '$extra_cache'"
153 cd "$extra_cache"
154
155 # Extra-cache should contain packages DB files (packages.info, etc.)
156 # to list extra-packages contained in the extra-cache
157 [ ! -f 'packages.info' ] && tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
158
159 if [ -f 'packages.info' ]; then
160 awk -F$'\t' -vp="$1" '$1==p{exit 1}' packages.info
161 if [ "$?" -eq 1 ]; then
162 _ 'Package "%s" already in the cache' "$1" >&2
163 echo -n "$(pwd)/"
164 awk -F$'\t' -vp="$1" '$1==p{printf "%s-%s.tazpkg\n", $1, $2; exit}' packages.info
165 exit 0
166 fi
167 fi
168 fi
169
170 mirror="$(cat "$PKGS_DB/mirror")"
171 debug "mirror='$mirror'"
172
173
174 # Try converted extra-packages first
175 # FIXME: Workaround to get packages filenames (even better to have names and versions separate)
176 converted="$(wget -O - http://mirror1.slitaz.org/packages/extra/ 2>/dev/null | \
177 tr \'\" $'\n' | grep "$1-.*\.tazpkg$" | sort -u)"
178 debug "converted='$converted'"
179 if [ -n "$converted" ]; then
180 case "$mirror" in
181 http://*|https://*|ftp://*)
182 # Default 'http://mirror.slitaz.org/packages/cooking/'
183 # -> 'http://mirror.slitaz.org/packages/extra/'
184 debug "wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/extra/$converted'"
185 wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/extra/$converted";;
186 esac
187 if [ -f "$converted" ]; then
188 echo "$extra_cache/$converted"; exit 0
189 fi
190 fi
191
192
193 # Fails to download converted extra-package; now try to download and execute get-script
194 cd ..
195 case "$mirror" in
196 http://*|https://*|ftp://*)
197 # Default 'http://mirror.slitaz.org/packages/cooking/'
198 # -> 'http://mirror.slitaz.org/packages/get/'
199 debug "wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/get/$1'"
200 wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/get/$1";;
201 esac
202
203 if [ ! -f "$1" ]; then
204 # TODO: extra package or extra package script? :) Too complicated...
205 die 'Unable to download extra package "%s".' "$(boldify $1)"
206 fi
207
208 # Extra-package script downloaded in the /var/cache/tazpkg/extra/
209
210 unset_receipt
211 PACKAGE="$1"
212 script="$(realpath $1)"
213 tmp_dir="$(mktemp -d)"; cd "$tmp_dir"
214 # Run script
215 set -e
216 . "$script"
217 set +e
218
219 if [ ! -d "$PACKAGE-$VERSION" ]; then
220 abort_package 'Could not download "%s" from "%s". Exiting.' "${TARBALL:-$PACKAGE}" "${WGET_URL:-$WEB_SITE}"
221 fi
222
223 if [ ! -s "$PACKAGE-$VERSION/receipt" ]; then
224 # Create receipt (if script not created it early) using variables from script
225 cat > "$PACKAGE-$VERSION/receipt" <<EOT
226 # SliTaz package receipt.
227
228 PACKAGE="$PACKAGE"
229 VERSION="${VERSION:-unknown}"
230 CATEGORY="${CATEGORY:-non-free}"
231 WEB_SITE="$WEB_SITE"
232 SHORT_DESC="${SHORT_DESC:-$PACKAGE}"
233 MAINTAINER="${MAINTAINER:-nobody@slitaz.org}"
234 LICENSE="$LICENSE"
235 TARBALL="$TARBALL"
236 WGET_URL="$WGET_URL"
237 CONFIG_FILES="$CONFIG_FILES"
238 SUGGESTED="$SUGGESTED"
239 PROVIDE="$PROVIDE"
240 DEPENDS="$DEPENDS"
241 HOST_ARCH="$HOST_ARCH"
242 TAGS="$TAGS"
243 EXTRA_SOURCE_FILES="$EXTRA_SOURCE_FILES"
244 EOT
245 fi
246
247 # Add dependencies which was found to already defined dependencies
248 DEPENDS="$(unset DEPENDS; . "$PACKAGE-$VERSION/receipt"; echo $DEPENDS)"
249 for i in $(find_depends "$PACKAGE-$VERSION/fs"); do
250 case " $DEPENDS " in
251 *\ $i\ *) continue;;
252 esac
253 sed -i "s/^DEPENDS=\"/&$i /" "$PACKAGE-$VERSION/receipt"
254 done
255
256 # Remove empty variables
257 sed -i '/=""$/d' "$PACKAGE-$VERSION/receipt"
258
259 tazpkg pack "$PACKAGE-$VERSION" gzip >&2
260
261 if [ -z "nocache" ]; then
262 # move package to the extra-cache for "get-install" command
263 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$extra_cache"
264 # Re-make extra packages database
265 tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
266 else
267 # move package to directory where "tazpkg get" command invoked
268 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$CUR_DIR"
269 fi
270
271 # Clean
272 rm -rf "$tmp_dir"
273
274 # Function output: path to package
275 echo "$CUR_DIR/$PACKAGE-$VERSION.tazpkg"
276 }
277
278
279 # Download package file to the cache
280
281 get_pkg() {
282 # input: $1 package name either "name" or "name-version"
283 # $2 (internal): empty or 'redo' (when recursive calling)
284 # action: download package from mirror to the cache directory
285 # output: full path to the downloaded package
286
287 debug "\nget_pkg('$1', '$2')"
288 local repo line namever pkgsum
289
290 IFS=$'\n'
291 for rep in $PRIORITY; do
292 [ ! -f "$rep/packages.info" ] && continue
293 # If found, output string "<name>-<ver>:<sum>"
294 line=$(awk -F$'\t' -vpkg="$1" \
295 '$1==pkg || $1"-"$2==pkg {printf "%s-%s:%s", $1, $2, $9; exit}' "$rep/packages.info")
296 if [ -n "$line" ]; then
297 namever=${line%:*}; pkgsum=${line#*:}
298 break
299 fi
300 done
301 unset IFS
302
303 debug " rep='$rep'\n namever='$namever'\n pkgsum='$pkgsum'"
304
305 if [ -z "$line" ]; then
306 _ 'Unable to find package "%s" in the mirrored packages list.' "$(boldify $1)" >&2
307 # Retry with "get-package"; prevent looping with 'redo'
308 if [ "$2" != 'redo' ]; then
309 get_pkg "get-$1" 'redo'; exit 0
310 else
311 # Retry with extra-package
312 get_pkg_extra "${1#get-}"
313 exit 0
314 fi
315 fi
316
317 # We need package "$namever.tazpkg" from "$rep" with "$pkgsum"
318
319 if [ -z "$nocache" ]; then
320 # Go to cache for "get-install" command; don't move for "get" command
321 debug "cd '$(get_cache_path "$rep")'"
322 cd "$(get_cache_path "$rep")"
323 fi
324
325 # Check if package already downloaded
326 if [ -f "$namever.tazpkg" ]; then
327 [ -z "$nocache" ] && _ 'Package "%s" already in the cache' "$namever" >&2
328
329 # Check if downloading complete, resume it not complete
330 if ! tail -c 2k "$namever.tazpkg" | fgrep -q '00000000TRAILER'; then
331 _ 'Continuing package "%s" download' "$namever" >&2
332 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
333 fi
334 else
335 # Package absent in the cache, "cold" downloading
336 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
337 fi
338
339 # Make sure we downloaded what we want with checksum
340
341 if [ "$($CHECKSUM "$namever.tazpkg" | cut -d' ' -f1)" != "$pkgsum" ]; then
342 _ 'Checksum error for "%s"' "$namever.tazpkg" >&2
343 rm "$namever.tazpkg"
344
345 # Recharge DBs and try again; prevent looping with 'redo'
346 if [ "$2" != 'redo' ]; then
347 tazpkg recharge >&2
348 get_pkg "$1" 'redo'; exit 0
349 else
350 # Give up
351 # TODO: try another mirror?
352 die 'Please wait until the mirror synchronization is complete and try again.'
353 fi
354 fi
355
356 # Output: path to downloaded package
357 printf '%s/%s.tazpkg\n' "$(pwd)" "$namever"
358 }
359
360
361
362
363 if [ -n "$extra" ]; then
364 # When '--extra' option given, extra-package has priority over 'regular' packages
365 get_pkg_extra "$1"
366 else
367 # Try to get 'package', then 'get-package', then extra 'package'
368 get_pkg "$1"
369 fi