cookutils view modules/compressor @ rev 872

modules/compressor: recompress *.gz files
author Aleksej Bobylev <al.bobylev@gmail.com>
date Thu Jan 26 18:10:16 2017 +0200 (2017-01-26)
parents 088359ca2353
children f816ea3e8dcc
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
6
7 . /usr/lib/slitaz/libcook.sh
8
9
10 # Compressor cache stuff
11
12 comp_cache_root='/var/cache/cook'
13 mkdir -p "$comp_cache_root"
14 cache_stat=$(mktemp)
15
16 # Cache notes.
17 # Do not do the same job twice. Getting the file from the cache is much faster
18 # than compressing the file one more time. In addition, this cache is trying not
19 # to take extra space, using the hardlinks. Although the files from the cache
20 # without reference to itself should be removed periodically.
21
22
23
24
25 #
26 # Functions
27 #
28
29
30 # Display time.
31
32 disp_time() {
33 div=$(( ($1 + 30) / 60))
34 case $div in
35 0) min='';;
36 # L10n: 'm' is for minutes (approximate time)
37 *) min=$(_n ' ~ %dm' "$div");;
38 esac
39
40 # L10n: 's' is for seconds (cooking time)
41 _ '%ds%s' "$1" "$min"
42 }
43
44
45 # Compressor mini summary
46
47 comp_summary() {
48 # "$time0" "$size0" "$size1"
49 time1=$(date +%s)
50 status
51 [ "$2" -eq 0 ] && return
52 time=$(($time1 - $1))
53 saving=$(( ($2 - $3) / 1024 ))
54 cache_msg=''
55 if [ -s "$cache_stat" ]; then
56 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
57 echo -n > $cache_stat
58 fi
59 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
60 "$(disp_time $time)" "$2" "$3" "$saving" "$cache_msg"
61 }
62
63
64 # Calculating different sizes
65
66 sizes() {
67 case $1 in
68 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
69 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
70 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
71 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
72 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
73 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
74 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
75 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
76 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
77 esac | awk '{s+=$5}END{print s}'
78 }
79
80
81 # Query cache for already existing compressed file; substitute original file with the cached
82 # compressed one if so.
83 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
84
85 query_cache() {
86 md5=$(md5sum "$2")
87 cachefile="$comp_cache_root/$1/${md5%% *}"
88 echo "$cachefile"
89 if [ -f "$cachefile" ]; then
90 ln -f "$cachefile" "$2"
91 echo '+' >> "$cache_stat"
92 else
93 echo '-' >> "$cache_stat"
94 false
95 fi
96 }
97
98
99 # Store compressed file to the cache
100 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
101
102 store_cache() {
103 mkdir -p "${1%/*}"
104 mv "$2" "$1"
105 ln "$1" "$2"
106 }
107
108
109 # Function to compress all man pages
110 # Compressing can be disabled with COOKOPTS="!manz"
111
112 compress_manpages() {
113 time0=$(date +%s)
114 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
115 manpath="$install/usr/share/man"
116 [ -d "$manpath" ] || return
117 size0=$(sizes man); [ -z "$size0" ] && return
118
119 tazpkg -gi advancecomp --quiet --cookmode
120
121 action 'Compressing man pages...'
122
123 # We'll use only Gzip compression, so decompress other formats first
124 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
125 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
126
127 # Fast compress with gzip
128 find $manpath -type f -name '*.[1-9]*' -exec gzip \{\} \;
129
130 # Fix symlinks
131 for i in $(find $install/usr/share/man -type l); do
132 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
133 link=$(echo $i | sed 's|\.[gbx]z2*$||')
134 rm $i; ln -s $dest.gz $link.gz
135 done
136
137 # Recompress with advdef (it can't compress, only recompress)
138 for i in $(find $install/usr/share/man -type f); do
139 if ! cached_path=$(query_cache mangz "$i"); then
140 advdef -z4q "$i"
141 store_cache "$cached_path" "$i"
142 fi
143 done
144
145 comp_summary "$time0" "$size0" "$(sizes man)"
146 }
147
148
149 # Function to recompress all gzip archives
150 # Recompressing can be disabled with COOKOPTS="!gz"
151
152 recompress_gz() {
153 time0=$(date +%s)
154 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
155 size0=$(sizes gz); [ -z "$size0" ] && return
156
157 tazpkg -gi advancecomp --quiet --cookmode
158
159 action 'Recompressing gzip files...'
160
161 # Recompress with advdef
162 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
163 if ! cached_path=$(query_cache gz "$i"); then
164 advdef -z4q "$i"
165 store_cache "$cached_path" "$i"
166 fi
167 done
168
169 comp_summary "$time0" "$size0" "$(sizes gz)"
170 }
171
172
173 # Function used after compile_rules() to compress all png images
174 # Compressing can be disabled with COOKOPTS="!pngz"
175
176 compress_png() {
177 time0=$(date +%s)
178 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
179 size0=$(sizes png); [ -z "$size0" ] && return
180
181 use_pq=true
182 use_op=true
183 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
184 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
185 $use_pq && tazpkg -gi pngquant --quiet --cookmode
186 $use_op && tazpkg -gi optipng --quiet --cookmode
187
188 action 'Compressing png images...'
189
190 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
191 [ -z "$oplevel" ] && oplevel='2'
192
193 cache_section="png$oplevel"
194 $use_pq && cache_section="${cache_section}p"
195 $use_op && cache_section="${cache_section}o"
196
197 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
198
199 for i in $(find $install -type f -name '*.png'); do
200 if ! cached_path=$(query_cache $cache_section "$i"); then
201 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"
202 $use_op && optipng -quiet -strip all -o$oplevel "$i"
203 store_cache "$cached_path" "$i"
204 fi
205 done
206
207 comp_summary "$time0" "$size0" "$(sizes png)"
208 }
209
210
211 # Function used after compile_rules() to compress all svg images
212 # Compressing can be disabled with COOKOPTS="!svgz"
213
214 compress_svg() {
215 time0=$(date +%s)
216 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
217 size0=$(sizes svg); [ -z "$size0" ] && return
218
219 tazpkg -gi svgcleaner --quiet --cookmode
220
221 action 'Compressing svg images...'
222
223 cleaner_log="$(mktemp)"
224 for i in $(find $install -type f -name '*.svg'); do
225 echo -n "$i: " >> "$cleaner_log"
226 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"
227 done
228
229 comp_summary "$time0" "$size0" "$(sizes svg)"
230
231 sed -i '/: $/d' "$cleaner_log"
232 if [ -s "$cleaner_log" ]; then
233 _ 'Cleaner warnings and errors:'
234 awk '{printf " %s\n", $0;}' "$cleaner_log"
235 echo
236 fi
237 rm "$cleaner_log"
238 }
239
240
241 # Function used after compile_rules() to shrink all *.ui and *.glade files:
242 # remove insignificant spaces and comments
243 # Compressing can be disabled with COOKOPTS="!uiz"
244
245 compress_ui() {
246 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
247 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
248
249 tazpkg -gi xmlstarlet --quiet --cookmode
250
251 action 'Compressing ui files...'
252
253 size0=$(sizes xml)
254 time0=$(date +%s)
255 temp_ui="$(mktemp)"
256 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
257 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"
258 cat "$temp_ui" > "$ui"
259 done
260
261 comp_summary "$time0" "$size0" "$(sizes xml)"
262 rm "$temp_ui"
263 }
264
265
266 # Get list of supported locales...
267
268 get_supported_locales() {
269 lpc='/slitaz-i18n/stuff/locale-pack.conf'
270 if [ -e "$WOK$lpc" ]; then
271 # ... from package in the local wok
272 . "$WOK$lpc"
273 else
274 # ... from Hg
275 temp_conf=$(mktemp)
276 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
277 if [ -s $temp_conf ]; then
278 . $temp_conf
279 else
280 # Give up and use hardcoded list
281 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
282 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
283 fi
284 rm $temp_conf
285 fi
286 echo $LOCALE_PACK
287 }
288
289
290 # Fix common errors and warnings in the .desktop files
291 # Fixing can be disabled with COOKOPTS="!fixdesktops"
292
293 fix_desktop_files() {
294 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
295 [ -z "$(find $install -type f -name '*.desktop')" ] && return
296
297 size0=$(sizes des)
298 time0=$(date +%s)
299
300 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
301 tazpkg -gi desktop-file-utils-extra --quiet --cookmode
302 fi
303
304 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
305 # Default value is "" (empty). That means for us that we'll use the full
306 # list of supported locales here.
307 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
308
309 for desktop in $(find $install -type f -name '*.desktop'); do
310 cp "$desktop" "$desktop.orig"
311
312 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
313 sdft "$desktop" -i
314
315 # Fix common errors in .desktop file
316 fix-desktop-file "$desktop"
317
318 # Strip unsupported locales from .desktop file
319 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
320 sdft "$desktop" -i -k "$LOCALE"
321
322 # Extra-strip
323 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
324 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
325
326 if [ -n "$QA" ]; then
327 # Check the rest of errors, warnings and tips
328 _ 'QA: Checking %s...' "$(basename $desktop)"
329 diff "$desktop.orig" "$desktop"
330 desktop-file-validate "$desktop" | busybox fold -s
331 echo
332 fi
333
334 rm "$desktop.orig"
335 done
336
337 comp_summary "$time0" "$size0" "$(sizes des)"
338 }
339
340
341 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really added
342 # to the translation (msgid = msgstr)
343 # Normalization can be disabled with COOKOPTS="!monorm"
344
345 normalize_mo() {
346 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
347 [ -z "$(find $install -type f -name '*.mo')" ] && return
348
349 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
350 tazpkg -gi gettext --quiet --cookmode
351 # Gconv modules (convert to UTF-8)
352 tazpkg -gi glibc-locale --quiet --cookmode
353
354 action 'Normalizing mo files...'
355
356 size0=$(sizes mo1)
357 time0=$(date +%s)
358
359 # Process all existing *.mo files
360 for mo in $(find "$install" -type f -name '*.mo'); do
361 tmpfile="$(mktemp)"
362
363 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'
364 # add newline
365 echo >> "$tmpfile"
366
367 # get Plural-Forms
368 awk '
369 BEGIN { skip = ""; }
370 {
371 if (! skip) {
372 s = $0;
373 gsub(/^[^\"]*\"/, "", s);
374 gsub(/\"$/, "", s);
375 printf("%s", s);
376 }
377 if (! $0) skip = "yes";
378 }
379 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
380
381 if ! grep -q 'msgid_plural' "$tmpfile"; then
382 echo > "$tmpfile.pf"
383 fi
384
385 # main
386 awk -v pf="$(cat "$tmpfile.pf")" '
387 function clean() {
388 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
389 }
390
391 function getstring() {
392 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
393 s = $0;
394 gsub(/^[^\"]*\"/, "", s);
395 gsub(/\"$/, "", s);
396 return s;
397 }
398
399 BEGIN {
400 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
401 if (pf)
402 printf("\"%s\\n\"\n", pf);
403 printf("\n");
404 skip = 1;
405 clean();
406 }
407
408 {
409 # Skip the entire header
410 if (!skip) {
411 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
412 mode = $1;
413 if ($1 == "msgstr[0]") mode = "msgstr0";
414 if ($1 == "msgstr[1]") mode = "msgstr1";
415 if ($1 == "msgstr[2]") mode = "msgstr2";
416 if ($1 == "msgstr[3]") mode = "msgstr3";
417 if ($1 == "msgstr[4]") mode = "msgstr4";
418 if ($1 == "msgstr[5]") mode = "msgstr5";
419
420 if (mode == "msgctxt") msgctxt = msgctxt getstring();
421 if (mode == "msgid") msgid = msgid getstring();
422 if (mode == "msgstr") msgstr = msgstr getstring();
423 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
424 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
425 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
426 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
427 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
428 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
429 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
430
431 if (! $0) {
432 if (msgid != msgstr) {
433 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
434 printf("msgid \"%s\"\n", msgid);
435 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
436 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
437 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
438 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
439 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
440 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
441 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
442 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
443 printf("\n");
444 }
445 clean();
446 }
447 }
448 if ($0 == "") skip = "";
449 }
450 ' "$tmpfile" > "$tmpfile.awk"
451
452 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"
453
454 if [ -s "$tmpfile.mo" ]; then
455 rm "$mo"; mv "$tmpfile.mo" "$mo"
456 else
457 _ 'Error processing %s' "$mo"
458 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
459 fi
460
461 # Clean
462 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
463 done
464
465 comp_summary "$time0" "$size0" "$(sizes mo1)"
466 }
467
468
469 # Strip locale definitions: normalize whitespace and remove comments
470 # Stripping can be disabled with COOKOPTS="!locdef"
471
472 strip_locale_def() {
473 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
474 [ ! -d "$install/usr/share/i18n/locales" ] && return
475 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
476
477 action 'Stripping locale definitions...'
478 size0=$(sizes loc)
479 time0=$(date +%s)
480
481 for i in $(find $install/usr/share/i18n/locales -type f); do
482 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
483 done
484
485 comp_summary "$time0" "$size0" "$(sizes loc)"
486 }
487
488
489 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
490 # as removing unneeded files like in Python packages. Cross compiled binaries
491 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
492 # Stripping can be disabled with COOKOPTS="!strip"
493
494 strip_package() {
495 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
496
497 case "$ARCH" in
498 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
499 *) export STRIP='strip' ;;
500 esac
501 action 'Executing strip on all files...'
502 size0=0
503 size1=0
504 time0=$(date +%s)
505
506 # Strip executable files
507 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
508 if [ -d "$dir" ]; then
509 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
510 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
511 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
512 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
513 fi
514 done
515
516 # Strip shared and static libraries
517 # Remove Python *.pyc and *.pyo, Perl perllocal.pod and .packlist
518 oldsize=$(find $fs -type f \( \
519 -name '*.so*' -o -name '*.a' -o \
520 -name '*.pyc' -o -name '*.pyo' -o \
521 -name 'perllocal.pod' -o -name '.packlist' \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
522
523 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
524 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;
525 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
526 find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \) -delete 2>/dev/null
527
528 newsize=$(find $fs -type f \( \
529 -name '*.so*' -o -name '*.a' -o \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
530
531 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
532 }
533
534
535 # Strip unsupported locales (.mo files)
536
537 strip_mo_i18n() {
538 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
539
540 [ ! -d "$fs/usr/share/locale" ] && return
541 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
542
543 action 'Thin out translation files...'
544 size0=$(sizes mo2)
545 time0=$(date +%s)
546
547 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
548 # Default value is "" (empty). That means for us that we'll use the full
549 # list of supported locales here.
550 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
551
552 # Existing locales
553 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
554
555 # Thin out the list of existing locales. At the end there will be only locales that need
556 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
557 for keep_locale in $LOCALE; do
558 elocales=${elocales//$keep_locale}
559 done
560
561 # Remove the unsupported locales
562 for rem_locale in $elocales; do
563 [ -d "$fs/usr/share/locale/$rem_locale" ] &&
564 rm -r "$fs/usr/share/locale/$rem_locale"
565 done
566
567 comp_summary "$time0" "$size0" "$(sizes mo2)"
568 }
569
570
571
572
573 case $1 in
574 install)
575 # Compressors working in the $install
576 case "$ARCH" in
577 arm*) ;;
578 *)
579 recompress_gz
580 compress_manpages
581 compress_png
582 compress_svg
583 compress_ui
584 ;;
585 esac
586
587 fix_desktop_files
588 normalize_mo
589 strip_locale_def
590 ;;
591 fs)
592 # Compressors working in the $fs
593 strip_package
594 strip_mo_i18n
595 ;;
596 esac
597
598 # Clean
599 rm "$cache_stat"
600 find $comp_cache_root -type f -links -2 -delete