cookutils diff modules/compressor @ rev 865
Split modules/compressor (~500 lines of code) from cook.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Wed Jan 11 11:53:04 2017 +0200 (2017-01-11) |
| parents | |
| children | 5f6be706ab4f |
line diff
1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 1.2 +++ b/modules/compressor Wed Jan 11 11:53:04 2017 +0200 1.3 @@ -0,0 +1,506 @@ 1.4 +#!/bin/sh 1.5 +# 1.6 +# compressor - module of the SliTaz Cook 1.7 +# Copyright (C) SliTaz GNU/Linux - GNU GPL v3 1.8 +# 1.9 + 1.10 +. /usr/lib/slitaz/libcook.sh 1.11 + 1.12 + 1.13 +# 1.14 +# Functions 1.15 +# 1.16 + 1.17 + 1.18 +# Display time. 1.19 + 1.20 +disp_time() { 1.21 + div=$(( ($1 + 30) / 60)) 1.22 + case $div in 1.23 + 0) min='';; 1.24 + # L10n: 'm' is for minutes (approximate time) 1.25 + *) min=$(_n ' ~ %dm' "$div");; 1.26 + esac 1.27 + 1.28 + # L10n: 's' is for seconds (cooking time) 1.29 + _ '%ds%s' "$1" "$min" 1.30 +} 1.31 + 1.32 + 1.33 +# Compressor mini summary 1.34 + 1.35 +comp_summary() { 1.36 + # "$time0" "$size0" "$size1" 1.37 + time1=$(date +%s) 1.38 + status 1.39 + [ "$2" -eq 0 ] && return 1.40 + time=$(($time1 - $1)) 1.41 + saving=$(( ($2 - $3) / 1024 )) 1.42 + _ ' Time: %s. Size: %s B -> %s B. Save: %s KB' "$(disp_time $time)" "$2" "$3" "$saving" 1.43 +} 1.44 + 1.45 + 1.46 +# Calculating different sizes 1.47 + 1.48 +sizes() { 1.49 + case $1 in 1.50 + man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;; 1.51 + png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;; 1.52 + svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;; 1.53 + xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;; 1.54 + des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;; 1.55 + mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;; 1.56 + loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;; 1.57 + mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;; 1.58 + esac | awk '{s+=$5}END{print s}' 1.59 +} 1.60 + 1.61 + 1.62 +# Function to compress all man pages 1.63 +# Compressing can be disabled with COOKOPTS="!manz" 1.64 + 1.65 +compress_manpages() { 1.66 + time0=$(date +%s) 1.67 + [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return 1.68 + manpath="$install/usr/share/man" 1.69 + [ -d "$manpath" ] || return 1.70 + size0=$(sizes man); [ -z "$size0" ] && return 1.71 + 1.72 + action 'Compressing man pages...' 1.73 + 1.74 + # We'll use only Gzip compression, so decompress other formats first 1.75 + find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \; 1.76 + find $manpath -type f -name '*.xz' -exec unxz \{\} \; 1.77 + 1.78 + # Fast compress with gzip 1.79 + find $manpath -type f -name '*.[1-9]*' -exec gzip \{\} \; 1.80 + 1.81 + # Fix symlinks 1.82 + for i in $(find $install/usr/share/man -type l); do 1.83 + dest=$(readlink $i | sed 's|\.[gbx]z2*$||') 1.84 + link=$(echo $i | sed 's|\.[gbx]z2*$||') 1.85 + rm $i; ln -s $dest.gz $link.gz 1.86 + done 1.87 + 1.88 + # Recompress with advdef (it can't compress, only recompress) 1.89 + tazpkg -gi advancecomp --quiet 1.90 + for i in $(find $install/usr/share/man -type f); do 1.91 + advdef -z4q $i 1.92 + done 1.93 + 1.94 + comp_summary "$time0" "$size0" "$(sizes man)" 1.95 +} 1.96 + 1.97 + 1.98 +# Function used after compile_rules() to compress all png images 1.99 +# Compressing can be disabled with COOKOPTS="!pngz" 1.100 + 1.101 +compress_png() { 1.102 + time0=$(date +%s) 1.103 + [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return 1.104 + size0=$(sizes png); [ -z "$size0" ] && return 1.105 + 1.106 + action 'Compressing png images...' 1.107 + 1.108 + use_pq=true 1.109 + use_op=true 1.110 + [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false 1.111 + [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false 1.112 + 1.113 + $use_pq && tazpkg -gi pngquant --quiet 1.114 + $use_op && tazpkg -gi optipng --quiet 1.115 + 1.116 + oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|') 1.117 + [ -z "$oplevel" ] && oplevel='2' 1.118 + [ "$oplevel" == '8' ] && oplevel='7 -zm1-9' 1.119 + 1.120 + for i in $(find $install -type f -name '*.png'); do 1.121 + $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i" 1.122 + $use_op && optipng -quiet -strip all -o$oplevel "$i" 1.123 + done 1.124 + 1.125 + comp_summary "$time0" "$size0" "$(sizes png)" 1.126 +} 1.127 + 1.128 + 1.129 +# Function used after compile_rules() to compress all svg images 1.130 +# Compressing can be disabled with COOKOPTS="!svgz" 1.131 + 1.132 +compress_svg() { 1.133 + time0=$(date +%s) 1.134 + [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return 1.135 + size0=$(sizes svg); [ -z "$size0" ] && return 1.136 + 1.137 + action 'Compressing svg images...' 1.138 + 1.139 + tazpkg -gi svgcleaner --quiet 1.140 + cleaner_log="$(mktemp)" 1.141 + for i in $(find $install -type f -name '*.svg'); do 1.142 + echo -n "$i: " >> "$cleaner_log" 1.143 + svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log" 1.144 + done 1.145 + 1.146 + comp_summary "$time0" "$size0" "$(sizes svg)" 1.147 + 1.148 + sed -i '/: $/d' "$cleaner_log" 1.149 + if [ -s "$cleaner_log" ]; then 1.150 + _ 'Cleaner warnings and errors:' 1.151 + awk '{printf " %s\n", $0;}' "$cleaner_log" 1.152 + echo 1.153 + fi 1.154 + rm "$cleaner_log" 1.155 +} 1.156 + 1.157 + 1.158 +# Function used after compile_rules() to shrink all *.ui and *.glade files: 1.159 +# remove insignificant spaces and comments 1.160 +# Compressing can be disabled with COOKOPTS="!uiz" 1.161 + 1.162 +compress_ui() { 1.163 + [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return 1.164 + [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return 1.165 + 1.166 + action 'Compressing ui files...' 1.167 + size0=$(sizes xml) 1.168 + time0=$(date +%s) 1.169 + tazpkg -gi xmlstarlet --quiet 1.170 + temp_ui="$(mktemp)" 1.171 + for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do 1.172 + xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui" 1.173 + cat "$temp_ui" > "$ui" 1.174 + done 1.175 + 1.176 + comp_summary "$time0" "$size0" "$(sizes xml)" 1.177 + rm "$temp_ui" 1.178 +} 1.179 + 1.180 + 1.181 +# Get list of supported locales... 1.182 + 1.183 +get_supported_locales() { 1.184 + lpc='/slitaz-i18n/stuff/locale-pack.conf' 1.185 + if [ -e "$WOK$lpc" ]; then 1.186 + # ... from package in the local wok 1.187 + . "$WOK$lpc" 1.188 + else 1.189 + # ... from Hg 1.190 + temp_conf=$(mktemp) 1.191 + wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc" 1.192 + if [ -s $temp_conf ]; then 1.193 + . $temp_conf 1.194 + else 1.195 + # Give up and use hardcoded list 1.196 + LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \ 1.197 + pt_BR ro ru sl sv tr uk zh_CN zh_TW" 1.198 + fi 1.199 + rm $temp_conf 1.200 + fi 1.201 + echo $LOCALE_PACK 1.202 +} 1.203 + 1.204 + 1.205 +# Fix common errors and warnings in the .desktop files 1.206 +# Fixing can be disabled with COOKOPTS="!fixdesktops" 1.207 + 1.208 +fix_desktop_files() { 1.209 + [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return 1.210 + [ -z "$(find $install -type f -name '*.desktop')" ] && return 1.211 + 1.212 + size0=$(sizes des) 1.213 + time0=$(date +%s) 1.214 + 1.215 + if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then 1.216 + tazpkg -gi desktop-file-utils-extra --quiet 1.217 + fi 1.218 + 1.219 + # The variable $LOCALE is set in cook.conf and may be overridden in the receipt. 1.220 + # Default value is "" (empty). That means for us that we'll use the full 1.221 + # list of supported locales here. 1.222 + [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales) 1.223 + 1.224 + for desktop in $(find $install -type f -name '*.desktop'); do 1.225 + cp "$desktop" "$desktop.orig" 1.226 + 1.227 + # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`) 1.228 + sdft "$desktop" -i 1.229 + 1.230 + # Fix common errors in .desktop file 1.231 + fix-desktop-file "$desktop" 1.232 + 1.233 + # Strip unsupported locales from .desktop file 1.234 + [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] && 1.235 + sdft "$desktop" -i -k "$LOCALE" 1.236 + 1.237 + # Extra-strip 1.238 + [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] && 1.239 + sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o 1.240 + 1.241 + if [ -n "$QA" ]; then 1.242 + # Check the rest of errors, warnings and tips 1.243 + _ 'QA: Checking %s...' "$(basename $desktop)" 1.244 + diff "$desktop.orig" "$desktop" 1.245 + desktop-file-validate "$desktop" | busybox fold -s 1.246 + echo 1.247 + fi 1.248 + 1.249 + rm "$desktop.orig" 1.250 + done 1.251 + 1.252 + comp_summary "$time0" "$size0" "$(sizes des)" 1.253 +} 1.254 + 1.255 + 1.256 +# Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really added 1.257 +# to the translation (msgid = msgstr) 1.258 +# Normalization can be disabled with COOKOPTS="!monorm" 1.259 + 1.260 +normalize_mo() { 1.261 + [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return 1.262 + [ -z "$(find $install -type f -name '*.mo')" ] && return 1.263 + 1.264 + action 'Normalizing mo files...' 1.265 + size0=$(sizes mo1) 1.266 + time0=$(date +%s) 1.267 + 1.268 + # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt 1.269 + tazpkg -gi gettext --quiet 1.270 + # Gconv modules (convert to UTF-8) 1.271 + tazpkg -gi glibc-locale --quiet 1.272 + 1.273 + # Process all existing *.mo files 1.274 + for mo in $(find "$install" -type f -name '*.mo'); do 1.275 + tmpfile="$(mktemp)" 1.276 + 1.277 + msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8' 1.278 + # add newline 1.279 + echo >> "$tmpfile" 1.280 + 1.281 + # get Plural-Forms 1.282 + awk ' 1.283 + BEGIN { skip = ""; } 1.284 + { 1.285 + if (! skip) { 1.286 + s = $0; 1.287 + gsub(/^[^\"]*\"/, "", s); 1.288 + gsub(/\"$/, "", s); 1.289 + printf("%s", s); 1.290 + } 1.291 + if (! $0) skip = "yes"; 1.292 + } 1.293 + ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf" 1.294 + 1.295 + if ! grep -q 'msgid_plural' "$tmpfile"; then 1.296 + echo > "$tmpfile.pf" 1.297 + fi 1.298 + 1.299 + # main 1.300 + awk -v pf="$(cat "$tmpfile.pf")" ' 1.301 + function clean() { 1.302 + mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = ""; 1.303 + } 1.304 + 1.305 + function getstring() { 1.306 + # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes 1.307 + s = $0; 1.308 + gsub(/^[^\"]*\"/, "", s); 1.309 + gsub(/\"$/, "", s); 1.310 + return s; 1.311 + } 1.312 + 1.313 + BEGIN { 1.314 + printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n"); 1.315 + if (pf) 1.316 + printf("\"%s\\n\"\n", pf); 1.317 + printf("\n"); 1.318 + skip = 1; 1.319 + clean(); 1.320 + } 1.321 + 1.322 + { 1.323 + # Skip the entire header 1.324 + if (!skip) { 1.325 + if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural") 1.326 + mode = $1; 1.327 + if ($1 == "msgstr[0]") mode = "msgstr0"; 1.328 + if ($1 == "msgstr[1]") mode = "msgstr1"; 1.329 + if ($1 == "msgstr[2]") mode = "msgstr2"; 1.330 + if ($1 == "msgstr[3]") mode = "msgstr3"; 1.331 + if ($1 == "msgstr[4]") mode = "msgstr4"; 1.332 + if ($1 == "msgstr[5]") mode = "msgstr5"; 1.333 + 1.334 + if (mode == "msgctxt") msgctxt = msgctxt getstring(); 1.335 + if (mode == "msgid") msgid = msgid getstring(); 1.336 + if (mode == "msgstr") msgstr = msgstr getstring(); 1.337 + if (mode == "msgid_plural") msgid_plural = msgid_plural getstring(); 1.338 + if (mode == "msgstr0") msgstr0 = msgstr0 getstring(); 1.339 + if (mode == "msgstr1") msgstr1 = msgstr1 getstring(); 1.340 + if (mode == "msgstr2") msgstr2 = msgstr2 getstring(); 1.341 + if (mode == "msgstr3") msgstr3 = msgstr3 getstring(); 1.342 + if (mode == "msgstr4") msgstr4 = msgstr4 getstring(); 1.343 + if (mode == "msgstr5") msgstr5 = msgstr5 getstring(); 1.344 + 1.345 + if (! $0) { 1.346 + if (msgid != msgstr) { 1.347 + if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt); 1.348 + printf("msgid \"%s\"\n", msgid); 1.349 + if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural); 1.350 + if (msgstr) printf("msgstr \"%s\"\n", msgstr); 1.351 + if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0); 1.352 + if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1); 1.353 + if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2); 1.354 + if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3); 1.355 + if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4); 1.356 + if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5); 1.357 + printf("\n"); 1.358 + } 1.359 + clean(); 1.360 + } 1.361 + } 1.362 + if ($0 == "") skip = ""; 1.363 + } 1.364 + ' "$tmpfile" > "$tmpfile.awk" 1.365 + 1.366 + msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 1.367 + 1.368 + if [ -s "$tmpfile.mo" ]; then 1.369 + rm "$mo"; mv "$tmpfile.mo" "$mo" 1.370 + else 1.371 + _ 'Error processing %s' "$mo" 1.372 + [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo" 1.373 + fi 1.374 + 1.375 + # Clean 1.376 + rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk" 1.377 + done 1.378 + 1.379 + comp_summary "$time0" "$size0" "$(sizes mo1)" 1.380 +} 1.381 + 1.382 + 1.383 +# Strip locale definitions: normalize whitespace and remove comments 1.384 +# Stripping can be disabled with COOKOPTS="!locdef" 1.385 + 1.386 +strip_locale_def() { 1.387 + [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return 1.388 + [ ! -d "$install/usr/share/i18n/locales" ] && return 1.389 + [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return 1.390 + 1.391 + action 'Stripping locale definitions...' 1.392 + size0=$(sizes loc) 1.393 + time0=$(date +%s) 1.394 + 1.395 + for i in $(find $install/usr/share/i18n/locales -type f); do 1.396 + sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i 1.397 + done 1.398 + 1.399 + comp_summary "$time0" "$size0" "$(sizes loc)" 1.400 +} 1.401 + 1.402 + 1.403 +# Find and strip: --strip-all (-s) or --strip-debug on static libs as well 1.404 +# as removing unneeded files like in Python packages. Cross compiled binaries 1.405 +# must be stripped with cross-tools aka $ARCH-slitaz-*-strip 1.406 +# Stripping can be disabled with COOKOPTS="!strip" 1.407 + 1.408 +strip_package() { 1.409 + [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return 1.410 + 1.411 + case "$ARCH" in 1.412 + arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;; 1.413 + *) export STRIP='strip' ;; 1.414 + esac 1.415 + action 'Executing strip on all files...' 1.416 + size0=0 1.417 + size1=0 1.418 + time0=$(date +%s) 1.419 + 1.420 + # Strip executable files 1.421 + for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do 1.422 + if [ -d "$dir" ]; then 1.423 + oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}') 1.424 + find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \; 1.425 + newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}') 1.426 + size0=$((size0 + oldsize)); size1=$((size1 + newsize)) 1.427 + fi 1.428 + done 1.429 + 1.430 + # Strip shared and static libraries 1.431 + # Remove Python *.pyc and *.pyo, Perl perllocal.pod and .packlist 1.432 + oldsize=$(find $fs -type f \( \ 1.433 + -name '*.so*' -o -name '*.a' -o \ 1.434 + -name '*.pyc' -o -name '*.pyo' -o \ 1.435 + -name 'perllocal.pod' -o -name '.packlist' \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}') 1.436 + 1.437 + find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \; 1.438 + find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \; 1.439 + find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null 1.440 + find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \) -delete 2>/dev/null 1.441 + 1.442 + newsize=$(find $fs -type f \( \ 1.443 + -name '*.so*' -o -name '*.a' -o \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}') 1.444 + 1.445 + comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))" 1.446 +} 1.447 + 1.448 + 1.449 +# Strip unsupported locales (.mo files) 1.450 + 1.451 +strip_mo_i18n() { 1.452 + [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return 1.453 + 1.454 + [ ! -d "$fs/usr/share/locale" ] && return 1.455 + [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return 1.456 + 1.457 + action 'Thin out translation files...' 1.458 + size0=$(sizes mo2) 1.459 + time0=$(date +%s) 1.460 + 1.461 + # The variable $LOCALE is set in cook.conf and may be overridden in the receipt. 1.462 + # Default value is "" (empty). That means for us that we'll use the full 1.463 + # list of supported locales here. 1.464 + [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales) 1.465 + 1.466 + # Existing locales 1.467 + elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') " 1.468 + 1.469 + # Thin out the list of existing locales. At the end there will be only locales that need 1.470 + # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...) 1.471 + for keep_locale in $LOCALE; do 1.472 + elocales=${elocales//$keep_locale} 1.473 + done 1.474 + 1.475 + # Remove the unsupported locales 1.476 + for rem_locale in $elocales; do 1.477 + [ -d "$fs/usr/share/locale/$rem_locale" ] && 1.478 + rm -r "$fs/usr/share/locale/$rem_locale" 1.479 + done 1.480 + 1.481 + comp_summary "$time0" "$size0" "$(sizes mo2)" 1.482 +} 1.483 + 1.484 + 1.485 + 1.486 + 1.487 +case $1 in 1.488 + install) 1.489 + # Compressors working in the $install 1.490 + case "$ARCH" in 1.491 + arm*) ;; 1.492 + *) 1.493 + compress_manpages 1.494 + compress_png 1.495 + compress_svg 1.496 + compress_ui 1.497 + ;; 1.498 + esac 1.499 + 1.500 + fix_desktop_files 1.501 + normalize_mo 1.502 + strip_locale_def 1.503 + ;; 1.504 + fs) 1.505 + # Compressors working in the $fs 1.506 + strip_package 1.507 + strip_mo_i18n 1.508 + ;; 1.509 +esac