cookutils: 5f6be706ab4f modules/compressor

cookutils view modules/compressor @ rev 867

cook, modules/compressor: implement tazpkg --cookmode; cooker.cgi: info2html(): pagination, syntax_highlighter(): distinguish local/cached installed packages (vital for the system update).

author	Aleksej Bobylev <al.bobylev@gmail.com>
date	Thu Jan 12 04:04:52 2017 +0200 (2017-01-12)
parents	c6bbdbfdc285
children	9e870e53cac9

line source

1 #!/bin/sh

2 #

3 # compressor - module of the SliTaz Cook

4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3

5 #

7 . /usr/lib/slitaz/libcook.sh

10 #

11 # Functions

12 #

15 # Display time.

17 disp_time() {

18 div=$(( ($1 + 30) / 60))

19 case $div in

20 0) min='';;

21 # L10n: 'm' is for minutes (approximate time)

22 *) min=$(_n ' ~ %dm' "$div");;

23 esac

25 # L10n: 's' is for seconds (cooking time)

26 _ '%ds%s' "$1" "$min"

27 }

30 # Compressor mini summary

32 comp_summary() {

33 # "$time0" "$size0" "$size1"

34 time1=$(date +%s)

35 status

36 [ "$2" -eq 0 ] && return

37 time=$(($time1 - $1))

38 saving=$(( ($2 - $3) / 1024 ))

39 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB' "$(disp_time $time)" "$2" "$3" "$saving"

40 }

43 # Calculating different sizes

45 sizes() {

46 case $1 in

47 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;

48 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;

49 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;

50 xml) find $install -type f $ -name '*.ui' -o -name '*.glade' $ -exec ls -l \{\} \; ;;

51 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;

52 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;

53 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;

54 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;

55 esac | awk '{s+=$5}END{print s}'

56 }

59 # Function to compress all man pages

60 # Compressing can be disabled with COOKOPTS="!manz"

62 compress_manpages() {

63 time0=$(date +%s)

64 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return

65 manpath="$install/usr/share/man"

66 [ -d "$manpath" ] || return

67 size0=$(sizes man); [ -z "$size0" ] && return

69 tazpkg -gi advancecomp --quiet --cookmode

71 action 'Compressing man pages...'

73 # We'll use only Gzip compression, so decompress other formats first

74 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;

75 find $manpath -type f -name '*.xz' -exec unxz \{\} \;

77 # Fast compress with gzip

78 find $manpath -type f -name '*.[1-9]*' -exec gzip \{\} \;

80 # Fix symlinks

81 for i in $(find $install/usr/share/man -type l); do

82 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')

83 link=$(echo $i | sed 's|\.[gbx]z2*$||')

84 rm $i; ln -s $dest.gz $link.gz

85 done

87 # Recompress with advdef (it can't compress, only recompress)

88 for i in $(find $install/usr/share/man -type f); do

89 advdef -z4q $i

90 done

92 comp_summary "$time0" "$size0" "$(sizes man)"

93 }

96 # Function used after compile_rules() to compress all png images

97 # Compressing can be disabled with COOKOPTS="!pngz"

99 compress_png() {

100 time0=$(date +%s)

101 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return

102 size0=$(sizes png); [ -z "$size0" ] && return

103

104 use_pq=true

105 use_op=true

106 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false

107 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false

108 $use_pq && tazpkg -gi pngquant --quiet --cookmode

109 $use_op && tazpkg -gi optipng --quiet --cookmode

110

111 action 'Compressing png images...'

112

113 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op$[0-8]$.*|\1|')

114 [ -z "$oplevel" ] && oplevel='2'

115 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'

116

117 for i in $(find $install -type f -name '*.png'); do

118 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"

119 $use_op && optipng -quiet -strip all -o$oplevel "$i"

120 done

121

122 comp_summary "$time0" "$size0" "$(sizes png)"

123 }

124

125

126 # Function used after compile_rules() to compress all svg images

127 # Compressing can be disabled with COOKOPTS="!svgz"

128

129 compress_svg() {

130 time0=$(date +%s)

131 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return

132 size0=$(sizes svg); [ -z "$size0" ] && return

133

134 tazpkg -gi svgcleaner --quiet --cookmode

135

136 action 'Compressing svg images...'

137

138 cleaner_log="$(mktemp)"

139 for i in $(find $install -type f -name '*.svg'); do

140 echo -n "$i: " >> "$cleaner_log"

141 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"

142 done

143

144 comp_summary "$time0" "$size0" "$(sizes svg)"

145

146 sed -i '/: $/d' "$cleaner_log"

147 if [ -s "$cleaner_log" ]; then

148 _ 'Cleaner warnings and errors:'

149 awk '{printf " %s\n", $0;}' "$cleaner_log"

150 echo

151 fi

152 rm "$cleaner_log"

153 }

154

155

156 # Function used after compile_rules() to shrink all *.ui and *.glade files:

157 # remove insignificant spaces and comments

158 # Compressing can be disabled with COOKOPTS="!uiz"

159

160 compress_ui() {

161 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return

162 [ -z "$(find $install -type f $ -name '*.ui' -o -name '*.glade' $ )" ] && return

163

164 tazpkg -gi xmlstarlet --quiet --cookmode

165

166 action 'Compressing ui files...'

167

168 size0=$(sizes xml)

169 time0=$(date +%s)

170 temp_ui="$(mktemp)"

171 for ui in $(find $install -type f $ -name '*.ui' -o -name '*.glade' $ ); do

172 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"

173 cat "$temp_ui" > "$ui"

174 done

175

176 comp_summary "$time0" "$size0" "$(sizes xml)"

177 rm "$temp_ui"

178 }

179

180

181 # Get list of supported locales...

182

183 get_supported_locales() {

184 lpc='/slitaz-i18n/stuff/locale-pack.conf'

185 if [ -e "$WOK$lpc" ]; then

186 # ... from package in the local wok

187 . "$WOK$lpc"

188 else

189 # ... from Hg

190 temp_conf=$(mktemp)

191 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"

192 if [ -s $temp_conf ]; then

193 . $temp_conf

194 else

195 # Give up and use hardcoded list

196 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \

197 pt_BR ro ru sl sv tr uk zh_CN zh_TW"

198 fi

199 rm $temp_conf

200 fi

201 echo $LOCALE_PACK

202 }

203

204

205 # Fix common errors and warnings in the .desktop files

206 # Fixing can be disabled with COOKOPTS="!fixdesktops"

207

208 fix_desktop_files() {

209 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return

210 [ -z "$(find $install -type f -name '*.desktop')" ] && return

211

212 size0=$(sizes des)

213 time0=$(date +%s)

214

215 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then

216 tazpkg -gi desktop-file-utils-extra --quiet --cookmode

217 fi

218

219 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.

220 # Default value is "" (empty). That means for us that we'll use the full

221 # list of supported locales here.

222 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)

223

224 for desktop in $(find $install -type f -name '*.desktop'); do

225 cp "$desktop" "$desktop.orig"

226

227 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)

228 sdft "$desktop" -i

229

230 # Fix common errors in .desktop file

231 fix-desktop-file "$desktop"

232

233 # Strip unsupported locales from .desktop file

234 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&

235 sdft "$desktop" -i -k "$LOCALE"

236

237 # Extra-strip

238 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&

239 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o

240

241 if [ -n "$QA" ]; then

242 # Check the rest of errors, warnings and tips

243 _ 'QA: Checking %s...' "$(basename $desktop)"

244 diff "$desktop.orig" "$desktop"

245 desktop-file-validate "$desktop" | busybox fold -s

246 echo

247 fi

248

249 rm "$desktop.orig"

250 done

251

252 comp_summary "$time0" "$size0" "$(sizes des)"

253 }

254

255

256 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really added

257 # to the translation (msgid = msgstr)

258 # Normalization can be disabled with COOKOPTS="!monorm"

259

260 normalize_mo() {

261 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return

262 [ -z "$(find $install -type f -name '*.mo')" ] && return

263

264 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt

265 tazpkg -gi gettext --quiet --cookmode

266 # Gconv modules (convert to UTF-8)

267 tazpkg -gi glibc-locale --quiet --cookmode

268

269 action 'Normalizing mo files...'

270

271 size0=$(sizes mo1)

272 time0=$(date +%s)

273

274 # Process all existing *.mo files

275 for mo in $(find "$install" -type f -name '*.mo'); do

276 tmpfile="$(mktemp)"

277

278 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'

279 # add newline

280 echo >> "$tmpfile"

281

282 # get Plural-Forms

283 awk '

284 BEGIN { skip = ""; }

285 {

286 if (! skip) {

287 s = $0;

288 gsub(/^[^\"]*\"/, "", s);

289 gsub(/\"$/, "", s);

290 printf("%s", s);

291 }

292 if (! $0) skip = "yes";

293 }

294 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"

295

296 if ! grep -q 'msgid_plural' "$tmpfile"; then

297 echo > "$tmpfile.pf"

298 fi

299

300 # main

301 awk -v pf="$(cat "$tmpfile.pf")" '

302 function clean() {

303 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";

304 }

305

306 function getstring() {

307 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes

308 s = $0;

309 gsub(/^[^\"]*\"/, "", s);

310 gsub(/\"$/, "", s);

311 return s;

312 }

313

314 BEGIN {

315 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");

316 if (pf)

317 printf("\"%s\\n\"\n", pf);

318 printf("\n");

319 skip = 1;

320 clean();

321 }

322

323 {

324 # Skip the entire header

325 if (!skip) {

326 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")

327 mode = $1;

328 if ($1 == "msgstr[0]") mode = "msgstr0";

329 if ($1 == "msgstr[1]") mode = "msgstr1";

330 if ($1 == "msgstr[2]") mode = "msgstr2";

331 if ($1 == "msgstr[3]") mode = "msgstr3";

332 if ($1 == "msgstr[4]") mode = "msgstr4";

333 if ($1 == "msgstr[5]") mode = "msgstr5";

334

335 if (mode == "msgctxt") msgctxt = msgctxt getstring();

336 if (mode == "msgid") msgid = msgid getstring();

337 if (mode == "msgstr") msgstr = msgstr getstring();

338 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();

339 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();

340 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();

341 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();

342 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();

343 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();

344 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();

345

346 if (! $0) {

347 if (msgid != msgstr) {

348 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);

349 printf("msgid \"%s\"\n", msgid);

350 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);

351 if (msgstr) printf("msgstr \"%s\"\n", msgstr);

352 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);

353 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);

354 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);

355 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);

356 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);

357 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);

358 printf("\n");

359 }

360 clean();

361 }

362 }

363 if ($0 == "") skip = "";

364 }

365 ' "$tmpfile" > "$tmpfile.awk"

366

367 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"

368

369 if [ -s "$tmpfile.mo" ]; then

370 rm "$mo"; mv "$tmpfile.mo" "$mo"

371 else

372 _ 'Error processing %s' "$mo"

373 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"

374 fi

375

376 # Clean

377 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"

378 done

379

380 comp_summary "$time0" "$size0" "$(sizes mo1)"

381 }

382

383

384 # Strip locale definitions: normalize whitespace and remove comments

385 # Stripping can be disabled with COOKOPTS="!locdef"

386

387 strip_locale_def() {

388 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return

389 [ ! -d "$install/usr/share/i18n/locales" ] && return

390 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return

391

392 action 'Stripping locale definitions...'

393 size0=$(sizes loc)

394 time0=$(date +%s)

395

396 for i in $(find $install/usr/share/i18n/locales -type f); do

397 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i

398 done

399

400 comp_summary "$time0" "$size0" "$(sizes loc)"

401 }

402

403

404 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well

405 # as removing unneeded files like in Python packages. Cross compiled binaries

406 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip

407 # Stripping can be disabled with COOKOPTS="!strip"

408

409 strip_package() {

410 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return

411

412 case "$ARCH" in

413 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;

414 *) export STRIP='strip' ;;

415 esac

416 action 'Executing strip on all files...'

417 size0=0

418 size1=0

419 time0=$(date +%s)

420

421 # Strip executable files

422 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do

423 if [ -d "$dir" ]; then

424 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')

425 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;

426 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')

427 size0=$((size0 + oldsize)); size1=$((size1 + newsize))

428 fi

429 done

430

431 # Strip shared and static libraries

432 # Remove Python *.pyc and *.pyo, Perl perllocal.pod and .packlist

433 oldsize=$(find $fs -type f \( \

434 -name '*.so*' -o -name '*.a' -o \

435 -name '*.pyc' -o -name '*.pyo' -o \

436 -name 'perllocal.pod' -o -name '.packlist' \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')

437

438 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;

439 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;

440 find $fs -type f $ -name '*.pyc' -o -name '*.pyo' $ -delete 2>/dev/null

441 find $fs -type f $ -name 'perllocal.pod' -o -name '.packlist' $ -delete 2>/dev/null

442

443 newsize=$(find $fs -type f \( \

444 -name '*.so*' -o -name '*.a' -o \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')

445

446 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"

447 }

448

449

450 # Strip unsupported locales (.mo files)

451

452 strip_mo_i18n() {

453 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return

454

455 [ ! -d "$fs/usr/share/locale" ] && return

456 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return

457

458 action 'Thin out translation files...'

459 size0=$(sizes mo2)

460 time0=$(date +%s)

461

462 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.

463 # Default value is "" (empty). That means for us that we'll use the full

464 # list of supported locales here.

465 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)

466

467 # Existing locales

468 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "

469

470 # Thin out the list of existing locales. At the end there will be only locales that need

471 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)

472 for keep_locale in $LOCALE; do

473 elocales=${elocales//$keep_locale}

474 done

475

476 # Remove the unsupported locales

477 for rem_locale in $elocales; do

478 [ -d "$fs/usr/share/locale/$rem_locale" ] &&

479 rm -r "$fs/usr/share/locale/$rem_locale"

480 done

481

482 comp_summary "$time0" "$size0" "$(sizes mo2)"

483 }

484

485

486

487

488 case $1 in

489 install)

490 # Compressors working in the $install

491 case "$ARCH" in

492 arm*) ;;

493 *)

494 compress_manpages

495 compress_png

496 compress_svg

497 compress_ui

498 ;;

499 esac

500

501 fix_desktop_files

502 normalize_mo

503 strip_locale_def

504 ;;

505 fs)

506 # Compressors working in the $fs

507 strip_package

508 strip_mo_i18n

509 ;;

510 esac

SliTaz Repositories

cookutils view modules/compressor @ rev 867