tazpkg annotate modules/mkdb @ rev 955

modules/search: allow search file with dash at start: tazpkg -sf "-spi"
author Aleksej Bobylev <al.bobylev@gmail.com>
date Fri Dec 22 00:02:54 2017 +0200 (2017-12-22)
parents 8a73a58ed3cb
children 03544b89ac81
rev   line source
al@824 1 #!/bin/sh
al@824 2 # TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg
al@824 3 # mkdb - TazPkg module
al@824 4 # Make TazPkg database for folder with *.tazpkg packages
al@824 5
al@824 6
al@824 7 # Input: $1 - path to folder contains *.tazpkg packages
al@824 8 # Output files in the $1 folder:
al@824 9 # packages.info
al@824 10 # packages.equiv
al@824 11 # descriptions.txt
al@824 12 # files.list.lzma
al@824 13 # IDs
al@824 14 # Do nothing if database already exists; force rebuild it with --forced option.
al@824 15
al@824 16 # DB format:
al@824 17 # ==========
al@824 18
al@824 19 # packages.info
al@824 20 # -------------
al@824 21 # Record is line; fields are tab-separated. Fields description:
al@824 22 # 1: package name
al@824 23 # 2: version with extra-version
al@824 24 # 3: category
al@824 25 # 4: short description
al@824 26 # 5: upstream web site
al@824 27 # 6: tags (space-separated)
al@824 28 # 7: packed and unpacked sizes (space-separated) in human readable format
al@824 29 # 8: depends
al@950 30 # 9: "release checksum"
al@950 31 #10: provide
al@824 32
al@824 33 # packages.equiv
al@824 34 # --------------
al@824 35 # This DB file used before package installation
al@824 36 # Record is line. Separator is "="
al@824 37 # Field 1 is package name to install (pkg1)
al@824 38 # Field 2 is space-separated list of items in the special format:
al@824 39 # a) pkg2:pkg3
al@824 40 # If pkg2 is installed, then install pkg3 instead of pkg1. Example:
al@824 41 # busybox=pam:busybox-pam
al@824 42 # If 'pam' is installed, then install 'busybox-pam' instead of 'busybox'
al@824 43 # b) pkg2
al@824 44 # If pkg2 already installed, then pkg1 will not be installed. Example:
al@824 45 # mysql=mariadb
al@824 46 # If 'mariadb' already installed, then 'mysql' will not be installed
al@824 47 # Complex rule example:
al@824 48 # ssh=pam:openssh-pam openssh pam:dropbear-pam dropbear
al@824 49
al@824 50 # descriptions.txt
al@824 51 # ----------------
al@824 52 # Field is line; record separator is empty line.
al@824 53 # First field is package name, rest - description itself.
al@824 54 # Empty lines in the description appended with space (" ") to avoid mess
al@824 55 # with end of record.
al@824 56
al@824 57 # files.list.lzma
al@824 58 # ---------------
al@824 59 # It is "files.list" compressed using lzma due to it's big size.
al@824 60 # Format of the files.list: record is line; field separator is ": ".
al@824 61 # First field is package name, second field is file path.
al@824 62 # There are DB records for all files installed with the package.
al@824 63
al@950 64 # packages.md5
al@950 65 #-------------
al@950 66 # Record is line; field separator is " ".
al@950 67 # First field is md5sum of the package file; second field is package file name.
al@950 68 # Actually, the type of checksum defined in variable $SUM and defaults to "md5".
al@950 69
al@824 70
al@844 71 # Connect function libraries
al@824 72 . /lib/libtaz.sh
al@824 73
al@844 74 # Get TazPkg working environment
al@844 75 . @@MODULES@@/getenv
al@824 76
al@844 77
al@824 78
al@824 79
al@824 80 # Exit if input folder not specified
al@824 81 [ -z "$1" ] && die 'Input folder not specified'
al@824 82
al@824 83 # Exit if input folder not exists
al@844 84 folder=$(realpath "$root$1") || exit 1
al@824 85
al@824 86 # Exit if folder is not writable
al@824 87 [ ! -w "$folder" ] && die 'You are not allowed to write to the folder "%s"' "$folder"
al@824 88
al@824 89 # Exit if input folder does not contain packages
al@824 90 [ -z "$(find "$folder" -maxdepth 1 -name '*.tazpkg')" ] && \
al@824 91 die 'Folder "%s" does not contain packages' "$folder"
al@824 92
al@824 93
al@824 94 # DB file names
al@824 95 DBi="$folder/packages.info"
al@824 96 DBe="$folder/packages.equiv"
al@950 97 DBm="$folder/packages.$SUM"
al@824 98 DBd="$folder/descriptions.txt"
al@824 99 DBf="$folder/files.list"
al@824 100
al@824 101 # Pre-remove DB if --forced and DB exists
al@824 102 if [ -n "$forced" ]; then
al@824 103 [ -e "$DBi" ] && rm "$DBi"
al@824 104 [ -e "$DBe" ] && rm "$DBe"
al@950 105 [ -e "$DBm" ] && rm "$DBm"
al@824 106 [ -e "$DBd" ] && rm "$DBd"
al@824 107 [ -e "$DBf.lzma" ] && rm "$DBf.lzma"
al@824 108 fi
al@824 109
al@824 110 if [ -s "$DBi" ]; then
al@824 111 _ 'Packages DB already exists.' >&2
al@824 112 exit 1
al@824 113 fi
al@824 114
al@824 115 # Random temporary folder
al@824 116 tempd="$(mktemp -d)"
al@824 117
al@824 118 # Make temporary list of packages checksum (usually md5sum)
al@824 119 _n 'Calculate %s...' "$CHECKSUM"
al@950 120 cd "$folder"; $CHECKSUM *.tazpkg | tee "$tempd/$SUM" > "$DBm"
al@824 121 status
al@824 122
al@824 123 cd "$tempd"
al@824 124
al@824 125 # Loop for every package
al@824 126 while read pkgsum pkgfile; do
al@824 127 # Current processed package
al@824 128 echo -n "$pkgfile"
al@824 129
al@950 130 # Extract $CHECKSUM, receipt, description.txt from package
al@950 131 # (description.txt may absent, no error produced)
al@950 132 cpio -F "$folder/$pkgfile" -i $CHECKSUM receipt description.txt >/dev/null 2>&1
al@950 133
al@950 134 # Make "release checksum"
al@950 135 cp $CHECKSUM rsum_file
al@950 136 md5sum receipt >> rsum_file
al@950 137 [ -e "description.txt" ] && md5sum description.txt >> rsum_file
al@950 138 rsum=$(md5sum rsum_file | awk '{print $1}')
al@824 139
al@824 140 # Unset variables that may absent in the receipt
al@824 141 unset EXTRAVERSION TAGS DEPENDS PROVIDE
al@824 142 # Get values
al@950 143 . ./receipt
al@824 144
al@950 145 # Clean
al@950 146 rm -f $CHECKSUM receipt description.txt rsum_file 2>/dev/null
al@824 147
al@824 148 # Make packages.info
al@824 149 echo -en "$PACKAGE\t$VERSION$EXTRAVERSION\t$CATEGORY\t" >> "$DBi"
al@824 150 echo -en "$SHORT_DESC\t$WEB_SITE\t$TAGS\t" >> "$DBi"
al@824 151 echo -en "$PACKED_SIZE $UNPACKED_SIZE\t" | sed 's|\.0||g' >> "$DBi"
al@950 152 echo -n $DEPENDS$'\t' >> "$DBi"
al@950 153 echo -e "$rsum\t$PROVIDE" >> "$DBi"
al@824 154
al@824 155
al@824 156 # Make packages.equiv
al@824 157 for i in $PROVIDE; do
al@824 158 # Example from busybox-pam package:
al@824 159 # PACKAGE="busybox-pam", PROVIDE="busybox:pam"
al@824 160 case $i in
al@824 161 # DEST="pam:"
al@824 162 *:*) DEST="${i#*:}:";;
al@824 163 *) DEST='';;
al@824 164 esac
al@824 165 # PKG="busybox"
al@824 166 PKG="${i%:*}"
al@824 167 if grep -qs ^$PKG= "$DBe"; then
al@824 168 # Append existing record
al@824 169 sed -i "s|^$PKG=|\0 $DEST$PACKAGE|" "$DBe"
al@824 170 else
al@824 171 # Add new record
al@824 172 echo "$PKG=$DEST$PACKAGE" >> "$DBe"
al@824 173 fi
al@824 174 done
al@824 175
al@824 176
al@824 177 # Make descriptions.txt
al@824 178 if cpio -F "$folder/$pkgfile" -t 2>/dev/null | fgrep -q 'description.txt'; then
al@824 179 # Extract description.txt from package
al@824 180 cpio -F "$folder/$pkgfile" -i description.txt >/dev/null 2>&1
al@824 181 # Append descriptions DB
al@824 182 echo "$PACKAGE" >> "$DBd"
al@824 183 cat description.txt | sed 's|^$| |' >> "$DBd"
al@824 184 echo >> "$DBd"
al@824 185 rm description.txt
al@824 186 fi
al@824 187
al@824 188
al@824 189 # Make files.list
al@824 190 if cpio -F "$folder/$pkgfile" -t 2>/dev/null | fgrep -q 'files.list'; then
al@824 191 # Extract files.list from package
al@824 192 cpio -F "$folder/$pkgfile" -i files.list >/dev/null 2>&1
al@824 193 # Append files list DB
al@824 194 sed "s|.*|$PACKAGE: \0|" files.list >> "$DBf"
al@824 195 rm files.list
al@824 196 fi
al@824 197
al@824 198 # End line with the status
al@824 199 status
al@824 200 done < "$tempd/$SUM"
al@824 201
al@824 202
al@824 203 # Sort DB alphabetically
al@824 204 sort -o "$tempd/pi" "$DBi"; mv -f "$tempd/pi" "$DBi"
al@824 205
al@824 206 # Create empty files if they not exists
al@824 207 touch "$DBi" "$DBe" "$DBd" "$DBf"
al@824 208
al@824 209 # Compress files.list using lzma
al@824 210 sort -k2 -o "$DBf.sorted" "$DBf"
al@824 211 lzma e "$DBf.sorted" "$DBf.lzma"
al@824 212 rm "$DBf" "$DBf.sorted"
al@824 213
al@824 214 # Make DB readable for all
al@824 215 chmod a+r "$DBi" "$DBe" "$DBd" "$DBf.lzma"
al@824 216
al@824 217
al@824 218 # Make files for DB recharge
al@824 219 # --------------------------
al@824 220
al@824 221 cd "$folder"
al@824 222
al@824 223 # Make IDs: md5 and timestamp
al@824 224 ( md5sum "$tempd/$SUM" | cut -d' ' -f1 | tr '\n' ' '; date -ur "$DBi" +%s ) > IDs
al@824 225
al@824 226
al@824 227 # Make files-list.md5: decide whether to download files.list.lzma or not
al@824 228 md5sum "$DBf.lzma" | cut -d' ' -f1 | tr -d $'\n' > files-list.md5
al@824 229
al@824 230 # Make bundle to fast recharge
al@824 231 [ -f 'bundle.tar.lzma' ] && rm 'bundle.tar.lzma'
al@846 232 tar -chaf bundle.tar.lzma \
al@824 233 files-list.md5 packages.info descriptions.txt packages.equiv
al@824 234
al@824 235 # Clean up
al@824 236 rm files-list.md5
al@824 237 rm -r "$tempd"
al@824 238