SliTaz Repositories

1 #!/bin/sh

2

3 # Make repo.json for Repology using SliTaz Cooking packages

4 # Aleksey Bobylev <al.bobylev@gmail.com>

5 #

6 # 2018-08-11: initial, working on my localhost

7 # 2018-09-04: fix paths for use at Tank

8 # 2018-09-08: fix space in src, fix md5sum when EXTRAVERSION used, skip empty src, skip broken, store log

9 # 2019-01-15: block parallel execution

10 # 2019-01-23: drop duplicate lines in DB (if any), discard empty repo.json, use absolute paths

11

12 mirror='/home/slitaz/packages'

13 local_repo='/home/slitaz/wok-hg'

14

15 my_log="/home/slitaz/log/repo-cooking.$(date '+%Y%m%d_%H%M%S')"

16 block='/home/slitaz/log/repo-cooking.block'

17

18 local_bin=/root/bin

19

20 # packages.info

21 # 1: name

22 # 2: version

23 # 3: category

24 # 4: short description

25 # 5: home

26 # 6: tags

27 # 7: size_packed size_unpacked

28 # 8: dependencies

29 # 9: checksum

30 # 10: maintainer

31 # 11: license

32 # 12: src URL

33 # 13: wanted

34 # 14: build dependencies

35 # 15: package md5sum

36 # 16: version without extraversion

37

38 timestamp1=$(date '+%s')

39 date_for_json=$(date '+%F %T')

40

41 if [ -e "$block" ]; then

42 echo "$0 already executed. Exit" | tee "$my_log"

43 exit 1

44 fi

45 touch "$block"

46

47 ls ${my_log%.*}* | sed '1,10d;s|.*|rm -f &|' | sh # keep only 10 log files

48

49

50 tempd=$(mktemp -d)

51

52 sort -u $mirror/packages.info > $tempd/packages.info # drop duplicate lines

53 sort -u $mirror/packages.md5 > $tempd/packages.md5 #

54

55 # get base settings

56 c_version=$(awk -F$'\t' -vp='cookutils' '{if ($1 == p) {print $2; exit}}' $tempd/packages.info) #'

57 cp $mirror/cookutils-$c_version.tazpkg $tempd/

58

59 cd $tempd

60 tazpkg extract $tempd/cookutils-$c_version.tazpkg 2>/dev/null >&2

61 cp $tempd/cookutils-$c_version/fs/etc/slitaz/cook.conf $tempd/

62 rm $tempd/cookutils-$c_version.tazpkg

63 rm -r $tempd/cookutils-$c_version/

64

65 # define some useful variables

66 . $tempd/cook.conf

67

68

69 ls $local_repo \

70 | while read pkg_i; do

71 i=$(( i + 1 ))

72 [ -f "$local_repo/$pkg_i/receipt" ] || continue

73

74 unset MAINTAINER LICENSE WGET_URL WANTED BUILD_DEPENDS VERSION EXTRAVERSION

75

76 case $i in

77 *00) echo $i;;

78 *0) echo -n $i;;

79 esac

80 echo -n '.'

81

82 . "$local_repo/$pkg_i/receipt"

83

84 MAINTAINER=$(echo $MAINTAINER | sed 's|.*<||; s|>||')

85

86 LICENSE=$(echo $LICENSE)

87

88 WGET_URL=$(echo $WGET_URL | sed 's#^git|git#git#; s#^git|https*#git#; s#^svn|https*#svn#; s#^subversion|svn#svn#; s#^subversion|https*#svn#; s#^svn|svn#svn#; s#^hg|#hg://#; s#^mercurial|#hg://#; s#^bzr|#bzr://#; s#^cvs|#cvs://#')

89

90 WANTED=$(echo $WANTED)

91 case $pkg_i in

92 locale-*-extra) WANTED='slitaz-i18n-extra';;

93 locale-*) WANTED='slitaz-i18n';;

94 esac

95

96 BUILD_DEPENDS=$(echo $BUILD_DEPENDS)

97

98 MD5SUM="$(fgrep " $pkg_i-$VERSION" $tempd/packages.md5 | cut -d' ' -f1)"

99 if [ $(echo "$MD5SUM" | wc -l) -ne 1 ]; then

100 echo "$pkg_i: error: multiple md5sum" >>$my_log

101 fi

102

103 sed -i "/^$pkg_i / s|$| $MAINTAINER $LICENSE $WGET_URL $WANTED $BUILD_DEPENDS $MD5SUM $VERSION|" $tempd/packages.info

104 done

105 echo

106 echo

107

108 timestamp2=$(date '+%s')

109 echo "Stage 1: $(( timestamp2 - timestamp1 )) s" >>$my_log

110 echo "Packages: $(wc -l ./packages.info)" >>$my_log

111 echo >>$my_log

112

113

114

115 {

116 echo '{"repo":"SliTaz Cooking","date":"'$date_for_json'","items":['

117

118 comma_up=''

119

120 { ls $local_repo; echo '@the_end@'; } \

121 | while read pkg_i; do

122 echo "$pkg_i" >&2

123

124 case $pkg_i in

125 @the_end@) echo ']}'; break;;

126 get-*) continue;;

127 esac

128

129 pkg_line=$(sed -n "/^$pkg_i /p" $tempd/packages.info)

130

131 wanted=$(echo "$pkg_line" | cut -d$'\t' -f 13)

132 [ -z "$wanted" ] || continue

133

134 echo "$pkg_i" >> $tempd/packages.txt

135

136 ver=$( echo "$pkg_line" | cut -d$'\t' -f 16)

137 home=$( echo "$pkg_line" | cut -d$'\t' -f 5)

138 maintainer=$(echo "$pkg_line" | cut -d$'\t' -f 10)

139 license=$( echo "$pkg_line" | cut -d$'\t' -f 11 | sed 's| |","|g')

140 if [ -z "$license" ]; then

141 echo "$pkg_i: license is empty" >>$my_log

142 license='unknown'

143 fi

144 src=$( echo "$pkg_line" | cut -d$'\t' -f 12 | sed 's| .*||')

145 bdeps=$( echo "$pkg_line" | cut -d$'\t' -f 14 | sed 's| |","|g')

146

147 # skip

148 if [ -z "$ver" ]; then

149 echo "$pkg_i: ver empty, skip" >>$my_log

150 continue

151 fi

152

153 echo "$comma_up{\"meta\":\"$pkg_i\", \"ver\":\"$ver\", \"maintainer\":\"$maintainer\", \"home\":\"$home\", \"license\":[\"$license\"]"

154

155 [ -z "$src" ] || echo ", \"src\":\"$src\""

156 [ -z "$bdeps" ] || echo ", \"bdeps\":[\"$bdeps\"]"

157

158 echo ", \"pkgs\":["

159

160 comma_up=','

161 comma=''

162

163 want=$(awk -F$'\t' -vp="$pkg_i" '{if ($13 == p) {print $1}}' $tempd/packages.info) #'

164 for pkg_j in $pkg_i $want; do

165 echo " - $pkg_j" >> $tempd/packages.txt

166

167 pkg_line=$(sed -n "/^$pkg_j /p" $tempd/packages.info)

168

169 echo $comma; comma=','

170

171 ver_sub=$(echo "$pkg_line" | cut -d$'\t' -f 16)

172 if [ "$ver" != "$ver_sub" ]; then

173 echo "$pkg_i > $pkg_j: version mismatch '$ver' -> '$ver_sub'" >>$my_log

174 fi

175

176 ver_ext=$( echo "$pkg_line" | cut -d$'\t' -f 2)

177 cat=$( echo "$pkg_line" | cut -d$'\t' -f 3)

178 desc=$( echo "$pkg_line" | cut -d$'\t' -f 4 | sed 's|"|\\"|g')

179 tags=$( echo "$pkg_line" | cut -d$'\t' -f 6 | sed 's| |","|g')

180 pkg_size=$( echo "$pkg_line" | cut -d$'\t' -f 7 | cut -d' ' -f1)

181 installed_size=$(echo "$pkg_line" | cut -d$'\t' -f 7 | cut -d' ' -f2)

182 deps=$( echo "$pkg_line" | cut -d$'\t' -f 8 | sed 's| |","|g')

183 pkg_md5sum=$( echo "$pkg_line" | cut -d$'\t' -f 15)

184

185 echo "{\"name\":\"$pkg_j\", \"cat\":\"$cat\", \"desc\":\"$desc\""

186

187 [ -z "$tags" ] || echo ",\"tags\":[\"$tags\"]"

188

189 [ -z "$deps" ] || echo ",\"deps\":[\"$deps\"]"

190

191 echo ",\"pkg_size\":\"$pkg_size\", \"installed_size\":\"$installed_size\", \"pkg_md5sum\":\"$pkg_md5sum\", \"get\":\"http://mirror.slitaz.org/packages/cooking/$pkg_j-$ver_ext.tazpkg\""

192

193 echo '}'

194 done

195

196 echo ']}'

197 done

198 } > $tempd/repo-dirt.json

199

200 cat $tempd/repo-dirt.json | $local_bin/jq '.' > $tempd/repo.json

201

202 if [ -s $tempd/repo.json ]; then

203 gzip $tempd/repo.json

204 $local_bin/advdef -z4 $tempd/repo.json.gz

205 mv $tempd/repo.json.gz $mirror

206 else

207 echo "ERROR! New repo.json was discarded" >>$my_log

208 fi

209

210 rm -rf $tempd

211

212 timestamp3=$(date '+%s')

213

214 echo -e "\nStage 2: $(( timestamp3 - timestamp2 )) s" >>$my_log

215 echo "Finished: $(date '+%F %T')" >>$my_log

216 rm "$block"