tazlito rev 516

tazlito: speedup deduplicate
author Pascal Bellard <pascal.bellard@slitaz.org>
date Mon Jul 01 12:00:20 2019 +0200 (2019-07-01)
parents e2ff6056f93a
children c38700f27d76
files tazlito
line diff
     1.1 --- a/tazlito	Thu May 23 11:09:57 2019 +0200
     1.2 +++ b/tazlito	Mon Jul 01 12:00:20 2019 +0200
     1.3 @@ -846,22 +846,24 @@
     1.4  # Deduplicate files (MUST be on the same filesystem).
     1.5  
     1.6  deduplicate() {
     1.7 -	find "${@:-.}" -xdev -type f -size +0c -exec stat -c '%s-%a-%u-%g %i %h %n' {} \; | sort | \
     1.8 +	find "${@:-.}" -xdev -type f ! -type l -size +0c -exec stat -c '%s-%a-%u-%g %i %h %n' {} \; | sort | \
     1.9  		(
    1.10 -			save=0; hardlinks=0; old_attr=""; old_inode=""; old_link=""; old_file=""
    1.11 +			save=0; hardlinks=0; old_attr=""; old_inode=""; old_link=""; old_file=""; hinode=""
    1.12  			while read attr inode link file; do
    1.13 -				[ -L "$file" ] && continue
    1.14 -				if [ "$attr" = "$old_attr" -a "$inode" != "$old_inode" ]; then
    1.15 -					if cmp "$file" "$old_file" >/dev/null 2>&1 ; then
    1.16 -						rm -f "$file"
    1.17 -						if ln "$old_file" "$file" 2>/dev/null; then
    1.18 -							inode="$old_inode"
    1.19 -							[ "$link" -eq 1 ] && hardlinks=$(($hardlinks+1)) &&
    1.20 -								save="$(($save+(${attr%%-*}+512)/1024))"
    1.21 -						else
    1.22 -							cp -a "$old_file" "$file"
    1.23 -						fi
    1.24 +				if [ "$attr" = "$old_attr" -a "$inode" != "$old_inode" ] &&
    1.25 +				   { [ "$inode" = "$hinode" ] || cmp "$file" "$old_file" >/dev/null 2>&1; } ; then
    1.26 +					rm -f "$file"
    1.27 +					if ln "$old_file" "$file" 2>/dev/null; then
    1.28 +						hinode="$inode"
    1.29 +						inode="$old_inode"
    1.30 +						[ "$link" -eq 1 ] && hardlinks=$(($hardlinks+1)) &&
    1.31 +							save="$(($save+(${attr%%-*}+512)/1024))"
    1.32 +						continue
    1.33 +					else
    1.34 +						cp -p "$old_file" "$file"
    1.35  					fi
    1.36 +				else
    1.37 +					hinode=""
    1.38  				fi
    1.39  				old_attr="$attr" ; old_inode="$inode" ; old_file="$file"
    1.40  			done