From 70d425c1d341e5b3d36e612560f922dbd3b20e63 Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Fri, 6 Apr 2012 20:51:17 +0200 Subject: pkgdelta: rework option/argument parser Signed-off-by: Florian Pritz Signed-off-by: Dan McGee --- scripts/pkgdelta.sh.in | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) (limited to 'scripts/pkgdelta.sh.in') diff --git a/scripts/pkgdelta.sh.in b/scripts/pkgdelta.sh.in index 5fcc5be4..511910ba 100644 --- a/scripts/pkgdelta.sh.in +++ b/scripts/pkgdelta.sh.in @@ -38,11 +38,14 @@ m4_include(library/output_format.sh) # print usage instructions usage() { printf "pkgdelta (pacman) %s\n\n" "$myver" - printf -- "$(gettext "Usage: pkgdelta [-q] \n")" + printf -- "$(gettext "Usage: pkgdelta [options] \n")" printf -- "$(gettext "\ pkgdelta will create a delta file between two packages.\n\ This delta file can then be added to a database using repo-add.\n\n")" printf -- "$(gettext "Example: pkgdelta pacman-3.0.0.pkg.tar.gz pacman-3.0.1.pkg.tar.gz")\n" + echo + printf -- "$(gettext "Options:\n")" + printf -- " -q ""$(gettext "quiet\n")" } version() { @@ -123,32 +126,38 @@ create_xdelta() return 0 } -case "$1" in - -h|--help) usage; exit 0 ;; - -V|--version) version; exit 0 ;; - -q|--quiet) QUIET=1; shift ;; -esac - -if (( $# != 2 )); then +declare -a args + +# parse arguments +while (( $# )); do + case "$1" in + -h|--help) usage; exit 0 ;; + -V|--version) version; exit 0 ;; + -q|--quiet) QUIET=1;; + --) shift; args+=("$@"); break 2 ;; + -*) echo "invalid option -- '$1'"; usage; exit 1 ;; + *) args+=("$1");; + esac + shift +done + +if (( ${#args[@]} != 2 )); then usage exit 1 fi -if [[ ! -f $1 ]]; then - error "$(gettext "File '%s' does not exist")" "$1" - exit 1 -fi - -if [[ ! -f $2 ]]; then - error "$(gettext "File '%s' does not exist")" "$2" - exit 1 -fi +for i in "${args[@]}"; do + if [[ ! -f $i ]]; then + error "$(gettext "File '%s' does not exist")" "$i" + exit 1 + fi +done if ! type xdelta3 &>/dev/null; then error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" exit 1 fi -create_xdelta "$1" "$2" +create_xdelta "${args[@]}" # vim: set ts=2 sw=2 noet: -- cgit v1.2.3-54-g00ecf From 839953f1113c70d13560bfd05c5e52139c7f562f Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Fri, 6 Apr 2012 23:37:32 +0200 Subject: pkgdelta: implement requirments for delta generation Big deltas or deltas for very small packages are not needed so we should check that and not generate any. Signed-off-by: Florian Pritz Signed-off-by: Dan McGee --- doc/pkgdelta.8.txt | 12 +++++++++++- scripts/pkgdelta.sh.in | 52 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) (limited to 'scripts/pkgdelta.sh.in') diff --git a/doc/pkgdelta.8.txt b/doc/pkgdelta.8.txt index 3804c5d9..5913dba6 100644 --- a/doc/pkgdelta.8.txt +++ b/doc/pkgdelta.8.txt @@ -11,7 +11,7 @@ pkgdelta - package delta generation utility Synopsis -------- -'pkgdelta' [-q] +'pkgdelta' [options] Description @@ -27,6 +27,16 @@ significantly. Options ------- +*--max-delta-size *:: + Only create delta files if the delta is smaller than ratio * package_size. + Possible values: 0.0 to 2.0. + Recommended values: 0.2 to 0.9. + Default value: 0.7 + +*--min-pkg-size *:: + Minimal size of the package file in bytes to be considered for delta creation. + Default value: 1048576 bytes = 1MiB + *-q, \--quiet*:: Be quiet. Do not output anything but warnings and errors. diff --git a/scripts/pkgdelta.sh.in b/scripts/pkgdelta.sh.in index 511910ba..66ee35ec 100644 --- a/scripts/pkgdelta.sh.in +++ b/scripts/pkgdelta.sh.in @@ -30,6 +30,13 @@ myver='@PACKAGE_VERSION@' QUIET=0 +# minimal of package before deltas are generated (bytes) +min_pkg_size=$((1024*1024)) + +# percent of new package above which the delta will be discarded +max_delta_size=70 + + # ensure we have a sane umask set umask 0022 @@ -46,6 +53,8 @@ This delta file can then be added to a database using repo-add.\n\n")" echo printf -- "$(gettext "Options:\n")" printf -- " -q ""$(gettext "quiet\n")" + printf -- " --min-pkg-size ""$(gettext "minimal of package before deltas are generated (bytes)\n")" + printf -- " --max-delta-size ""$(gettext "percent of new package above which the delta will be discarded\n")" } version() { @@ -56,6 +65,10 @@ This is free software; see the source for copying conditions.\n\ There is NO WARRANTY, to the extent permitted by law.\n")" } +isnumeric() { + [[ $1 != *[!0-9]* ]] +} + read_pkginfo() { pkgname= pkgver= arch= @@ -96,6 +109,13 @@ create_xdelta() newver="$pkgver" newarch="$arch" + pkgsize="$(@SIZECMD@ -L "$newfile")" + + if ((pkgsize < min_pkg_size)); then + msg "$(gettext "Skipping delta creation for small package: %s - size %s")" "$newname" "$pkgsize" + return 0 + fi + if [[ $oldname != "$newname" ]]; then error "$(gettext "The package names don't match : '%s' and '%s'")" "$oldname" "$newname" return 1 @@ -119,10 +139,19 @@ create_xdelta() if (( ret )); then error "$(gettext "Delta could not be created.")" return 1 - else - msg "$(gettext "Generated delta : '%s'")" "$deltafile" - (( QUIET )) && echo "$deltafile" fi + + deltasize="$(@SIZECMD@ -L "$deltafile")" + + if ((max_delta_size * pkgsize / 100 < deltasize)); then + msg "$(gettext "Delta package larger than maximum size. Removing.")" + rm -f "$deltafile" + return 0 + fi + + msg "$(gettext "Generated delta : '%s'")" "$deltafile" + (( QUIET )) && echo "$deltafile" + return 0 } @@ -134,6 +163,23 @@ while (( $# )); do -h|--help) usage; exit 0 ;; -V|--version) version; exit 0 ;; -q|--quiet) QUIET=1;; + --min-pkg-size) + if ! isnumeric "$2"; then + echo "invalid argument '$2' for option -- '$1'" + exit 1 + fi + min_pkg_size=$2 + shift + ;; + --max-delta-size) + arg=$(echo "$2" | awk '{print $1 * 100}') + if ! isnumeric "$arg" || (($arg > 200)); then + echo "invalid argument '$2' for option -- '$1'" + exit 1 + fi + max_delta_size=$arg + shift + ;; --) shift; args+=("$@"); break 2 ;; -*) echo "invalid option -- '$1'"; usage; exit 1 ;; *) args+=("$1");; -- cgit v1.2.3-54-g00ecf From 133d174e60951dcca39d4c4063a85de75f52f6a8 Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Sat, 7 Apr 2012 14:18:28 +0200 Subject: pkgdelta/repo-add: quoting fixes This removes some unnecessary quotes and adds quotes in a few places to hopefully work correctly if the tempdir has spaces. Signed-off-by: Florian Pritz Signed-off-by: Dan McGee --- scripts/pkgdelta.sh.in | 2 +- scripts/repo-add.sh.in | 94 +++++++++++++++++++++++++------------------------- 2 files changed, 48 insertions(+), 48 deletions(-) (limited to 'scripts/pkgdelta.sh.in') diff --git a/scripts/pkgdelta.sh.in b/scripts/pkgdelta.sh.in index 66ee35ec..41d399b0 100644 --- a/scripts/pkgdelta.sh.in +++ b/scripts/pkgdelta.sh.in @@ -132,7 +132,7 @@ create_xdelta() fi msg "$(gettext "Generating delta from version %s to version %s")" "$oldver" "$newver" - deltafile="$(dirname $newfile)/$pkgname-${oldver}_to_${newver}-$arch.delta" + deltafile=$(dirname "$newfile")/$pkgname-${oldver}_to_${newver}-$arch.delta local ret=0 xdelta3 -q -f -s "$oldfile" "$newfile" "$deltafile" || ret=$? diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index 2bb9c83f..5724022a 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -108,7 +108,7 @@ format_entry() { find_pkgentry() { local pkgname=$1 local pkgentry - for pkgentry in $tmpdir/tree/$pkgname*; do + for pkgentry in "$tmpdir/tree/$pkgname"*; do name=${pkgentry##*/} if [[ ${name%-*-*} = $pkgname ]]; then echo $pkgentry @@ -129,31 +129,31 @@ get_delta_pkgname() { # write a delta entry # arg1 - path to delta file db_write_delta() { - deltafile="$1" - pkgname="$(get_delta_pkgname $deltafile)" + deltafile=$1 + pkgname=$(get_delta_pkgname "$deltafile") - pkgentry=$(find_pkgentry $pkgname) + pkgentry=$(find_pkgentry "$pkgname") if [[ -z $pkgentry ]]; then error "$(gettext "No database entry for package '%s'.")" "$pkgname" return 1 fi - deltas="$pkgentry/deltas" + deltas=$pkgentry/deltas if [[ ! -f $deltas ]]; then - echo -e "%DELTAS%" >$deltas + echo -e "%DELTAS%" >"$deltas" fi # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile")" - md5sum="${md5sum##* }" + md5sum=$(openssl dgst -md5 "$deltafile") + md5sum=${md5sum##* } csize=$(@SIZECMD@ -L "$deltafile") - oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') - newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') + oldfile=$(xdelta3 printhdr "$deltafile" | grep "XDELTA filename (source)" | sed 's/.*: *//') + newfile=$(xdelta3 printhdr "$deltafile" | grep "XDELTA filename (output)" | sed 's/.*: *//') - if grep -q "$oldfile.*$newfile" $deltas; then - sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup + if grep -q "$oldfile.*$newfile" "$deltas"; then + sed -i.backup "/$oldfile.*$newfile/d" "$deltas" && rm -f "$deltas.backup" fi msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" - echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas + echo "${deltafile##*/} $md5sum $csize $oldfile $newfile" >> "$deltas" return 0 } # end db_write_delta @@ -161,20 +161,20 @@ db_write_delta() { # remove a delta entry # arg1 - path to delta file db_remove_delta() { - deltafile="$1" + deltafile=$1 filename=${deltafile##*/} - pkgname="$(get_delta_pkgname $deltafile)" + pkgname=$(get_delta_pkgname "$deltafile") - pkgentry=$(find_pkgentry $pkgname) + pkgentry=$(find_pkgentry "$pkgname") if [[ -z $pkgentry ]]; then return 1 fi - deltas="$pkgentry/deltas" + deltas=$pkgentry/deltas if [[ ! -f $deltas ]]; then return 1 fi - if grep -q "$filename" $deltas; then - sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup + if grep -q "$filename" "$deltas"; then + sed -i.backup "/$filename/d" "$deltas" && rm -f "$deltas.backup" msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" return 0 fi @@ -192,7 +192,7 @@ check_gpg() { # sign the package database once repackaged create_signature() { (( ! SIGN )) && return - local dbfile="$1" + local dbfile=$1 local ret=0 msg "$(gettext "Signing database...")" @@ -212,7 +212,7 @@ create_signature() { # verify the existing package database signature verify_signature() { (( ! VERIFY )) && return - local dbfile="$1" + local dbfile=$1 local ret=0 msg "$(gettext "Verifying database signature...")" @@ -232,7 +232,7 @@ verify_signature() { verify_repo_extension() { local repofile=$1 - case "$repofile" in + case $repofile in *.@(db|files).tar.gz) TAR_OPT="z" ;; *.@(db|files).tar.bz2) TAR_OPT="j" ;; *.@(db|files).tar.xz) TAR_OPT="J" ;; @@ -250,7 +250,7 @@ verify_repo_extension() { # arg1 - path to package db_write_entry() { # blank out all variables - local pkgfile="$1" + local pkgfile=$1 local -a _groups _licenses _replaces _depends _conflicts _provides _optdepends local pkgname pkgver pkgdesc csize size url arch builddate packager \ md5sum sha256sum pgpsig pgpsigsize @@ -263,7 +263,7 @@ db_write_entry() { # normalize whitespace with an extglob declare "$var=${val//+([[:space:]])/ }" - case "$var" in + case $var in group) _groups+=("$group") ;; license) _licenses+=("$license") ;; replaces) _replaces+=("$replaces") ;; @@ -284,10 +284,10 @@ db_write_entry() { warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" else if (( DELTA )); then - pkgentry=$(find_pkgentry $pkgname) + pkgentry=$(find_pkgentry "$pkgname") if [[ -n $pkgentry ]]; then - local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) - local oldfile="$(dirname $1)/$oldfilename" + local oldfilename=$(grep -A1 FILENAME "$pkgentry/desc" | tail -n1) + local oldfile="$(dirname "$1")/$oldfilename" fi fi fi @@ -307,10 +307,10 @@ db_write_entry() { # compute checksums msg2 "$(gettext "Computing checksums...")" - md5sum="$(openssl dgst -md5 "$pkgfile")" - md5sum="${md5sum##* }" - sha256sum="$(openssl dgst -sha256 "$pkgfile")" - sha256sum="${sha256sum##* }" + md5sum=$(openssl dgst -md5 "$pkgfile") + md5sum=${md5sum##* } + sha256sum=$(openssl dgst -sha256 "$pkgfile") + sha256sum=${sha256sum##* } # remove an existing entry if it exists, ignore failures db_remove_entry "$pkgname" @@ -366,17 +366,17 @@ db_write_entry() { if (( WITHFILES )); then msg2 "$(gettext "Creating '%s' db entry...")" 'files' local files_path="$tmpdir/tree/$pkgname-$pkgver/files" - echo "%FILES%" >$files_path - bsdtar --exclude='^.*' -tf "$pkgfile" >>$files_path + echo "%FILES%" >"$files_path" + bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_path" fi # create a delta file if (( DELTA )); then if [[ -n $oldfilename ]]; then if [[ -f $oldfile ]]; then - delta=$(pkgdelta -q $oldfile $1) + delta=$(pkgdelta -q "$oldfile" "$1") if [[ -f $delta ]]; then - db_write_delta $delta + db_write_delta "$delta" fi else warning "$(gettext "Old package file not found: %s")" "$oldfilename" @@ -392,7 +392,7 @@ db_write_entry() { db_remove_entry() { local pkgname=$1 local notfound=1 - local pkgentry=$(find_pkgentry $pkgname) + local pkgentry=$(find_pkgentry "$pkgname") while [[ -n $pkgentry ]]; do notfound=0 if [[ -f $pkgentry/deltas ]]; then @@ -400,8 +400,8 @@ db_remove_entry() { fi msg2 "$(gettext "Removing existing entry '%s'...")" \ "${pkgentry##*/}" - rm -rf $pkgentry - pkgentry=$(find_pkgentry $pkgname) + rm -rf "$pkgentry" + pkgentry=$(find_pkgentry "$pkgname") done return $notfound } # end db_remove_entry @@ -432,7 +432,7 @@ check_repo_db() { repodir=${repodir%/*}/ fi - if [[ ! -d "$repodir" ]]; then + if [[ ! -d $repodir ]]; then error "$(gettext "%s does not exist or is not a directory.")" "$repodir" exit 1 fi @@ -442,7 +442,7 @@ check_repo_db() { CLEAN_LOCK=1 else error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" + [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat "$LOCKFILE")" exit 1 fi @@ -460,7 +460,7 @@ check_repo_db() { msg "$(gettext "Extracting database to a temporary location...")" bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir/tree" else - case "$cmd" in + case $cmd in repo-remove) error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" exit 1 @@ -562,7 +562,7 @@ if ! type gettext &>/dev/null; then } fi -case "$1" in +case $1 in -h|--help) usage; exit 0;; -V|--version) version; exit 0;; esac @@ -582,7 +582,7 @@ fi tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ error "$(gettext "Cannot create temp directory for database building.")"; \ exit 1) -mkdir $tmpdir/tree +mkdir "$tmpdir/tree" trap 'clean_up' EXIT for signal in TERM HUP QUIT; do @@ -595,7 +595,7 @@ declare -a args success=0 # parse arguments while (( $# )); do - case "$1" in + case $1 in -q|--quiet) QUIET=1;; -d|--delta) DELTA=1;; -f|--files) WITHFILES=1;; @@ -614,7 +614,7 @@ while (( $# )); do -k|--key) check_gpg shift - GPGKEY="$1" + GPGKEY=$1 if ! gpg --list-key ${GPGKEY} &>/dev/null; then error "$(gettext "The key ${GPGKEY} does not exist in your keyring.")" exit 1 @@ -643,7 +643,7 @@ verify_repo_extension "$REPO_DB_FILE" >/dev/null check_repo_db for arg in "${args[@]:1}"; do - case "$cmd" in + case $cmd in repo-add) add "$arg" ;; repo-remove) remove "$arg" ;; esac && success=1 @@ -676,7 +676,7 @@ if (( success )); then fi [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" [[ -f $tmpdir/$filename.sig ]] && mv "$tmpdir/$filename.sig" "$REPO_DB_FILE.sig" - dblink="${REPO_DB_FILE%.tar*}" + dblink=${REPO_DB_FILE%.tar*} target=${REPO_DB_FILE##*/} rm -f "$dblink" "$dblink.sig" ln -s "$target" "$dblink" 2>/dev/null || \ -- cgit v1.2.3-54-g00ecf