From 3ce1ce586c0d8079e8ce9ebe5fd1c2f8b66dfb92 Mon Sep 17 00:00:00 2001 From: Valerie R Young Date: Wed, 8 Jun 2016 19:09:56 -0400 Subject: reproducible debian: bug fix: only debian html belongs in tests.r-b.org/debian --- bin/reproducible_build.sh | 52 +++++++++++++------------- bin/reproducible_common.py | 71 ++++++++++++++++++------------------ bin/reproducible_common.sh | 43 +++++++++++----------- bin/reproducible_html_breakages.py | 2 +- bin/reproducible_html_dashboard.sh | 34 ++++++++--------- bin/reproducible_html_dd_list.py | 2 +- bin/reproducible_html_indexes.py | 4 +- bin/reproducible_html_live_status.py | 4 +- bin/reproducible_html_notes.py | 8 ++-- bin/reproducible_html_packages.py | 10 ++--- bin/reproducible_html_pkg_sets.sh | 16 ++++---- bin/reproducible_maintenance.sh | 12 +++--- 12 files changed, 130 insertions(+), 128 deletions(-) (limited to 'bin') diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index b48c644a..efce0b14 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -14,12 +14,12 @@ common_init "$@" set -e create_results_dirs() { - mkdir -vp $BASE/dbd/${SUITE}/${ARCH} - mkdir -vp $BASE/dbdtxt/${SUITE}/${ARCH} - mkdir -vp $BASE/logs/${SUITE}/${ARCH} - mkdir -vp $BASE/logdiffs/${SUITE}/${ARCH} - mkdir -vp $BASE/rbuild/${SUITE}/${ARCH} - mkdir -vp $BASE/buildinfo/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/dbd/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/dbdtxt/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/logs/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/rbuild/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH} } handle_race_condition() { @@ -44,9 +44,9 @@ save_artifacts() { local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5) local ARTIFACTS="artifacts/r00t-me/${SRCPACKAGE}_${SUITE}_tmp-${random}" local URL="$DEBIAN_URL/$ARTIFACTS/" - local HEADER="$BASE/$ARTIFACTS/.HEADER.html" - mkdir -p $BASE/$ARTIFACTS - cp -r $TMPDIR/* $BASE/$ARTIFACTS/ + local HEADER="$DEBIAN_BASE/$ARTIFACTS/.HEADER.html" + mkdir -p $DEBIAN_BASE/$ARTIFACTS + cp -r $TMPDIR/* $DEBIAN_BASE/$ARTIFACTS/ echo | tee -a ${RBUILDLOG} local msg="Artifacts from this build have been preserved. They will be available for 24h only, so download them now.\n" msg="${msg}WARNING: You shouldn't trust packages downloaded from this host, they can contain malware or the worst of your fears, packaged nicely in debian format.\n" @@ -54,7 +54,7 @@ save_artifacts() { printf "$msg" | tee -a $BUILDLOG echo "

" > $HEADER printf "$msg" | sed 's#$#
#g' >> $HEADER - echo "Package page: $REPRODUCIBLE_URL/${SUITE}/${ARCH}/${SRCPACKAGE}
" >> $HEADER + echo "Package page: $REPRODUCIBLE_URL/${SUITE}/${ARCH}/${SRCPACKAGE}
" >> $HEADER echo "

" >> $HEADER chmod 644 $HEADER echo | tee -a ${RBUILDLOG} @@ -135,12 +135,12 @@ update_db_and_html() { update_rbuildlog() { chmod 644 $RBUILDLOG - mv $RBUILDLOG $BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log - RBUILDLOG=$BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log + mv $RBUILDLOG $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log + RBUILDLOG=$DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log } diff_copy_buildlogs() { - local DIFF="$BASE/logdiffs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.diff" + local DIFF="$DEBIAN_BASE/logdiffs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.diff" if [ -f b1/build.log ] ; then if [ -f b2/build.log ] ; then printf "Diff of the two buildlogs:\n\n--\n" | tee -a $DIFF @@ -150,15 +150,15 @@ diff_copy_buildlogs() { fi echo -e "\nCompressing the 2nd log..." gzip -9vn $DIFF - gzip -9cvn b2/build.log > $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz - chmod 644 $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz + gzip -9cvn b2/build.log > $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz + chmod 644 $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz elif [ $FTBFS -eq 0 ] ; then echo "Warning: No second build log, what happened?" | tee -a $RBUILDLOG fi set -x # # to debug diffoscope/schroot problems echo "Compressing the 1st log..." - gzip -9cvn b1/build.log > $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz - chmod 644 $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz + gzip -9cvn b1/build.log > $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz + chmod 644 $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz else echo "Error: No first build log, not even looking for the second" | tee -a $RBUILDLOG fi @@ -207,21 +207,21 @@ handle_ftbfs() { local nodevar="NODE$BUILD" local node="" eval node=\$$nodevar - if [ ! -f "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ] ; then + if [ ! -f "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ] ; then continue fi - if zgrep -F "E: pbuilder-satisfydepends failed." "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then + if zgrep -F "E: pbuilder-satisfydepends failed." "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then handle_depwait return fi for NEEDLE in '^tar:.*Cannot write: No space left on device' 'fatal error: error writing to .* No space left on device' './configure: line .* printf: write error: No space left on device' 'cat: write error: No space left on device' '^dpkg-deb.*No space left on device' '^cp: (erreur|impossible).*No space left on device' '^tee: .* No space left on device' '^zip I/O error: No space left on device' '^mkdir .*: No space left on device' ; do - if zgrep -e "$NEEDLE" "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then + if zgrep -e "$NEEDLE" "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then handle_enospace $node return fi done # notify about unkown diskspace issues where we are not 100% sure yet those are diskspace issues - if zgrep -e "No space left on device" "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then + if zgrep -e "No space left on device" "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then MESSAGE="${BUILD_URL}console for ${SRCPACKAGE} (ftbfs in $SUITE/$ARCH) _probably_ had a diskspace issue on $node. Please check, tune handle_ftbfs() and reschedule the package." echo $MESSAGE | tee -a /var/log/jenkins/reproducible-diskspace-issues.log irc_message debian-reproducible "$MESSAGE" @@ -238,18 +238,18 @@ handle_ftbr() { local FTBRmessage="$@" echo | tee -a ${RBUILDLOG} echo "$(date -u) - ${SRCPACKAGE} failed to build reproducibly in ${SUITE} on ${ARCH}." | tee -a ${RBUILDLOG} - cp b1/${BUILDINFO} $BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 || true # will fail if there is no .buildinfo + cp b1/${BUILDINFO} $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 || true # will fail if there is no .buildinfo if [ ! -z "$FTRmessage" ] ; then echo "$(date -u) - ${FTBRmessage}." | tee -a ${RBUILDLOG} fi if [ -f ./${DBDREPORT} ] ; then - mv ./${DBDREPORT} $BASE/dbd/${SUITE}/${ARCH}/ + mv ./${DBDREPORT} $DEBIAN_BASE/dbd/${SUITE}/${ARCH}/ else echo "$(date -u) - $DIFFOSCOPE produced no output (which is strange)." | tee -a $RBUILDLOG fi if [ -f ./$DBDTXT ] ; then - mv ./$DBDTXT $BASE/dbdtxt/$SUITE/$ARCH/ - gzip -9n $BASE/dbdtxt/$SUITE/$ARCH/$DBDTXT + mv ./$DBDTXT $DEBIAN_BASE/dbdtxt/$SUITE/$ARCH/ + gzip -9n $DEBIAN_BASE/dbdtxt/$SUITE/$ARCH/$DBDTXT fi calculate_build_duration update_db_and_html "unreproducible" @@ -257,7 +257,7 @@ handle_ftbr() { handle_reproducible() { if [ ! -f ./${DBDREPORT} ] && [ -f b1/${BUILDINFO} ] ; then - cp b1/${BUILDINFO} $BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 + cp b1/${BUILDINFO} $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 figlet ${SRCPACKAGE} echo | tee -a ${RBUILDLOG} echo "$DIFFOSCOPE found no differences in the changes files, and a .buildinfo file also exists." | tee -a ${RBUILDLOG} diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py index 58f77731..b08fedb4 100755 --- a/bin/reproducible_common.py +++ b/bin/reproducible_common.py @@ -42,22 +42,23 @@ defaultsuite = 'unstable' defaultarch = 'amd64' BIN_PATH = '/srv/jenkins/bin' -BASE = '/var/lib/jenkins/userContent/reproducible/debian' +BASE = '/var/lib/jenkins/userContent/reproducible' +DEBIAN_BASE = '/var/lib/jenkins/userContent/reproducible/debian' REPRODUCIBLE_JSON = BASE + '/reproducible.json' REPRODUCIBLE_TRACKER_JSON = BASE + '/reproducible-tracker.json' REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db' -DBD_URI = '/dbd' -DBDTXT_URI = '/dbdtxt' -LOGS_URI = '/logs' -DIFFS_URI = '/logdiffs' -NOTES_URI = '/notes' -ISSUES_URI = '/issues' -RB_PKG_URI = '/rb-pkg' -RBUILD_URI = '/rbuild' -HISTORY_URI = '/history' -BUILDINFO_URI = '/buildinfo' +DBD_URI = '/debian/dbd' +DBDTXT_URI = '/debian/dbdtxt' +LOGS_URI = '/debian/logs' +DIFFS_URI = '/debian/logdiffs' +NOTES_URI = '/debian/notes' +ISSUES_URI = '/debian/issues' +RB_PKG_URI = '/debian/rb-pkg' +RBUILD_URI = '/debian/rbuild' +HISTORY_URI = '/debian/history' +BUILDINFO_URI = '/debian/buildinfo' DBD_PATH = BASE + DBD_URI DBDTXT_PATH = BASE + DBDTXT_URI LOGS_PATH = BASE + LOGS_URI @@ -173,56 +174,56 @@ html_head_page = Template((tab*2).join((""" $project_links @@ -273,10 +274,10 @@ filter_query = '' for issue in filtered_issues: if filter_query == '': filter_query = 'n.issues LIKE "%' + issue + '%"' - filter_html = '' + issue + '' + filter_html = '' + issue + '' else: filter_query += ' OR n.issues LIKE "%' + issue + '%"' - filter_html += ' or ' + issue + '' + filter_html += ' or ' + issue + '' @atexit.register @@ -326,20 +327,20 @@ def convert_into_hms_string(duration): def _gen_pkg_sets_link(suite, arch): html = '' if suite != 'experimental': - html = '
  • package sets
    • ' + html = '
  • package sets
    • ' return html def _gen_arch_links(suite, arch): html = '
  • ' for a in ARCHS: - html += ' ' + a + '  ' + html += ' ' + a + '  ' html += '
    • ' return html def _gen_suite_links(suite, arch): html = '
  • ' for s in SUITES: - html += ' ' + s + '  ' + html += ' ' + s + '  ' html += '
    • ' return html diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index f92301e1..eaa77743 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -71,8 +71,9 @@ NUM_CPU=$(grep -c '^processor' /proc/cpuinfo) # we only this array for html creation but we cannot declare them in a function declare -A SPOKENTARGET -BASE="/var/lib/jenkins/userContent/reproducible/debian" -mkdir -p "$BASE" +BASE="/var/lib/jenkins/userContent/reproducible" +DEBIAN_BASE="/var/lib/jenkins/userContent/reproducible/debian" +mkdir -p "$DEBIAN_BASE" # to hold reproducible temporary files/directories without polluting /tmp TEMPDIR="/tmp/reproducible" @@ -80,7 +81,7 @@ mkdir -p "$TEMPDIR" # create subdirs for suites for i in $SUITES ; do - mkdir -p "$BASE/$i" + mkdir -p "$DEBIAN_BASE/$i" done # table names and image names @@ -186,7 +187,7 @@ set_icon() { write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() - write_page "\"${STATE_TARGET_NAME}" + write_page "\"${STATE_TARGET_NAME}" } write_page_header() { @@ -233,16 +234,16 @@ write_page_header() { continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} - BASEURL="/$SUITE/$ARCH" + BASEURL="/debian/$SUITE/$ARCH" local i for i in $GLOBALVIEWS ; do if [ "$TARGET" = "$i" ] ; then - BASEURL="" + BASEURL="/debian" fi done for i in ${SUITEVIEWS} ; do if [ "$TARGET" = "$i" ] ; then - BASEURL="/$SUITE" + BASEURL="/debian/$SUITE" fi done # prepare unsorted lists @@ -253,7 +254,7 @@ write_page_header() { fi # prepare links if [ "$TARGET" = "scheduled" ] ; then - write_page "
  • ${SPOKEN_TARGET}
    • " + write_page "
  • ${SPOKEN_TARGET}
    • " elif [ "$TARGET" = "notify" ] ; then write_page "
  • ${SPOKEN_TARGET}
    • " elif [ "$TARGET" = "arch" ] ; then @@ -262,7 +263,7 @@ write_page_header() { if [ "$1" = "suite_arch_stats" ] && [ "$i" = "$ARCH" ] ; then CLASS=" class=\"active\"" fi - write_page " $i  " + write_page " $i  " CLASS="" done write_page "" @@ -272,7 +273,7 @@ write_page_header() { if [ "$1" = "suite_arch_stats" ] && [ "$i" = "$SUITE" ] ; then CLASS=" class=\"active\"" fi - write_page " $i  " + write_page " $i  " CLASS="" done write_page "" @@ -564,7 +565,7 @@ publish_page() { else TARGET=$1/$PAGE fi - cp $PAGE $BASE/$TARGET + cp $PAGE $DEBIAN_BASE/$TARGET rm $PAGE echo "Enjoy $DEBIAN_URL/$TARGET" } @@ -696,12 +697,12 @@ get_filesize() { } cleanup_pkg_files() { - rm -vf $BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz} - rm -vf $BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz} - rm -vf $BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.html - rm -vf $BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.txt{,.gz} - rm -vf $BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo - rm -vf $BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz} + rm -vf $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz} + rm -vf $DEBIAN_BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz} + rm -vf $DEBIAN_BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.html + rm -vf $DEBIAN_BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.txt{,.gz} + rm -vf $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo + rm -vf $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz} } # @@ -806,10 +807,10 @@ create_png_from_table() { mkdir -p $DIR echo "Generating $2." /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT - mv $2 $BASE/$DIR + mv $2 $DEBIAN_BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) # create empty dummy png if there havent been any results ever - elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then + elif [ ! -f $DEBIAN_BASE/$DIR/$(basename $2) ] ; then DIR=$(dirname $2) mkdir -p $DIR echo "Creating $2 dummy." @@ -817,9 +818,9 @@ create_png_from_table() { if [ "$3" != "" ] ; then local THUMB="${TABLE[1]}_${3}-thumbnail.png" convert $2 -adaptive-resize 160x80 ${THUMB} - mv ${THUMB} $BASE/$DIR + mv ${THUMB} $DEBIAN_BASE/$DIR fi - mv $2 $BASE/$DIR + mv $2 $DEBIAN_BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) fi rm ${TABLE[$1]}.csv diff --git a/bin/reproducible_html_breakages.py b/bin/reproducible_html_breakages.py index 7bfc03dd..05f8d6ba 100755 --- a/bin/reproducible_html_breakages.py +++ b/bin/reproducible_html_breakages.py @@ -326,7 +326,7 @@ if __name__ == '__main__': html += '

    COOL!!! Everything is GOOD and not a single issue was ' html += 'detected. Enjoy!

    ' title = 'Breakage on the Debian pages of tests.reproducible-builds.org' - destfile = BASE + '/index_breakages.html' + destfile = DEBIAN_BASE + '/index_breakages.html' desturl = DEBIAN_URL + '/index_breakages.html' write_html_page(title, html, destfile, style_note=True) log.info('Breackages page created at ' + desturl) diff --git a/bin/reproducible_html_dashboard.sh b/bin/reproducible_html_dashboard.sh index ba3dfff3..fb1d5654 100755 --- a/bin/reproducible_html_dashboard.sh +++ b/bin/reproducible_html_dashboard.sh @@ -140,9 +140,9 @@ update_suite_arch_stats() { PREFIX=$SUITE/$ARCH fi # force regeneration of the image if it exists - if [ -f $BASE/$PREFIX/${TABLE[$i]}.png ] ; then + if [ -f $DEBIAN_BASE/$PREFIX/${TABLE[$i]}.png ] ; then echo "Touching $PREFIX/${TABLE[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $BASE/$PREFIX/${TABLE[$i]}.png + touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$PREFIX/${TABLE[$i]}.png fi done fi @@ -239,7 +239,7 @@ update_bug_stats() { local i=0 for i in 3 7 8 9 ; do echo "Touching ${TABLE[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $BASE/${TABLE[$i]}.png + touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/${TABLE[$i]}.png done fi fi @@ -433,10 +433,10 @@ create_suite_arch_stats_page() { write_icon write_page "$COUNT_BLACKLISTED ($PERCENT_BLACKLISTED%) blacklisted packages neither.

    " write_page "

    " - write_page " \"${MAINLABEL[0]}\"" + write_page " \"${MAINLABEL[0]}\"" for i in 0 2 ; do # recreate png once a day - if [ ! -f $BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] ; then + if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] ; then create_png_from_table $i $SUITE/$ARCH/${TABLE[$i]}.png fi done @@ -453,7 +453,7 @@ write_meta_pkg_graphs_links () { for i in $(seq 1 ${#META_PKGSET[@]}) ; do THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "\"$LABEL\"" + write_page "\"$LABEL\"" done write_page "

    " } @@ -472,7 +472,7 @@ create_dashboard_page() { # write suite graphs for ARCH in ${ARCHS} ; do for SUITE in $SUITES ; do - write_page " \"$SUITE/$ARCH" + write_page " \"$SUITE/$ARCH" done SUITE="unstable" if [ "$ARCH" = "amd64" ] ; then @@ -489,7 +489,7 @@ create_dashboard_page() { done write_page "" ARCH="amd64" - write_page "identified distinct and categorized issues$ISSUES" + write_page "identified distinct and categorized issues$ISSUES" write_page "total number of identified issues in packages$COUNT_ISSUES" write_page "packages with notes about these issues$NOTES" @@ -502,7 +502,7 @@ create_dashboard_page() { SUITE="unstable" gather_suite_arch_stats RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - TD_PKG_NOISSUES="$TD_PKG_NOISSUES$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%" + TD_PKG_NOISSUES="$TD_PKG_NOISSUES$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%" RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") TD_PKG_FTBR="$TD_PKG_FTBR$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%" RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") @@ -532,7 +532,7 @@ create_dashboard_page() { wc -l)" fi RESULT=$(cat /srv/reproducible-results/modified_in_sid.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh - write_page "packages modified in our toolchain (in unstable)$(echo $RESULT)" + write_page "packages modified in our toolchain (in unstable)$(echo $RESULT)" RESULT=$(cat /srv/reproducible-results/modified_in_exp.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh write_page "  - (in experimental)$(echo $RESULT)" RESULT=$(cat /srv/reproducible-results/binnmus_needed.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh @@ -545,9 +545,9 @@ create_dashboard_page() { write_page "

    " # do other global graphs for i in 8 9 3 7 4 5 ; do - write_page " \"${MAINLABEL[$i]}\"" + write_page " \"${MAINLABEL[$i]}\"" # redo pngs once a day - if [ ! -f $BASE/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $BASE/${TABLE[$i]}.png ] ; then + if [ ! -f $DEBIAN_BASE/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/${TABLE[$i]}.png ] ; then create_png_from_table $i ${TABLE[$i]}.png fi done @@ -557,7 +557,7 @@ create_dashboard_page() { write_page "
    There are some problems in this test setup itself too. And there is documentation too, in case you missed the link at the top. Feedback is very much appreciated.

    " # the end write_page_footer - cp $PAGE $BASE/reproducible.html + cp $PAGE $DEBIAN_BASE/reproducible.html publish_page } @@ -574,8 +574,8 @@ create_performance_page() { # arch performance stats write_page "

    " for ARCH in ${ARCHS} ; do - write_page " \"${MAINLABEL[1]}\"" - if [ ! -f $BASE/${TABLE[1]}_$ARCH.png ] || [ $DUMMY_FILE -nt $BASE/${TABLE[1]}_$ARCH.png ] ; then + write_page " \"${MAINLABEL[1]}\"" + if [ ! -f $DEBIAN_BASE/${TABLE[1]}_$ARCH.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/${TABLE[1]}_$ARCH.png ] ; then create_png_from_table 1 ${TABLE[1]}_$ARCH.png fi done @@ -585,14 +585,14 @@ create_performance_page() { write_page "

    " for ARCH in ${ARCHS} ; do for SUITE in $SUITES ; do - write_page " \"age" + write_page " \"age" done write_page "

    " done # the end write_page "Daily individual build node performance stats are available as well as views of oldest results for" for ARCH in ${ARCHS} ; do - write_page " $ARCH" + write_page " $ARCH" done write_page ".

    " write_page_footer diff --git a/bin/reproducible_html_dd_list.py b/bin/reproducible_html_dd_list.py index ae077d47..174c8124 100755 --- a/bin/reproducible_html_dd_list.py +++ b/bin/reproducible_html_dd_list.py @@ -72,6 +72,6 @@ for suite in SUITES: html += '\n' html += '

    ' title = 'Maintainers of unreproducible packages in ' + suite - destfile = BASE + '/' + suite + '/index_dd-list.html' + destfile = DEBIAN_BASE + '/' + suite + '/index_dd-list.html' write_html_page(title, html, destfile, suite, arch, style_note=True) log.info('%s/%s/index_dd-list.html published', DEBIAN_URL, suite) diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py index 7dd01aea..f0eda28b 100755 --- a/bin/reproducible_html_indexes.py +++ b/bin/reproducible_html_indexes.py @@ -522,11 +522,11 @@ def build_page(page, suite=None, arch=None): html += html1 footnote = True if footnote1 else footnote if gpage: - destfile = BASE + '/index_' + page + '.html' + destfile = DEBIAN_BASE + '/index_' + page + '.html' desturl = DEBIAN_URL + '/index_' + page + '.html' suite = defaultsuite # used for the links generated by write_html_page else: - destfile = BASE + '/' + suite + '/' + arch + '/index_' + page + '.html' + destfile = DEBIAN_BASE + '/' + suite + '/' + arch + '/index_' + page + '.html' desturl = DEBIAN_URL + '/' + suite + '/' + arch + '/index_' + \ page + '.html' write_html_page(title=title, body=html, destfile=destfile, suite=suite, arch=arch, style_note=True) diff --git a/bin/reproducible_html_live_status.py b/bin/reproducible_html_live_status.py index 7ffe5217..fa401def 100755 --- a/bin/reproducible_html_live_status.py +++ b/bin/reproducible_html_live_status.py @@ -50,7 +50,7 @@ def generate_schedule(arch): html += link_package(pkg, row[1], row[2], bugs) html += ''+convert_into_status_html(str(row[4]))+''+duration+'' + avg_duration + '\n' html += '

    \n' - destfile = BASE + '/index_' + arch + '_scheduled.html' + destfile = DEBIAN_BASE + '/index_' + arch + '_scheduled.html' desturl = DEBIAN_URL + '/index_' + arch + '_scheduled.html' write_html_page(title=title, body=html, destfile=destfile, arch=arch, style_note=True, refresh_every=60) log.info("Page generated at " + desturl) @@ -120,7 +120,7 @@ def generate_oldies(arch): html += link_package(pkg, row[0], row[1], bugs) html += ''+convert_into_status_html(str(row[3]))+'' + row[4] + '\n' html += '

    \n' - destfile = BASE + '/index_' + arch + '_oldies.html' + destfile = DEBIAN_BASE + '/index_' + arch + '_oldies.html' desturl = DEBIAN_URL + '/index_' + arch + '_oldies.html' write_html_page(title=title, body=html, destfile=destfile, arch=arch, style_note=True, refresh_every=60) log.info("Page generated at " + desturl) diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py index 9cb00517..15f6068a 100755 --- a/bin/reproducible_html_notes.py +++ b/bin/reproducible_html_notes.py @@ -263,7 +263,7 @@ def gen_html_issue(issue, suite): if suite_links != '': suite_links += ' / ' if i != suite: - suite_links += '' + i + '' + suite_links += '' + i + '' else: suite_links += '' + i + '' # check for url: @@ -361,7 +361,7 @@ def iterate_over_notes(notes): write_html_page(title=title, body=html, destfile=destfile, noheader=True) - desturl = DEBIAN_URL + NOTES_URI + '/' + package + '_note.html' + desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' log.debug("Note created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' note pages.') @@ -381,7 +381,7 @@ def iterate_over_issues(issues): write_html_page(title=title, body=html, destfile=destfile, style_note=True) - desturl = DEBIAN_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html' + desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html' log.debug("Issue created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' issue pages for ' + suite) @@ -417,7 +417,7 @@ def index_issues(issues): ' issues.

    ' html += tab*2 + '

    ' + NOTESGIT_DESCRIPTION + '

    ' title = 'Known issues related to reproducible builds' - destfile = BASE + '/index_issues.html' + destfile = DEBIAN_BASE + '/index_issues.html' desturl = DEBIAN_URL + '/index_issues.html' write_html_page(title=title, body=html, destfile=destfile) log.info('Issues index now available at ' + desturl) diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py index 78461b2a..caa32413 100755 --- a/bin/reproducible_html_packages.py +++ b/bin/reproducible_html_packages.py @@ -16,7 +16,7 @@ html_package_page = Template((tab*2).join(("""

    $package

    \n' @@ -283,7 +283,7 @@ def gen_packages_html(packages, no_clean=False): project_links=project_links, default_view=default_view) destfile = RB_PKG_PATH + '/' + suite + '/' + arch + '/' + pkg + '.html' - desturl = DEBIAN_URL + RB_PKG_URI + '/' + suite + \ + desturl = REPRODUCIBLE_URL + RB_PKG_URI + '/' + suite + \ '/' + arch + '/' + pkg + '.html' title = pkg + ' - reproducible build results' write_html_page(title=title, body=html, destfile=destfile, diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh index 72389dc6..7cf5fec3 100755 --- a/bin/reproducible_html_pkg_sets.sh +++ b/bin/reproducible_html_pkg_sets.sh @@ -80,7 +80,7 @@ update_meta_pkg_stats() { echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE/$ARCH on $DATE." fi echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png + touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png fi done } @@ -105,12 +105,12 @@ create_pkg_sets_navigation() { else CLASS="" fi - if [ -f $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then + if [ -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" write_page "
  • " - write_page "${META_PKGSET[$i]}" - write_page "\"$LABEL\"" + write_page "${META_PKGSET[$i]}" + write_page "\"$LABEL\"" write_page "
    • " fi done @@ -149,19 +149,19 @@ create_pkg_sets_pages() { PNG=${TABLE[6]}_${META_PKGSET[$i]}.png THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" # redo pngs once a day - if [ ! -f $BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then + if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $DEBIAN_BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]} - convert $BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $BASE/$SUITE/$ARCH/$THUMB + convert $DEBIAN_BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $DEBIAN_BASE/$SUITE/$ARCH/$THUMB fi LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH" - write_page "

    \"$LABEL\"" + write_page "

    \"$LABEL\"" write_page "
    The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:" write_page " (this set on " for LINKARCH in ${ARCHS} ; do if [ "$LINKARCH" = "$ARCH" ] ; then continue else - write_page "$LINKARCH " + write_page "$LINKARCH " fi done write_page ")
     
    " diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index 90ca5e30..b8a2fbb4 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -171,7 +171,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then # (ignore "*None.rbuild.log" because these are build which were just started) # this job runs every 4h echo "$(date -u) - Rescheduling failed builds due to network issues." - FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true) + FAILED_BUILDS=$(find $DEBIAN_BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true) if [ ! -z "$FAILED_BUILDS" ] ; then echo echo "The following builds have failed due to network problems and will be rescheduled now:" @@ -203,7 +203,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then # (ignore "*None.rbuild.log" because these are build which were just started) # this job runs every 4h echo "$(date -u) - Rescheduling failed builds due to diffoscope schroot issues." - FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -F 'E: 10mount: error: Directory' {} \; || true) + FAILED_BUILDS=$(find $DEBIAN_BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -F 'E: 10mount: error: Directory' {} \; || true) if [ ! -z "$FAILED_BUILDS" ] ; then echo echo "Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled now:" @@ -274,7 +274,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then QUERY="DELETE FROM removed_packages WHERE name='$PKGNAME' AND suite='$SUITE' AND architecture='$ARCH'" sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" - cd $BASE + cd $DEBIAN_BASE find rb-pkg/$SUITE/$ARCH rbuild/$SUITE/$ARCH dbd/$SUITE/$ARCH dbdtxt/$SUITE/$ARCH buildinfo/$SUITE/$ARCH logs/$SUITE/$ARCH logdiffs/$SUITE/$ARCH -name "${PKGNAME}_*" | xargs -r rm -v || echo "Warning: couldn't delete old files from ${PKGNAME} in $SUITE/$ARCH" done cd - > /dev/null @@ -366,17 +366,17 @@ fi # remove artifacts older than a day echo "$(date -u) - Checking for artifacts older than a day." -ARTIFACTS=$(find $BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec ls -lad {} \; 2>/dev/null|| true) +ARTIFACTS=$(find $DEBIAN_BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec ls -lad {} \; 2>/dev/null|| true) if [ ! -z "$ARTIFACTS" ] ; then echo echo "Removed old artifacts:" - find $BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec rm -rv {} \; + find $DEBIAN_BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec rm -rv {} \; echo fi # find + chmod files with bad permissions echo "$(date -u) - Checking for files with bad permissions." -BADPERMS=$(find $BASE/{buildinfo,dbd,rbuild,artifacts,unstable,experimental,testing,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true) +BADPERMS=$(find $DEBIAN_BASE/{buildinfo,dbd,rbuild,artifacts,unstable,experimental,testing,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true) if [ ! -z "$BADPERMS" ] ; then DIRTY=true echo -- cgit v1.2.3-70-g09d2