diff options
| -rwxr-xr-x | bin/reproducible_build.sh | 52 | ||||
| -rwxr-xr-x | bin/reproducible_common.py | 71 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 43 | ||||
| -rwxr-xr-x | bin/reproducible_html_breakages.py | 2 | ||||
| -rwxr-xr-x | bin/reproducible_html_dashboard.sh | 34 | ||||
| -rwxr-xr-x | bin/reproducible_html_dd_list.py | 2 | ||||
| -rwxr-xr-x | bin/reproducible_html_indexes.py | 4 | ||||
| -rwxr-xr-x | bin/reproducible_html_live_status.py | 4 | ||||
| -rwxr-xr-x | bin/reproducible_html_notes.py | 8 | ||||
| -rwxr-xr-x | bin/reproducible_html_packages.py | 10 | ||||
| -rwxr-xr-x | bin/reproducible_html_pkg_sets.sh | 16 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 12 | ||||
| -rw-r--r-- | hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf | 52 |
13 files changed, 156 insertions, 154 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index b48c644a..efce0b14 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -14,12 +14,12 @@ common_init "$@" set -e create_results_dirs() { - mkdir -vp $BASE/dbd/${SUITE}/${ARCH} - mkdir -vp $BASE/dbdtxt/${SUITE}/${ARCH} - mkdir -vp $BASE/logs/${SUITE}/${ARCH} - mkdir -vp $BASE/logdiffs/${SUITE}/${ARCH} - mkdir -vp $BASE/rbuild/${SUITE}/${ARCH} - mkdir -vp $BASE/buildinfo/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/dbd/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/dbdtxt/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/logs/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/rbuild/${SUITE}/${ARCH} + mkdir -vp $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH} } handle_race_condition() { @@ -44,9 +44,9 @@ save_artifacts() { local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5) local ARTIFACTS="artifacts/r00t-me/${SRCPACKAGE}_${SUITE}_tmp-${random}" local URL="$DEBIAN_URL/$ARTIFACTS/" - local HEADER="$BASE/$ARTIFACTS/.HEADER.html" - mkdir -p $BASE/$ARTIFACTS - cp -r $TMPDIR/* $BASE/$ARTIFACTS/ + local HEADER="$DEBIAN_BASE/$ARTIFACTS/.HEADER.html" + mkdir -p $DEBIAN_BASE/$ARTIFACTS + cp -r $TMPDIR/* $DEBIAN_BASE/$ARTIFACTS/ echo | tee -a ${RBUILDLOG} local msg="Artifacts from this build have been preserved. They will be available for 24h only, so download them now.\n" msg="${msg}WARNING: You shouldn't trust packages downloaded from this host, they can contain malware or the worst of your fears, packaged nicely in debian format.\n" @@ -54,7 +54,7 @@ save_artifacts() { printf "$msg" | tee -a $BUILDLOG echo "<p>" > $HEADER printf "$msg" | sed 's#$#<br />#g' >> $HEADER - echo "Package page: <a href=\"$REPRODUCIBLE_URL/${SUITE}/${ARCH}/${SRCPACKAGE}\">$REPRODUCIBLE_URL/${SUITE}/${ARCH}/${SRCPACKAGE}</a><br />" >> $HEADER + echo "Package page: <a href=\"$DEBIAN_URL/${SUITE}/${ARCH}/${SRCPACKAGE}\">$REPRODUCIBLE_URL/${SUITE}/${ARCH}/${SRCPACKAGE}</a><br />" >> $HEADER echo "</p>" >> $HEADER chmod 644 $HEADER echo | tee -a ${RBUILDLOG} @@ -135,12 +135,12 @@ update_db_and_html() { update_rbuildlog() { chmod 644 $RBUILDLOG - mv $RBUILDLOG $BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log - RBUILDLOG=$BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log + mv $RBUILDLOG $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log + RBUILDLOG=$DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log } diff_copy_buildlogs() { - local DIFF="$BASE/logdiffs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.diff" + local DIFF="$DEBIAN_BASE/logdiffs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.diff" if [ -f b1/build.log ] ; then if [ -f b2/build.log ] ; then printf "Diff of the two buildlogs:\n\n--\n" | tee -a $DIFF @@ -150,15 +150,15 @@ diff_copy_buildlogs() { fi echo -e "\nCompressing the 2nd log..." gzip -9vn $DIFF - gzip -9cvn b2/build.log > $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz - chmod 644 $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz + gzip -9cvn b2/build.log > $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz + chmod 644 $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build2.log.gz elif [ $FTBFS -eq 0 ] ; then echo "Warning: No second build log, what happened?" | tee -a $RBUILDLOG fi set -x # # to debug diffoscope/schroot problems echo "Compressing the 1st log..." - gzip -9cvn b1/build.log > $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz - chmod 644 $BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz + gzip -9cvn b1/build.log > $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz + chmod 644 $DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build1.log.gz else echo "Error: No first build log, not even looking for the second" | tee -a $RBUILDLOG fi @@ -207,21 +207,21 @@ handle_ftbfs() { local nodevar="NODE$BUILD" local node="" eval node=\$$nodevar - if [ ! -f "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ] ; then + if [ ! -f "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ] ; then continue fi - if zgrep -F "E: pbuilder-satisfydepends failed." "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then + if zgrep -F "E: pbuilder-satisfydepends failed." "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then handle_depwait return fi for NEEDLE in '^tar:.*Cannot write: No space left on device' 'fatal error: error writing to .* No space left on device' './configure: line .* printf: write error: No space left on device' 'cat: write error: No space left on device' '^dpkg-deb.*No space left on device' '^cp: (erreur|impossible).*No space left on device' '^tee: .* No space left on device' '^zip I/O error: No space left on device' '^mkdir .*: No space left on device' ; do - if zgrep -e "$NEEDLE" "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then + if zgrep -e "$NEEDLE" "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then handle_enospace $node return fi done # notify about unkown diskspace issues where we are not 100% sure yet those are diskspace issues - if zgrep -e "No space left on device" "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then + if zgrep -e "No space left on device" "$DEBIAN_BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then MESSAGE="${BUILD_URL}console for ${SRCPACKAGE} (ftbfs in $SUITE/$ARCH) _probably_ had a diskspace issue on $node. Please check, tune handle_ftbfs() and reschedule the package." echo $MESSAGE | tee -a /var/log/jenkins/reproducible-diskspace-issues.log irc_message debian-reproducible "$MESSAGE" @@ -238,18 +238,18 @@ handle_ftbr() { local FTBRmessage="$@" echo | tee -a ${RBUILDLOG} echo "$(date -u) - ${SRCPACKAGE} failed to build reproducibly in ${SUITE} on ${ARCH}." | tee -a ${RBUILDLOG} - cp b1/${BUILDINFO} $BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 || true # will fail if there is no .buildinfo + cp b1/${BUILDINFO} $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 || true # will fail if there is no .buildinfo if [ ! -z "$FTRmessage" ] ; then echo "$(date -u) - ${FTBRmessage}." | tee -a ${RBUILDLOG} fi if [ -f ./${DBDREPORT} ] ; then - mv ./${DBDREPORT} $BASE/dbd/${SUITE}/${ARCH}/ + mv ./${DBDREPORT} $DEBIAN_BASE/dbd/${SUITE}/${ARCH}/ else echo "$(date -u) - $DIFFOSCOPE produced no output (which is strange)." | tee -a $RBUILDLOG fi if [ -f ./$DBDTXT ] ; then - mv ./$DBDTXT $BASE/dbdtxt/$SUITE/$ARCH/ - gzip -9n $BASE/dbdtxt/$SUITE/$ARCH/$DBDTXT + mv ./$DBDTXT $DEBIAN_BASE/dbdtxt/$SUITE/$ARCH/ + gzip -9n $DEBIAN_BASE/dbdtxt/$SUITE/$ARCH/$DBDTXT fi calculate_build_duration update_db_and_html "unreproducible" @@ -257,7 +257,7 @@ handle_ftbr() { handle_reproducible() { if [ ! -f ./${DBDREPORT} ] && [ -f b1/${BUILDINFO} ] ; then - cp b1/${BUILDINFO} $BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 + cp b1/${BUILDINFO} $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/ > /dev/null 2>&1 figlet ${SRCPACKAGE} echo | tee -a ${RBUILDLOG} echo "$DIFFOSCOPE found no differences in the changes files, and a .buildinfo file also exists." | tee -a ${RBUILDLOG} diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py index 58f77731..b08fedb4 100755 --- a/bin/reproducible_common.py +++ b/bin/reproducible_common.py @@ -42,22 +42,23 @@ defaultsuite = 'unstable' defaultarch = 'amd64' BIN_PATH = '/srv/jenkins/bin' -BASE = '/var/lib/jenkins/userContent/reproducible/debian' +BASE = '/var/lib/jenkins/userContent/reproducible' +DEBIAN_BASE = '/var/lib/jenkins/userContent/reproducible/debian' REPRODUCIBLE_JSON = BASE + '/reproducible.json' REPRODUCIBLE_TRACKER_JSON = BASE + '/reproducible-tracker.json' REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db' -DBD_URI = '/dbd' -DBDTXT_URI = '/dbdtxt' -LOGS_URI = '/logs' -DIFFS_URI = '/logdiffs' -NOTES_URI = '/notes' -ISSUES_URI = '/issues' -RB_PKG_URI = '/rb-pkg' -RBUILD_URI = '/rbuild' -HISTORY_URI = '/history' -BUILDINFO_URI = '/buildinfo' +DBD_URI = '/debian/dbd' +DBDTXT_URI = '/debian/dbdtxt' +LOGS_URI = '/debian/logs' +DIFFS_URI = '/debian/logdiffs' +NOTES_URI = '/debian/notes' +ISSUES_URI = '/debian/issues' +RB_PKG_URI = '/debian/rb-pkg' +RBUILD_URI = '/debian/rbuild' +HISTORY_URI = '/debian/history' +BUILDINFO_URI = '/debian/buildinfo' DBD_PATH = BASE + DBD_URI DBDTXT_PATH = BASE + DBDTXT_URI LOGS_PATH = BASE + LOGS_URI @@ -173,56 +174,56 @@ html_head_page = Template((tab*2).join((""" <ul class=\"menu\"> <li>$suite/$arch:<ul class="children"> <li>Notes:<ul class="children"> - <li><a href="/$suite/$arch/index_notes.html">packages with notes</a></li> - <li><a href="/$suite/$arch/index_no_notes.html">packages without notes</a></li> + <li><a href="debian/$suite/$arch/index_notes.html">packages with notes</a></li> + <li><a href="debian/$suite/$arch/index_no_notes.html">packages without notes</a></li> </ul></li> <li>Package states:<ul class="children"> <li> - <a href="/$suite/$arch/index_reproducible.html" target="_parent"> + <a href="debian/$suite/$arch/index_reproducible.html" target="_parent"> <img src="/static/weather-clear.png" alt="reproducible icon" /> </a> - <a href="/$suite/$arch/index_FTBR.html" target="_parent"> + <a href="debian/$suite/$arch/index_FTBR.html" target="_parent"> <img src="/static/weather-showers-scattered.png" alt="FTBR icon" /> </a> - <a href="/$suite/$arch/index_FTBFS.html" target="_parent"> + <a href="debian/$suite/$arch/index_FTBFS.html" target="_parent"> <img src="/static/weather-storm.png" alt="FTBFS icon" /> </a> - <a href="/$suite/$arch/index_depwait.html" target="_parent"> + <a href="debian/$suite/$arch/index_depwait.html" target="_parent"> <img src="/static/weather-snow.png" alt="depwait icon" /> </a> - <a href="/$suite/$arch/index_not_for_us.html" target="_parent"> + <a href="debian/$suite/$arch/index_not_for_us.html" target="_parent"> <img src="/static/weather-few-clouds-night.png" alt="not_for_us icon" /> </a> - <a href="/$suite/$arch/index_404.html" target="_parent"> + <a href="debian/$suite/$arch/index_404.html" target="_parent"> <img src="/static/weather-severe-alert.png" alt="404 icon" /> </a> - <a href="/$suite/$arch/index_blacklisted.html" target="_parent"> + <a href="debian/$suite/$arch/index_blacklisted.html" target="_parent"> <img src="/static/error.png" alt="blacklisted icon" /> </a> </li> </ul></li> $link_pkgsets <li>Recently tested:<ul class="children"> - <li><a href="/$suite/$arch/index_last_24h.html">packages tested in the last 24h</a></li> - <li><a href="/$suite/$arch/index_last_48h.html">packages tested in the last 48h</a></li> + <li><a href="debian/$suite/$arch/index_last_24h.html">packages tested in the last 24h</a></li> + <li><a href="debian/$suite/$arch/index_last_48h.html">packages tested in the last 48h</a></li> </ul></li> - <li><a href="/$suite/$arch/index_all_abc.html">all tested packages (sorted alphabetically)</a></li> + <li><a href="debian/$suite/$arch/index_all_abc.html">all tested packages (sorted alphabetically)</a></li> </ul></li> <li>Architectures:<ul class="children"> $link_archs - <li><a href="/index_${arch}_scheduled.html">currently scheduled</a></li> + <li><a href="debian/index_${arch}_scheduled.html">currently scheduled</a></li> </ul></li> <li>Suites:<ul class="children"> $link_suites - <li><a href="/$suite/index_dd-list.html">maintainers of unreproducible packages</a></li> + <li><a href="debian/$suite/index_dd-list.html">maintainers of unreproducible packages</a></li> </ul></li> <li><a href="%s">Debian dashboard</a> <ul class="children"> - <li><a href="/index_issues.html">issues</a></li> - <li><a href="/index_repositories.html">repositories overview</a></li> - <li><a href="/index_notify.html" title="notify icon">⚑ packages with enabled notifications</a></li> - <li><a href="/index_performance.html">performance stats</a></li> - <li><a href="/index_variations.html">variations tested</a></li> + <li><a href="debian/index_issues.html">issues</a></li> + <li><a href="debian/index_repositories.html">repositories overview</a></li> + <li><a href="debian/index_notify.html" title="notify icon">⚑ packages with enabled notifications</a></li> + <li><a href="debian/index_performance.html">performance stats</a></li> + <li><a href="debian/index_variations.html">variations tested</a></li> </ul></li> </ul> $project_links @@ -273,10 +274,10 @@ filter_query = '' for issue in filtered_issues: if filter_query == '': filter_query = 'n.issues LIKE "%' + issue + '%"' - filter_html = '<a href="' + DEBIAN_URL + ISSUES_URI + '/$suite/' + issue + '_issue.html">' + issue + '</a>' + filter_html = '<a href="' + REPRODUCIBLE_URL + ISSUES_URI + '/$suite/' + issue + '_issue.html">' + issue + '</a>' else: filter_query += ' OR n.issues LIKE "%' + issue + '%"' - filter_html += ' or <a href="' + DEBIAN_URL + ISSUES_URI + '/$suite/' + issue + '_issue.html">' + issue + '</a>' + filter_html += ' or <a href="' + REPRODUCIBLE_URL + ISSUES_URI + '/$suite/' + issue + '_issue.html">' + issue + '</a>' @atexit.register @@ -326,20 +327,20 @@ def convert_into_hms_string(duration): def _gen_pkg_sets_link(suite, arch): html = '' if suite != 'experimental': - html = '<li><a href="/' + suite + '/' + arch + '/index_pkg_sets.html">package sets</a></li>' + html = '<li><a href="/debian/' + suite + '/' + arch + '/index_pkg_sets.html">package sets</a></li>' return html def _gen_arch_links(suite, arch): html = '<li>' for a in ARCHS: - html += ' <a href="/' + suite + '/index_suite_' + a + '_stats.html\">' + a + '</a> ' + html += ' <a href="/debian/' + suite + '/index_suite_' + a + '_stats.html\">' + a + '</a> ' html += '</li>' return html def _gen_suite_links(suite, arch): html = '<li>' for s in SUITES: - html += ' <a href="/' + s + '/index_suite_' + arch + '_stats.html">' + s + '</a> ' + html += ' <a href="/debian/' + s + '/index_suite_' + arch + '_stats.html">' + s + '</a> ' html += '</li>' return html diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index f92301e1..eaa77743 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -71,8 +71,9 @@ NUM_CPU=$(grep -c '^processor' /proc/cpuinfo) # we only this array for html creation but we cannot declare them in a function declare -A SPOKENTARGET -BASE="/var/lib/jenkins/userContent/reproducible/debian" -mkdir -p "$BASE" +BASE="/var/lib/jenkins/userContent/reproducible" +DEBIAN_BASE="/var/lib/jenkins/userContent/reproducible/debian" +mkdir -p "$DEBIAN_BASE" # to hold reproducible temporary files/directories without polluting /tmp TEMPDIR="/tmp/reproducible" @@ -80,7 +81,7 @@ mkdir -p "$TEMPDIR" # create subdirs for suites for i in $SUITES ; do - mkdir -p "$BASE/$i" + mkdir -p "$DEBIAN_BASE/$i" done # table names and image names @@ -186,7 +187,7 @@ set_icon() { write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() - write_page "<a href=\"/$SUITE/$ARCH/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>" + write_page "<a href=\"/debian/$SUITE/$ARCH/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>" } write_page_header() { @@ -233,16 +234,16 @@ write_page_header() { continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} - BASEURL="/$SUITE/$ARCH" + BASEURL="/debian/$SUITE/$ARCH" local i for i in $GLOBALVIEWS ; do if [ "$TARGET" = "$i" ] ; then - BASEURL="" + BASEURL="/debian" fi done for i in ${SUITEVIEWS} ; do if [ "$TARGET" = "$i" ] ; then - BASEURL="/$SUITE" + BASEURL="/debian/$SUITE" fi done # prepare unsorted lists @@ -253,7 +254,7 @@ write_page_header() { fi # prepare links if [ "$TARGET" = "scheduled" ] ; then - write_page "<li><a href=\"/index_${ARCH}_scheduled.html\">${SPOKEN_TARGET}</a></li>" + write_page "<li><a href=\"/debian/index_${ARCH}_scheduled.html\">${SPOKEN_TARGET}</a></li>" elif [ "$TARGET" = "notify" ] ; then write_page "<li><a href=\"$BASEURL/index_${TARGET}.html\" title=\"notify icon\">${SPOKEN_TARGET}</a></li>" elif [ "$TARGET" = "arch" ] ; then @@ -262,7 +263,7 @@ write_page_header() { if [ "$1" = "suite_arch_stats" ] && [ "$i" = "$ARCH" ] ; then CLASS=" class=\"active\"" fi - write_page " <a href=\"/$SUITE/index_suite_${i}_stats.html\"$CLASS>$i</a> " + write_page " <a href=\"debian/$SUITE/index_suite_${i}_stats.html\"$CLASS>$i</a> " CLASS="" done write_page "</li>" @@ -272,7 +273,7 @@ write_page_header() { if [ "$1" = "suite_arch_stats" ] && [ "$i" = "$SUITE" ] ; then CLASS=" class=\"active\"" fi - write_page " <a href=\"/$i/index_suite_${ARCH}_stats.html\"$CLASS>$i</a> " + write_page " <a href=\"debian/$i/index_suite_${ARCH}_stats.html\"$CLASS>$i</a> " CLASS="" done write_page "</li>" @@ -564,7 +565,7 @@ publish_page() { else TARGET=$1/$PAGE fi - cp $PAGE $BASE/$TARGET + cp $PAGE $DEBIAN_BASE/$TARGET rm $PAGE echo "Enjoy $DEBIAN_URL/$TARGET" } @@ -696,12 +697,12 @@ get_filesize() { } cleanup_pkg_files() { - rm -vf $BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz} - rm -vf $BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz} - rm -vf $BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.html - rm -vf $BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.txt{,.gz} - rm -vf $BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo - rm -vf $BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz} + rm -vf $DEBIAN_BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz} + rm -vf $DEBIAN_BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz} + rm -vf $DEBIAN_BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.html + rm -vf $DEBIAN_BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diffoscope.txt{,.gz} + rm -vf $DEBIAN_BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo + rm -vf $DEBIAN_BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz} } # @@ -806,10 +807,10 @@ create_png_from_table() { mkdir -p $DIR echo "Generating $2." /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" $WIDTH $HEIGHT - mv $2 $BASE/$DIR + mv $2 $DEBIAN_BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) # create empty dummy png if there havent been any results ever - elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then + elif [ ! -f $DEBIAN_BASE/$DIR/$(basename $2) ] ; then DIR=$(dirname $2) mkdir -p $DIR echo "Creating $2 dummy." @@ -817,9 +818,9 @@ create_png_from_table() { if [ "$3" != "" ] ; then local THUMB="${TABLE[1]}_${3}-thumbnail.png" convert $2 -adaptive-resize 160x80 ${THUMB} - mv ${THUMB} $BASE/$DIR + mv ${THUMB} $DEBIAN_BASE/$DIR fi - mv $2 $BASE/$DIR + mv $2 $DEBIAN_BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) fi rm ${TABLE[$1]}.csv diff --git a/bin/reproducible_html_breakages.py b/bin/reproducible_html_breakages.py index 7bfc03dd..05f8d6ba 100755 --- a/bin/reproducible_html_breakages.py +++ b/bin/reproducible_html_breakages.py @@ -326,7 +326,7 @@ if __name__ == '__main__': html += '<p><b>COOL!!!</b> Everything is GOOD and not a single issue was ' html += 'detected. <i>Enjoy!</i></p>' title = 'Breakage on the Debian pages of tests.reproducible-builds.org' - destfile = BASE + '/index_breakages.html' + destfile = DEBIAN_BASE + '/index_breakages.html' desturl = DEBIAN_URL + '/index_breakages.html' write_html_page(title, html, destfile, style_note=True) log.info('Breackages page created at ' + desturl) diff --git a/bin/reproducible_html_dashboard.sh b/bin/reproducible_html_dashboard.sh index ba3dfff3..fb1d5654 100755 --- a/bin/reproducible_html_dashboard.sh +++ b/bin/reproducible_html_dashboard.sh @@ -140,9 +140,9 @@ update_suite_arch_stats() { PREFIX=$SUITE/$ARCH fi # force regeneration of the image if it exists - if [ -f $BASE/$PREFIX/${TABLE[$i]}.png ] ; then + if [ -f $DEBIAN_BASE/$PREFIX/${TABLE[$i]}.png ] ; then echo "Touching $PREFIX/${TABLE[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $BASE/$PREFIX/${TABLE[$i]}.png + touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$PREFIX/${TABLE[$i]}.png fi done fi @@ -239,7 +239,7 @@ update_bug_stats() { local i=0 for i in 3 7 8 9 ; do echo "Touching ${TABLE[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $BASE/${TABLE[$i]}.png + touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/${TABLE[$i]}.png done fi fi @@ -433,10 +433,10 @@ create_suite_arch_stats_page() { write_icon write_page "$COUNT_BLACKLISTED ($PERCENT_BLACKLISTED%) blacklisted packages neither.</p>" write_page "<p>" - write_page " <a href=\"/$SUITE/$ARCH/${TABLE[0]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[0]}.png\" alt=\"${MAINLABEL[0]}\"></a>" + write_page " <a href=\"/debian/$SUITE/$ARCH/${TABLE[0]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[0]}.png\" alt=\"${MAINLABEL[0]}\"></a>" for i in 0 2 ; do # recreate png once a day - if [ ! -f $BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] ; then + if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] ; then create_png_from_table $i $SUITE/$ARCH/${TABLE[$i]}.png fi done @@ -453,7 +453,7 @@ write_meta_pkg_graphs_links () { for i in $(seq 1 ${#META_PKGSET[@]}) ; do THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/$SUITE/$ARCH/$THUMB\" class=\"metaoverview\" alt=\"$LABEL\"></a>" + write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/$SUITE/$ARCH/$THUMB\" class=\"metaoverview\" alt=\"$LABEL\"></a>" done write_page "</center></p>" } @@ -472,7 +472,7 @@ create_dashboard_page() { # write suite graphs for ARCH in ${ARCHS} ; do for SUITE in $SUITES ; do - write_page " <a href=\"/$SUITE/$ARCH\"><img src=\"/$SUITE/$ARCH/${TABLE[0]}.png\" class=\"overview\" alt=\"$SUITE/$ARCH stats\"></a>" + write_page " <a href=\"/debian/$SUITE/$ARCH\"><img src=\"/$SUITE/$ARCH/${TABLE[0]}.png\" class=\"overview\" alt=\"$SUITE/$ARCH stats\"></a>" done SUITE="unstable" if [ "$ARCH" = "amd64" ] ; then @@ -489,7 +489,7 @@ create_dashboard_page() { done write_page "</tr>" ARCH="amd64" - write_page "<tr><td class=\"left\">identified <a href=\"/index_issues.html\">distinct and categorized issues</a></td><td>$ISSUES</td><td colspan=\"$AC\"></td></tr>" + write_page "<tr><td class=\"left\">identified <a href=\"/debian/index_issues.html\">distinct and categorized issues</a></td><td>$ISSUES</td><td colspan=\"$AC\"></td></tr>" write_page "<tr><td class=\"left\">total number of identified issues in packages</td><td>$COUNT_ISSUES</td><td colspan=\"$AC\"></td></tr>" write_page "<tr><td class=\"left\">packages with notes about these issues</td><td>$NOTES</td><td colspan=\"$AC\"></td></tr>" @@ -502,7 +502,7 @@ create_dashboard_page() { SUITE="unstable" gather_suite_arch_stats RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - TD_PKG_NOISSUES="$TD_PKG_NOISSUES<td><a href=\"/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>" + TD_PKG_NOISSUES="$TD_PKG_NOISSUES<td><a href=\"/debian/$SUITE/$ARCH/index_no_notes.html\">$RESULT</a> / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>" RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") TD_PKG_FTBR="$TD_PKG_FTBR<td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td>" RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") @@ -532,7 +532,7 @@ create_dashboard_page() { wc -l)</td><td colspan=\"$AC\"></td></tr>" fi RESULT=$(cat /srv/reproducible-results/modified_in_sid.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh - write_page "<tr><td class=\"left\">packages <a href=\"/index_repositories.html\">modified in our toolchain</a> (in unstable)</td><td>$(echo $RESULT)</td><td colspan=\"$AC\"></td></tr>" + write_page "<tr><td class=\"left\">packages <a href=\"/debian/index_repositories.html\">modified in our toolchain</a> (in unstable)</td><td>$(echo $RESULT)</td><td colspan=\"$AC\"></td></tr>" RESULT=$(cat /srv/reproducible-results/modified_in_exp.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh write_page "<tr><td class=\"left\"> - (in experimental)</td><td>$(echo $RESULT)</td><td colspan=\"$AC\"></td></tr>" RESULT=$(cat /srv/reproducible-results/binnmus_needed.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh @@ -545,9 +545,9 @@ create_dashboard_page() { write_page "</p><p style=\"clear:both;\">" # do other global graphs for i in 8 9 3 7 4 5 ; do - write_page " <a href=\"/${TABLE[$i]}.png\"><img src=\"/${TABLE[$i]}.png\" class="halfview" alt=\"${MAINLABEL[$i]}\"></a>" + write_page " <a href=\"/debian/${TABLE[$i]}.png\"><img src=\"/${TABLE[$i]}.png\" class="halfview" alt=\"${MAINLABEL[$i]}\"></a>" # redo pngs once a day - if [ ! -f $BASE/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $BASE/${TABLE[$i]}.png ] ; then + if [ ! -f $DEBIAN_BASE/${TABLE[$i]}.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/${TABLE[$i]}.png ] ; then create_png_from_table $i ${TABLE[$i]}.png fi done @@ -557,7 +557,7 @@ create_dashboard_page() { write_page "<br />There are <a href=\"$BASEURL/index_breakages.html\">some problems in this test setup itself</a> too. And there is <a href=\"https://jenkins.debian.net/userContent/about.html#_reproducible_builds_jobs\">documentation</a> too, in case you missed the link at the top. Feedback is very much appreciated.</p>" # the end write_page_footer - cp $PAGE $BASE/reproducible.html + cp $PAGE $DEBIAN_BASE/reproducible.html publish_page } @@ -574,8 +574,8 @@ create_performance_page() { # arch performance stats write_page "<p style=\"clear:both;\">" for ARCH in ${ARCHS} ; do - write_page " <a href=\"/${TABLE[1]}_$ARCH.png\"><img src=\"/${TABLE[1]}_$ARCH.png\" class=\"overview\" alt=\"${MAINLABEL[1]}\"></a>" - if [ ! -f $BASE/${TABLE[1]}_$ARCH.png ] || [ $DUMMY_FILE -nt $BASE/${TABLE[1]}_$ARCH.png ] ; then + write_page " <a href=\"/debian/${TABLE[1]}_$ARCH.png\"><img src=\"/${TABLE[1]}_$ARCH.png\" class=\"overview\" alt=\"${MAINLABEL[1]}\"></a>" + if [ ! -f $DEBIAN_BASE/${TABLE[1]}_$ARCH.png ] || [ $DUMMY_FILE -nt $DEBIAN_BASE/${TABLE[1]}_$ARCH.png ] ; then create_png_from_table 1 ${TABLE[1]}_$ARCH.png fi done @@ -585,14 +585,14 @@ create_performance_page() { write_page "</p><p style=\"clear:both;\">" for ARCH in ${ARCHS} ; do for SUITE in $SUITES ; do - write_page " <a href=\"/$SUITE/$ARCH/${TABLE[2]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[2]}.png\" class=\"overview\" alt=\"age of oldest reproducible build result in $SUITE/$ARCH\"></a>" + write_page " <a href=\"/debian/$SUITE/$ARCH/${TABLE[2]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[2]}.png\" class=\"overview\" alt=\"age of oldest reproducible build result in $SUITE/$ARCH\"></a>" done write_page "</p><p style=\"clear:both;\">" done # the end write_page "Daily <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_nodes_info/lastBuild/console\">individual build node performance stats</a> are available as well as views of oldest results for" for ARCH in ${ARCHS} ; do - write_page " <a href=\"/index_${ARCH}_oldies.html\">$ARCH</a>" + write_page " <a href=\"/debian/index_${ARCH}_oldies.html\">$ARCH</a>" done write_page ".</p>" write_page_footer diff --git a/bin/reproducible_html_dd_list.py b/bin/reproducible_html_dd_list.py index ae077d47..174c8124 100755 --- a/bin/reproducible_html_dd_list.py +++ b/bin/reproducible_html_dd_list.py @@ -72,6 +72,6 @@ for suite in SUITES: html += '\n' html += '</pre></p>' title = 'Maintainers of unreproducible packages in ' + suite - destfile = BASE + '/' + suite + '/index_dd-list.html' + destfile = DEBIAN_BASE + '/' + suite + '/index_dd-list.html' write_html_page(title, html, destfile, suite, arch, style_note=True) log.info('%s/%s/index_dd-list.html published', DEBIAN_URL, suite) diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py index 7dd01aea..f0eda28b 100755 --- a/bin/reproducible_html_indexes.py +++ b/bin/reproducible_html_indexes.py @@ -522,11 +522,11 @@ def build_page(page, suite=None, arch=None): html += html1 footnote = True if footnote1 else footnote if gpage: - destfile = BASE + '/index_' + page + '.html' + destfile = DEBIAN_BASE + '/index_' + page + '.html' desturl = DEBIAN_URL + '/index_' + page + '.html' suite = defaultsuite # used for the links generated by write_html_page else: - destfile = BASE + '/' + suite + '/' + arch + '/index_' + page + '.html' + destfile = DEBIAN_BASE + '/' + suite + '/' + arch + '/index_' + page + '.html' desturl = DEBIAN_URL + '/' + suite + '/' + arch + '/index_' + \ page + '.html' write_html_page(title=title, body=html, destfile=destfile, suite=suite, arch=arch, style_note=True) diff --git a/bin/reproducible_html_live_status.py b/bin/reproducible_html_live_status.py index 7ffe5217..fa401def 100755 --- a/bin/reproducible_html_live_status.py +++ b/bin/reproducible_html_live_status.py @@ -50,7 +50,7 @@ def generate_schedule(arch): html += link_package(pkg, row[1], row[2], bugs) html += '</code></td><td>'+convert_into_status_html(str(row[4]))+'</td><td>'+duration+'</td><td>' + avg_duration + '</td></tr>\n' html += '</table></p>\n' - destfile = BASE + '/index_' + arch + '_scheduled.html' + destfile = DEBIAN_BASE + '/index_' + arch + '_scheduled.html' desturl = DEBIAN_URL + '/index_' + arch + '_scheduled.html' write_html_page(title=title, body=html, destfile=destfile, arch=arch, style_note=True, refresh_every=60) log.info("Page generated at " + desturl) @@ -120,7 +120,7 @@ def generate_oldies(arch): html += link_package(pkg, row[0], row[1], bugs) html += '</code></td><td>'+convert_into_status_html(str(row[3]))+'</td><td>' + row[4] + '</td></tr>\n' html += '</table></p>\n' - destfile = BASE + '/index_' + arch + '_oldies.html' + destfile = DEBIAN_BASE + '/index_' + arch + '_oldies.html' desturl = DEBIAN_URL + '/index_' + arch + '_oldies.html' write_html_page(title=title, body=html, destfile=destfile, arch=arch, style_note=True, refresh_every=60) log.info("Page generated at " + desturl) diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py index 9cb00517..15f6068a 100755 --- a/bin/reproducible_html_notes.py +++ b/bin/reproducible_html_notes.py @@ -263,7 +263,7 @@ def gen_html_issue(issue, suite): if suite_links != '': suite_links += ' / ' if i != suite: - suite_links += '<a href="' + DEBIAN_URL + ISSUES_URI + '/' + i + '/' + issue + '_issue.html">' + i + '</a>' + suite_links += '<a href="' + REPRODUCIBLE_URL + ISSUES_URI + '/' + i + '/' + issue + '_issue.html">' + i + '</a>' else: suite_links += '<em>' + i + '</em>' # check for url: @@ -361,7 +361,7 @@ def iterate_over_notes(notes): write_html_page(title=title, body=html, destfile=destfile, noheader=True) - desturl = DEBIAN_URL + NOTES_URI + '/' + package + '_note.html' + desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' log.debug("Note created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' note pages.') @@ -381,7 +381,7 @@ def iterate_over_issues(issues): write_html_page(title=title, body=html, destfile=destfile, style_note=True) - desturl = DEBIAN_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html' + desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + suite + '/' + issue + '_issue.html' log.debug("Issue created: " + desturl) i = i + 1 log.info('Created ' + str(i) + ' issue pages for ' + suite) @@ -417,7 +417,7 @@ def index_issues(issues): '</b> issues.</p>' html += tab*2 + '<p>' + NOTESGIT_DESCRIPTION + '</p>' title = 'Known issues related to reproducible builds' - destfile = BASE + '/index_issues.html' + destfile = DEBIAN_BASE + '/index_issues.html' desturl = DEBIAN_URL + '/index_issues.html' write_html_page(title=title, body=html, destfile=destfile) log.info('Issues index now available at ' + desturl) diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py index 78461b2a..caa32413 100755 --- a/bin/reproducible_html_packages.py +++ b/bin/reproducible_html_packages.py @@ -16,7 +16,7 @@ html_package_page = Template((tab*2).join((""" <h2 class="package-name">$package</h2> <ul class="menu"> <li><ul class="children"> - <li>$version <a href="/index_notify.html" target="_parent"> + <li>$version <a href="debian/index_notify.html" target="_parent"> <span class="notification" title="Notifications for this package are enabled. Every reproducibility related status change will be emailed to the maintainers">$notify_maintainer</span></a></li> <li>$suite/$arch </li> <li>$status </li> @@ -181,17 +181,17 @@ def gen_suites_links(package, current_suite, current_arch): prefix = '' suffix = '\n' else: - prefix = '<a href="/{}/{}/index_{}.html">'.format(s, a, status) + prefix = '<a href="/debian/{}/{}/index_{}.html">'.format(s, a, status) suffix = '</a>\n' icon_html = prefix + '<img src="/static/{icon}" alt="{spokenstatus}" title="{spokenstatus}"/>' + suffix html += icon_html.format(icon=icon, status=status, spokenstatus=spokenstatus) if ( s == current_suite and a == current_arch ): html += (tab*2 + ' {}').format(version) else: - html += (tab*2 + ' <a href="{}/{}/{}/{}.html" target="_parent"' + \ + html += (tab*2 + ' <a href="/debian/{}/{}/{}/{}.html" target="_parent"' + \ ' title="{}: {}{}">{}</a>').format(RB_PKG_URI, s, a, package.name, spokenstatus, version, build_date, version) - html += ' in <a href="/{}/{}/" target="_parent">{}</a>\n'.format(s, a, s) + html += ' in <a href="/debian/{}/{}/" target="_parent">{}</a>\n'.format(s, a, s) html += '</li>\n' html += tab + '</ul></li>' html += '</ul>\n' @@ -283,7 +283,7 @@ def gen_packages_html(packages, no_clean=False): project_links=project_links, default_view=default_view) destfile = RB_PKG_PATH + '/' + suite + '/' + arch + '/' + pkg + '.html' - desturl = DEBIAN_URL + RB_PKG_URI + '/' + suite + \ + desturl = REPRODUCIBLE_URL + RB_PKG_URI + '/' + suite + \ '/' + arch + '/' + pkg + '.html' title = pkg + ' - reproducible build results' write_html_page(title=title, body=html, destfile=destfile, diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh index 72389dc6..7cf5fec3 100755 --- a/bin/reproducible_html_pkg_sets.sh +++ b/bin/reproducible_html_pkg_sets.sh @@ -80,7 +80,7 @@ update_meta_pkg_stats() { echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE/$ARCH on $DATE." fi echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png + touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png fi done } @@ -105,12 +105,12 @@ create_pkg_sets_navigation() { else CLASS="" fi - if [ -f $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then + if [ -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" write_page "<li>" - write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"$CLASS>${META_PKGSET[$i]}</a>" - write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/userContent/$SUITE/$ARCH/$THUMB\" class=\"setview\" alt=\"$LABEL\" title=\"${META_PKGSET[$i]}\" name=\"${META_PKGSET[$i]}\"></a>" + write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"$CLASS>${META_PKGSET[$i]}</a>" + write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/userContent/$SUITE/$ARCH/$THUMB\" class=\"setview\" alt=\"$LABEL\" title=\"${META_PKGSET[$i]}\" name=\"${META_PKGSET[$i]}\"></a>" write_page "</li>" fi done @@ -149,19 +149,19 @@ create_pkg_sets_pages() { PNG=${TABLE[6]}_${META_PKGSET[$i]}.png THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" # redo pngs once a day - if [ ! -f $BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then + if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $DEBIAN_BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]} - convert $BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $BASE/$SUITE/$ARCH/$THUMB + convert $DEBIAN_BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $DEBIAN_BASE/$SUITE/$ARCH/$THUMB fi LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH" - write_page "<p><a href=\"/userContent/$SUITE/$ARCH/$PNG\"><img src=\"/userContent/$SUITE/$ARCH/$PNG\" class=\"overview\" alt=\"$LABEL\"></a>" + write_page "<p><a href=\"/debian/$SUITE/$ARCH/$PNG\"><img src=\"/userContent/$SUITE/$ARCH/$PNG\" class=\"overview\" alt=\"$LABEL\"></a>" write_page "<br />The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:" write_page " (this set on " for LINKARCH in ${ARCHS} ; do if [ "$LINKARCH" = "$ARCH" ] ; then continue else - write_page "<a href=\"/$SUITE/$LINKARCH/pkg_set_${META_PKGSET[$i]}.html\">$LINKARCH</a> " + write_page "<a href=\"/debian/$SUITE/$LINKARCH/pkg_set_${META_PKGSET[$i]}.html\">$LINKARCH</a> " fi done write_page ") <br /> <br />" diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index 90ca5e30..b8a2fbb4 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -171,7 +171,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then # (ignore "*None.rbuild.log" because these are build which were just started) # this job runs every 4h echo "$(date -u) - Rescheduling failed builds due to network issues." - FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true) + FAILED_BUILDS=$(find $DEBIAN_BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true) if [ ! -z "$FAILED_BUILDS" ] ; then echo echo "The following builds have failed due to network problems and will be rescheduled now:" @@ -203,7 +203,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then # (ignore "*None.rbuild.log" because these are build which were just started) # this job runs every 4h echo "$(date -u) - Rescheduling failed builds due to diffoscope schroot issues." - FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -F 'E: 10mount: error: Directory' {} \; || true) + FAILED_BUILDS=$(find $DEBIAN_BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -F 'E: 10mount: error: Directory' {} \; || true) if [ ! -z "$FAILED_BUILDS" ] ; then echo echo "Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled now:" @@ -274,7 +274,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then QUERY="DELETE FROM removed_packages WHERE name='$PKGNAME' AND suite='$SUITE' AND architecture='$ARCH'" sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" - cd $BASE + cd $DEBIAN_BASE find rb-pkg/$SUITE/$ARCH rbuild/$SUITE/$ARCH dbd/$SUITE/$ARCH dbdtxt/$SUITE/$ARCH buildinfo/$SUITE/$ARCH logs/$SUITE/$ARCH logdiffs/$SUITE/$ARCH -name "${PKGNAME}_*" | xargs -r rm -v || echo "Warning: couldn't delete old files from ${PKGNAME} in $SUITE/$ARCH" done cd - > /dev/null @@ -366,17 +366,17 @@ fi # remove artifacts older than a day echo "$(date -u) - Checking for artifacts older than a day." -ARTIFACTS=$(find $BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec ls -lad {} \; 2>/dev/null|| true) +ARTIFACTS=$(find $DEBIAN_BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec ls -lad {} \; 2>/dev/null|| true) if [ ! -z "$ARTIFACTS" ] ; then echo echo "Removed old artifacts:" - find $BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec rm -rv {} \; + find $DEBIAN_BASE/artifacts/* -maxdepth 1 -type d -mtime +1 -exec rm -rv {} \; echo fi # find + chmod files with bad permissions echo "$(date -u) - Checking for files with bad permissions." -BADPERMS=$(find $BASE/{buildinfo,dbd,rbuild,artifacts,unstable,experimental,testing,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true) +BADPERMS=$(find $DEBIAN_BASE/{buildinfo,dbd,rbuild,artifacts,unstable,experimental,testing,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true) if [ ! -z "$BADPERMS" ] ; then DIRTY=true echo diff --git a/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf b/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf index 7b187592..4f01a075 100644 --- a/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf +++ b/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf @@ -364,23 +364,23 @@ Use common-debian-service-https-redirect reproducible-builds.org RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/issues/unstable/$1 -f - RewriteRule ^/issues/([a-z0-9.+-_]+) /issues/unstable/$1 [R=302,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/issues/unstable/$1 -f + RewriteRule ^/issues/([a-z0-9.+-_]+) /debian/issues/unstable/$1 [R=302,L] # redirect rb.d.n/$PKG → rb.d.n/rb-pkg/unstable/amd64/$PKG.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/rb-pkg/unstable/amd64/$1.html -f - RewriteRule ^/([a-z0-9.+-]+) /rb-pkg/unstable/amd64/$1.html [R=302,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/rb-pkg/unstable/amd64/$1.html -f + RewriteRule ^/([a-z0-9.+-]+) /debian/rb-pkg/unstable/amd64/$1.html [R=302,L] # redirect rb.d.n/redirect/?SrcPkg=$PKG → rb.d.n/rb-pkg/unstable/amd64/$PKG.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d RewriteCond %{QUERY_STRING} ^(\w+)=([a-z0-9.+-]+)$ - RewriteCond /var/lib/jenkins/userContent/reproducible/rb-pkg/unstable/amd64/%2.html -f - RewriteRule ^/redirect /rb-pkg/unstable/amd64/%2.html? [R=302,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/rb-pkg/unstable/amd64/%2.html -f + RewriteRule ^/redirect /debian/rb-pkg/unstable/amd64/%2.html? [R=302,L] # the following two rules are fallbacks for the previous two redirects and should only catch packages which are only in experimental @@ -388,48 +388,48 @@ Use common-debian-service-https-redirect reproducible-builds.org RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/rb-pkg/experimental/amd64/$1.html -f - RewriteRule ^/([a-z0-9.+-]+) /rb-pkg/experimental/amd64/$1.html [R=302,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/rb-pkg/experimental/amd64/$1.html -f + RewriteRule ^/([a-z0-9.+-]+) /debian/rb-pkg/experimental/amd64/$1.html [R=302,L] # redirect rb.d.n/redirect/?SrcPkg=$PKG → rb.d.n/rb-pkg/experimental/amd64/$PKG.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d RewriteCond %{QUERY_STRING} ^(\w+)=([a-z0-9.+-]+)$ - RewriteCond /var/lib/jenkins/userContent/reproducible/rb-pkg/experimental/amd64/%2.html -f - RewriteRule ^/redirect /rb-pkg/experimental/amd64/%2.html? [R=302,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/rb-pkg/experimental/amd64/%2.html -f + RewriteRule ^/redirect /debian/rb-pkg/experimental/amd64/%2.html? [R=302,L] # redirect rb.d.n/$suite/(amd64|armhf|i386)/$PKG → rb.d.n/rb-pkg/$suite/$arch/$PKG.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/rb-pkg/$1/$2/$3.html -f - RewriteRule ^/(unstable|testing|experimental)/([a-z0-9]+)/([a-z0-9.+-]+) /rb-pkg/$1/$2/$3.html [R=302,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/rb-pkg/$1/$2/$3.html -f + RewriteRule ^/(unstable|testing|experimental)/([a-z0-9]+)/([a-z0-9.+-]+) /debian/rb-pkg/$1/$2/$3.html [R=302,L] # redirect rb.d.n/rb-pkg/$PKG.html → rb.d.n/rb-pkg/unstable/amd64/$PKG.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/rb-pkg/unstable/amd64/$1 -f - RewriteRule ^/rb-pkg/([a-z0-9.+-]+) /rb-pkg/unstable/amd64/$1 [R=301,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/rb-pkg/unstable/amd64/$1 -f + RewriteRule ^/rb-pkg/([a-z0-9.+-]+) /debian/rb-pkg/unstable/amd64/$1 [R=301,L] # the same for /dbd/ RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/dbd/unstable/amd64/$1 -f - RewriteRule ^/dbd/([a-z0-9.+-_]+) /dbd/unstable/amd64/$1 [R=301,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/dbd/unstable/amd64/$1 -f + RewriteRule ^/dbd/([a-z0-9.+-_]+) /debian/dbd/unstable/amd64/$1 [R=301,L] # the same for /rbuild/ RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/rbuild/unstable/amd64/$1 -f - RewriteRule ^/rbuild/([a-z0-9.+-_]+) /rbuild/unstable/amd64/$1 [R=301,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/rbuild/unstable/amd64/$1 -f + RewriteRule ^/rbuild/([a-z0-9.+-_]+) /debian/rbuild/unstable/amd64/$1 [R=301,L] # the same for /buildinfo/ RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-d - RewriteCond /var/lib/jenkins/userContent/reproducible/buildinfo/unstable/amd64/$1 -f - RewriteRule ^/buildinfo/([a-z0-9.+-_]+) /buildinfo/unstable/amd64/$1 [R=301,L] + RewriteCond /var/lib/jenkins/userContent/reproducible/debian/buildinfo/unstable/amd64/$1 -f + RewriteRule ^/buildinfo/([a-z0-9.+-_]+) /debian/buildinfo/unstable/amd64/$1 [R=301,L] # redirect some rb.d.n/index_*.html to the suite/arch relative one RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_FILENAME} !-f @@ -444,23 +444,23 @@ Use common-debian-service-https-redirect reproducible-builds.org RewriteCond %{REQUEST_URI} ^/index_last_24h.html$ [or] RewriteCond %{REQUEST_URI} ^/index_last_48h.html$ [or] RewriteCond %{REQUEST_URI} ^/index_all_abc.html$ - RewriteRule ^/?(.+) /unstable/amd64/$1 [R=301,L] + RewriteRule ^/?(.+) /debian/unstable/amd64/$1 [R=301,L] # redirect (/testing|unstable|/experimental) to (/testing|/unstable|/experimental)/index_suite_amd64_stats.html # note: the missing slash in the RewriteRule is wanted to avoid a double slash RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_URI} ^/(testing|unstable|experimental)(/|)$ - RewriteRule ^/(.*) /$1/index_suite_amd64_stats.html [R,L] + RewriteRule ^/(.*) /debian/$1/index_suite_amd64_stats.html [R,L] # redirect (/testing|unstable|/experimental)/(amd64|armhf|i386) to (/testing|/unstable|/experimental)/index_suite_(amd64|armhf|i386)_stats.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_URI} ^/(testing|unstable|experimental)/(amd64|armhf|i386)(/|)$ - RewriteRule ^/([a-z0-9]+)/([a-z0-9]+) /$1/index_suite_$2_stats.html [R,L] + RewriteRule ^/([a-z0-9]+)/([a-z0-9]+) /debian/$1/index_suite_$2_stats.html [R,L] # redirect (/(amd64|armhf|i386) to (/testing|/unstable|/experimental)/index_suite_(amd64|armhf|i386)_stats.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_URI} ^/(amd64|armhf|i386)(/|)$ - RewriteRule ^/([a-z0-9]+) /unstable/index_suite_$1_stats.html [R,L] + RewriteRule ^/([a-z0-9]+) /debian/unstable/index_suite_$1_stats.html [R,L] # redirect /coreboot/ to coreboot/coreboot.html # note: the missing slash in the RewriteRule is wanted to avoid a double slash @@ -500,12 +500,12 @@ Use common-debian-service-https-redirect reproducible-builds.org # redirect /issues/ to /index_issues.html RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_URI} ^/issues(/|)$ - RewriteRule ^/(.*) /index_issues.html [R,L] + RewriteRule ^/(.*) /debian/index_issues.html [R,L] # temporary redirect until the html is rewritten RewriteCond %{HTTP_HOST} tests\.reproducible-builds\.org RewriteCond %{REQUEST_URI} ^/debian(/|)$ - RewriteRule ^/?(.*) /reproducible.html [R,L] + RewriteRule ^/?(.*) /debian/reproducible.html [R,L] <Proxy *> Require all granted |