$2

From 7406d5ea8bcd5a7eb058b32d6e6460bb9f761018 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Fri, 17 Oct 2014 22:04:53 +0200 Subject: reproducible: refactor, add page with scheduled packages --- bin/reproducible_common.sh | 61 ++++++++++++++++++++++------------ bin/reproducible_html_indexes.sh | 3 +- bin/reproducible_schedule_on_demand.sh | 2 -- bin/reproducible_scheduler.sh | 11 +++--- 4 files changed, 49 insertions(+), 28 deletions(-) (limited to 'bin') diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 4808e57d..7321bf70 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -111,23 +111,24 @@ mkdir -p /var/lib/jenkins/userContent/rb-pkg/ init_html() { SUITE=sid - ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted" MAINVIEW="stats" - ALLVIEWS="last_24h last_48h all_abc" - SPOKENTARGET["last_24h"]="packages tested in the last 24h" - SPOKENTARGET["last_48h"]="packages tested in the last 48h" - SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" - SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" - SPOKENTARGET["stats"]="various statistics about reproducible builds" - SPOKENTARGET["notes"]="packages with notes" - SPOKENTARGET["issues"]="known issues related to reproducible builds" + ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted" + ALLVIEWS="issues notes scheduled last_24h last_48h all_abc dd-list stats" SPOKENTARGET["reproducible"]="packages which built reproducibly" - SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and don't create a .buildinfo file" + SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and do'nt create a .buildinfo file" SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and create a .buildinfo file" SPOKENTARGET["FTBFS"]="packages which failed to build from source" SPOKENTARGET["404"]="packages where the sources failed to downloaded" SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'" SPOKENTARGET["blacklisted"]="packages which have been blacklisted" + SPOKENTARGET["issues"]="known issues related to reproducible builds" + SPOKENTARGET["notes"]="packages with notes" + SPOKENTARGET["scheduled"]="packages currently scheduled for testing" + SPOKENTARGET["last_24h"]="packages tested in the last 24h" + SPOKENTARGET["last_48h"]="packages tested in the last 48h" + SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" + SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" + SPOKENTARGET["stats"]="various statistics about reproducible builds" # query some data we need everywhere AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources") COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages") @@ -184,7 +185,7 @@ write_page_header() { write_page "$2" write_page "

$2

" if [ "$1" = "$MAINVIEW" ] ; then - write_page "

These pages are updated every six hours. Results are obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on!

" + write_page "

These pages contain results obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on!

" fi write_page "

$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!" if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then @@ -202,9 +203,11 @@ write_page_header() { write_icon write_page "" done - for TARGET in issues notes $ALLVIEWS dd-list stats ; do - if [ "$TARGET" = "issues" ] || [ "$TARGET" = "stats" ]; then + for TARGET in $ALLVIEWS ; do + if [ "$TARGET" = "issues" ] || [ "$TARGET" = "stats" ] ; then SPOKEN_TARGET=$TARGET + elif [ "$TARGET" = "scheduled" ] ; then + SPOKEN_TARGET="currently scheduled" else SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} fi @@ -265,8 +268,12 @@ link_packages() { for PKG in $@ ; do if $BUILDINFO_SIGNS ; then set_package_star + if ! $BUILDINFO_ON_PAGE && [ ! -z "$STAR" ] ; then + BUILDINFO_ON_PAGE=true + fi + fi - write_page " ${LINKTARGET[$PKG]} $STAR" + write_page " ${LINKTARGET[$PKG]}$STAR" done } @@ -298,7 +305,6 @@ process_packages() { BUILD_DATE=$(echo $RESULT|cut -d "|" -f1) # version with epoch removed EVERSION=$(echo $RESULT | cut -d "|" -f2 | cut -d ":" -f2) - set_package_star # only build $PKG pages if they don't exist or are older than $BUILD_DATE or have a note PKG_FILE="/var/lib/jenkins/userContent/rb-pkg/${PKG}.html" OLD_FILE=$(find $(dirname ${PKG_FILE}) -name $(basename ${PKG_FILE}) ! -newermt "$BUILD_DATE" 2>/dev/null || true) @@ -311,6 +317,7 @@ process_packages() { if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then NOTES_LINK=" notes " fi + set_package_star init_pkg_page "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "$STAR" append2pkg_page "${NOTES_LINK}" if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then @@ -339,12 +346,6 @@ process_packages() { fi finish_pkg_page "$MAINLINK" fi - if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then - set_package_class - LINKTARGET[$PKG]="$PKG$STAR" - else - LINKTARGET[$PKG]="$PKG" - fi done } @@ -359,3 +360,21 @@ gather_stats() { PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) } + +update_html_schedule() { + gather_stats + SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM sources_scheduled ORDER BY date_scheduled DESC" | xargs echo) + COUNT_SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources_scheduled ORDER BY date_scheduled DESC" | xargs echo) + VIEW=scheduled + BUILDINFO_SIGNS=true + PAGE=index_${VIEW}.html + echo "$(date) - starting to write $PAGE page." + write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}" + write_page "

${COUNT_SCHEDULED} packages are currently scheduled for testing: " + force_package_targets $SCHEDULED + link_packages $SCHEDULED + write_page "

" + write_page_meta_sign + write_page_footer + publish_page +} diff --git a/bin/reproducible_html_indexes.sh b/bin/reproducible_html_indexes.sh index f6ed419f..d253f5c0 100755 --- a/bin/reproducible_html_indexes.sh +++ b/bin/reproducible_html_indexes.sh @@ -45,7 +45,7 @@ echo "$(date) - processing $COUNT_TOTAL packages... this will take a while." force_package_targets ${BAD["all"]} force_package_targets ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED -for VIEW in $ALLVIEWS ; do +for VIEW in last_24h last_48h all_abc ; do BUILDINFO_SIGNS=true PAGE=index_${VIEW}.html echo "$(date) - starting to write $PAGE page." @@ -163,3 +163,4 @@ for STATE in $ALLSTATES ; do publish_page done +update_html_schedule diff --git a/bin/reproducible_schedule_on_demand.sh b/bin/reproducible_schedule_on_demand.sh index 1002890e..ca8e5bfd 100755 --- a/bin/reproducible_schedule_on_demand.sh +++ b/bin/reproducible_schedule_on_demand.sh @@ -39,8 +39,6 @@ check_candidates() { fi } - - # # main # diff --git a/bin/reproducible_scheduler.sh b/bin/reproducible_scheduler.sh index 33c543d7..c0ee4d39 100755 --- a/bin/reproducible_scheduler.sh +++ b/bin/reproducible_scheduler.sh @@ -149,12 +149,14 @@ schedule_packages() { # set +x update_apt -SCHEDULED=$(sqlite3 ${PACKAGES_DB} 'SELECT count(name) FROM sources_scheduled') -if [ $SCHEDULED -gt 250 ] ; then - echo "$SCHEDULED packages scheduled, nothing to do." +init_html +COUNT_SCHEDULED=$(sqlite3 ${PACKAGES_DB} 'SELECT count(name) FROM sources_scheduled') +if [ $COUNT_SCHEDULED -gt 250 ] ; then + update_html_schedule + echo "$COUNT_SCHEDULED packages scheduled, nothing to do." exit 0 else - echo "$SCHEDULED packages currently scheduled, scheduling some more..." + echo "$COUNT_SCHEDULED packages currently scheduled, scheduling some more..." fi update_sources_table @@ -193,6 +195,7 @@ MESSAGE="$MESSAGE and $AMOUNT packages with the same version again, for a total # finally schedule_packages +update_html_schedule echo echo "$MESSAGE" kgb-client --conf /srv/jenkins/kgb/debian-reproducible.conf --relay-msg "$MESSAGE" -- cgit v1.2.3-54-g00ecf