+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# define db
+PACKAGES_DB=/var/lib/jenkins/reproducible.db
+INIT=/var/lib/jenkins/reproducible.init
+if [ ! -f $PACKAGES_DB ] ; then
+ echo "$PACKAGES_DB doesn't exist, no builds possible."
+ exit 1
+fi
+
+# this needs sid entries in sources.list:
+grep deb-src /etc/apt/sources.list | grep sid
+# try apt-get update twice, else fail gracefully, aka not.
+sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true )
+
+# update sources table in db
+update_sources_table() {
+ TMPFILE=$(mktemp)
+ curl $MIRROR/dists/sid/main/source/Sources.xz > $TMPFILE
+ CSVFILE=$(mktemp)
+ (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE
+ sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources"
+ echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB}
+ # update amount of available packages (for doing statistics later)
+ P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | sort -u | wc -l)
+ sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")"
+ rm $CSVFILE # $TMPFILE is still being used
+}
+
+set +x
+if [ $1 = "unknown" ] ; then
+ update_sources_table
+ AMOUNT=$2
+ REAL_AMOUNT=0
+ GUESSES=$(echo "${AMOUNT}*3" | bc)
+ PACKAGES=""
+ CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo)
+ for PKG in $CANDIDATES ; do
+ if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
+ continue
+ fi
+ RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
+ if [ "$RESULT" = "" ] ; then
+ PACKAGES="${PACKAGES} $PKG"
+ fi
+ done
+elif [ $1 = "known" ] ; then
+ update_sources_table
+ AMOUNT=$2
+ PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo)
+else
+ # CANDIDATES is defined in that file
+ . /srv/jenkins/bin/reproducible_candidates.sh
+ PACKAGES=""
+ AMOUNT=$2
+ REAL_AMOUNT=0
+ for i in $(seq 0 ${#CANDIDATES[@]}) ; do
+ if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
+ continue
+ fi
+ PKG=${CANDIDATES[$i]}
+ RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
+ if [ "$RESULT" = "" ] ; then
+ PACKAGES="${PACKAGES} $PKG"
+ let "REAL_AMOUNT=REAL_AMOUNT+1"
+ fi
+ done
+fi
+AMOUNT=0
+for PKG in $PACKAGES ; do
+ let "AMOUNT=AMOUNT+1"
+done
+echo "============================================================================="
+echo "The following $AMOUNT source packages will be scheduled: ${PACKAGES}"
+echo "============================================================================="
+echo
+rm -f $TMPFILE
+
diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh
index 996eaefd..4efb0a3e 100755
--- a/bin/reproducible_stats.sh
+++ b/bin/reproducible_stats.sh
@@ -38,65 +38,108 @@ PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc)
PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc)
GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc)
-htmlecho() {
+write_index() {
echo "$1" >> index.html
}
+mkdir -p /var/lib/jenkins/userContent/rb-pkg/
+write_pkg_frameset() {
+ FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html"
+ cat > $FRAMESET <<-EOF
+
+
+
+
+
+
+EOF
+}
+
+init_navi_frame() {
+ NAVI="/var/lib/jenkins/userContent/rb-pkg/$1_navigation.html"
+ echo "" > $NAVI
+ echo "$1: " >> $NAVI
+}
+
+append2navi_frame() {
+ echo "$2" >> $NAVI
+}
+
+finish_navi_frame() {
+ echo "
" >> $NAVI
+}
+
link_packages() {
for PKG in $@ ; do
VERSION=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT version FROM source_packages WHERE name = \"$PKG\"")
# remove epoch
EVERSION=$(echo $VERSION | cut -d ":" -f2)
- htmlecho $PKG
# FIXME: remove unused code once all diffp.log and pbuilder.log files are gone
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] || [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] || [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] || [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] || [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ]; then
- htmlecho " ("
+ write_index " $PKG "
+ MAINLINK=""
+ init_navi_frame "$PKG"
if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
- htmlecho " buildinfo "
+ append2navi_frame " buildinfo "
+ MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
fi
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
- htmlecho " debbindiff "
+ append2navi_frame " debbindiff "
+ MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
elif [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] ; then
- htmlecho " diffp "
+ append2navi_frame " diffp "
+ MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log"
fi
if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
- htmlecho " rbuild "
+ append2navi_frame " rbuild "
+ if [ "$MAINLINK" = "" ] ; then
+ MAINLINK="$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log"
+ fi
fi
if [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] ; then
- htmlecho " pbuilder "
+ append2navi_frame " pbuilder "
+ if [ "$MAINLINK" = "" ] ; then
+ MAINLINK="$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log"
+ fi
fi
- htmlecho ") "
+ finish_navi_frame
+ write_pkg_frameset "$PKG" "$MAINLINK"
+ else
+ write_index " $PKG "
fi
done
}
-htmlecho "" > index.html
-htmlecho "Statistics for reproducible builds
"
-htmlecho "Results were obtained by several jobs running on jenkins.debian.net. This page is updated after each job run.
"
-htmlecho "$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!
"
-htmlecho "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: "
+write_index "" > index.html
+write_index "Statistics for reproducible builds
"
+write_index "Results were obtained by several jobs running on jenkins.debian.net. This page is updated after each job run.
"
+write_index "$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!
"
+write_index "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: "
link_packages $BAD
-htmlecho "
"
-htmlecho
-htmlecho "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: "
+write_index "
"
+write_index
+write_index "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: "
link_packages $UGLY
-htmlecho "
"
+write_index "
"
if [ $COUNT_SOURCELESS -gt 0 ] ; then
- htmlecho "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. $SOURCELESS
"
+ write_index "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. $SOURCELESS
"
fi
if [ $COUNT_NOTFORUS -gt 0 ] ; then
- htmlecho "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': $NOTFORUS
"
+ write_index "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': $NOTFORUS
"
fi
if [ $COUNT_BLACKLISTED -gt 0 ] ; then
- htmlecho "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED
"
+ write_index "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED
"
fi
-htmlecho "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: "
+write_index "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: "
link_packages $GOOD
-htmlecho "
"
-htmlecho "
Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields."
-htmlecho "
$(echo $BAD | dd-list -i)
"
-htmlecho "
Static URL for this page. Last modified: $(date)"
-htmlecho "
"
+write_index "