#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 # # included by all reproducible_*.sh scripts # # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init MAINNODE="jenkins" # host which contains reproducible.db if [ -f $PACKAGES_DB ] && [ -f $INIT ] ; then if [ -f ${PACKAGES_DB}.lock ] ; then for i in $(seq 0 200) ; do sleep 15 echo "sleeping 15s, $PACKAGES_DB is locked." if [ ! -f ${PACKAGES_DB}.lock ] ; then break fi done if [ -f ${PACKAGES_DB}.lock ] ; then echo "${PACKAGES_DB}.lock still exist, exiting." exit 1 fi fi elif [ ! -f ${PACKAGES_DB} ] && [ "$HOSTNAME" = "$MAINNODE" ] ; then echo "Warning: $PACKAGES_DB doesn't exist, creating it now." /srv/jenkins/bin/reproducible_db_maintenance.py # 60 seconds timeout when trying to get a lock cat > $INIT <<-EOF .timeout 60000 EOF fi # common variables REPRODUCIBLE_URL=https://reproducible.debian.net REPRODUCIBLE_DOT_ORG_URL=https://reproducible-builds.org # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} # suites being tested SUITES="testing unstable experimental" # architectures being tested ARCHS="armhf amd64" # define build nodes in use BUILD_NODES="profitbricks-build1-amd64.debian.net profitbricks-build2-amd64.debian.net wbq0-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net bpi0-armhf-rb.debian.net hb0-armhf-rb.debian.net" # these also needs to be defined in bin/jenkins_master_wrapper.sh BUILD_ENV_VARS="ARCH NUM_CPU CPU_MODEL DATETIME KERNEL1 KERNEL2" # these also needs to be defined in bin/reproducible_info.sh # existing usertags USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness buildinfo cpu signatures environment umask ftbfs locale" # number of cores to be used NUM_CPU=$(grep -c '^processor' /proc/cpuinfo) # we only need them for html creation but we cannot declare them in a function declare -A SPOKENTARGET BASE="/var/lib/jenkins/userContent/reproducible" mkdir -p "$BASE" # to hold reproducible temporary files/directories without polluting /tmp TEMPDIR="/tmp/reproducible" mkdir -p "$TEMPDIR" # create subdirs for suites for i in $SUITES ; do mkdir -p "$BASE/$i" done # tables for stats TABLE[0]=stats_pkg_state TABLE[1]=stats_builds_per_day TABLE[2]=stats_builds_age TABLE[3]=stats_bugs TABLE[4]=stats_notes TABLE[5]=stats_issues TABLE[6]=stats_meta_pkg_state TABLE[7]=stats_bugs_state # known package sets META_PKGSET[1]="essential" META_PKGSET[2]="required" META_PKGSET[3]="build-essential" META_PKGSET[4]="build-essential-depends" META_PKGSET[5]="popcon_top1337-installed-sources" META_PKGSET[6]="key_packages" META_PKGSET[7]="installed_on_debian.org" META_PKGSET[8]="had_a_DSA" META_PKGSET[9]="cii-census" META_PKGSET[10]="gnome" META_PKGSET[11]="gnome_build-depends" META_PKGSET[12]="kde" META_PKGSET[13]="kde_build-depends" META_PKGSET[14]="xfce" META_PKGSET[15]="xfce_build-depends" META_PKGSET[16]="tails" META_PKGSET[17]="tails_build-depends" META_PKGSET[18]="grml" META_PKGSET[19]="grml_build-depends" META_PKGSET[20]="maint_pkg-perl-maintainers" META_PKGSET[21]="maint_pkg-java-maintainers" META_PKGSET[22]="maint_pkg-haskell-maintainers" META_PKGSET[23]="maint_pkg-ruby-extras-maintainers" META_PKGSET[24]="maint_pkg-golang-maintainers" META_PKGSET[25]="maint_pkg-php-pear" META_PKGSET[26]="maint_pkg-javascript-devel" META_PKGSET[27]="maint_debian-boot" META_PKGSET[28]="maint_debian-ocaml" META_PKGSET[29]="maint_debian-x" META_PKGSET[30]="maint_lua" schedule_packages() { LC_USER="$REQUESTER" \ LOCAL_CALL="true" \ /srv/jenkins/bin/reproducible_remote_scheduler.py \ --message "$REASON" \ --no-notify \ --suite "$SUITE" \ --architecture "$ARCH" \ $@ } write_page() { echo "$1" >> $PAGE } set_icon() { # icons taken from tango-icon-theme (0.8.90-5) # licenced under http://creativecommons.org/licenses/publicdomain/ STATE_TARGET_NAME="$1" case "$1" in reproducible) ICON=weather-clear.png ;; unreproducible|FTBR) ICON=weather-showers-scattered.png ;; FTBFS) ICON=weather-storm.png ;; depwait) ICON=weather-snow.png ;; 404) ICON=weather-severe-alert.png ;; not_for_us|"not for us") ICON=weather-few-clouds-night.png STATE_TARGET_NAME="not_for_us" ;; blacklisted) ICON=error.png ;; *) ICON="" esac } write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() write_page "\"${STATE_TARGET_NAME}" } write_page_header() { rm -f $PAGE MAINVIEW="stats" ALLSTATES="reproducible FTBR FTBFS depwait not_for_us 404 blacklisted" ALLVIEWS="issues notes no_notes scheduled last_24h last_48h all_abc notify dd-list pkg_sets suite_stats arch repositories stats" GLOBALVIEWS="issues scheduled notify repositories stats" SUITEVIEWS="dd-list suite_stats" SPOKENTARGET["issues"]="issues" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["no_notes"]="packages without notes" SPOKENTARGET["scheduled"]="currently scheduled" SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["notify"]="⚑" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" SPOKENTARGET["pkg_sets"]="package sets" SPOKENTARGET["suite_stats"]="suite: $SUITE" SPOKENTARGET["repositories"]="repositories overview" SPOKENTARGET["stats"]="reproducible stats" write_page "" write_page "" write_page "" write_page "$2" if [ "$1" != "$MAINVIEW" ] ; then write_page "

$2

" else write_page "

$2

" write_page "

These pages are showing the prospects of reproducible builds of Debian packages." write_page " The results shown were obtained from several jobs running on" write_page " jenkins.debian.net." write_page " Thanks to Profitbricks for donating the virtual machines this is running on!

" fi if [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then write_page "

Join #debian-reproducible on OFTC," write_page " or send us an email," write_page " to get support for making sure your packages build reproducibly too. Also, we care about free software in general," write_page " so if you are an upstream developer or working on another distribution, we'd love to hear from you!" write_page " Besides Debian we are also testing coreboot, OpenWrt, NetBSD and FreeBSD now, though not as thoroughly as Debian (yet?) - and there are plans to test Fedora too." write_page " As we think that reproducible builds should become the norm, we have started to write How to make your software reproducible. As always we appreciate feedback on this document, just please don't consider it to be finished, comprehensive or correct yet.

" fi write_page "" if [ "$1" = "$MAINVIEW" ] ; then LATEST=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status IN ('unreproducible') AND s.suite = 'unstable' AND s.architecture = 'amd64' AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1) write_page "
https://reproducible.debian.net/" write_page "" write_page "" write_page "
" fi write_page "
" } write_page_intro() { write_page "

Reproducible builds enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived. There is a lot more information about reproducible builds on the Debian wiki and on https://reproducible.debian.net. The wiki explains in more depth why this is useful, what common issues exist and which workarounds and solutions are known.
" local BUILD_ENVIRONMENT=" in a Debian environment" local BRANCH="master" if [ "$1" = "coreboot" ] ; then write_page " Reproducible Coreboot is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using diffoscope. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.

" local PROJECTNAME="$1" local PROJECTURL="https://review.coreboot.org/p/coreboot.git" elif [ "$1" = "OpenWrt" ] ; then write_page " Reproducible OpenWrt is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few varitations added and then the resulting images and packages from the two builds are compared using diffoscope, which currently cannot detect .bin files as squashfs filesystems. Thus the resulting diffoscope output is not nearly as clear as it could be - hopefully this limitation will be overcome soon. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.

" local PROJECTNAME="openwrt" local PROJECTURL="git://git.openwrt.org/openwrt.git" elif [ "$1" = "NetBSD" ] ; then write_page " Reproducible NetBSD is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few varitations added and then the resulting files from the two builds are compared using diffoscope. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.

" local PROJECTNAME="netbsd" local PROJECTURL="https://github.com/jsonn/src" elif [ "$1" = "FreeBSD" ] ; then write_page " Reproducible FreeBSD is an effort to apply this to FreeBSD. Thus FreeBSD is build twice, with a few varitations added and then the resulting filesystems from the two builds are put into a compressed tar archive, which is then compared using diffoscope. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.

" local PROJECTNAME="freebsd" local PROJECTURL="https://github.com/freebsd/freebsd.git" local BUILD_ENVIRONMENT=", which via ssh triggers a build on a FreeBSD 10.1 system" local BRANCH="release/10.2.0" fi write_page "

There is a weekly run jenkins job to test the $BRANCH branch of $PROJECTNAME.git. Currently this job is triggered more often though, because this is still under development and brand new. The jenkins job is running reproducible_$PROJECTNAME.sh$BUILD_ENVIRONMENT and this script is solely responsible for creating this page. Feel invited to join #debian-reproducible (on irc.oftc.net) to request job runs whenever sensible. Patches and other feedback are very much appreciated - if you want to help, please start by looking at the ToDo list for $1, you might find something easy to contribute.

" } write_page_footer() { write_page "

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014-2015 Holger Levsen and others, GPL2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library." if [ "$1" = "coreboot" ] ; then write_page "The Coreboot logo is Copyright © 2008 by Konsult Stuge and coresystems GmbH and can be freely used to refer to the Coreboot project." elif [ "$1" = "NetBSD" ] ; then write_page "NetBSD® is a registered trademark of The NetBSD Foundation, Inc." elif [ "$1" = "FreeBSD" ] ; then write_page "FreeBSD is a registered trademark of The FreeBSD Foundation. The FreeBSD logo and The Power to Serve are trademarks of The FreeBSD Foundation." fi write_page "

" } write_page_meta_sign() { write_page "

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.
" write_page "A # sign after the name of a package indicates that a bug is filed against it. Likewise, a + sign indicates there is a patch available, a P means a pending bug while # indicates a closed bug. In cases of several bugs, the symbol is repeated.

" } write_explaination_table() { write_page "

" write_page "" if [ "$1" = "debian" ] ; then write_page "" write_page "" else write_page "" write_page "" fi if [ "$1" != "FreeBSD" ] ; then write_page "" fi write_page "" write_page "" write_page "" if [ "$1" != "FreeBSD" ] ; then write_page "" else write_page "" fi if [ "$1" = "debian" ] ; then write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" else write_page "" write_page "" write_page "" if [ "$1" != "FreeBSD" ] ; then write_page "" fi fi if [ "$1" != "FreeBSD" ] ; then if [ "$1" = "debian" ] ; then write_page "" else write_page "" fi write_page "" else write_page "" write_page "" fi if [ "$1" = "debian" ] ; then write_page "" else write_page "" fi write_page "" if [ "$1" != "FreeBSD" ] ; then write_page "" if [ "$1" = "debian" ] ; then write_page "" else write_page "" fi else write_page "" write_page "" fi if [ "$1" = "debian" ] ; then write_page "" else write_page "" fi write_page "
variationfirst buildsecond build
hostnameone of: $(for i in $BUILD_NODES ; do echo '
  ' ; echo $i | cut -d '.' -f1 ; done)		i-capture-the-hostname
domainname$(hostname -d)i-capture-the-domainname
hostname is not yet varied between rebuilds of $1.
domainname is not yet varied between rebuilds of $1.
env CAPTURE_ENVIRONMENTnot setCAPTURE_ENVIRONMENT=\"I capture the environment\"
env TZTZ=\"/usr/share/zoneinfo/Etc/GMT+12\"TZ=\"/usr/share/zoneinfo/Etc/GMT-14\"
env LANGLANG=\"en_GB.UTF-8\"LANG=\"fr_CH.UTF-8\"
env LC_ALLnot setLC_ALL=\"fr_CH.UTF-8\"
env PATHPATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:\"PATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path\"
env PATH is not yet varied between rebuilds of $1.\"is not yet varied between rebuilds of $1.\"
env BUILDUSERIDBUILDUSERID=\"1111\"BUILDUSERID=\"2222\"
env BUILDUSERNAMEBUILDUSERNAME=\"pbuilder1\"BUILDUSERNAME=\"pbuilder2\"
env USERUSER=\"pbuilder1\"USER=\"pbuilder2\"
uiduid=1111uid=2222
gidgid=1111gid=2222
env DEB_BUILD_OPTIONSDEB_BUILD_OPTIONS=\"parallel=XXX\"
  XXX for amd64: 16 or 8
  XXX for armhf: 4 or 2		DEB_BUILD_OPTIONS=\"parallel=YYY\"
  YYY for amd64: XXX-1
  YYY for armhf: 2 or 4 - the opposite of the first build)
UTS namespaceshared with the hostmodified using /usr/bin/unshare --uts
env USER is not yet varied between rebuilds of $1.
uid is not yet varied between rebuilds of $1.
gid is not yet varied between rebuilds of $1.
UTS namespace is not yet varied between rebuilds of $1.
kernel version, modified using /usr/bin/linux64 --uname-2.6one of: $(cat /srv/reproducible-results/node-information/* | grep KERNEL1 | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '
  ')		one of: $(cat /srv/reproducible-results/node-information/* | grep KERNEL2 | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '
  ')
kernel version, modified using /usr/bin/linux64 --uname-2.6$(uname -sr)$(/usr/bin/linux64 --uname-2.6 uname -sr)
umask00220002
FreeBSD kernel version is not yet varied between rebuilds of $1.
umask is not yet varied between rebuilds of $1.
CPU typeone of: $(cat /srv/reproducible-results/node-information/* | grep CPU_MODEL | cut -d '=' -f2- | sort -u | tr '\n' '\0' | xargs -0 -n1 echo '
  ')		on armhf: sometimes varied (depending on the build job)
on amd64: same for both builds (currently, work in progress)
CPU type$(cat /proc/cpuinfo|grep 'model name'|head -1|cut -d ":" -f2-)same for both builds
year, month, datetoday ($DATE)same for both builds (currently, work in progress)
hour, minutehour and minute will probably vary between two builds...but this is not enforced systematically... (currently, work in progress)
filesystemtmpfsvaried using disorderfs (manpage)
Filesystemtmpfssame for both builds (currently, this could be varied using disorderfs)
hour, minutehour and minute will probably vary between two builds...but this is not enforced systematically...)
filesystem of the build directoryufssame for both builds
everything else...is likely the same. So far, this is just about the prospects of reproducible builds of Debian - there will be more variations in the wild.
everything else...is likely the same. There will be more variations in the wild.

" } publish_page() { if [ "$1" = "" ] ; then TARGET=$PAGE else TARGET=$1/$PAGE fi cp $PAGE $BASE/$TARGET rm $PAGE echo "Enjoy $REPRODUCIBLE_URL/$TARGET" } link_packages() { set +x local i for (( i=1; i<$#+1; i=i+400 )) ; do local string='[' local delimiter='' local j for (( j=0; j<400; j++)) ; do local item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' cd /srv/jenkins/bin DATA=" $(python3 -c "from reproducible_common import link_packages; \ print(link_packages(${string}, '$SUITE', '$ARCH'))" 2> /dev/null)" cd - > /dev/null write_page "$DATA" done if "$DEBUG" ; then set -x ; fi } gen_package_html() { cd /srv/jenkins/bin python3 -c "import reproducible_html_packages as rep pkg = rep.Package('$1', no_notes=True) rep.gen_packages_html([pkg], no_clean=True)" || echo "Warning: cannot update html pages for $1" cd - > /dev/null } calculate_build_duration() { END=$(date +'%s') DURATION=$(( $END - $START )) } print_out_duration() { local HOUR=$(echo "$DURATION/3600"|bc) local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc) local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc) echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s." | tee -a ${RBUILDLOG} } irc_message() { local MESSAGE="$@" kgb-client --conf /srv/jenkins/kgb/debian-reproducible.conf --relay-msg "$MESSAGE" || true # don't fail the whole job } call_diffoscope() { mkdir -p $TMPDIR/$1/$(dirname $2) local TMPLOG=(mktemp --tmpdir=$TMPDIR) local msg="" set +e # remember to also modify the retry diffoscope call 15 lines below ( timeout $TIMEOUT schroot \ --directory $TMPDIR \ -c source:jenkins-reproducible-${DBDSUITE}-diffoscope \ diffoscope -- \ --html $TMPDIR/$1/$2.html \ $TMPDIR/b1/$1/$2 \ $TMPDIR/b2/$1/$2 2>&1 \ ) 2>&1 >> $TMPLOG RESULT=$? LOG_RESULT=$(grep '^E: 15binfmt: update-binfmts: unable to open' $TMPLOG || true) if [ ! -z "$LOG_RESULT" ] ; then rm -f $TMPLOG $TMPDIR/$1/$2.html echo "$(date -u) - schroot jenkins-reproducible-${DBDSUITE}-diffoscope not available, will sleep 2min and retry." sleep 2m # remember to also modify the retry diffoscope call 15 lines above ( timeout $TIMEOUT schroot \ --directory $TMPDIR \ -c source:jenkins-reproducible-${DBDSUITE}-diffoscope \ diffoscope -- \ --html $TMPDIR/$1/$2.html \ $TMPDIR/b1/$1/$2 \ $TMPDIR/b2/$1/$2 2>&1 \ ) 2>&1 >> $TMPLOG RESULT=$? fi if ! "$DEBUG" ; then set +x ; fi set -e cat $TMPLOG # print dbd output rm -f $TMPLOG case $RESULT in 0) echo "$(date -u) - $1/$2 is reproducible, yay!" ;; 1) echo "$(date -u) - $DIFFOSCOPE found issues, please investigate $1/$2" ;; 2) msg="$(date -u) - $DIFFOSCOPE had trouble comparing the two builds. Please investigate $1/$2" ;; 124) if [ ! -s $TMPDIR/$1.html ] ; then msg="$(date -u) - $DIFFOSCOPE produced no output for $1/$2 and was killed after running into timeout after ${TIMEOUT}..." else msg="$DIFFOSCOPE was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1/$2.html" fi ;; *) msg="$(date -u) - Something weird happened when running $DIFFOSCOPE on $1/$2 (which exited with $RESULT) and I don't know how to handle it." ;; esac if [ ! -z "$msg" ] ; then echo $msg | tee -a $TMPDIR/$1/$2.html fi } get_filesize() { local BYTESIZE="$(du -h -b $1 | cut -f1)" # numbers below 16384K are understood and more meaningful than 16M... if [ $BYTESIZE -gt 16777216 ] ; then SIZE="$(echo $BYTESIZE/1048576|bc)M" elif [ $BYTESIZE -gt 1024 ] ; then SIZE="$(echo $BYTESIZE/1024|bc)K" else SIZE="$BYTESIZE bytes" fi } cleanup_pkg_files() { rm -vf $BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz} rm -vf $BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz} rm -vf $BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.debbindiff.html rm -vf $BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.debbindiff.txt{,.gz} rm -vf $BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo rm -vf $BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz} } # # create the png (and query the db to populate a csv file...) # create_png_from_table() { echo "Checking whether to update $2..." # $1 = id of the stats table # $2 = image file name # $3 = meta package set, only sensible if $1=6 echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv # prepare query WHERE_EXTRA="WHERE suite = '$SUITE'" if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then # TABLE[3+4+5] don't have a suite column: WHERE_EXTRA="" elif [ $1 -eq 6 ] ; then # 6 is special too: WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'" fi if [ $1 -eq 0 ] || [ $1 -eq 2 ] ; then # TABLE[0+2] have a architecture column: WHERE_EXTRA="$WHERE_EXTRA AND architecture = \"$ARCH\"" if [ $1 -eq 2 ] && [ "$ARCH" = "armhf" ] ; then # armhf was only build since 2015-08-30 WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2015-08-30'" fi fi # run query if [ $1 -eq 1 ] ; then # not sure if it's worth to generate the following query... WHERE_EXTRA="AND architecture='$ARCH'" if [ "$ARCH" = "armhf" ] ; then # armhf was only build since 2015-08-30 WHERE2_EXTRA="WHERE s.datum >= '2015-08-30'" fi sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum, COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='testing' $WHERE_EXTRA),0) AS 'reproducible_testing', COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS 'reproducible_unstable', COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS 'reproducible_experimental', (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing' $WHERE_EXTRA) AS unreproducible_testing, (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS unreproducible_unstable, (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS unreproducible_experimental, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing' $WHERE_EXTRA) AS FTBFS_testing, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS FTBFS_unstable, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS FTBFS_experimental, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing' $WHERE_EXTRA) AS other_testing, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS other_unstable, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS other_experimental FROM stats_builds_per_day AS s $WHERE2_EXTRA GROUP BY s.datum" >> ${TABLE[$1]}.csv elif [ $1 -eq 2 ] ; then # just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible) sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv elif [ $1 -eq 7 ] ; then sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv else sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv fi # this is a gross hack: normally we take the number of colors a table should have... # for the builds_age table we only want one color, but different ones, so this hack: COLORS=${COLOR[$1]} if [ $1 -eq 2 ] ; then case "$SUITE" in testing) COLORS=40 ;; unstable) COLORS=41 ;; experimental) COLORS=42 ;; esac fi # only generate graph if the query returned data if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then echo "Updating $2..." DIR=$(dirname $2) mkdir -p $DIR echo "Generating $2." /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" mv $2 $BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) # create empty dummy png if there havent been any results ever elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then DIR=$(dirname $2) mkdir -p $DIR echo "Creating $2 dummy." convert -size 1920x960 xc:#aaaaaa -depth 8 $2 if [ "$3" != "" ] ; then local THUMB="${TABLE[1]}_${3}-thumbnail.png" convert $2 -adaptive-resize 160x80 ${THUMB} mv ${THUMB} $BASE/$DIR fi mv $2 $BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) fi rm ${TABLE[$1]}.csv }