diff options
| -rwxr-xr-x | bin/reproducible_common.sh | 4 | ||||
| -rwxr-xr-x | bin/reproducible_create_meta_pkg_sets.sh | 297 | ||||
| -rwxr-xr-x | bin/reproducible_html_graphs.sh | 8 |
3 files changed, 158 insertions, 151 deletions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index fec3f114..c2d7409c 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -143,8 +143,8 @@ write_page_header() { write_page "</li>" done for TARGET in $ALLVIEWS ; do - if [ "$TARGET" = "pkg_sets" ] && [ "$SUITE" != "sid" ] ; then - # FIXME: no pkg_sets are tested in experimental + if [ "$TARGET" = "pkg_sets" ] && [ "$SUITE" = "experimental" ] ; then + # no pkg_sets are tested in experimental continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} diff --git a/bin/reproducible_create_meta_pkg_sets.sh b/bin/reproducible_create_meta_pkg_sets.sh index 7052db31..f08a4538 100755 --- a/bin/reproducible_create_meta_pkg_sets.sh +++ b/bin/reproducible_create_meta_pkg_sets.sh @@ -11,26 +11,6 @@ common_init "$@" . /srv/jenkins/bin/reproducible_common.sh ARCH=amd64 -SUITE=sid - -DISTNAME="$SUITE-$ARCH" -TPATH=/srv/reproducible-results/meta_pkgsets-$SUITE -CHPATH=/srv/reproducible-results/chdist-$SUITE -mkdir -p $TPATH $CHPATH - -# delete possibly existing dist -cd $CHPATH -rm -rf $DISTNAME -cd - - -# the "[arch=$ARCH]" is a workaround until #774685 is fixed -chdist --data-dir=$CHPATH --arch=$ARCH create $DISTNAME "[arch=$ARCH]" $MIRROR $SUITE main -chdist --data-dir=$CHPATH --arch=$ARCH apt-get $DISTNAME update - -PACKAGES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_binary-${ARCH}_Packages) -SOURCES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_source_Sources) -TMPFILE=$(mktemp) -TMPFILE2=$(mktemp) # helper functions convert_into_source_packages_only() { @@ -50,7 +30,6 @@ convert_into_source_packages_only() { cut -d " " -f1 ${TMPFILE2} | cut -d ":" -f1 | sort -u > $TMPFILE rm ${TMPFILE2} } - convert_from_deb822_into_source_packages_only() { # given a Packages file in deb822 format on standard input, the # following perl "oneliner" outputs the associated (unversioned) @@ -88,135 +67,163 @@ update_if_similar() { fi } +update_pkg_sets() { + # the essential package set + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[1]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then + chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FEssential yes > $TMPFILE + convert_from_deb822_into_source_packages_only + update_if_similar ${META_PKGSET[1]}.pkgset + fi -# -# main -# - -# the essential package set -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[1]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then - chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FEssential yes > $TMPFILE - convert_from_deb822_into_source_packages_only - update_if_similar ${META_PKGSET[1]}.pkgset -fi - -# the required package set -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[2]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[2]}.pkgset ] ; then - chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FPriority required > $TMPFILE - convert_from_deb822_into_source_packages_only - update_if_similar ${META_PKGSET[2]}.pkgset -fi - -# build-essential -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[3]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then - chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FBuild-Essential yes --or -FPackage build-essential \) > ${TMPFILE2} - # here we want the installable set: - schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE - convert_from_deb822_into_source_packages_only - update_if_similar ${META_PKGSET[3]}.pkgset -fi - -# popcon top 1337 installed sources -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[4]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[4]}.pkgset ] ; then - SQL_QUERY="SELECT popcon_src.source FROM popcon_src ORDER BY popcon_src.insts DESC LIMIT 1337;" - PGPASSWORD=public-udd-mirror \ - psql -U public-udd-mirror \ - -h public-udd-mirror.xvm.mit.edu -p 5432 \ - -t \ - udd -c"${SQL_QUERY}" > $TMPFILE - update_if_similar ${META_PKGSET[4]}.pkgset -fi - -# installed on one or more .debian.org machines -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[5]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[5]}.pkgset ] ; then - # one day we will get a proper data provider from DSA... - # (so far it was a manual "dpkg --get-selections" on all machines - # converted into a list of source packages...) - cat /srv/jenkins/bin/reproducible_installed_on_debian.org > $TMPFILE - update_if_similar ${META_PKGSET[5]}.pkgset -fi - -# packages which had a DSA -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[6]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[6]}.pkgset ] ; then - svn export svn://svn.debian.org/svn/secure-testing/data/DSA/list ${TMPFILE2} - grep "^\[" ${TMPFILE2} | grep "DSA-" | cut -d " " -f5|sort -u > $TMPFILE - convert_into_source_packages_only - update_if_similar ${META_PKGSET[6]}.pkgset -fi - -# gnome and everything it depends on -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[7]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[7]}.pkgset ] ; then - chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FPriority required --or -FPackage gnome \) > ${TMPFILE2} - schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE - convert_from_deb822_into_source_packages_only - update_if_similar ${META_PKGSET[7]}.pkgset -fi - -# The build-depends of X tasks can be solved once dose-ceve is able to read -# Debian source packages (possible in dose3 git but needs a new dose3 release -# and upload to unstable) -# -# Ignoring parsing issues, the current method is unable to resolve virtual -# build dependencies -# -# The current method also ignores Build-Depends-Indep and Build-Depends-Arch - -# all build depends of gnome -rm -f $TMPFILE -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[8]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[8]}.pkgset ] ; then - for PKG in $(cat $TPATH/${META_PKGSET[7]}.pkgset) ; do - grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE - done - convert_into_source_packages_only - update_if_similar ${META_PKGSET[8]}.pkgset -fi - -# tails -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[9]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[9]}.pkgset ] ; then - curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.binpkgs > $TMPFILE - curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.srcpkgs >> $TMPFILE - convert_into_source_packages_only - update_if_similar ${META_PKGSET[9]}.pkgset -fi - -# all build depends of tails -rm -f $TMPFILE -if [ -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[10]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[10]}.pkgset ] ; then - for PKG in $(cat $TPATH/${META_PKGSET[9]}.pkgset) ; do - grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE - done - convert_into_source_packages_only - update_if_similar ${META_PKGSET[10]}.pkgset -fi - -# grml -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[11]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[11]}.pkgset ] ; then - curl http://grml.org/files/grml64-full_latest/dpkg.selections | cut -f1 > $TMPFILE - if ! grep '<title>404 Not Found</title>' $TMPFILE ; then + # the required package set + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[2]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[2]}.pkgset ] ; then + chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X -FPriority required > $TMPFILE + convert_from_deb822_into_source_packages_only + update_if_similar ${META_PKGSET[2]}.pkgset + fi + + # build-essential + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[3]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[3]}.pkgset ] ; then + chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FBuild-Essential yes --or -FPackage build-essential \) > ${TMPFILE2} + # here we want the installable set: + schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE + convert_from_deb822_into_source_packages_only + update_if_similar ${META_PKGSET[3]}.pkgset + fi + + # popcon top 1337 installed sources + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[4]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[4]}.pkgset ] ; then + SQL_QUERY="SELECT popcon_src.source FROM popcon_src ORDER BY popcon_src.insts DESC LIMIT 1337;" + PGPASSWORD=public-udd-mirror \ + psql -U public-udd-mirror \ + -h public-udd-mirror.xvm.mit.edu -p 5432 \ + -t \ + udd -c"${SQL_QUERY}" > $TMPFILE + update_if_similar ${META_PKGSET[4]}.pkgset + fi + + # installed on one or more .debian.org machines + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[5]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[5]}.pkgset ] ; then + # one day we will get a proper data provider from DSA... + # (so far it was a manual "dpkg --get-selections" on all machines + # converted into a list of source packages...) + cat /srv/jenkins/bin/reproducible_installed_on_debian.org > $TMPFILE + update_if_similar ${META_PKGSET[5]}.pkgset + fi + + # packages which had a DSA + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[6]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[6]}.pkgset ] ; then + svn export svn://svn.debian.org/svn/secure-testing/data/DSA/list ${TMPFILE2} + grep "^\[" ${TMPFILE2} | grep "DSA-" | cut -d " " -f5|sort -u > $TMPFILE convert_into_source_packages_only - update_if_similar ${META_PKGSET[11]}.pkgset - else - echo "Warning: could not download grml's latest dpkg.selections file, skipping pkg set..." + update_if_similar ${META_PKGSET[6]}.pkgset fi -fi -# all build depends of grml -rm -f $TMPFILE -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[12]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[12]}.pkgset ] ; then - for PKG in $(cat $TPATH/${META_PKGSET[11]}.pkgset) ; do - grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE - done - convert_into_source_packages_only - update_if_similar ${META_PKGSET[12]}.pkgset -fi + # gnome and everything it depends on + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[7]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[7]}.pkgset ] ; then + chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FPriority required --or -FPackage gnome \) > ${TMPFILE2} + schroot --directory /tmp -c source:jenkins-dpkg-jessie -- dose-deb-coinstall --deb-native-arch=$ARCH --bg=$PACKAGES --fg=${TMPFILE2} > $TMPFILE + convert_from_deb822_into_source_packages_only + update_if_similar ${META_PKGSET[7]}.pkgset + fi + + # The build-depends of X tasks can be solved once dose-ceve is able to read + # Debian source packages (possible in dose3 git but needs a new dose3 release + # and upload to unstable) + # + # Ignoring parsing issues, the current method is unable to resolve virtual + # build dependencies + # + # The current method also ignores Build-Depends-Indep and Build-Depends-Arch + + # all build depends of gnome + rm -f $TMPFILE + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[8]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[8]}.pkgset ] ; then + for PKG in $(cat $TPATH/${META_PKGSET[7]}.pkgset) ; do + grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE + done + convert_into_source_packages_only + update_if_similar ${META_PKGSET[8]}.pkgset + fi + + # tails + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[9]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[9]}.pkgset ] ; then + curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.binpkgs > $TMPFILE + curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.srcpkgs >> $TMPFILE + convert_into_source_packages_only + update_if_similar ${META_PKGSET[9]}.pkgset + fi + + # all build depends of tails + rm -f $TMPFILE + if [ -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[10]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[10]}.pkgset ] ; then + for PKG in $(cat $TPATH/${META_PKGSET[9]}.pkgset) ; do + grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE + done + convert_into_source_packages_only + update_if_similar ${META_PKGSET[10]}.pkgset + fi + + # grml + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[11]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[11]}.pkgset ] ; then + curl http://grml.org/files/grml64-full_latest/dpkg.selections | cut -f1 > $TMPFILE + if ! grep '<title>404 Not Found</title>' $TMPFILE ; then + convert_into_source_packages_only + update_if_similar ${META_PKGSET[11]}.pkgset + else + echo "Warning: could not download grml's latest dpkg.selections file, skipping pkg set..." + fi + fi + + # all build depends of grml + rm -f $TMPFILE + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[12]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[12]}.pkgset ] ; then + for PKG in $(cat $TPATH/${META_PKGSET[11]}.pkgset) ; do + grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE + done + convert_into_source_packages_only + update_if_similar ${META_PKGSET[12]}.pkgset + fi + + # pkg-perl-maintainers + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[13]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[13]}.pkgset ] ; then + grep-dctrl -sPackage -n -FMaintainer pkg-perl-maintainers@lists.alioth.debian.org $SOURCES > $TMPFILE + update_if_similar ${META_PKGSET[13]}.pkgset + fi +} + +TMPFILE=$(mktemp) +TMPFILE2=$(mktemp) +for SUITE in $SUITES ; do + if [ "$SUITE" = "experimental" ] ; then + # no pkg sets in experimental + continue + fi + echo + echo "$(date) - Creating meta package sets for $SUITE now." + echo + + DISTNAME="$SUITE-$ARCH" + TPATH=/srv/reproducible-results/meta_pkgsets-$SUITE + CHPATH=/srv/reproducible-results/chdist-$SUITE + mkdir -p $TPATH $CHPATH + + # delete possibly existing dist + cd $CHPATH + rm -rf $DISTNAME + cd - + + # the "[arch=$ARCH]" is a workaround until #774685 is fixed + chdist --data-dir=$CHPATH --arch=$ARCH create $DISTNAME "[arch=$ARCH]" $MIRROR $SUITE main + chdist --data-dir=$CHPATH --arch=$ARCH apt-get $DISTNAME update -# pkg-perl-maintainers -if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[13]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[13]}.pkgset ] ; then - grep-dctrl -sPackage -n -FMaintainer pkg-perl-maintainers@lists.alioth.debian.org $SOURCES > $TMPFILE - update_if_similar ${META_PKGSET[13]}.pkgset -fi + PACKAGES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_binary-${ARCH}_Packages) + SOURCES=$(ls $CHPATH/$DISTNAME/var/lib/apt/lists/*_dists_${SUITE}_main_source_Sources) + # finally + update_pkg_sets + echo "$(date) - All meta package sets for $SUITE created successfully." +done -# finally rm -f $TMPFILE ${TMPFILE2} -echo "All meta package sets created successfully." +echo diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh index 05116ea6..c8dba35b 100755 --- a/bin/reproducible_html_graphs.sh +++ b/bin/reproducible_html_graphs.sh @@ -478,8 +478,8 @@ create_main_stats_page() { write_page "</p><p>" # write meta pkg graphs per suite for SUITE in $SUITES ; do - if [ "$SUITE" != "sid" ] ; then - # FIXME: no pkg sets in experimental + if [ "$SUITE" = "experimental" ] ; then + # no pkg sets in experimental continue fi for i in $(seq 1 ${#META_PKGSET[@]}) ; do @@ -536,8 +536,8 @@ for SUITE in $SUITES ; do update_suite_stats gather_suite_stats create_suite_stats_page - if [ "$SUITE" != "sid" ] ; then - # FIXME: should be: no pkg sets in experimental + if [ "$SUITE" = "experimental" ] ; then + # no pkg sets in experimental continue fi update_meta_pkg_stats |