diff options
| -rwxr-xr-x | bin/reproducible_common.sh | 33 | ||||
| -rwxr-xr-x | bin/reproducible_create_meta_pkg_sets.sh | 89 |
2 files changed, 67 insertions, 55 deletions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 2b6d9564..55da7b1e 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -64,22 +64,23 @@ META_PKGSET[2]="required" META_PKGSET[3]="build-essential" META_PKGSET[4]="build-essential-depends" META_PKGSET[5]="popcon_top1337-installed-sources" -META_PKGSET[6]="installed_on_debian.org" -META_PKGSET[7]="had_a_DSA" -META_PKGSET[8]="gnome" -META_PKGSET[9]="gnome_build-depends" -META_PKGSET[10]="kde" -META_PKGSET[11]="kde_build-depends" -META_PKGSET[12]="xfce" -META_PKGSET[13]="xfce_build-depends" -META_PKGSET[14]="tails" -META_PKGSET[15]="tails_build-depends" -META_PKGSET[16]="grml" -META_PKGSET[17]="grml_build-depends" -META_PKGSET[18]="maint_pkg-perl-maintainers" -META_PKGSET[19]="maint_pkg-java-maintainers" -META_PKGSET[20]="maint_pkg-haskell-maintainers" -META_PKGSET[21]="maint_pkg-ruby-extras-maintainers" +META_PKGSET[6]="key_packages" +META_PKGSET[7]="installed_on_debian.org" +META_PKGSET[8]="had_a_DSA" +META_PKGSET[9]="gnome" +META_PKGSET[10]="gnome_build-depends" +META_PKGSET[11]="kde" +META_PKGSET[12]="kde_build-depends" +META_PKGSET[13]="xfce" +META_PKGSET[14]="xfce_build-depends" +META_PKGSET[15]="tails" +META_PKGSET[16]="tails_build-depends" +META_PKGSET[17]="grml" +META_PKGSET[18]="grml_build-depends" +META_PKGSET[19]="maint_pkg-perl-maintainers" +META_PKGSET[20]="maint_pkg-java-maintainers" +META_PKGSET[21]="maint_pkg-haskell-maintainers" +META_PKGSET[22]="maint_pkg-ruby-extras-maintainers" schedule_packages() { # these packages are manually scheduled, so should have high priority, diff --git a/bin/reproducible_create_meta_pkg_sets.sh b/bin/reproducible_create_meta_pkg_sets.sh index 78084581..cf00c766 100755 --- a/bin/reproducible_create_meta_pkg_sets.sh +++ b/bin/reproducible_create_meta_pkg_sets.sh @@ -152,30 +152,41 @@ update_pkg_sets() { update_if_similar ${META_PKGSET[5]}.pkgset fi - # installed on one or more .debian.org machines + # key packages (same for all suites) if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[6]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[6]}.pkgset ] ; then + SQL_QUERY="SELECT source FROM key_packages;" + PGPASSWORD=public-udd-mirror \ + psql -U public-udd-mirror \ + -h public-udd-mirror.xvm.mit.edu -p 5432 \ + -t \ + udd -c"${SQL_QUERY}" > $TMPFILE + update_if_similar ${META_PKGSET[6]}.pkgset + fi + + # installed on one or more .debian.org machines + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[7]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[7]}.pkgset ] ; then # one day we will get a proper data provider from DSA... # (so far it was a manual "dpkg --get-selections" on all machines # converted into a list of source packages...) cat /srv/jenkins/bin/reproducible_installed_on_debian.org > $TMPFILE - update_if_similar ${META_PKGSET[6]}.pkgset + update_if_similar ${META_PKGSET[7]}.pkgset fi # packages which had a DSA - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[7]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[7]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[8]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[8]}.pkgset ] ; then svn export svn://svn.debian.org/svn/secure-testing/data/DSA/list ${TMPFILE2} grep "^\[" ${TMPFILE2} | grep "DSA-" | cut -d " " -f5|sort -u > $TMPFILE convert_into_source_packages_only - update_if_similar ${META_PKGSET[7]}.pkgset + update_if_similar ${META_PKGSET[8]}.pkgset fi # gnome and everything it depends on - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[8]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[8]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[9]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[9]}.pkgset ] ; then chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FPriority required --or -FPackage gnome \) > ${TMPFILE2} - get_installable_set ${META_PKGSET[8]}.pkgset + get_installable_set ${META_PKGSET[9]}.pkgset if [ -f $TMPFILE ] ; then convert_from_deb822_into_source_packages_only - update_if_similar ${META_PKGSET[8]}.pkgset + update_if_similar ${META_PKGSET[9]}.pkgset fi fi @@ -190,76 +201,76 @@ update_pkg_sets() { # all build depends of gnome rm -f $TMPFILE - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[9]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[9]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[10]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[10]}.pkgset ] ; then for PKG in $(cat $TPATH/${META_PKGSET[8]}.pkgset) ; do grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE done convert_into_source_packages_only - update_if_similar ${META_PKGSET[9]}.pkgset + update_if_similar ${META_PKGSET[10]}.pkgset fi # kde and everything it depends on - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[10]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[10]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[11]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[11]}.pkgset ] ; then chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FPriority required --or -FPackage kde-full \) > ${TMPFILE2} - get_installable_set ${META_PKGSET[10]}.pkgset + get_installable_set ${META_PKGSET[11]}.pkgset if [ -f $TMPFILE ] ; then convert_from_deb822_into_source_packages_only - update_if_similar ${META_PKGSET[10]}.pkgset + update_if_similar ${META_PKGSET[11]}.pkgset fi fi # all build depends of kde rm -f $TMPFILE - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[11]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[11]}.pkgset ] ; then - for PKG in $(cat $TPATH/${META_PKGSET[10]}.pkgset) ; do + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[12]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[12]}.pkgset ] ; then + for PKG in $(cat $TPATH/${META_PKGSET[11]}.pkgset) ; do grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE done convert_into_source_packages_only - update_if_similar ${META_PKGSET[11]}.pkgset + update_if_similar ${META_PKGSET[12]}.pkgset fi # xfce and everything it depends on - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[12]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[12]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[13]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[13]}.pkgset ] ; then chdist --data-dir=$CHPATH grep-dctrl-packages $DISTNAME -X \( -FPriority required --or -FPackage xfce4 \) > ${TMPFILE2} - get_installable_set ${META_PKGSET[12]}.pkgset + get_installable_set ${META_PKGSET[13]}.pkgset if [ -f $TMPFILE ] ; then convert_from_deb822_into_source_packages_only - update_if_similar ${META_PKGSET[12]}.pkgset + update_if_similar ${META_PKGSET[13]}.pkgset fi fi # all build depends of xfce rm -f $TMPFILE - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[13]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[13]}.pkgset ] ; then - for PKG in $(cat $TPATH/${META_PKGSET[12]}.pkgset) ; do + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[14]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[14]}.pkgset ] ; then + for PKG in $(cat $TPATH/${META_PKGSET[13]}.pkgset) ; do grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE done convert_into_source_packages_only - update_if_similar ${META_PKGSET[13]}.pkgset + update_if_similar ${META_PKGSET[14]}.pkgset fi # tails - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[14]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[14]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[15]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[15]}.pkgset ] ; then curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.binpkgs > $TMPFILE curl http://nightly.tails.boum.org/build_Tails_ISO_feature-jessie/latest.iso.srcpkgs >> $TMPFILE convert_into_source_packages_only - update_if_similar ${META_PKGSET[14]}.pkgset + update_if_similar ${META_PKGSET[15]}.pkgset fi # all build depends of tails rm -f $TMPFILE - if [ -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[15]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[15]}.pkgset ] ; then - for PKG in $(cat $TPATH/${META_PKGSET[14]}.pkgset) ; do + if [ -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[16]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[16]}.pkgset ] ; then + for PKG in $(cat $TPATH/${META_PKGSET[15]}.pkgset) ; do grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE done convert_into_source_packages_only - update_if_similar ${META_PKGSET[15]}.pkgset + update_if_similar ${META_PKGSET[16]}.pkgset fi # grml - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[16]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[16]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[17]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[17]}.pkgset ] ; then curl http://grml.org/files/grml64-full_latest/dpkg.selections | cut -f1 > $TMPFILE if ! grep '<title>404 Not Found</title>' $TMPFILE ; then convert_into_source_packages_only - update_if_similar ${META_PKGSET[16]}.pkgset + update_if_similar ${META_PKGSET[17]}.pkgset else echo "Warning: could not download grml's latest dpkg.selections file, skipping pkg set..." fi @@ -267,38 +278,38 @@ update_pkg_sets() { # all build depends of grml rm -f $TMPFILE - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[17]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[17]}.pkgset ] ; then - for PKG in $(cat $TPATH/${META_PKGSET[16]}.pkgset) ; do + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[18]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[18]}.pkgset ] ; then + for PKG in $(cat $TPATH/${META_PKGSET[17]}.pkgset) ; do grep-dctrl -sBuild-Depends -n -X -FPackage $PKG $SOURCES | sed "s#([^()]*)##g ; s#\[[^][]*\]##g ; s#,##g" >> $TMPFILE done convert_into_source_packages_only - update_if_similar ${META_PKGSET[17]}.pkgset + update_if_similar ${META_PKGSET[18]}.pkgset fi # pkg-perl-maintainers - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[18]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[18]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[19]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[19]}.pkgset ] ; then grep-dctrl -sPackage -n -FMaintainer pkg-perl-maintainers@lists.alioth.debian.org $SOURCES > $TMPFILE - update_if_similar ${META_PKGSET[18]}.pkgset + update_if_similar ${META_PKGSET[19]}.pkgset fi # pkg-java-maintainers - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[19]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[19]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[20]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[20]}.pkgset ] ; then grep-dctrl -sPackage -n -FMaintainer pkg-java-maintainers@lists.alioth.debian.org $SOURCES > $TMPFILE grep-dctrl -FDepends -n default-jdk -sPackage $PACKAGES >> $TMPFILE - update_if_similar ${META_PKGSET[19]}.pkgset + update_if_similar ${META_PKGSET[20]}.pkgset fi # pkg-haskell-maintainers - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[20]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[20]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[21]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[21]}.pkgset ] ; then grep-dctrl -sPackage -n -FMaintainer pkg-haskell-maintainers@lists.alioth.debian.org $SOURCES > $TMPFILE grep-dctrl -FDepends -n ghc -sPackage $PACKAGES >> $TMPFILE - update_if_similar ${META_PKGSET[20]}.pkgset + update_if_similar ${META_PKGSET[21]}.pkgset fi # pkg-ruby-extras-maintainers - if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[21]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[21]}.pkgset ] ; then + if [ ! -z $(find $TPATH -maxdepth 1 -mtime +0 -name ${META_PKGSET[22]}.pkgset) ] || [ ! -f $TPATH/${META_PKGSET[22]}.pkgset ] ; then grep-dctrl -sPackage -n -FMaintainer pkg-ruby-extras-maintainers@lists.alioth.debian.org $SOURCES > $TMPFILE - update_if_similar ${META_PKGSET[21]}.pkgset + update_if_similar ${META_PKGSET[22]}.pkgset fi } |