From bcc697c4ebb0f7ae0098607bd0b38773ee7bdf59 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Tue, 31 Mar 2015 13:18:29 +0200 Subject: fix language: s#maintainance#maintenance#g --- README | 2 +- TODO | 3 +- bin/g-i-installation.sh | 2 +- bin/maintainance.sh | 208 --------------------------------- bin/maintenance.sh | 208 +++++++++++++++++++++++++++++++++ bin/reproducible_maintainance.sh | 224 ------------------------------------ bin/reproducible_maintenance.sh | 224 ++++++++++++++++++++++++++++++++++++ job-cfg/chroot-installation.yaml.py | 32 +++--- job-cfg/d-i.yaml | 6 +- job-cfg/g-i-installation.yaml | 10 +- job-cfg/reproducible.yaml | 8 +- job-cfg/schroot.yaml | 4 +- job-cfg/self.yaml | 30 ++--- 13 files changed, 480 insertions(+), 481 deletions(-) delete mode 100755 bin/maintainance.sh create mode 100755 bin/maintenance.sh delete mode 100755 bin/reproducible_maintainance.sh create mode 100755 bin/reproducible_maintenance.sh diff --git a/README b/README index 453bab81..dd072213 100644 --- a/README +++ b/README @@ -69,7 +69,7 @@ Installation tests with g-i, the graphical version of d-i, the debian-installer. Installation tests inside chroot environments. -* 'chroot-installation_maintainance_$distro': +* 'chroot-installation_maintenance_$distro': ** make sure chroots have been cleaned up properly ** runs daily at 05:00 UTC and triggers the $distro specific bootstrap job on success ** wheezy is only triggered on the 4th day and 18th of each month (as it was released on the 4th) diff --git a/TODO b/TODO index 316c7379..e793aed9 100644 --- a/TODO +++ b/TODO @@ -18,7 +18,6 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian == General ToDo -* rename maintainance jobs into maintenance jobs * replace amd64 in scripts with $HOSTARCH * put kgb-client.conf in git and sed passwords from filesystem into it... * move this setup to jenkins.d.o @@ -130,7 +129,7 @@ properties: ** meld bin/reproducible_setup_schroot.sh into bin/schroot-create.sh and alias the former to the latter ** "fork" etc/schroot/default into etc/schroot/reproducible ** use one css for all, not two minimally different ones -** restore the "find packages which have been removed from sid" part of reproducible_maintainance.sh +** restore the "find packages which have been removed from sid" part of reproducible_maintenance.sh ** repo-comparison: check for binaries without source ** turn job-cfg/g-i.yaml into .yaml.py diff --git a/bin/g-i-installation.sh b/bin/g-i-installation.sh index 87ea5299..22cd382d 100755 --- a/bin/g-i-installation.sh +++ b/bin/g-i-installation.sh @@ -1080,7 +1080,7 @@ monitor_system() { echo "ERROR: An installation step failed." >> $GOCR exit 1 elif [ ! -z "$ROOT_PROBLEM" ] ; then - echo "ERROR: System is hanging at boot and waiting for root maintainance." >> $GOCR + echo "ERROR: System is hanging at boot and waiting for root maintenance." >> $GOCR exit 1 elif [ ! -z "$BUILD_LTSP_PROBLEM" ] ; then echo "ERROR: The failing step is: Build LTSP chroot." >> $GOCR diff --git a/bin/maintainance.sh b/bin/maintainance.sh deleted file mode 100755 index 89bf5fd1..00000000 --- a/bin/maintainance.sh +++ /dev/null @@ -1,208 +0,0 @@ -#!/bin/bash - -# Copyright 2012-2015 Holger Levsen -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -check_for_mounted_chroots() { - CHROOT_PATTERN="/chroots/${1}-*" - OUTPUT=$(mktemp) - ls $CHROOT_PATTERN 2>/dev/null > $OUTPUT || true - if [ -s $OUTPUT ] ; then - figlet "Warning:" - echo - echo "Probably manual cleanup needed:" - echo - echo "$ ls -la $CHROOT_PATTERN" - # List the processes using the partition - echo - fuser -mv $CHROOT_PATTERN - cat $OUTPUT - rm $OUTPUT - exit 1 - fi - rm $OUTPUT -} - -chroot_checks() { - check_for_mounted_chroots $1 - report_disk_usage /chroots - report_disk_usage /schroots - echo "WARNING: should remove directories in /(s)chroots which are older than a month." -} - -compress_old_jenkins_logs() { - local COMPRESSED - # compress logs to save space - COMPRESSED=$(find /var/lib/jenkins/jobs/*/builds/ -maxdepth 2 -mindepth 2 -mtime +1 -name log -exec gzip -9 -v {} \;) - if [ ! -z "$COMPRESSED" ] ; then - echo "Logs have been compressed:" - echo - echo "$COMPRESSED" - echo - fi -} - -remove_old_rebootstrap_logs() { - local OLDSTUFF - # delete old html logs to save space - OLDSTUFF=$(find /var/lib/jenkins/jobs/rebootstrap_* -maxdepth 3 -mtime +7 -name log_content.html -exec rm -v {} \;) - if [ ! -z "$OLDSTUFF" ] ; then - echo "Old html logs have been deleted:" - echo - echo "$OLDSTUFF" - echo - fi -} - -report_old_directories() { - # find and warn about old temp directories - if [ -z "$3" ] ; then - OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 -exec ls -lad {} \;) - else - # if $3 is given, ignore it - OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 ! -path "$3*" -exec ls -lad {} \;) - fi - if [ ! -z "$OLDSTUFF" ] ; then - echo "Warning: old temp directories found in $REP_RESULTS" - echo - echo "$OLDSTUFF" - echo "Please cleanup manually." - echo - fi -} - -report_disk_usage() { - if [ -z "$WATCHED_JOBS" ] ; then - echo "File system usage for all ${1} jobs:" - else - echo "File system usage for all ${1} jobs (including those currently running):" - fi - du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#" - echo - if [ ! -z "$WATCHED_JOBS" ] ; then - TMPFILE=$(mktemp) - for JOB in $(cat $WATCHED_JOBS) ; do - du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true - done - if [ -s $TMPFILE ] ; then - echo - echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:" - cat $TMPFILE - echo - fi - rm $TMPFILE - fi -} - -report_filetype_usage() { - OUTPUT=$(mktemp) - for JOB in $(cat $WATCHED_JOBS) ; do - if [ "$2" != "bak" ] && [ "$2" != "png" ] ; then - find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT - else - # find archived .bak + .png files too - find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT - fi - done - if [ -s $OUTPUT ] ; then - echo "File system use in $1 for $2 files:" - cat $OUTPUT - if [ "$3" = "warn" ] ; then - echo "Warning: there are $2 files and there should not be any." - fi - echo - fi - rm $OUTPUT -} - -report_squid_usage() { - REPORT=/var/www/calamaris/calamaris.txt - if [ -z $1 ] ; then - cat $REPORT - else - head -31 $REPORT - fi -} - -wait4idle() { - echo "Waiting until no $1.sh process runs.... $(date)" - while [ $(ps fax | grep -c $1.sh) -gt 1 ] ; do - sleep 30 - done - echo "Done waiting: $(date)" -} - -general_maintainance() { - uptime - - echo - # ignore unreadable /media fuse mountpoints from guestmount - df -h 2>/dev/null || true - - echo - for DIR in /var/cache/apt/archives/ /var/spool/squid/ /var/cache/pbuilder/build/ /var/lib/jenkins/jobs/ /chroots /schroots ; do - sudo du -shx $DIR 2>/dev/null - done - JOB_PREFIXES=$(ls -1 /var/lib/jenkins/jobs/|cut -d "_" -f1|sort -f -u) - for PREFIX in $JOB_PREFIXES ; do - report_disk_usage $PREFIX - done - - echo - vnstat - - (df 2>/dev/null || true ) | grep tmpfs > /dev/null || ( echo ; echo "Warning: no tmpfs mounts in use. Please investigate the host system." ; exit 1 ) -} - -# -# if $1 is empty, we do general maintainance, else for some subgroup of all jobs -# -if [ -z $1 ] ; then - general_maintainance - compress_old_jenkins_logs - report_squid_usage brief -else - case $1 in - chroot-installation*) wait4idle $1 - report_disk_usage $1 - chroot_checks $1 - ;; - g-i-installation) ACTIVE_JOBS=$(mktemp) - WATCHED_JOBS=$(mktemp) - RUNNING=$(mktemp) - ps fax > $RUNNING - cd /var/lib/jenkins/jobs - for GIJ in g-i-installation_* ; do - if grep -q "$GIJ/workspace" $RUNNING ; then - echo "$GIJ" >> $ACTIVE_JOBS - echo "Ignoring $GIJ job as it's currently running." - else - echo "$GIJ" >> $WATCHED_JOBS - fi - done - echo - report_disk_usage $1 - report_filetype_usage $1 png warn - report_filetype_usage $1 bak warn - report_filetype_usage $1 raw warn - report_filetype_usage $1 iso warn - echo "WARNING: there is no check / handling on stale lvm volumes" - rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING - ;; - d-i) report_old_directories /srv/d-i 7 /srv/d-i/workspace - ;; - squid) report_squid_usage - ;; - rebootstrap) remove_old_rebootstrap_logs - ;; - *) ;; - esac -fi - -echo -echo "No (big) problems found, all seems good." -figlet "Ok." diff --git a/bin/maintenance.sh b/bin/maintenance.sh new file mode 100755 index 00000000..c123a9bd --- /dev/null +++ b/bin/maintenance.sh @@ -0,0 +1,208 @@ +#!/bin/bash + +# Copyright 2012-2015 Holger Levsen +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +check_for_mounted_chroots() { + CHROOT_PATTERN="/chroots/${1}-*" + OUTPUT=$(mktemp) + ls $CHROOT_PATTERN 2>/dev/null > $OUTPUT || true + if [ -s $OUTPUT ] ; then + figlet "Warning:" + echo + echo "Probably manual cleanup needed:" + echo + echo "$ ls -la $CHROOT_PATTERN" + # List the processes using the partition + echo + fuser -mv $CHROOT_PATTERN + cat $OUTPUT + rm $OUTPUT + exit 1 + fi + rm $OUTPUT +} + +chroot_checks() { + check_for_mounted_chroots $1 + report_disk_usage /chroots + report_disk_usage /schroots + echo "WARNING: should remove directories in /(s)chroots which are older than a month." +} + +compress_old_jenkins_logs() { + local COMPRESSED + # compress logs to save space + COMPRESSED=$(find /var/lib/jenkins/jobs/*/builds/ -maxdepth 2 -mindepth 2 -mtime +1 -name log -exec gzip -9 -v {} \;) + if [ ! -z "$COMPRESSED" ] ; then + echo "Logs have been compressed:" + echo + echo "$COMPRESSED" + echo + fi +} + +remove_old_rebootstrap_logs() { + local OLDSTUFF + # delete old html logs to save space + OLDSTUFF=$(find /var/lib/jenkins/jobs/rebootstrap_* -maxdepth 3 -mtime +7 -name log_content.html -exec rm -v {} \;) + if [ ! -z "$OLDSTUFF" ] ; then + echo "Old html logs have been deleted:" + echo + echo "$OLDSTUFF" + echo + fi +} + +report_old_directories() { + # find and warn about old temp directories + if [ -z "$3" ] ; then + OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 -exec ls -lad {} \;) + else + # if $3 is given, ignore it + OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 ! -path "$3*" -exec ls -lad {} \;) + fi + if [ ! -z "$OLDSTUFF" ] ; then + echo "Warning: old temp directories found in $REP_RESULTS" + echo + echo "$OLDSTUFF" + echo "Please cleanup manually." + echo + fi +} + +report_disk_usage() { + if [ -z "$WATCHED_JOBS" ] ; then + echo "File system usage for all ${1} jobs:" + else + echo "File system usage for all ${1} jobs (including those currently running):" + fi + du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#" + echo + if [ ! -z "$WATCHED_JOBS" ] ; then + TMPFILE=$(mktemp) + for JOB in $(cat $WATCHED_JOBS) ; do + du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true + done + if [ -s $TMPFILE ] ; then + echo + echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:" + cat $TMPFILE + echo + fi + rm $TMPFILE + fi +} + +report_filetype_usage() { + OUTPUT=$(mktemp) + for JOB in $(cat $WATCHED_JOBS) ; do + if [ "$2" != "bak" ] && [ "$2" != "png" ] ; then + find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT + else + # find archived .bak + .png files too + find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT + fi + done + if [ -s $OUTPUT ] ; then + echo "File system use in $1 for $2 files:" + cat $OUTPUT + if [ "$3" = "warn" ] ; then + echo "Warning: there are $2 files and there should not be any." + fi + echo + fi + rm $OUTPUT +} + +report_squid_usage() { + REPORT=/var/www/calamaris/calamaris.txt + if [ -z $1 ] ; then + cat $REPORT + else + head -31 $REPORT + fi +} + +wait4idle() { + echo "Waiting until no $1.sh process runs.... $(date)" + while [ $(ps fax | grep -c $1.sh) -gt 1 ] ; do + sleep 30 + done + echo "Done waiting: $(date)" +} + +general_maintenance() { + uptime + + echo + # ignore unreadable /media fuse mountpoints from guestmount + df -h 2>/dev/null || true + + echo + for DIR in /var/cache/apt/archives/ /var/spool/squid/ /var/cache/pbuilder/build/ /var/lib/jenkins/jobs/ /chroots /schroots ; do + sudo du -shx $DIR 2>/dev/null + done + JOB_PREFIXES=$(ls -1 /var/lib/jenkins/jobs/|cut -d "_" -f1|sort -f -u) + for PREFIX in $JOB_PREFIXES ; do + report_disk_usage $PREFIX + done + + echo + vnstat + + (df 2>/dev/null || true ) | grep tmpfs > /dev/null || ( echo ; echo "Warning: no tmpfs mounts in use. Please investigate the host system." ; exit 1 ) +} + +# +# if $1 is empty, we do general maintenance, else for some subgroup of all jobs +# +if [ -z $1 ] ; then + general_maintenance + compress_old_jenkins_logs + report_squid_usage brief +else + case $1 in + chroot-installation*) wait4idle $1 + report_disk_usage $1 + chroot_checks $1 + ;; + g-i-installation) ACTIVE_JOBS=$(mktemp) + WATCHED_JOBS=$(mktemp) + RUNNING=$(mktemp) + ps fax > $RUNNING + cd /var/lib/jenkins/jobs + for GIJ in g-i-installation_* ; do + if grep -q "$GIJ/workspace" $RUNNING ; then + echo "$GIJ" >> $ACTIVE_JOBS + echo "Ignoring $GIJ job as it's currently running." + else + echo "$GIJ" >> $WATCHED_JOBS + fi + done + echo + report_disk_usage $1 + report_filetype_usage $1 png warn + report_filetype_usage $1 bak warn + report_filetype_usage $1 raw warn + report_filetype_usage $1 iso warn + echo "WARNING: there is no check / handling on stale lvm volumes" + rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING + ;; + d-i) report_old_directories /srv/d-i 7 /srv/d-i/workspace + ;; + squid) report_squid_usage + ;; + rebootstrap) remove_old_rebootstrap_logs + ;; + *) ;; + esac +fi + +echo +echo "No (big) problems found, all seems good." +figlet "Ok." diff --git a/bin/reproducible_maintainance.sh b/bin/reproducible_maintainance.sh deleted file mode 100755 index efdd9985..00000000 --- a/bin/reproducible_maintainance.sh +++ /dev/null @@ -1,224 +0,0 @@ -#!/bin/bash - -# Copyright 2014-2015 Holger Levsen -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -DIRTY=false - -# prepare backup -REP_RESULTS=/srv/reproducible-results -mkdir -p $REP_RESULTS/backup -cd $REP_RESULTS/backup - -# keep 30 days and the 1st of the month -DAY=(date -d "30 day ago" '+%d') -DATE=$(date -d "30 day ago" '+%Y-%m-%d') -if [ "$DAY" != "01" ] && [ -f reproducible_$DATE.db.xz ] ; then - rm -f reproducible_$DATE.db.xz -fi - -# actually do the backup -DATE=$(date '+%Y-%m-%d') -if [ ! -f reproducible_$DATE.db.xz ] ; then - cp -v $PACKAGES_DB . - DATE=$(date '+%Y-%m-%d') - mv -v reproducible.db reproducible_$DATE.db - xz reproducible_$DATE.db -fi - -# provide copy for external backups -cp -v $PACKAGES_DB /var/lib/jenkins/userContent/ - -# delete old temp directories -OLDSTUFF=$(find $REP_RESULTS -maxdepth 1 -type d -name "tmp.*" -mtime +2 -exec ls -lad {} \;) -if [ ! -z "$OLDSTUFF" ] ; then - echo - echo "Warning: old temp directories found in $REP_RESULTS" - find $REP_RESULTS -maxdepth 1 -type d -name "tmp.*" -mtime +2 -exec rm -rv {} \; - echo "These old directories have been deleted." - echo - DIRTY=true -fi - -# find old schroots -OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -name "reproducible-*-*" -mtime +2 -exec ls -lad {} \;) -if [ ! -z "$OLDSTUFF" ] ; then - echo - echo "Warning: old schroots found in /schroots, which have been deleted:" - find /schroots/ -maxdepth 1 -type d -name "reproducible-*-*" -mtime +2 -exec sudo rm -rf {} \; - echo "$OLDSTUFF" - echo - DIRTY=true -fi - -# find and warn about pbuild leftovers -OLDSTUFF=$(find /var/cache/pbuilder/result/ -mtime +1 -exec ls -lad {} \;) -if [ ! -z "$OLDSTUFF" ] ; then - # delete known files, see #777537 - cd /var/cache/pbuilder/result/ - echo "Attempting file detection..." - for i in $(find . -maxdepth 1 -mtime +1 -type f -exec basename {} \;) ; do - case $i in - stderr|stdout) rm -v $i - ;; - seqan-*.bed) rm -v $i # leftovers reported in #766741 - ;; - bootlogo) rm -v $i - ;; - org.daisy.paper.CustomPaperCollection.obj) rm -v $i - ;; - debian-faq.pdf.gz|debian-faq.ps.gz|debian-faq.txt.gz) rm -v $i - ;; - sumo_doxygen_lastrun.log) rm -v $i - ;; - *) ;; - esac - done - cd - - # report the rest - OLDSTUFF=$(find /var/cache/pbuilder/result/ -mtime +1 -exec ls -lad {} \;) - if [ ! -z "$OLDSTUFF" ] ; then - echo "Warning: old files or directories found in /var/cache/pbuilder/result/" - echo "$OLDSTUFF" - echo "Please cleanup manually." - fi - echo - DIRTY=true -fi - -# find failed builds due to network problems and reschedule them -# only grep through the last 5h (300 minutes) of builds... -# (ignore "*None.rbuild.log" because these are build which were just started) -# this job runs every 4h -FAILED_BUILDS=$(find /var/lib/jenkins/userContent/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec egrep -l -e 'E: Failed to fetch.*(Connection failed|Size mismatch|Cannot initiate the connection to)' {} \; || true) -if [ ! -z "$FAILED_BUILDS" ] ; then - echo - echo "Warning: the following failed builds have been found" - echo "$FAILED_BUILDS" - echo - echo "Rescheduling packages: " - for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f7 | sort -u) ; do - CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/" | cut -d "/" -f9 | cut -d "_" -f1) ; do echo -n "$PKG " ; done) - check_candidates - if [ $TOTAL -ne 0 ] ; then - echo " - in $SUITE: $CANDIDATES" - # '0' here means the artifacts will not be saved - schedule_packages 0 $PACKAGE_IDS - fi - done - DIRTY=true -fi - -# find+terminate processes which should not be there -HAYSTACK=$(mktemp) -RESULT=$(mktemp) -PBUIDS="1234 1111 2222" -ps axo pid,user,size,pcpu,cmd > $HAYSTACK -for i in $PBUIDS ; do - for PROCESS in $(pgrep -u $i -P 1 || true) ; do - # faked-sysv comes and goes... - grep ^$PROCESS $HAYSTACK | grep -v faked-sysv >> $RESULT 2> /dev/null || true - done -done -if [ -s $RESULT ] ; then - echo - echo "Warning: processes found which should not be there, maybe killing them now:" - cat $RESULT - echo - for PROCESS in $(cat $RESULT | cut -d " " -f1 | xargs echo) ; do - AGE=$(ps -p $PROCESS -o etimes= || echo 0) - # a single build may only take half a day, so... - if [ $AGE -gt 43200 ] ; then - sudo kill -9 $PROCESS 2>&1 - echo "'kill -9 $PROCESS' done." - else - echo "Did not kill $PROCESS as it is only $AGE seconds old." - fi - done - echo - DIRTY=true -fi -rm $HAYSTACK $RESULT - -# find packages which build didnt end correctly -QUERY=" - SELECT s.id, s.name, p.date_scheduled, p.date_build_started - FROM schedule AS p JOIN sources AS s ON p.package_id=s.id - WHERE p.date_scheduled != '' - AND p.date_build_started != '' - AND p.date_build_started < datetime('now', '-36 hours') - ORDER BY p.date_scheduled - " -PACKAGES=$(mktemp) -sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" > $PACKAGES 2> /dev/null || echo "Warning: SQL query '$QUERY' failed." -if grep -q '|' $PACKAGES ; then - echo - echo "Warning: packages found where the build was started more than 36h ago:" - echo "pkg_id|name|date_scheduled|date_build_started" - echo - cat $PACKAGES - echo - for PKG in $(cat $PACKAGES | cut -d "|" -f1) ; do - echo "sqlite3 ${PACKAGES_DB} \"DELETE FROM schedule WHERE package_id = '$PKG';\"" - sqlite3 -init $INIT ${PACKAGES_DB} "DELETE FROM schedule WHERE package_id = '$PKG';" - done - echo "Packages have been removed from scheduling." - echo - DIRTY=true -fi -rm $PACKAGES - -# find packages which have been removed from unstable -# commented out for now. This can't be done using the database anymore -QUERY="SELECT source_packages.name FROM source_packages - WHERE source_packages.name NOT IN - (SELECT sources.name FROM sources) - LIMIT 25" -#PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY") -PACKAGES='' -if [ ! -z "$PACKAGES" ] ; then - echo - echo "Removing these removed packages from database:" - echo $PACKAGES - echo - QUERY="DELETE FROM source_packages - WHERE source_packages.name NOT IN - (SELECT sources.name FROM sources) - LIMIT 25" - sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" - cd /var/lib/jenkins/userContent - for i in PACKAGES ; do - find rb-pkg/ rbuild/ notes/ dbd/ -name "${i}_*" -exec rm -v {} \; - done - cd - -fi - -# delete jenkins html logs from reproducible_builder_* jobs as they are mostly redundant -# (they only provide the extended value of parsed console output, which we dont need here.) -OLDSTUFF=$(find /var/lib/jenkins/jobs/reproducible_builder_* -maxdepth 3 -mtime +0 -name log_content.html -exec rm -v {} \; | wc -l) -if [ ! -z "$OLDSTUFF" ] ; then - echo - echo "Removed $OLDSTUFF jenkins html logs." - echo -fi - -# remove artifacts older than 3 days -ARTIFACTS=$(find /var/lib/jenkins/userContent/artifacts/* -maxdepth 1 -type d -mtime +3 -exec ls -lad {} \; || true) -if [ ! -z "$ARTIFACTS" ] ; then - echo - echo "Removed old artifacts:" - find /var/lib/jenkins/userContent/artifacts/* -maxdepth 1 -type d -mtime +3 -exec rm -rv {} \; - echo -fi - -if ! $DIRTY ; then - echo "Everything seems to be fine." - echo -fi diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh new file mode 100755 index 00000000..efdd9985 --- /dev/null +++ b/bin/reproducible_maintenance.sh @@ -0,0 +1,224 @@ +#!/bin/bash + +# Copyright 2014-2015 Holger Levsen +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +DIRTY=false + +# prepare backup +REP_RESULTS=/srv/reproducible-results +mkdir -p $REP_RESULTS/backup +cd $REP_RESULTS/backup + +# keep 30 days and the 1st of the month +DAY=(date -d "30 day ago" '+%d') +DATE=$(date -d "30 day ago" '+%Y-%m-%d') +if [ "$DAY" != "01" ] && [ -f reproducible_$DATE.db.xz ] ; then + rm -f reproducible_$DATE.db.xz +fi + +# actually do the backup +DATE=$(date '+%Y-%m-%d') +if [ ! -f reproducible_$DATE.db.xz ] ; then + cp -v $PACKAGES_DB . + DATE=$(date '+%Y-%m-%d') + mv -v reproducible.db reproducible_$DATE.db + xz reproducible_$DATE.db +fi + +# provide copy for external backups +cp -v $PACKAGES_DB /var/lib/jenkins/userContent/ + +# delete old temp directories +OLDSTUFF=$(find $REP_RESULTS -maxdepth 1 -type d -name "tmp.*" -mtime +2 -exec ls -lad {} \;) +if [ ! -z "$OLDSTUFF" ] ; then + echo + echo "Warning: old temp directories found in $REP_RESULTS" + find $REP_RESULTS -maxdepth 1 -type d -name "tmp.*" -mtime +2 -exec rm -rv {} \; + echo "These old directories have been deleted." + echo + DIRTY=true +fi + +# find old schroots +OLDSTUFF=$(find /schroots/ -maxdepth 1 -type d -name "reproducible-*-*" -mtime +2 -exec ls -lad {} \;) +if [ ! -z "$OLDSTUFF" ] ; then + echo + echo "Warning: old schroots found in /schroots, which have been deleted:" + find /schroots/ -maxdepth 1 -type d -name "reproducible-*-*" -mtime +2 -exec sudo rm -rf {} \; + echo "$OLDSTUFF" + echo + DIRTY=true +fi + +# find and warn about pbuild leftovers +OLDSTUFF=$(find /var/cache/pbuilder/result/ -mtime +1 -exec ls -lad {} \;) +if [ ! -z "$OLDSTUFF" ] ; then + # delete known files, see #777537 + cd /var/cache/pbuilder/result/ + echo "Attempting file detection..." + for i in $(find . -maxdepth 1 -mtime +1 -type f -exec basename {} \;) ; do + case $i in + stderr|stdout) rm -v $i + ;; + seqan-*.bed) rm -v $i # leftovers reported in #766741 + ;; + bootlogo) rm -v $i + ;; + org.daisy.paper.CustomPaperCollection.obj) rm -v $i + ;; + debian-faq.pdf.gz|debian-faq.ps.gz|debian-faq.txt.gz) rm -v $i + ;; + sumo_doxygen_lastrun.log) rm -v $i + ;; + *) ;; + esac + done + cd - + # report the rest + OLDSTUFF=$(find /var/cache/pbuilder/result/ -mtime +1 -exec ls -lad {} \;) + if [ ! -z "$OLDSTUFF" ] ; then + echo "Warning: old files or directories found in /var/cache/pbuilder/result/" + echo "$OLDSTUFF" + echo "Please cleanup manually." + fi + echo + DIRTY=true +fi + +# find failed builds due to network problems and reschedule them +# only grep through the last 5h (300 minutes) of builds... +# (ignore "*None.rbuild.log" because these are build which were just started) +# this job runs every 4h +FAILED_BUILDS=$(find /var/lib/jenkins/userContent/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec egrep -l -e 'E: Failed to fetch.*(Connection failed|Size mismatch|Cannot initiate the connection to)' {} \; || true) +if [ ! -z "$FAILED_BUILDS" ] ; then + echo + echo "Warning: the following failed builds have been found" + echo "$FAILED_BUILDS" + echo + echo "Rescheduling packages: " + for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f7 | sort -u) ; do + CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/" | cut -d "/" -f9 | cut -d "_" -f1) ; do echo -n "$PKG " ; done) + check_candidates + if [ $TOTAL -ne 0 ] ; then + echo " - in $SUITE: $CANDIDATES" + # '0' here means the artifacts will not be saved + schedule_packages 0 $PACKAGE_IDS + fi + done + DIRTY=true +fi + +# find+terminate processes which should not be there +HAYSTACK=$(mktemp) +RESULT=$(mktemp) +PBUIDS="1234 1111 2222" +ps axo pid,user,size,pcpu,cmd > $HAYSTACK +for i in $PBUIDS ; do + for PROCESS in $(pgrep -u $i -P 1 || true) ; do + # faked-sysv comes and goes... + grep ^$PROCESS $HAYSTACK | grep -v faked-sysv >> $RESULT 2> /dev/null || true + done +done +if [ -s $RESULT ] ; then + echo + echo "Warning: processes found which should not be there, maybe killing them now:" + cat $RESULT + echo + for PROCESS in $(cat $RESULT | cut -d " " -f1 | xargs echo) ; do + AGE=$(ps -p $PROCESS -o etimes= || echo 0) + # a single build may only take half a day, so... + if [ $AGE -gt 43200 ] ; then + sudo kill -9 $PROCESS 2>&1 + echo "'kill -9 $PROCESS' done." + else + echo "Did not kill $PROCESS as it is only $AGE seconds old." + fi + done + echo + DIRTY=true +fi +rm $HAYSTACK $RESULT + +# find packages which build didnt end correctly +QUERY=" + SELECT s.id, s.name, p.date_scheduled, p.date_build_started + FROM schedule AS p JOIN sources AS s ON p.package_id=s.id + WHERE p.date_scheduled != '' + AND p.date_build_started != '' + AND p.date_build_started < datetime('now', '-36 hours') + ORDER BY p.date_scheduled + " +PACKAGES=$(mktemp) +sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" > $PACKAGES 2> /dev/null || echo "Warning: SQL query '$QUERY' failed." +if grep -q '|' $PACKAGES ; then + echo + echo "Warning: packages found where the build was started more than 36h ago:" + echo "pkg_id|name|date_scheduled|date_build_started" + echo + cat $PACKAGES + echo + for PKG in $(cat $PACKAGES | cut -d "|" -f1) ; do + echo "sqlite3 ${PACKAGES_DB} \"DELETE FROM schedule WHERE package_id = '$PKG';\"" + sqlite3 -init $INIT ${PACKAGES_DB} "DELETE FROM schedule WHERE package_id = '$PKG';" + done + echo "Packages have been removed from scheduling." + echo + DIRTY=true +fi +rm $PACKAGES + +# find packages which have been removed from unstable +# commented out for now. This can't be done using the database anymore +QUERY="SELECT source_packages.name FROM source_packages + WHERE source_packages.name NOT IN + (SELECT sources.name FROM sources) + LIMIT 25" +#PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY") +PACKAGES='' +if [ ! -z "$PACKAGES" ] ; then + echo + echo "Removing these removed packages from database:" + echo $PACKAGES + echo + QUERY="DELETE FROM source_packages + WHERE source_packages.name NOT IN + (SELECT sources.name FROM sources) + LIMIT 25" + sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" + cd /var/lib/jenkins/userContent + for i in PACKAGES ; do + find rb-pkg/ rbuild/ notes/ dbd/ -name "${i}_*" -exec rm -v {} \; + done + cd - +fi + +# delete jenkins html logs from reproducible_builder_* jobs as they are mostly redundant +# (they only provide the extended value of parsed console output, which we dont need here.) +OLDSTUFF=$(find /var/lib/jenkins/jobs/reproducible_builder_* -maxdepth 3 -mtime +0 -name log_content.html -exec rm -v {} \; | wc -l) +if [ ! -z "$OLDSTUFF" ] ; then + echo + echo "Removed $OLDSTUFF jenkins html logs." + echo +fi + +# remove artifacts older than 3 days +ARTIFACTS=$(find /var/lib/jenkins/userContent/artifacts/* -maxdepth 1 -type d -mtime +3 -exec ls -lad {} \; || true) +if [ ! -z "$ARTIFACTS" ] ; then + echo + echo "Removed old artifacts:" + find /var/lib/jenkins/userContent/artifacts/* -maxdepth 1 -type d -mtime +3 -exec rm -rv {} \; + echo +fi + +if ! $DIRTY ; then + echo "Everything seems to be fine." + echo +fi diff --git a/job-cfg/chroot-installation.yaml.py b/job-cfg/chroot-installation.yaml.py index d17761f6..6c5b49a4 100755 --- a/job-cfg/chroot-installation.yaml.py +++ b/job-cfg/chroot-installation.yaml.py @@ -20,7 +20,7 @@ trigger_times = { 'squeeze': '30 16 25 * *', 'sid': '30 4 * * *' } targets = """ - maintainance + maintenance bootstrap gnome kde @@ -84,7 +84,7 @@ def get_targets_in_distro(distro, targets): # who gets mail for which target # def get_recipients(target): - if target == 'maintainance': + if target == 'maintenance': return 'qa-jenkins-scm@lists.alioth.debian.org' elif target == 'haskell': return 'jenkins+debian-haskell qa-jenkins-scm@lists.alioth.debian.org pkg-haskell-maintainers@lists.alioth.debian.org' @@ -103,7 +103,7 @@ def get_recipients(target): # views for different targets # def get_view(target, distro): - if target == 'maintainance': + if target == 'maintenance': return 'jenkins.d.n' elif target == 'haskell': return 'haskell' @@ -198,19 +198,19 @@ print(""" """) for base_distro in sorted(base_distros): for target in sorted(get_targets_in_distro(base_distro, targets)): - if target in ('bootstrap', 'maintainance'): + if target in ('bootstrap', 'maintenance'): action = target else: action = 'install_'+target # default job - if target == 'maintainance' or base_distro != oldstable: + if target == 'maintenance' or base_distro != oldstable: print("""- job-template: defaults: chroot-installation name: '{name}_%(base_distro)s_%(action)s'""" % dict(base_distro=base_distro, action=action)) # upgrade job - if base_distro in distro_upgrades and action != 'maintainance': + if base_distro in distro_upgrades and action != 'maintenance': print("""- job-template: defaults: chroot-installation name: '{name}_%(base_distro)s_%(action)s_upgrade_to_%(second_base)s'""" % @@ -218,7 +218,7 @@ for base_distro in sorted(base_distros): action=action, second_base=distro_upgrades[base_distro])) # upgrade job with upgrading apt+dpkg first - if base_distro in distro_upgrades and base_distro != oldstable and target[:10] != 'education-' and action != 'maintainance': + if base_distro in distro_upgrades and base_distro != oldstable and target[:10] != 'education-' and action != 'maintenance': print("""- job-template: defaults: chroot-installation name: '{name}_%(base_distro)s_%(action)s_upgrade_to_%(second_base)s_aptdpkg_first'""" % @@ -233,9 +233,9 @@ print(""" jobs:""") for base_distro in sorted(base_distros): for target in sorted(get_targets_in_distro(base_distro, targets)): - if target == 'maintainance': + if target == 'maintenance': description = 'Maintainance job for chroot-installation_'+base_distro+'_* jobs, do some cleanups and monitoring so that there is a predictable environment.' - shell = '/srv/jenkins/bin/maintainance.sh chroot-installation_'+base_distro + shell = '/srv/jenkins/bin/maintenance.sh chroot-installation_'+base_distro prio = 135 time = trigger_times[base_distro] if base_distro in distro_upgrades.values(): @@ -252,7 +252,7 @@ for base_distro in sorted(base_distros): time = '' trigger = '' for trigger_target in get_targets_in_distro(base_distro, targets): - if trigger_target not in ('maintainance', 'bootstrap'): + if trigger_target not in ('maintenance', 'bootstrap'): if trigger != '': trigger = trigger+', ' trigger = trigger+'chroot-installation_'+base_distro+'_install_'+trigger_target @@ -262,12 +262,12 @@ for base_distro in sorted(base_distros): prio = 130 time = '' trigger = '' - if target in ('bootstrap', 'maintainance'): + if target in ('bootstrap', 'maintenance'): action = target else: action = 'install_'+target # default job - if target == 'maintainance' or base_distro != oldstable: + if target == 'maintenance' or base_distro != oldstable: print(""" - '{name}_%(base_distro)s_%(action)s': my_shell: '%(shell)s' my_prio: '%(prio)s' @@ -286,13 +286,13 @@ for base_distro in sorted(base_distros): view=get_view(target, base_distro), description=description)) # upgrade job - if base_distro in distro_upgrades and action != 'maintainance': + if base_distro in distro_upgrades and action != 'maintenance': if target == 'bootstrap': shell = '/srv/jenkins/bin/chroot-installation.sh '+base_distro+' none '+distro_upgrades[base_distro] description = 'Debootstrap '+base_distro+', then upgrade to '+distro_upgrades[base_distro]+'.' trigger = '' for trigger_target in get_targets_in_distro(base_distro, targets): - if trigger_target not in ('maintainance', 'bootstrap'): + if trigger_target not in ('maintenance', 'bootstrap'): if trigger != '': trigger = trigger+', ' trigger = trigger+'chroot-installation_'+base_distro+'_install_'+trigger_target+'_upgrade_to_'+distro_upgrades[base_distro] @@ -318,12 +318,12 @@ for base_distro in sorted(base_distros): second_base=distro_upgrades[base_distro], description=description)) # upgrade job with upgrading apt+dpkg first - if base_distro in distro_upgrades and base_distro != oldstable and target[:10] != 'education-' and action != 'maintainance': + if base_distro in distro_upgrades and base_distro != oldstable and target[:10] != 'education-' and action != 'maintenance': description = 'Debootstrap '+base_distro+', then upgrade apt and dpkg to '+distro_upgrades[base_distro]+' and then everything else.' if target == 'bootstrap': trigger = '' for trigger_target in get_targets_in_distro(base_distro, targets): - if trigger_target not in ('maintainance', 'bootstrap'): + if trigger_target not in ('maintenance', 'bootstrap'): if trigger != '': trigger = trigger+', ' trigger = trigger+'chroot-installation_'+base_distro+'_install_'+trigger_target+'_upgrade_to_'+distro_upgrades[base_distro]+'_aptdpkg_first' diff --git a/job-cfg/d-i.yaml b/job-cfg/d-i.yaml index e8c084db..039c9006 100644 --- a/job-cfg/d-i.yaml +++ b/job-cfg/d-i.yaml @@ -323,7 +323,7 @@ - job-template: defaults: d-i - name: '{name}_maintainance' + name: '{name}_maintenance' description: 'Cleanup and monitor so that there is a predictable environment.{do_not_edit}' properties: - sidebar: @@ -343,7 +343,7 @@ triggers: - timed: '30 5 * * *' builders: - - shell: '/srv/jenkins/bin/maintainance.sh {name}' + - shell: '/srv/jenkins/bin/maintenance.sh {name}' publishers: - logparser: parse-rules: '/srv/jenkins/logparse/debian.rules' @@ -804,7 +804,7 @@ name: d-i do_not_edit: '

Job configuration source is d-i.yaml.' jobs: - - '{name}_maintainance' + - '{name}_maintenance' - '{name}_check_jenkins_jobs' - '{name}_manual': include: '/trunk/manual/debian/.* diff --git a/job-cfg/g-i-installation.yaml b/job-cfg/g-i-installation.yaml index c6289de6..486c015a 100644 --- a/job-cfg/g-i-installation.yaml +++ b/job-cfg/g-i-installation.yaml @@ -59,7 +59,7 @@ categories: - g-i-installation - defaults: - name: g-i-installation-maintainance + name: g-i-installation-maintenance description: 'Cleanup and monitor so that there is a predictable environment.{do_not_edit}' logrotate: daysToKeep: 90 @@ -69,7 +69,7 @@ triggers: - timed: "0 6 * * *" builders: - - shell: '/srv/jenkins/bin/maintainance.sh {name}' + - shell: '/srv/jenkins/bin/maintenance.sh {name}' publishers: - logparser: parse-rules: '/srv/jenkins/logparse/debian.rules' @@ -94,8 +94,8 @@ job-prio: '175' - job-template: - defaults: g-i-installation-maintainance - name: '{name}_maintainance' + defaults: g-i-installation-maintenance + name: '{name}_maintenance' - job-template: defaults: g-i-installation @@ -325,7 +325,7 @@ name: g-i-installation do_not_edit: '

Job configuration source is g-i-installation.yaml.' jobs: - - '{name}_maintainance' + - '{name}_maintenance' - '{name}_debian-edu_wheezy_minimal': my_title: 'Debian Edu (wheezy) minimal' my_shell: '/srv/jenkins/bin/g-i-installation.sh 25 http://ftp.skolelinux.org/cd-wheezy-amd64-i386-netinst/debian-edu-amd64-i386-NETINST-1.iso' diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 9035b123..6fd2d40a 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -122,7 +122,7 @@ - job-template: defaults: reproducible - name: '{name}_maintainance' + name: '{name}_maintenance' - job-template: defaults: reproducible @@ -224,10 +224,10 @@ - project: name: reproducible jobs: - - '{name}_maintainance': - my_description: 'Do some maintainance: check for old directories laying around, do backups, etc.' + - '{name}_maintenance': + my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.' my_timed: '5 0,4,8,12,16,20 * * *' - my_shell: '/srv/jenkins/bin/reproducible_maintainance.sh' + my_shell: '/srv/jenkins/bin/reproducible_maintenance.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' - '{name}_setup_pbuilder_testing': my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' diff --git a/job-cfg/schroot.yaml b/job-cfg/schroot.yaml index 1f5e6814..15f3f9d8 100644 --- a/job-cfg/schroot.yaml +++ b/job-cfg/schroot.yaml @@ -7,7 +7,7 @@ icon: /userContent/images/debian-swirl-24x24.png - sidebar: url: https://jenkins.debian.net/view/schroot-maint/ - text: schroot maintainance jobs + text: schroot maintenance jobs icon: /userContent/images/debian-jenkins-24x24.png - sidebar: url: http://www.profitbricks.com @@ -37,7 +37,7 @@ icon: /userContent/images/debian-swirl-24x24.png - sidebar: url: https://jenkins.debian.net/view/schroot-maint/ - text: schroot maintainance jobs + text: schroot maintenance jobs icon: /userContent/images/debian-jenkins-24x24.png - sidebar: url: http://www.profitbricks.com diff --git a/job-cfg/self.yaml b/job-cfg/self.yaml index ed6f9be2..e90bc0fb 100644 --- a/job-cfg/self.yaml +++ b/job-cfg/self.yaml @@ -1,5 +1,5 @@ - defaults: - name: self-maintainance + name: self-maintenance description: 'Cleanup and monitor so that there is a predictable environment.{do_not_edit}' logrotate: daysToKeep: 90 @@ -30,8 +30,8 @@ job-prio: '125' - job-template: - defaults: self-maintainance - name: '{name}_maintainance' + defaults: self-maintenance + name: '{name}_maintenance' publishers: - logparser: parse-rules: '/srv/jenkins/logparse/debian.rules' @@ -39,21 +39,21 @@ fail-on-error: 'true' - job-template: - defaults: self-maintainance - name: '{name}_sub-maintainance' - defaults: self-maintainance - name: '{name}_sub-maintainance-squid' - defaults: self-maintainance - name: 'rebootstrap_maintainance' + defaults: self-maintenance + name: '{name}_sub-maintenance' + defaults: self-maintenance + name: '{name}_sub-maintenance-squid' + defaults: self-maintenance + name: 'rebootstrap_maintenance' - project: name: self do_not_edit: '

Job configuration source is self.yaml.' jobs: - - '{name}_maintainance': - my_shell: '/srv/jenkins/bin/maintainance.sh' - - '{name}_sub-maintainance-squid': - my_shell: '/srv/jenkins/bin/maintainance.sh squid' - - 'rebootstrap_maintainance': - my_shell: '/srv/jenkins/bin/maintainance.sh rebootstrap' + - '{name}_maintenance': + my_shell: '/srv/jenkins/bin/maintenance.sh' + - '{name}_sub-maintenance-squid': + my_shell: '/srv/jenkins/bin/maintenance.sh squid' + - 'rebootstrap_maintenance': + my_shell: '/srv/jenkins/bin/maintenance.sh rebootstrap' -- cgit v1.2.3-54-g00ecf