diff options
| -rwxr-xr-x | update_jdn.sh | 467 |
1 files changed, 240 insertions, 227 deletions
diff --git a/update_jdn.sh b/update_jdn.sh index 02e37c2a..110b9cd1 100755 --- a/update_jdn.sh +++ b/update_jdn.sh @@ -7,6 +7,7 @@ BASEDIR=/root/jenkins.debian.net PVNAME=/dev/vdb # LVM physical volume for jobs VGNAME=jenkins01 # LVM volume group STAMP=/var/log/jenkins/update-jenkins.stamp +TMPFILE=$(mktemp) explain() { echo @@ -33,19 +34,21 @@ done mkdir -p /srv/workspace -if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then - echo "tmpfs /srv/workspace tmpfs defaults,size=100g 0 0" >> /etc/fstab -fi +if [ "$HOSTNAME" = "jenkins" ] ; then + if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then + echo "tmpfs /srv/workspace tmpfs defaults,size=100g 0 0" >> /etc/fstab + fi -if ! mountpoint -q /srv/workspace; then - if test -z "$(ls -A /srv/workspace)"; then - mount /srv/workspace - else - explain "mountpoint /srv/workspace is non-empty" + if ! mountpoint -q /srv/workspace; then + if test -z "$(ls -A /srv/workspace)"; then + mount /srv/workspace + else + explain "mountpoint /srv/workspace is non-empty" + fi fi fi -# make sure needed directories exists +# make sure needed directories exists - some directories will not be needed on all hosts... for directory in /schroots /srv/reproducible-results /srv/d-i /srv/live-build ; do if [ ! -d $directory ] ; then sudo mkdir $directory @@ -68,136 +71,136 @@ if ! test -h /chroots; then fi fi -if ! test -h /var/cache/pbuilder/build; then - rmdir /var/cache/pbuilder/build || rm -f /var/cache/pbuilder/build - if test -e /var/cache/pbuilder/build; then - explain "could not clear /var/cache/pbuilder/build" - else - ln -s /srv/workspace/pbuilder /var/cache/pbuilder/build +# only on Debian systems +if [ -f /etc/debian_version ] ; then + if ! test -h /var/cache/pbuilder/build; then + rmdir /var/cache/pbuilder/build || rm -f /var/cache/pbuilder/build + if test -e /var/cache/pbuilder/build; then + explain "could not clear /var/cache/pbuilder/build" + else + ln -s /srv/workspace/pbuilder /var/cache/pbuilder/build + fi fi -fi - -# -# install packages we need -# -if [ ./$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then - sudo apt-get install \ - apache2 \ - apt-file \ - apt-listchanges \ - bash-completion \ - bc \ - binfmt-support \ - bison \ - build-essential \ - calamaris \ - cmake \ - cron-apt \ - csvtool \ - cucumber \ - curl \ - debootstrap \ - devscripts \ - dnsmasq-base \ - dose-extra \ - dstat \ - etckeeper \ - figlet \ - flex \ - gawk \ - ghc \ - gocr \ - graphviz \ - haveged \ - iasl \ - imagemagick \ - ip2host \ - less \ - libapache2-mod-macro \ - libav-tools \ - libcap2-bin \ - libfile-touch-perl \ - libguestfs-tools \ - libjson-rpc-perl \ - libsikuli-script-java \ - libsoap-lite-perl \ - libvirt0 \ - libvirt-bin \ - libvirt-dev \ - libvpx1 \ - libxslt1-dev \ - linux-image-amd64 \ - mock \ - molly-guard \ - moreutils \ - mr \ - mtr-tiny \ - munin \ - munin-plugins-extra \ - ntp \ - openbios-ppc \ - openbios-sparc \ - openjdk-7-jre \ - ovmf \ - pigz \ - postgresql-client-9.4 \ - poxml \ - procmail \ - python3-debian \ - python3-psycopg2 \ - python3-yaml \ - python-arpy \ - python-hachoir-metadata \ - python-imaging \ - python-lzma \ - python-pip \ - python-rpy2 \ - python-setuptools \ - python-twisted \ - python-yaml \ - qemu \ - qemu-kvm \ - qemu-system-x86 \ - qemu-user-static \ - radvd \ - ruby-json \ - ruby-libvirt \ - ruby-packetfu \ - ruby-rjb \ - ruby-rspec \ - schroot \ - screen \ - seabios \ - shorewall \ - shorewall6 \ - sqlite3 \ - squid3 \ - subversion \ - subversion-tools \ - sudo \ - syslinux \ - tcpdump \ - unclutter \ - unzip \ - vim \ - virt-viewer \ - vncsnapshot \ - vnstat \ - x11-apps \ - x11vnc \ - xtightvncviewer \ - xvfb \ - zutils \ - sysvinit-core - - sudo apt-get install -t jessie-backports \ - pbuilder - # botch - - explain "Packages installed." -else - explain "No new packages to be installed." + # + # install packages we need + # + if [ ./$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then + sudo apt-get install \ + apache2 \ + apt-file \ + apt-listchanges \ + bash-completion \ + bc \ + binfmt-support \ + bison \ + build-essential \ + calamaris \ + cmake \ + cron-apt \ + csvtool \ + cucumber \ + curl \ + debootstrap \ + devscripts \ + dnsmasq-base \ + dose-extra \ + dstat \ + etckeeper \ + figlet \ + flex \ + gawk \ + ghc \ + gocr \ + graphviz \ + haveged \ + iasl \ + imagemagick \ + ip2host \ + less \ + libapache2-mod-macro \ + libav-tools \ + libcap2-bin \ + libfile-touch-perl \ + libguestfs-tools \ + libjson-rpc-perl \ + libsikuli-script-java \ + libsoap-lite-perl \ + libvirt0 \ + libvirt-bin \ + libvirt-dev \ + libvpx1 \ + libxslt1-dev \ + linux-image-amd64 \ + mock \ + molly-guard \ + moreutils \ + mr \ + mtr-tiny \ + munin \ + munin-plugins-extra \ + ntp \ + openbios-ppc \ + openbios-sparc \ + openjdk-7-jre \ + ovmf \ + pigz \ + postgresql-client-9.4 \ + poxml \ + procmail \ + python3-debian \ + python3-psycopg2 \ + python3-yaml \ + python-arpy \ + python-hachoir-metadata \ + python-imaging \ + python-lzma \ + python-pip \ + python-rpy2 \ + python-setuptools \ + python-twisted \ + python-yaml \ + qemu \ + qemu-kvm \ + qemu-system-x86 \ + qemu-user-static \ + radvd \ + ruby-json \ + ruby-libvirt \ + ruby-packetfu \ + ruby-rjb \ + ruby-rspec \ + schroot \ + screen \ + seabios \ + shorewall \ + shorewall6 \ + sqlite3 \ + squid3 \ + subversion \ + subversion-tools \ + sudo \ + syslinux \ + tcpdump \ + unclutter \ + unzip \ + vim \ + virt-viewer \ + vncsnapshot \ + vnstat \ + x11-apps \ + x11vnc \ + xtightvncviewer \ + xvfb \ + zutils \ + sysvinit-core + sudo apt-get install -t jessie-backports \ + pbuilder + # botch + explain "Packages installed." + else + explain "No new packages to be installed." + fi fi # @@ -209,23 +212,27 @@ sudo cp --preserve=mode,timestamps -r hosts/jenkins/etc/* /etc # # more configuration than a simple cp can do # -if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then - sudo a2enmod proxy - sudo a2enmod proxy_http - sudo a2enmod rewrite - sudo a2enmod ssl - sudo a2enmod headers - sudo a2enmod macro - sudo a2enmod filter -fi sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm -sudo a2ensite -q jenkins.debian.net -sudo a2enconf -q munin -sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.conf -# for reproducible.d.n url rewriting: -[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent -sudo service apache2 reload + +if [ "$HOSTNAME" = "jenkins" ] ; then + if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then + sudo a2enmod proxy + sudo a2enmod proxy_http + sudo a2enmod rewrite + sudo a2enmod ssl + sudo a2enmod headers + sudo a2enmod macro + sudo a2enmod filter + fi + sudo a2ensite -q jenkins.debian.net + sudo a2enconf -q munin + sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.conf + # for reproducible.d.n url rewriting: + [ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent + sudo service apache2 reload +fi + cd /etc/munin/plugins ; sudo rm -f postfix_* open_inodes df_inode interrupts irqstats threads proc_pri vmstat if_err_eth0 fw_forwarded_local fw_packets forks open_files users 2>/dev/null [ -L apache_accesses ] || for i in apache_accesses apache_volume ; do ln -s /usr/share/munin/plugins/$i $i ; done explain "Packages configured." @@ -246,48 +253,49 @@ chown -R jenkins:jenkins /var/lib/jenkins/.ssh chmod 700 /var/lib/jenkins/.ssh chmod 600 /var/lib/jenkins/.ssh/authorized_keys explain "Jenkins updated." -cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/ -TMPFILE=$(mktemp) -git log | grep ^Author| cut -d " " -f2-|sort -u > $TMPFILE -echo "----" >> $TMPFILE -cat THANKS.head > /var/lib/jenkins/userContent/THANKS -# samuel and lunar committed with several commiters, only display one -grep -v "samuel.thibault@ens-lyon.org" $TMPFILE | grep -v Lunar >> /var/lib/jenkins/userContent/THANKS -rm $TMPFILE -cp -pr userContent /var/lib/jenkins/ -cd /var/lib/jenkins/userContent/ -ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css" -[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README -[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO -[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL -[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING -diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS -mv THANKS .THANKS -rm TODO README INSTALL CONTRIBUTING -chown -R jenkins.jenkins /var/lib/jenkins/userContent -explain "Updated user content for Jenkins." -# -# run jenkins-job-builder to update jobs if needed -# (using sudo because /etc/jenkins_jobs is root:root 700) -# -cd /srv/jenkins/job-cfg -for metaconfig in *.yaml.py ; do -# there are both python2 and python3 scripts here - ./$metaconfig > $TMPFILE - if ! $(diff ${metaconfig%.py} $TMPFILE > /dev/null) ; then - cp $TMPFILE ${metaconfig%.py} - fi -done -for config in *.yaml ; do - if [ $config -nt $STAMP ] || [ ! -f $STAMP ] ; then - sudo jenkins-jobs update $config - else - echo "$config has not changed, nothing to do." - fi -done -explain "Jenkins jobs updated." -rm -f $TMPFILE +if [ "$HOSTNAME" = "jenkins" ] ; then + cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/ + git log | grep ^Author| cut -d " " -f2-|sort -u > $TMPFILE + echo "----" >> $TMPFILE + cat THANKS.head > /var/lib/jenkins/userContent/THANKS + # samuel and lunar committed with several commiters, only display one + grep -v "samuel.thibault@ens-lyon.org" $TMPFILE | grep -v Lunar >> /var/lib/jenkins/userContent/THANKS + rm $TMPFILE + cp -pr userContent /var/lib/jenkins/ + cd /var/lib/jenkins/userContent/ + ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css" + [ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README + [ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO + [ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL + [ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING + diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS + mv THANKS .THANKS + rm TODO README INSTALL CONTRIBUTING + chown -R jenkins.jenkins /var/lib/jenkins/userContent + explain "Updated user content for Jenkins." + + # + # run jenkins-job-builder to update jobs if needed + # (using sudo because /etc/jenkins_jobs is root:root 700) + # + cd /srv/jenkins/job-cfg + for metaconfig in *.yaml.py ; do + # there are both python2 and python3 scripts here + ./$metaconfig > $TMPFILE + if ! $(diff ${metaconfig%.py} $TMPFILE > /dev/null) ; then + cp $TMPFILE ${metaconfig%.py} + fi + done + for config in *.yaml ; do + if [ $config -nt $STAMP ] || [ ! -f $STAMP ] ; then + sudo jenkins-jobs update $config + else + echo "$config has not changed, nothing to do." + fi + done + explain "Jenkins jobs updated." +fi # # configure git for jenkins @@ -297,42 +305,46 @@ if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins sudo su - jenkins -c "git config --global user.name Jenkins" fi -# -# configure pbuilder for jenkins user -# -sudo chown jenkins /var/cache/pbuilder/result - -# -# creating LVM volume group for jobs -# -if [ "$PVNAME" = "" ]; then - figlet -f banner Error - explain "Set \$PVNAME to physical volume pathname." - exit 1 -else - if ! sudo pvs $PVNAME >/dev/null 2>&1; then - sudo pvcreate $PVNAME - fi - if ! sudo vgs $VGNAME >/dev/null 2>&1; then - sudo vgcreate $VGNAME $PVNAME - fi +if [ -f /etc/debian_version ] ; then + # + # configure pbuilder for jenkins user + # + sudo chown jenkins /var/cache/pbuilder/result fi -# -# generate the kgb-client configurations -# -cd $BASEDIR -KGB_SECRETS="/srv/jenkins/kgb/secrets.yml" -if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then - # the last condition is to assure the files are owned by the right user/team - if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then - sudo -u jenkins-adm "./deploy_kgb.py" - else - explain "kgb-client configuration unchanged, nothing to do." - fi -else - echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated" - echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm." +if [ "$HOSTNAME" = "jenkins" ] ; then + # + # creating LVM volume group for jobs + # + if [ "$PVNAME" = "" ]; then + figlet -f banner Error + explain "Set \$PVNAME to physical volume pathname." + exit 1 + else + if ! sudo pvs $PVNAME >/dev/null 2>&1; then + sudo pvcreate $PVNAME + fi + if ! sudo vgs $VGNAME >/dev/null 2>&1; then + sudo vgcreate $VGNAME $PVNAME + fi + fi + + # + # generate the kgb-client configurations + # + cd $BASEDIR + KGB_SECRETS="/srv/jenkins/kgb/secrets.yml" + if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then + # the last condition is to assure the files are owned by the right user/team + if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then + sudo -u jenkins-adm "./deploy_kgb.py" + else + explain "kgb-client configuration unchanged, nothing to do." + fi + else + echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated" + echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm." + fi fi # @@ -346,4 +358,5 @@ rgrep FIXME $BASEDIR/* | grep -v "rgrep FIXME" | grep -v echo # finally # touch $STAMP # so on the next run, only configs newer than this file will be updated +rm -f $TMPFILE explain "$(hostname -f) successfully updated." |