#!/usr/bin/python3 # # Copyright 2015 Philip Hands # written to generate something very similar to d-i.yaml so much of the # quoted text is Copyright Holger Levsen # Licensed under GPL-2 import sys import os from yaml import load, dump try: from yaml import CLoader as Loader, CDumper as Dumper except ImportError: from yaml import Loader, Dumper langs = { 'ca': 'Catalan', 'cs': 'Czech', 'de': 'German', 'en': 'English', 'fr': 'French', 'it': 'Italian', 'pt_BR': 'Brazilian Portuguese', 'da': 'Danish', 'el': 'Greek', 'es': 'Spanish', 'fi': 'Finnish', 'hu': 'Hungarian', 'ja': 'Japanese', 'ko': 'Korean', 'nl': 'Dutch', 'nn': 'Norwegian Nynorsk', 'pt': 'Portuguese', 'ro': 'Romanian', 'ru': 'Russian', 'sv': 'Swedish', 'tl': 'Tagalog', 'vi': 'Vietnamese', 'zh_CN': 'Chinese (zh_CN)', 'zh_TW': 'Chinese (zh_TW)', } non_pdf_langs = ['el', 'vi', 'ja', 'zh_CN', 'zh_TW'] non_po_langs = ['ca', 'cs', 'de', 'en', 'fr', 'it', 'pt_BR'] pkgs = """ anna apt-setup arcboot-installer base-installer bterm-unifont babelbox busybox cdebconf-entropy cdebconf-terminal cdebconf cdrom-checker cdrom-detect cdrom-retriever choose-mirror clock-setup console-setup debian-installer-launcher debian-installer-netboot-images debian-installer-utils debian-installer debootstrap desktop-chooser devicetype-detect dh-di efi-reader elilo-installer finish-install flash-kernel grub-installer hw-detect installation-locale installation-report iso-scan kbd-chooser kernel-wedge kickseed libdebian-installer lilo-installer live-installer localechooser lowmem lvmcfg main-menu mdcfg media-retriever mklibs mountmedia net-retriever netboot-assistant netcfg network-console nobootloader oldsys-preseed os-prober partconf partitioner partman-auto-crypto partman-auto-lvm partman-auto-raid partman-auto partman-base partman-basicfilesystems partman-basicmethods partman-btrfs partman-crypto partman-efi partman-ext3 partman-iscsi partman-jfs partman-lvm partman-md partman-multipath partman-nbd partman-newworld partman-partitioning partman-prep partman-target partman-ufs partman-xfs partman-zfs pkgsel prep-installer preseed quik-installer rescue rootskel-gtk rootskel s390-dasd s390-netdevice s390-sysconfig-writer sibyl-installer silo-installer tzsetup udpkg usb-discover user-setup win32-loader yaboot-installer zipl-installer """.split() def scm_svn(po, inc_regs=None): if inc_regs == None: inc_regs = os.path.join('/trunk/manual/', 'po' if po else '', '{lang}', '.*') return [{'svn': {'excludedCommitMessages': '', 'url': 'svn://anonscm.debian.org/svn/d-i/trunk', 'basedir': '.', 'workspaceupdater': 'update', 'includedRegions': inc_regs, 'excludedUsers': '', 'excludedRevprop': '', 'excludedRegions': '', 'viewvc-url': 'http://anonscm.debian.org/viewvc/d-i/trunk'}}] def svn_desc(po, fmt): s = 'Builds the {languagename} ' + fmt + ' version of the installation-guide for all architectures. ' s += 'Triggered by SVN commits to svn://anonscm.debian.org/svn/d-i/trunk/manual' s += '/po' if po else '' s += '/{lang}/. After successful build d-i_manual_{lang}_pdf is triggered.' return s def pdf_desc(): s = 'Builds the {languagename} pdf version of the installation-guide for all architectures. Triggered by successful build of d-i_manual_{lang}_html.' return s def instguide_desc(): return 'Builds the installation-guide package. Triggered by SVN commits to svn://anonscm.debian.org/svn/d-i/ matching these patterns:
{include}
' def lr(keep): return {'artifactDaysToKeep': -1, 'daysToKeep': keep, 'numToKeep': 30, 'artifactNumToKeep': -1} def publ_email(irc=None): r = ['jenkins+' + irc] if irc != None else [] r.append('qa-jenkins-scm@lists.alioth.debian.org') return {'email': {'recipients': ' '.join(r)}} def publ(fmt=None,trigger=False,irc=None): p = [] if trigger: p = [{'trigger': {'project': 'd-i_manual_{lang}_pdf', 'threshold': 'UNSTABLE'}}] p.extend([ {'logparser': {'parse-rules': '/srv/jenkins/logparse/debian-installer.rules', 'unstable-on-warning': 'true', 'fail-on-error': 'true'}}]) p.append(publ_email(irc=irc)) if fmt != None: p.append({'archive': {'artifacts': fmt + '/**/*.*', 'latest_only': True}}) return p def prop(type='manual', priority=None): p = [{'sidebar': {'url': 'https://jenkins.debian.net/userContent/about.html', 'text': 'About jenkins.debian.net', 'icon': '/userContent/images/debian-swirl-24x24.png'}}, {'sidebar': {'url': 'https://jenkins.debian.net/view/d-i_' + type + '/', 'text': 'debian-installer ' + type + ' jobs', 'icon': '/userContent/images/debian-jenkins-24x24.png'}}, {'sidebar': {'url': 'http://www.profitbricks.co.uk', 'text': 'Sponsored by Profitbricks', 'icon': '/userContent/images/profitbricks-24x24.png'}}] if priority != None: p.append( {'priority': {'job-prio': str(priority)}} ) return p def jtmpl(act, target, fmt=None, po=False): n = ['{name}', act, target] d = [ 'd-i', act ] if fmt: n.append(fmt) d.append(fmt) if po: d.append('po2xml') return {'job-template': {'name': '_'.join(n), 'defaults': '-'.join(d)}} def jobspec_svn(key, name, desc=None, defaults=None, priority=120, logkeep=None, trigger=None, publishers=None, lang=None, fmt=None, po=False, inc_regs=None ): j = {'scm': scm_svn(po=po,inc_regs=inc_regs), 'project-type': 'freestyle', 'builders': [{'shell': '/srv/jenkins/bin/d-i_manual.sh' + (' ' + lang if lang else '') + (' ' + fmt if fmt else '') + (' po2xml' if po else '')}], 'properties': prop(priority=priority), 'name': name} j['publishers'] = publishers if publishers != None else publ(fmt=fmt,trigger=trigger,irc='debian-boot') if desc != None: j['description'] = desc() else: if fmt != None: j['description'] = svn_desc(po=po,fmt=fmt) j['description'] += ' {do_not_edit}' if defaults != None: j['defaults'] = defaults if trigger != None: j['triggers'] = [{'pollscm': 'H/' + str(trigger) + ' * * * *'}] if logkeep != None: j['logrotate'] = lr(logkeep) return { key : j } def templs_jobs(): templates = [] jobs = [ '{name}_maintenance', '{name}_check_jenkins_jobs', {'{name}_manual': {'include': ( '/trunk/manual/debian/.*\n' '/trunk/manual/po/.*\n' '/trunk/manual/doc/.*\n' '/trunk/manual/scripts/.*' )}}] def tj_append(t, j): templates.append(t) jobs.append(j) # this is a bit contrived: in order to only need to go through each loop once # we're producing the template and it's job at the same time in a tuple, then # using a local function to append those results onto their apropriate lists # This is done mostly as a stepping ston to discarding the teplates (assuming # that is possible, which we'll find out...) [tj_append(t,j) for (t,j) in [ ( jtmpl(act='manual',target=l,fmt=f,po=(l not in non_po_langs)), {'_'.join(['{name}','manual',l,f]): {'lang': l, 'languagename': langs[l]}} ) for f in ['html', 'pdf'] for l in sorted(langs.keys()) if not (f=='pdf' and l in non_pdf_langs)]] [tj_append(t,j) for (t,j) in [ ( jtmpl(act=act,target=pkg), {'_'.join(['{name}',act,pkg]): {'gitrepo': 'git://git.debian.org/git/d-i/' + pkg}} ) for act in ['build'] for pkg in pkgs]] return (templates, jobs) # -- here we build the data to be dumped as yaml data = [] data.append( {'defaults': { 'name': 'd-i', 'logrotate': lr(90), 'project-type': 'freestyle', 'properties': prop(type='misc')}}) data.extend( [jobspec_svn(key='defaults', name='d-i-manual-' + fmt + ('-po2xml' if po else ''), fmt=fmt, lang='{lang}', trigger=None if fmt == 'pdf' else 15 if not po else 30, desc=pdf_desc if fmt == 'pdf' else None, po=po, logkeep=90) for fmt in ['html', 'pdf'] for po in [False, True]]) data.extend( [{'defaults': { 'name': n, 'description': 'Builds debian packages in sid from git '+ bdsc +', triggered by pushes to
{gitrepo}
{do_not_edit}', 'triggers': [{'pollscm': trg}], 'scm': [{'git': {'url': '{gitrepo}', 'branches': [br]}}], 'builders': [{'shell': '/srv/jenkins/bin/d-i_build.sh'}], 'project-type': 'freestyle', 'properties': prop(type='packages', priority=99), 'logrotate': lr(90), 'publishers': publ(irc=irc)}} for (n,bdsc,br,trg,irc) in [('d-i-build', 'master branch', 'origin/master', 'H/6 * * * *', None), # irc should be 'debian-boot' but disabled due to gcc5 transition ('d-i-pu-build', 'pu/ branches', 'origin/pu/**' , 'H/10 * * * *', None)]]) # same data.append( jobspec_svn(key='job-template', defaults='d-i', name='{name}_manual', desc=instguide_desc, trigger=15, priority=125, publishers=[publ_email()], inc_regs='{include}')) data.append( {'job-template': { 'defaults': 'd-i', 'name': '{name}_check_jenkins_jobs', 'description': 'Checks daily for missing jenkins jobs. {do_not_edit}', 'triggers': [{'timed': '23 0 * * *'}], 'builders': [{'shell': '/srv/jenkins/bin/d-i_check_jobs.sh'}], 'publishers': [{'logparser': {'parse-rules': '/srv/jenkins/logparse/debian.rules', 'unstable-on-warning': 'true', 'fail-on-error': 'true'}}, publ_email()]}}) data.append( {'job-template': { 'defaults': 'd-i', 'name': '{name}_maintenance', 'description': 'Cleanup and monitor so that there is a predictable environment.{do_not_edit}', 'triggers': [{'timed': '30 5 * * *'}], 'builders': [{'shell': '/srv/jenkins/bin/maintenance.sh {name}'}], 'properties': prop(priority=150), 'publishers': [{'logparser': {'parse-rules': '/srv/jenkins/logparse/debian.rules', 'unstable-on-warning': 'true', 'fail-on-error': 'true'}}, publ_email('debian-boot')]}}) (templs, jobs) = templs_jobs() # let's see if we can be rather more efficient with the yaml -- test just the pu stuff templs.append(jtmpl(act='pu-build',target='{pkg}')) jobs.append({'_'.join(['{name}','pu-build','{pkg}']): {'gitrepo': 'git://git.debian.org/git/d-i/{pkg}'}}) data.extend(templs) data.append( {'project': { 'name': 'd-i', 'do_not_edit': '

Job configuration source is d-i.yaml.py.', 'pkg': pkgs, 'jobs': jobs}}) sys.stdout.write( dump(data, Dumper=Dumper) )