diff options
| -rwxr-xr-x | job-cfg/chroot-installation.yaml.py | 571 |
1 files changed, 285 insertions, 286 deletions
diff --git a/job-cfg/chroot-installation.yaml.py b/job-cfg/chroot-installation.yaml.py index 96dbb95b..ea0aec2a 100755 --- a/job-cfg/chroot-installation.yaml.py +++ b/job-cfg/chroot-installation.yaml.py @@ -1,98 +1,127 @@ #!/usr/bin/python -base_distros = """ - squeeze - wheezy - jessie - stretch - sid - """.split() +import sys +import os +from string import join +from yaml import load, dump +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper -distro_upgrades = { 'squeeze': 'wheezy', - 'wheezy': 'jessie', - 'jessie': 'stretch', - 'stretch': 'sid' } + +base_distros = [ + 'squeeze', + 'wheezy', + 'jessie', + 'stretch', + 'sid', + ] + +distro_upgrades = { + 'squeeze': 'wheezy', + 'wheezy': 'jessie', + 'jessie': 'stretch', + 'stretch': 'sid', + } oldoldstable = 'squeeze' # ftp.de.debian.org runs mirror updates at 03:25, 09:25, 15:25 and 21:25 UTC and usually they run 10m... -trigger_times = { 'squeeze': '30 16 25 * *', - 'wheezy': '30 16 1,15 * *', - 'jessie': '30 10 * * 1,4', - 'stretch': '30 10 */2 * *', - 'sid': '30 4 * * *' } +trigger_times = { + 'squeeze': '30 16 25 * *', + 'wheezy': '30 16 1,15 * *', + 'jessie': '30 10 * * 1,4', + 'stretch': '30 10 */2 * *', + 'sid': '30 4 * * *', + } -targets = """ - maintenance - bootstrap - gnome - kde - kde-full - cinnamon - lxde - xfce - full_desktop - qt4 - qt5 - haskell - developer - education-tasks - education-menus - education-astronomy - education-chemistry - education-common - education-desktop-gnome - education-desktop-kde - education-desktop-lxde - education-desktop-mate - education-desktop-other - education-desktop-sugar - education-desktop-xfce - education-development - education-electronics - education-geography - education-graphics - education-language - education-laptop - education-logic-games - education-main-server - education-mathematics - education-misc - education-music - education-networked - education-physics - education-services - education-standalone - education-thin-client - education-thin-client-server - education-workstation - """.split() +all_targets = [ + 'gnome', + 'kde', + 'kde-full', + 'cinnamon', + 'lxde', + 'xfce', + 'full_desktop', + 'qt4', + 'qt5', + 'haskell', + 'developer', + 'education-tasks', + 'education-menus', + 'education-astronomy', + 'education-chemistry', + 'education-common', + 'education-desktop-gnome', + 'education-desktop-kde', + 'education-desktop-lxde', + 'education-desktop-mate', + 'education-desktop-other', + 'education-desktop-sugar', + 'education-desktop-xfce', + 'education-development', + 'education-electronics', + 'education-geography', + 'education-graphics', + 'education-language', + 'education-laptop', + 'education-logic-games', + 'education-main-server', + 'education-mathematics', + 'education-misc', + 'education-music', + 'education-networked', + 'education-physics', + 'education-services', + 'education-standalone', + 'education-thin-client', + 'education-thin-client-server', + 'education-workstation', + ] # # not all packages are available in all distros # -def get_targets_in_distro(distro, targets): - targets_in_distro = [] - for target in targets: +def is_target_in_distro(distro, target): # haskell, cinnamon, qt5 and edu tests not in squeeze if distro == 'squeeze' and ( target == 'haskell' or target[:10] == 'education-' or target == 'cinnamon' or target == 'qt5' ): - continue + return False # qt5, education-desktop-mate and cinnamon weren't in wheezy if distro == 'wheezy' and ( target == 'education-desktop-mate' or target == 'cinnamon' or target == 'qt5' ): - continue + return False # sugar has been removed from jessie and thus education-desktop-sugar has been removed from jessie and sid - it's also not yet available in stretch again... if (distro == 'sid' or distro == 'jessie' or distro == 'stretch') and ( target == 'education-desktop-sugar' ): - continue - targets_in_distro.append(target) - return targets_in_distro + return False + return True + +# +# return the list of targets, filtered to be those present in 'distro' +# +def get_targets_in_distro(distro): + return [t for t in all_targets if is_target_in_distro(distro, t)] + +# +# given a target, returns a list of ([dist], key) tuples, so we can handle the +# edu packages having views that are distro dependant +# +# this groups all the distros that have matching views +# +def get_dists_per_key(target,get_distro_key): + dists_per_key = {} + for distro in base_distros: + if is_target_in_distro(distro, target): + key = get_distro_key(distro) + if key not in dists_per_key.keys(): + dists_per_key[key] = [] + dists_per_key[key].append(distro) + return dists_per_key # # who gets mail for which target # def get_recipients(target): - if target == 'maintenance': - return 'qa-jenkins-scm@lists.alioth.debian.org' - elif target == 'haskell': + if target == 'haskell': return 'jenkins+debian-haskell qa-jenkins-scm@lists.alioth.debian.org pkg-haskell-maintainers@lists.alioth.debian.org' elif target == 'gnome': return 'jenkins+debian-qa pkg-gnome-maintainers@lists.alioth.debian.org qa-jenkins-scm@lists.alioth.debian.org' @@ -109,9 +138,7 @@ def get_recipients(target): # views for different targets # def get_view(target, distro): - if target == 'maintenance': - return 'jenkins.d.n' - elif target == 'haskell': + if target == 'haskell': return 'haskell' elif target[:10] == 'education-': if distro in ('squeeze', 'wheezy'): @@ -124,18 +151,19 @@ def get_view(target, distro): # # special descriptions used for some targets # -spoken_names = {} -spoken_names = { 'gnome': 'GNOME', - 'kde': 'KDE plasma desktop', - 'kde-full': 'complete KDE desktop', - 'cinnamon': 'Cinnamon', - 'lxde': 'LXDE', - 'xfce': 'Xfce', - 'qt4': 'Qt4 cross-platform C++ application framework', - 'qt5': 'Qt5 cross-platform C++ application framework', - 'full_desktop': 'four desktop environments and the most commonly used applications and packages', - 'haskell': 'all Haskell related packages', - 'developer': 'four desktop environments and the most commonly used applications and packages - and the build depends for all of these' } +spoken_names = { + 'gnome': 'GNOME', + 'kde': 'KDE plasma desktop', + 'kde-full': 'complete KDE desktop', + 'cinnamon': 'Cinnamon', + 'lxde': 'LXDE', + 'xfce': 'Xfce', + 'qt4': 'Qt4 cross-platform C++ application framework', + 'qt5': 'Qt5 cross-platform C++ application framework', + 'full_desktop': 'four desktop environments and the most commonly used applications and packages', + 'haskell': 'all Haskell related packages', + 'developer': 'four desktop environments and the most commonly used applications and packages - and the build depends for all of these', + } def get_spoken_name(target): if target[:10] == 'education-': return 'the Debian Edu metapackage '+target @@ -145,209 +173,180 @@ def get_spoken_name(target): return target # +# This structure contains the differences between the default, upgrade and upgrade_apt+dpkg_first jobs +# +jobspecs = [ + { 'j_ext': '', + 'd_ext': '', + 's_ext': '', + 'dist_func': (lambda d: d), + 'distfilter': (lambda d: tuple(set(d) - set([oldoldstable]))), + 'skiptaryet': (lambda t: False) + }, + { 'j_ext': '_upgrade_to_{dist2}', + 'd_ext': ', then upgrade to {dist2}', + 's_ext': ' {dist2}', + 'dist_func': (lambda d: [{dist: {'dist2': distro_upgrades[dist]}} for dist in d]), + 'distfilter': (lambda d: tuple(set(d) & set(distro_upgrades))), + 'skiptaryet': (lambda t: False) + }, + { 'j_ext': '_upgrade_to_{dist2}_aptdpkg_first', + 'd_ext': ', then upgrade apt and dpkg to {dist2} and then everything else', + 's_ext': ' {dist2}', + 'dist_func': (lambda d: [{dist: {'dist2': distro_upgrades[dist]}} for dist in d]), + 'distfilter': (lambda d: tuple((set(d) & set(distro_upgrades)) - set([oldoldstable]))), + 'skiptaryet': (lambda t: t[:10] == 'education-') + }, +] + +# # nothing to edit below # -print(""" -- defaults: - name: chroot-installation - description: '{my_description}{do_not_edit}' - logrotate: - daysToKeep: 90 - numToKeep: 30 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - triggers: - - timed: '{my_time}' - builders: - - shell: '{my_shell}' - publishers: - - trigger: - project: '{my_trigger}' - - logparser: - parse-rules: '/srv/jenkins/logparse/debian.rules' - unstable-on-warning: 'false' - fail-on-error: 'false' - - email-ext: - recipients: '{my_recipients}' - first-failure: true - fixed: true - subject: '$BUILD_STATUS: $JOB_NAME/$BUILD_NUMBER' - attach-build-log: false - body: 'See $BUILD_URL/console or just $BUILD_URL for more information.' - properties: - - sidebar: - url: https://jenkins.debian.net/userContent/about.html - text: About jenkins.debian.net - icon: /userContent/images/debian-swirl-24x24.png - - sidebar: - url: https://jenkins.debian.net/view/{my_view}/ - text: All {my_view} jobs - icon: /userContent/images/debian-jenkins-24x24.png - - sidebar: - url: http://www.profitbricks.co.uk - text: Sponsored by Profitbricks - icon: /userContent/images/profitbricks-24x24.png - - priority-sorter: - priority: '{my_prio}' - - throttle: - max-total: 6 - max-per-node: 6 - enabled: true - option: category - categories: - - chroot-installation - wrappers: - - timeout: - timeout: 360 +data = [] +jobs = [] -""") -for base_distro in sorted(base_distros): - for target in sorted(get_targets_in_distro(base_distro, targets)): - if target in ('bootstrap', 'maintenance'): - action = target - else: - action = 'install_'+target - # default job - if target == 'maintenance' or base_distro != oldoldstable: - print("""- job-template: - defaults: chroot-installation - name: '{name}_%(base_distro)s_%(action)s'""" % - dict(base_distro=base_distro, - action=action)) - # upgrade job - if base_distro in distro_upgrades and action != 'maintenance': - print("""- job-template: - defaults: chroot-installation - name: '{name}_%(base_distro)s_%(action)s_upgrade_to_%(second_base)s'""" % - dict(base_distro=base_distro, - action=action, - second_base=distro_upgrades[base_distro])) - # upgrade job with upgrading apt+dpkg first - if base_distro in distro_upgrades and base_distro != oldoldstable and target[:10] != 'education-' and action != 'maintenance': - print("""- job-template: - defaults: chroot-installation - name: '{name}_%(base_distro)s_%(action)s_upgrade_to_%(second_base)s_aptdpkg_first'""" % - dict(base_distro=base_distro, - action=action, - second_base=distro_upgrades[base_distro])) +data.append( + { 'defaults': { 'builders': [{ 'shell': '{my_shell}'}], + 'description': '{my_description}{do_not_edit}', + 'logrotate': { 'artifactDaysToKeep': -1, + 'artifactNumToKeep': -1, + 'daysToKeep': 90, + 'numToKeep': 30}, + 'name': 'chroot-installation', + 'properties': [ { 'sidebar': { 'icon': '/userContent/images/debian-swirl-24x24.png', + 'text': 'About jenkins.debian.net', + 'url': 'https://jenkins.debian.net/userContent/about.html'}}, + { 'sidebar': { 'icon': '/userContent/images/debian-jenkins-24x24.png', + 'text': 'All {my_view} jobs', + 'url': 'https://jenkins.debian.net/view/{my_view}/'}}, + { 'sidebar': { 'icon': '/userContent/images/profitbricks-24x24.png', + 'text': 'Sponsored by Profitbricks', + 'url': 'http://www.profitbricks.co.uk'}}, + { 'priority-sorter': { 'priority': '{my_prio}'}}, + { 'throttle': { 'categories': [ 'chroot-installation'], + 'enabled': True, + 'max-per-node': 6, + 'max-total': 6, + 'option': 'category'}}], + 'publishers': [ { 'trigger': { 'project': '{my_trigger}'}}, + { 'logparser': { 'fail-on-error': 'false', + 'parse-rules': '/srv/jenkins/logparse/debian.rules', + 'unstable-on-warning': 'false'}}, + { 'email-ext': { 'attach-build-log': False, + 'body': 'See $BUILD_URL/console or just $BUILD_URL for more information.', + 'first-failure': True, + 'fixed': True, + 'recipients': '{my_recipients}', + 'subject': '$BUILD_STATUS: $JOB_NAME/$BUILD_NUMBER'}}], + 'triggers': [{ 'timed': '{my_time}'}], + 'wrappers': [{ 'timeout': { 'timeout': 360}}]}}) +data.append( + { 'job-template': { 'defaults': 'chroot-installation', + 'name': '{name}_{dist}_{action}'}}) +data.append( + { 'job-template': { 'defaults': 'chroot-installation', + 'name': '{name}_{dist}_install_{target}'}}) +data.append( + { 'job-template': { 'defaults': 'chroot-installation', + 'name': '{name}_{dist}_{action}_upgrade_to_{dist2}'}}) +data.append( + { 'job-template': { 'defaults': 'chroot-installation', + 'name': '{name}_{dist}_install_{target}_upgrade_to_{dist2}'}}) +data.append( + { 'job-template': { 'defaults': 'chroot-installation', + 'name': '{name}_{dist}_{action}_upgrade_to_{dist2}_aptdpkg_first'}}) +data.append( + { 'job-template': { 'defaults': 'chroot-installation', + 'name': '{name}_{dist}_install_{target}_upgrade_to_{dist2}_aptdpkg_first'}}) -print(""" -- project: - name: chroot-installation - do_not_edit: '<br><br>Job configuration source is <a href="http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/job-cfg/chroot-installation.yaml.py">chroot-installation.yaml.py</a>.' - jobs:""") +# maintenance jobs +maint_distros = [] for base_distro in sorted(base_distros): - for target in sorted(get_targets_in_distro(base_distro, targets)): - if target == 'maintenance': - description = 'Maintainance job for chroot-installation_'+base_distro+'_* jobs, do some cleanups and monitoring so that there is a predictable environment.' - shell = '/srv/jenkins/bin/maintenance.sh chroot-installation_'+base_distro - prio = 135 - time = trigger_times[base_distro] - if base_distro in distro_upgrades.values(): - trigger = 'chroot-installation_'+base_distro+'_bootstrap' - for item in distro_upgrades.items(): - if item[1]==base_distro and base_distro in distro_upgrades: - trigger = trigger+', chroot-installation_'+base_distro+'_bootstrap_upgrade_to_'+distro_upgrades[base_distro]+', chroot-installation_'+base_distro+'_bootstrap_upgrade_to_'+distro_upgrades[base_distro]+'_aptdpkg_first' - else: - trigger = 'chroot-installation_'+base_distro+'_bootstrap_upgrade_to_'+distro_upgrades[base_distro] - elif target == 'bootstrap': - description = 'Debootstrap '+base_distro+'.' - shell = '/srv/jenkins/bin/chroot-installation.sh '+base_distro - prio = 131 - time = '' - trigger = '' - for trigger_target in get_targets_in_distro(base_distro, targets): - if trigger_target not in ('maintenance', 'bootstrap'): - if trigger != '': - trigger = trigger+', ' - trigger = trigger+'chroot-installation_'+base_distro+'_install_'+trigger_target - else: - description = 'Debootstrap '+base_distro+', then install '+get_spoken_name(target)+'.' - shell = '/srv/jenkins/bin/chroot-installation.sh '+base_distro+' '+target - prio = 130 - time = '' - trigger = '' - if target in ('bootstrap', 'maintenance'): - action = target - else: - action = 'install_'+target - # default job - if target == 'maintenance' or base_distro != oldoldstable: - print(""" - '{name}_%(base_distro)s_%(action)s': - my_shell: '%(shell)s' - my_prio: '%(prio)s' - my_time: '%(time)s' - my_trigger: '%(trigger)s' - my_recipients: '%(recipients)s' - my_view: '%(view)s' - my_description: '%(description)s'""" % - dict(base_distro=base_distro, - action=action, - shell=shell, - prio=prio, - time=time, - trigger=trigger, - recipients=get_recipients(target), - view=get_view(target, base_distro), - description=description)) - # upgrade job - if base_distro in distro_upgrades and action != 'maintenance': - if target == 'bootstrap': - shell = '/srv/jenkins/bin/chroot-installation.sh '+base_distro+' none '+distro_upgrades[base_distro] - description = 'Debootstrap '+base_distro+', then upgrade to '+distro_upgrades[base_distro]+'.' - trigger = '' - for trigger_target in get_targets_in_distro(base_distro, targets): - if trigger_target not in ('maintenance', 'bootstrap'): - if trigger != '': - trigger = trigger+', ' - trigger = trigger+'chroot-installation_'+base_distro+'_install_'+trigger_target+'_upgrade_to_'+distro_upgrades[base_distro] - else: - shell = '/srv/jenkins/bin/chroot-installation.sh '+base_distro+' '+target+' '+distro_upgrades[base_distro] - description = 'Debootstrap '+base_distro+', then install '+get_spoken_name(target)+', then upgrade to '+distro_upgrades[base_distro]+'.' - trigger = '' - print(""" - '{name}_%(base_distro)s_%(action)s_upgrade_to_%(second_base)s': - my_shell: '%(shell)s' - my_prio: '%(prio)s' - my_time: '' - my_trigger: '%(trigger)s' - my_recipients: '%(recipients)s' - my_view: '%(view)s' - my_description: '%(description)s'""" % - dict(base_distro=base_distro, - action=action, - shell=shell, - prio=prio, - trigger=trigger, - recipients=get_recipients(target), - view=get_view(target, base_distro), - second_base=distro_upgrades[base_distro], - description=description)) - # upgrade job with upgrading apt+dpkg first - if base_distro in distro_upgrades and base_distro != oldoldstable and target[:10] != 'education-' and action != 'maintenance': - description = 'Debootstrap '+base_distro+', then upgrade apt and dpkg to '+distro_upgrades[base_distro]+' and then everything else.' - if target == 'bootstrap': - trigger = '' - for trigger_target in get_targets_in_distro(base_distro, targets): - if trigger_target not in ('maintenance', 'bootstrap'): - if trigger != '': - trigger = trigger+', ' - trigger = trigger+'chroot-installation_'+base_distro+'_install_'+trigger_target+'_upgrade_to_'+distro_upgrades[base_distro]+'_aptdpkg_first' - print(""" - '{name}_%(base_distro)s_%(action)s_upgrade_to_%(second_base)s_aptdpkg_first': - my_shell: '%(shell)s' - my_prio: '%(prio)s' - my_time: '' - my_trigger: '%(trigger)s' - my_recipients: '%(recipients)s' - my_view: '%(view)s' - my_description: '%(description)s'""" % - dict(base_distro=base_distro, - action=action, - shell=shell, - prio=prio, - trigger=trigger, - recipients=get_recipients(target), - view=get_view(target, base_distro), - second_base=distro_upgrades[base_distro], - description=description)) + dist2 = '' + if base_distro in distro_upgrades.values(): + trigger = 'chroot-installation_{dist}_bootstrap' + for item in distro_upgrades.items(): + if item[1]==base_distro and base_distro in distro_upgrades: + trigger = trigger+', chroot-installation_{dist}_bootstrap_upgrade_to_{dist2}, chroot-installation_{dist}_bootstrap_upgrade_to_{dist2}_aptdpkg_first' + dist2 = distro_upgrades[base_distro] + else: + trigger = 'chroot-installation_{dist}_bootstrap_upgrade_to_{dist2}' + dist2 = distro_upgrades[base_distro] + maint_distros.append({ base_distro: { + 'my_time': trigger_times[base_distro], + 'dist2': dist2, + 'my_trigger': trigger}}) +jobs.append({ '{name}_{dist}_{action}': { + 'action': 'maintenance', + 'dist': maint_distros, + 'my_description': 'Maintainance job for chroot-installation_{dist}_* jobs, do some cleanups and monitoring so that there is a predictable environment.', + 'my_prio': '135', + 'my_recipients': 'qa-jenkins-scm@lists.alioth.debian.org', + 'my_shell': '/srv/jenkins/bin/maintenance.sh chroot-installation_{dist}', + 'my_view': 'jenkins.d.n'}}) + + +# bootstrap jobs +js_dists_trigs = [{},{},{}] +for trigs, dists in get_dists_per_key('bootstrap',(lambda d: tuple(sorted(get_targets_in_distro(d))))).items(): + for jobindex, jobspec in enumerate(jobspecs): + js_dists = jobspec['distfilter'](dists) + if (js_dists): + js_disttrig = tuple((tuple(js_dists), trigs)) + js_dists_trigs[jobindex][js_disttrig] = True + + +for jobindex, jobspec in enumerate(jobspecs): + jobs.extend([{ '{name}_{dist}_{action}'+jobspec['j_ext']: { + 'action': 'bootstrap', + 'dist': list(dists) if jobspec['j_ext'] == '' else + [{dist: {'dist2': distro_upgrades[dist]}} for dist in dists], + 'my_trigger': join(['chroot-installation_{dist}_install_'+t+jobspec['j_ext'] + for t in list(trigs)], ', '), + 'my_description': 'Debootstrap {dist}'+jobspec['d_ext']+'.', + 'my_prio': 131, + 'my_time': '', + 'my_recipients': get_recipients('bootstrap'), + 'my_shell': '/srv/jenkins/bin/chroot-installation.sh {dist} none'+jobspec['s_ext'], + 'my_view': get_view('bootstrap', None), + }} + for (dists, trigs) in js_dists_trigs[jobindex].keys()]) + +# now all the other jobs +targets_per_distview = [{},{},{}] +for target in sorted(all_targets): + for view, dists in get_dists_per_key(target,(lambda d: get_view(target, d))).items(): + for jobindex, jobspec in enumerate(jobspecs): + if jobspec['skiptaryet'](target): + continue + + js_dists = jobspec['distfilter'](dists) + if (js_dists): + distview = tuple((tuple(js_dists), view)) + if distview not in targets_per_distview[jobindex].keys(): + targets_per_distview[jobindex][distview] = [] + targets_per_distview[jobindex][distview].append(target) + +for jobindex, jobspec in enumerate(jobspecs): + jobs.extend([{ '{name}_{dist}_install_{target}'+jobspec['j_ext']: { + 'dist': jobspec['dist_func'](list(dists)), + 'target': [{t: { + 'my_spokenname': get_spoken_name(t), + 'my_recipients': get_recipients(t)}} + for t in dv_targs], + 'my_description': 'Debootstrap {dist}, then install {my_spokenname}'+jobspec['d_ext']+'.', + 'my_shell': '/srv/jenkins/bin/chroot-installation.sh {dist} {target}'+jobspec['s_ext'], + 'my_view': view, + }} + for (dists, view), dv_targs in targets_per_distview[jobindex].items()]) + +data.append({'project': { + 'name': 'chroot-installation', + 'do_not_edit': '<br><br>Job configuration source is <a href="http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/job-cfg/chroot-installation.yaml.py">chroot-installation.yaml.py</a>.', + 'my_prio': '130', + 'my_trigger': '', + 'my_time': '', + 'jobs': jobs}}) +sys.stdout.write(dump(data, Dumper=Dumper)) |