path: root/job-cfg/reproducible.yaml

- defaults:
    name: reproducible
    project-type: freestyle
    properties:
      - sidebar: &sb1
          url: https://jenkins.debian.net/userContent/about.html
          text: About jenkins.debian.net
          icon: /userContent/images/debian-swirl-24x24.png
      - sidebar: &sb2
          url: https://jenkins.debian.net/view/reproducible
          text: reproducible builds jobs
          icon: /userContent/images/debian-jenkins-24x24.png
      - sidebar: &sb3
          url: http://www.profitbricks.co.uk
          text: Sponsored by Profitbricks
          icon: /userContent/images/profitbricks-24x24.png
    description: &desc '{my_description}<br>Job configuration source is <a href="http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/job-cfg/reproducible.yaml">reproducible.yaml</a>.<br>Results are available at <a href="https://reproducible.debian.net/">https://reproducible.debian.net</a>.'
    triggers:
      - timed: '{my_timed}'
    logrotate:
      daysToKeep: 90
      numToKeep: 30
      artifactDaysToKeep: -1
      artifactNumToKeep: -1
    builders:
      - shell: '{my_shell}'
    publishers:
      - logparser: &logparse
          parse-rules: '/srv/jenkins/logparse/reproducible.rules'
          unstable-on-warning: 'true'
          fail-on-error: 'true'
      - email:
          recipients: '{my_recipients}'
    node: '{my_node}'

- defaults:
    name: reproducible_git
    project-type: freestyle
    properties:
      - sidebar: *sb1
      - sidebar: *sb2
      - sidebar: *sb3
    description: *desc
    triggers:
      - timed: '{my_timed}'
      - pollscm: '*/6 * * * *'
    logrotate:
      daysToKeep: 90
      numToKeep: 20
      artifactDaysToKeep: -1
      artifactNumToKeep: -1
    builders:
      - shell: '{my_shell}'
    publishers:
      - logparser: *logparse
      - email:
          recipients: '{my_recipients}'
    scm:
      - git:
          url: '{my_gitrepo}'
          branches:
            - master
    node: '{my_node}'

- defaults:
    name: reproducible_builder
    project-type: freestyle
    description: *desc
    properties:
      - sidebar: *sb1
      - sidebar: *sb2
      - sidebar: *sb3
      - priority-sorter:
          priority: '150'
      - throttle:
          max-total: 58
          max-per-node: 58
          enabled: true
          option: category
          categories:
            - reproducible
    triggers:
      - timed: '{my_timed}'
    wrappers:
      - timeout:
          timeout: 1560
    logrotate:
      daysToKeep: 3
      numToKeep: 300
      artifactDaysToKeep: -1
      artifactNumToKeep: -1
    builders:
      - shell: '{my_shell}'
    publishers:
      - logparser: *logparse
      - email:
          recipients: '{my_recipients}'
          notify-every-unstable-build: false
    node: '{my_node}'

- defaults:
    name: reproducible_builder_trigger
    project-type: freestyle
    description: *desc
    properties:
      - sidebar: *sb1
      - sidebar: *sb2
      - sidebar: *sb3
      - priority-sorter:
          priority: '150'
      - throttle:
          max-total: 58
          max-per-node: 58
          enabled: true
          option: category
          categories:
            - reproducible
    triggers:
      - timed: '{my_timed}'
    wrappers:
      - timeout:
          timeout: 1560
    logrotate:
      daysToKeep: 3
      numToKeep: 300
      artifactDaysToKeep: -1
      artifactNumToKeep: -1
    builders:
      - shell: '{my_shell}'
    publishers:
      - logparser: *logparse
      - email:
          recipients: '{my_recipients}'
          notify-every-unstable-build: false
      - trigger:
          project: '{my_trigger}'
    node: '{my_node}'

- defaults:
    name: reproducible_other_projects
    project-type: freestyle
    description: *desc
    properties:
      - sidebar: *sb1
      - sidebar: *sb2
      - sidebar: *sb3
      - throttle:
          max-total: 9
          max-per-node: 9
          enabled: true
          option: category
          categories:
            - reproducible      # no special category yet
    triggers:
      - timed: '{my_timed}'
    logrotate:
      daysToKeep: 90
      numToKeep: 30
      artifactDaysToKeep: -1
      artifactNumToKeep: -1
    builders:
      - shell: '{my_shell}'
    publishers:
      - logparser:
          parse-rules: '/srv/jenkins/logparse/reproducible-extra.rules'
          unstable-on-warning: 'true'
          fail-on-error: 'true'
      - email:
          recipients: '{my_recipients}'
    node: '{my_node}'

- defaults:
    name: reproducible_other_projects_trigger
    project-type: freestyle
    description: *desc
    properties:
      - sidebar: *sb1
      - sidebar: *sb2
      - sidebar: *sb3
      - throttle:
          max-total: 9
          max-per-node: 9
          enabled: true
          option: category
          categories:
            - reproducible      # no special category yet
    triggers:
      - timed: '{my_timed}'
    logrotate:
      daysToKeep: 90
      numToKeep: 30
      artifactDaysToKeep: -1
      artifactNumToKeep: -1
    builders:
      - shell: '{my_shell}'
    publishers:
      - logparser:
          parse-rules: '/srv/jenkins/logparse/reproducible-extra.rules'
          unstable-on-warning: 'true'
          fail-on-error: 'true'
      - email:
          recipients: '{my_recipients}'
      - trigger:
          project: '{my_trigger}'
    node: '{my_node}'


- job-template:
    defaults: reproducible
    name: '{name}_{my_task}'

- job-template:
    defaults: reproducible
    name: '{name}_{my_task}_{my_arch}_{my_hname}'

- job-template:
    defaults: reproducible
    name: '{name}_setup_schroot_{my_distro}_{my_arch}_{my_hname}'

- job-template:
    defaults: reproducible_builder
    name: '{name}_builder_{my_arch}_{my_num}'

- job-template:
    defaults: reproducible_git
    name: '{name}_html_{my_reponame}'

- job-template:
    defaults: reproducible_other_projects
    name: '{name}_{otherproject}'

- job-template:
    defaults: reproducible_other_projects_trigger
    name: '{name}_{otherproject_triggering}'

- job-template:
    defaults: reproducible_builder_trigger
    name: '{name}_builder_{otherproject}_{my_num}'

- project:
    name: reproducible
    my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
    my_node: '{my_hname}'
    jobs:
        - '{name}_{my_task}_{my_arch}_{my_hname}':
            my_task:
                - 'maintenance':
                    my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
                    my_timed: '5 0,3,6,9,12,15,18,21 * * *'
            my_shell: '/srv/jenkins/bin/reproducible_maintenance.sh'
            my_hname:
                - 'jenkins':        { my_arch: 'amd64' }
            my_node: ''

        - '{name}_{my_task}_{my_arch}_{my_hname}':
            my_task:
                - 'maintenance':
                    my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
                    my_timed: '5 0,3,6,9,12,15,18,21 * * *'
            my_hname:
                - 'bpi0':           { my_arch: 'armhf' }
                - 'cbxi4pro0':      { my_arch: 'armhf' }
                - 'hb0':            { my_arch: 'armhf' }
                - 'wbq0':           { my_arch: 'armhf' }
                - 'odxu4':          { my_arch: 'armhf' }
                - 'wbd0':           { my_arch: 'armhf' }
                - 'rpi2b':          { my_arch: 'armhf' }
                - 'profitbricks1':  { my_arch: 'amd64' }
                - 'profitbricks2':  { my_arch: 'amd64' }
                - 'profitbricks3':  { my_arch: 'amd64' }
                - 'profitbricks5':  { my_arch: 'amd64' }
                - 'profitbricks6':  { my_arch: 'amd64' }
            my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'

        - '{name}_{my_task}_{my_arch}_{my_hname}':
            my_task:
                - 'setup_pbuilder_unstable':
                    my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
                    my_timed: '42 7 * * *'
                - 'setup_pbuilder_experimental':
                    my_description: 'Setup pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
                    my_timed: '33 7 * * *'
                - 'setup_schroot_testing':
                    my_description: 'Setup testing schroot for fetching source packages for the builder jobs.'
                    my_timed: 'H/60 1 * * *'
                - 'setup_schroot_experimental':
                    my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
                    my_timed: '33 1 * * *'

            my_hname:
                - 'profitbricks1':  { my_arch: 'amd64' }
                - 'profitbricks2':  { my_arch: 'amd64' }
                - 'profitbricks5':  { my_arch: 'amd64' }
                - 'profitbricks6':  { my_arch: 'amd64' }
            my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'

        - '{name}_{my_task}_{my_arch}_{my_hname}':
            my_task:
                - 'setup_pbuilder_testing':
                    my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
                    my_timed: '23 7 * * *'
                - 'setup_schroot_unstable':
                    my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
                    my_timed: '42 1 * * *'
            my_hname:
                - 'bpi0':           { my_arch: 'armhf' }
                - 'cbxi4pro0':      { my_arch: 'armhf' }
                - 'hb0':            { my_arch: 'armhf' }
                - 'wbq0':           { my_arch: 'armhf' }
                - 'odxu4':          { my_arch: 'armhf' }
                - 'wbd0':           { my_arch: 'armhf' }
                - 'rpi2b':          { my_arch: 'armhf' }
                - 'profitbricks1':  { my_arch: 'amd64' }
                - 'profitbricks2':  { my_arch: 'amd64' }
                - 'profitbricks5':  { my_arch: 'amd64' }
                - 'profitbricks6':  { my_arch: 'amd64' }
            my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'

        - '{name}_setup_schroot_{my_distro}_{my_arch}_{my_hname}':
            my_distro:
                - 'unstable_diffoscope':
                    my_description: 'Setup schroot for running diffoscope in a unstable environment (this is needed to be able to correctly investigate haskell binaries...)'
                    my_timed: '42 0 * * *'
                    my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-diffoscope unstable diffoscope locales-all'
                - 'testing_diffoscope':
                    my_description: 'Setup schroot for running diffoscope in a testing environment (this is needed to be able to correctly investigate haskell binaries...)'
                    my_timed: '23 0 * * *'
                    my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-testing-diffoscope testing diffoscope locales-all'
                - 'unstable':
                    my_timed: '42 1 * * *'
                    my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-{my_distro} {my_distro} botch'
                - 'testing':
                    my_timed: '42 1 * * *'
                - 'experimental':
                    my_timed: '33 1 * * *'
            my_description: 'Setup {my_distro} schroot for fetching source packages for the builder jobs.'
            my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-{my_distro} {my_distro}'
            my_hname:
                - 'jenkins':        { my_arch: 'amd64' }
            my_node: ''

        - '{name}_{my_task}_{my_arch}_{my_hname}':
            my_task:
                - 'setup_schroot_unstable_diffoscope':
                    my_description: 'Setup schroot for running diffoscope in a unstable environment'
                    my_timed: '42 0 * * *'
                    my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
            my_hname:
                - 'profitbricks3':  { my_arch: 'amd64' }

        - '{name}_{my_task}':
            my_task:
                - 'scheduler':
                    my_description: 'Schedule packages to be tested for reproducibility.'
                    my_timed: '10 * * * *'
                    my_shellext: ".py"
                - 'html_dashboard':
                    my_description: 'Generate HTML dashboard with graphs for reproducible builds.'
                    my_timed: '*/5 * * * *'
                - 'html_pkg_sets':
                    my_description: 'Generate HTML results (package sets with graphs) for reproducible builds.'
                    my_timed: '0 * * * *'
                - 'html_indexes':
                    my_description: 'Generate HTML results (indexes) for reproducible builds.'
                    my_timed: '*/15 * * * *'
                    my_shellext: ".py"
                - 'html_dd_list':
                    my_description: 'Generate HTML results (dd-list) for reproducible builds.'
                    my_timed: '55 */2 * * *'
                    my_shellext: ".py"
                - 'html_all_packages':
                    my_description: 'Generate HTML results (for all packages) for reproducible builds. This job is rather redudant and just run to give a fuzzy warm feeling all pages are good.'
                    my_timed: '37 13 * * 1'
                    my_shellext: ".py"
                - 'html_repository_comparison':
                    my_description: 'Generate HTML results (repository_comparison) for reproducible builds.'
                    my_timed: '0 1 * * *'
                - 'html_breakages':
                    my_description: 'Generate an HTML page with CI issues (packages with incoherent status or files that should or should not be there).'
                    my_timed: '30 0,12 * * *'
                    my_shellext: ".py"
                - 'html_rsync_remote_results':
                    my_description: 'rsync the results from some remote jobs to the main node.'
                    my_timed: ''
                - 'html_live_status':
                    my_description: 'Generate an HTML page showing the live status of all builds happening or scheduled by humans.'
                    my_timed: 'H/2 * * * *'
                    my_shellext: ".py"
                - 'create_meta_pkg_sets':
                    my_description: 'Create meta package sets for later consumption when generating HTML stats.'
                    my_timed: '0 6 * * H/3'
                - 'nodes_info':
                    my_description: 'Collect various information from build nodes used for reproducible builds.'
                    my_timed: '0 0 * * *'
                - 'json':
                    my_description: 'Generate https://reproducible.debian.net/userContent/reproducible.json for consumption by tracker.debian.org.'
                    my_timed: '1 * * * *'
                    my_shellext: ".py"
                - 'html_archlinux':
                    my_description: 'Build a simple webpage for Arch Linux reproducibility tests'
                    my_timed: ''
                - 'html_fedora-23_x86_64':
                    my_description: 'Build a simple webpage for Fedora 23 reproducibility tests'
                    my_timed: ''
                    my_shell: '/srv/jenkins/bin/reproducible_html_rpms.sh fedora-23 x86_64'

            my_shellext: ".sh"
            my_shell: '/srv/jenkins/bin/reproducible_{my_task}{my_shellext}'
            my_hname: ''

        - '{name}_html_{my_reponame}':
            my_reponame:
                - 'notes':
                    my_description: 'Generate HTML results ({my_reponame}) for reproducible builds, triggered by changes to the <a href="https://anonscm.debian.org/cgit/reproducible/{my_reponame}.git" target="_blank">{my_reponame}.git</a> repository.'
                    my_timed: '57 0 * * *'
                    my_shell: '/srv/jenkins/bin/reproducible_{my_reponame}.py && /srv/jenkins/bin/reproducible_html_{my_reponame}.py'
                - 'source-date-epoch-spec':
                    my_description: 'Generate the HTML of the reproducible builds SOURCE_DATE_EPOCH specification, triggered by changes to the <a href="https://anonscm.debian.org/cgit/reproducible/{my_reponame}.git" target="_blank">{my_reponame}.git</a> repository.'
                    my_timed: ''
                    my_shell: '/srv/jenkins/bin/reproducible_html_specs.sh {my_reponame}'
                    my_recipients: 'jenkins+debian-reproducible qa-jenkins-scm@lists.alioth.debian.org'
            my_gitrepo: 'git://git.debian.org/git/reproducible/{my_reponame}.git'
            my_hname: ''

        - '{name}_builder_{my_arch}_{my_num}':
            my_description: 'Try to reproducibly build a scheduled package. This is one of several builder jobs.'
            my_arch: 'amd64'
            my_num:
                -  '1': &bd_amd_1_5 { my_node1: 'profitbricks-build1', my_node2: 'profitbricks-build5' }
                -  '2': &bd_amd_2_6 { my_node1: 'profitbricks-build2', my_node2: 'profitbricks-build6' }
                -  '3': &bd_amd_1_6 { my_node1: 'profitbricks-build1', my_node2: 'profitbricks-build6' }
                -  '4': &bd_amd_2_5 { my_node1: 'profitbricks-build2', my_node2: 'profitbricks-build5' }
                -  '5': &bd_amd_6_2 { my_node1: 'profitbricks-build6', my_node2: 'profitbricks-build2' }
                -  '6': &bd_amd_5_1 { my_node1: 'profitbricks-build5', my_node2: 'profitbricks-build1' }
                -  '7': &bd_amd_5_2 { my_node1: 'profitbricks-build5', my_node2: 'profitbricks-build2' }
                -  '8': &bd_amd_6_1 { my_node1: 'profitbricks-build6', my_node2: 'profitbricks-build1' }
                -  '9': *bd_amd_1_5
                - '10': *bd_amd_2_6
                - '11': *bd_amd_1_6
                - '12': *bd_amd_2_5
                - '13': *bd_amd_6_2
                - '14': *bd_amd_5_1
                - '15': *bd_amd_5_2
                - '16': *bd_amd_6_1
                - '17': *bd_amd_1_5
                - '18': *bd_amd_2_6
                - '19': *bd_amd_1_6
                - '20': *bd_amd_2_5
                - '21': *bd_amd_6_2
                - '22': *bd_amd_5_1
                - '23': *bd_amd_5_2
                - '24': *bd_amd_6_1
                - '25': *bd_amd_1_5
                - '26': *bd_amd_2_6
                - '27': *bd_amd_1_6
                - '28': *bd_amd_2_5
                - '29': *bd_amd_6_2
                - '30': *bd_amd_5_1
                - '31': *bd_amd_5_2
                - '32': *bd_amd_6_1
            my_shell: '/srv/jenkins/bin/reproducible_build.sh {my_node1}-{my_arch} {my_node2}-{my_arch}'
            my_timed: '* * * * *'
            my_hname: ''

        # could have all archs in one block, but it seems cleaner this way
        - '{name}_builder_{my_arch}_{my_num}':
            my_description: 'Try to reproducibly build a scheduled package. This is one of several builder jobs.'
            my_arch: 'armhf'
            my_num:
                -  '1': { my_node1: 'bpi0-armhf-rb:2222',      my_node2: 'odxu4-armhf-rb:2229'     }
                -  '2': { my_node1: 'wbq0-armhf-rb:2225',      my_node2: 'cbxi4pro0-armhf-rb:2226' }
                -  '3': { my_node1: 'hb0-armhf-rb:2224',       my_node2: 'odxu4-armhf-rb:2229'     }
                -  '4': { my_node1: 'cbxi4pro0-armhf-rb:2226', my_node2: 'wbq0-armhf-rb:2225'      }
                -  '5': { my_node1: 'cbxi4pro0-armhf-rb:2226', my_node2: 'bpi0-armhf-rb:2222'      }
                -  '6': { my_node1: 'wbq0-armhf-rb:2225',      my_node2: 'cbxi4pro0-armhf-rb:2226' }
                -  '7': { my_node1: 'wbq0-armhf-rb:2225',      my_node2: 'odxu4-armhf-rb:2229'     }
                -  '8': { my_node1: 'hb0-armhf-rb:2224',       my_node2: 'wbq0-armhf-rb:2225'      }
                -  '9': { my_node1: 'odxu4-armhf-rb:2229',     my_node2: 'bpi0-armhf-rb:2222'      }
                - '10': { my_node1: 'odxu4-armhf-rb:2229',     my_node2: 'rpi2b-armhf-rb:2230'     }
                - '11': { my_node1: 'odxu4-armhf-rb:2229',     my_node2: 'wbd0-armhf-rb:2223'      }
                - '12': { my_node1: 'wbd0-armhf-rb:2223',      my_node2: 'cbxi4pro0-armhf-rb:2226' }
                - '13': { my_node1: 'cbxi4pro0-armhf-rb:2226', my_node2: 'rpi2b-armhf-rb:2230'     }
                - '14': { my_node1: 'rpi2b-armhf-rb:2230',     my_node2: 'wbq0-armhf-rb:2225'      }
                - '15': { my_node1: 'wbd0-armhf-rb:2223',      my_node2: 'hb0-armhf-rb:2224'       }
            my_shell: '/srv/jenkins/bin/reproducible_build.sh {my_node1} {my_node2}'
            my_timed: '* * * * *'
            my_hname: ''

        - '{name}_{otherproject_triggering}':
            otherproject_triggering:
                - 'coreboot':
                    my_description: 'Rebuild all coreboot ROMs twice (with variations) and investigate regarding reproducibility.'
                    my_timed: '42 23 * * 1'
                - 'openwrt':
                    my_description: 'Rebuild OpenWrt (some targets, some packages) twice (with variations) and investigate regarding reproducibility.'
                    my_timed: '42 23 * * 2'
                - 'netbsd':
                    my_description: 'Rebuild NetBSD (for some archs) twice (with variations) and investigate regarding reproducibility.'
                    my_timed: '42 23 * * 3'
            my_trigger: 'reproducible_html_rsync_remote_results'
            my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
            my_hname: 'profitbricks3'

        - '{name}_{otherproject}':
            otherproject:
                - 'freebsd':
                    # FIXME: should this job be called '{name}_freebsd_jenkins'?  If so, we should add the hname above
                    my_description: 'Rebuild FreeBSD twice (with variations) and investigate regarding reproducibility.'
                    my_timed: '42 23 * * 4'
                - 'setup_schroot_archlinux_jenkins':
                    # FIXME: it looks like this should be renamed to setup_archlinux_schroot, and the jenkins applied to all or none of the jobs here -- fil
                    my_description: 'Setup an Arch Linux schroot to test Arch Linux packages for reproducibility.'
                    my_timed: '42 23 1 1 *'
                    my_shell: '/srv/jenkins/bin/reproducible_setup_archlinux_schroot.sh'
                - 'setup_schroot_archlinux_profitbricks3':
                    my_description: 'Setup an Arch Linux schroot to test Arch Linux packages for reproducibility.'
                    my_timed: '42 23 1 1 *'
                    my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
                    my_hname: 'profitbricks3'
                - 'setup_mock_fedora-23_x86_64_jenkins':
                    my_description: 'Setup mock for fedora-23 on X86_64 to test .rpm packages for reproducibility.'
                    my_timed: '42 23 1 1 *'
                    my_shell: '/srv/jenkins/bin/reproducible_setup_mock.sh fedora-23 x86_64'
                - 'setup_mock_fedora-23_x86_64_profitbricks3':
                    my_description: 'Setup mock for fedora-23 on X86_64 to test .rpm packages for reproducibility.'
                    my_timed: '42 23 1 1 *'
                    my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
                    my_hname: 'profitbricks3'
            my_shell: '/srv/jenkins/bin/reproducible_{otherproject}.sh'
            my_hname: ''

        - '{name}_builder_{otherproject}_{my_num}':
            otherproject:
                - 'archlinux':
                    my_num: 1
                    op_descname: 'an Arch Linux'
                - 'archlinux':
                    my_num: 2
                    op_descname: 'an Arch Linux'
                - 'fedora-23_x86_64':
                    my_num: 1
                    op_descname: 'a Fedora 23'
                    my_shell: '/srv/jenkins/bin/reproducible_build_rpm.sh fedora-23 x86_64'
            my_description: 'Try to reproducibly build {op_descname} package.'
            my_shell: '/srv/jenkins/bin/reproducible_build_{otherproject}_pkg.sh'
            my_timed: '* * * * *'
            my_trigger: 'reproducible_html_{otherproject}'
            my_hname: ''