From 3e0dac83b499e1e6b9124659a28bf4324d42110e Mon Sep 17 00:00:00 2001 From: Mattia Rizzolo Date: Thu, 8 Jan 2015 00:21:29 +0100 Subject: reproducible: add reproducible_{common,html_notes}.py. "equivalent" of their homonyms .sh calling reproducible_html_notes.py in the same directory where packages.yml and issues.yml (from the git.debian.org:/git/reproducible/notes.git repo) is enough to fast build all notes html pages for every pacakge listed in the packages.yml file, a page for every issue listed in the issues.yml and a index_issues.html --- bin/reproducible_common.py | 191 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100755 bin/reproducible_common.py (limited to 'bin/reproducible_common.py') diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py new file mode 100755 index 00000000..0df8cdaf --- /dev/null +++ b/bin/reproducible_common.py @@ -0,0 +1,191 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# clean-notes: sort and clean the notes stored in notes.git +# Copyright © 2015 Mattia Rizzolo +# Licensed under GPL-2+ +# +# Depends: python3 + +import os +import re +import sys +import sqlite3 +import logging +import argparse +import datetime +from string import Template + +DEBUG = False +QUIET = False + +BIN_PATH = '/srv/jenkins/bin' +BASE = '/var/lib/jenkins' + +REPRODUCIBLE_DB = BASE + '/userContent/reproducible.db' +REPRODUCIBLE_JSON = BASE + '/userContent/reproducible.json' + +NOTES_URI = '/userContent/notes' +ISSUES_URI = '/userContent/issues' +RB_PKG_URI = '/usrtContent/rb-pkg' +NOTES_PATH = BASE + NOTES_URI +ISSUES_PATH = BASE + ISSUES_URI +RB_PKG_PATH = BASE + RB_PKG_URI + +REPRODUCIBLE_URL = 'https://reproducible.debian.net' +JENKINS_URL = 'https://jenkins.debian.net' + +parser = argparse.ArgumentParser() +group = parser.add_mutually_exclusive_group() +group.add_argument("-d", "--debug", action="store_true") +group.add_argument("-q", "--quiet", action="store_true") +args = parser.parse_args() +log_level = logging.INFO +if args.debug or DEBUG: + log_level = logging.DEBUG +if args.quiet or QUIET: + log_level = logging.ERROR +logging.basicConfig(level=log_level) + +logging.debug("BIN_PATH:\t" + BIN_PATH) +logging.debug("BASE:\t" + BASE) +logging.debug("NOTES_URI:\t" + NOTES_URI) +logging.debug("ISSUES_URI:\t" + ISSUES_URI) +logging.debug("NOTES_PATH:\t" + NOTES_PATH) +logging.debug("ISSUES_PATH:\t" + ISSUES_PATH) +logging.debug("RB_PKG_URI:\t" + RB_PKG_URI) +logging.debug("RB_PKG_PATH:\t" + RB_PKG_PATH) +logging.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB) +logging.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON) +logging.debug("JENKINS_URL:\t\t" + JENKINS_URL) +logging.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL) + +html_header = Template(""" +\n + + +$page_title\n + +""") +html_footer = Template("""

There is more +information about jenkins.debian.net +and about reproducible +builds of Debian available elsewhere. Last update: $date. +Copyright 2014-%s Holger Levsen, +GPL-2 licensed. The weather icons are public domain and have been taken from +the Tango +Icon Library.

+""" % (JENKINS_URL, datetime.datetime.now().strftime('%Y'))) + +html_head_page = Template("""
+

$page_title

+

$count_total packages have been attempted to be build so far, that's +$percent_total% of $amount source packages in Debian sid +currently. Out of these, $count_good packagea ($percent_good%) +could be built +reproducible!

+
""") + +html_foot_page = Template(""" +

A package name displayed with a bold font is an +indication that this package has a note. Visited packages are linked in green, +those which have not been visited are linked in blue.

""") + + +url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)') + + +def write_html_page(title, body, destfile, noheader=False, nofooter=False): + now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') + html = '' + html += html_header.substitute(page_title=title) + if not noheader: + html += html_head_page.substitute( + page_title=title, + count_total=count_total, + amount=amount, + percent_total=percent_total, + count_good=count_good, + percent_good=percent_good) + html += body + if not nofooter: + html += html_foot_page.substitute() + html += html_footer.substitute(date=now) + os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True) + with open(destfile, 'w') as fd: + fd.write(html) + +def init_conn(): + return sqlite3.connect(REPRODUCIBLE_DB) + +def query_db(query): + cursor = conn.cursor() + cursor.execute(query) + return cursor.fetchall() + +# do the db querying +conn = init_conn() +amount = int(query_db('SELECT count(name) FROM sources')[0][0]) +count_total = int(query_db('SELECT COUNT(name) FROM source_packages')[0][0]) +count_good = int(query_db( + 'SELECT COUNT(name) FROM source_packages WHERE status="reproducible"')[0][0]) +percent_total = round(((count_total/amount)*100), 2) +percent_good = round(((count_good/count_total)*100), 2) +logging.info('Total packages in Sid:\t' + str(amount)) +logging.info('Total tested packages:\t' + str(count_total)) +logging.info('Total reproducible packages:\t' + str(count_good)) +logging.info('That means that out of the ' + str(percent_total) + '% of ' + + 'the Sid tested packages the ' + str(percent_good) + '% are ' + + 'reproducible!') + + -- cgit v1.2.3-70-g09d2