summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2017-01-12 12:55:20 +0100
committerHolger Levsen <holger@layer-acht.org>2017-01-12 12:55:20 +0100
commit93a522cbaffa3f59107b06e1ed587296bb6d9aaa (patch)
tree5757e7cabbd08a5a6ecf8de379fadcdfcab5998e
parent35ebb278789ec4ada5e3b2f84bca7f2b99b6317f (diff)
downloadjenkins.debian.net-93a522cbaffa3f59107b06e1ed587296bb6d9aaa.tar.xz
reproducible Debian: add a graph with diffoscope crashes and timeouts
Signed-off-by: Holger Levsen <holger@layer-acht.org>
-rwxr-xr-xbin/reproducible_db_maintenance.py9
-rwxr-xr-xbin/reproducible_html_breakages.py52
2 files changed, 60 insertions, 1 deletions
diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py
index bace9618..649d13dd 100755
--- a/bin/reproducible_db_maintenance.py
+++ b/bin/reproducible_db_maintenance.py
@@ -591,6 +591,15 @@ schema_updates = {
"ALTER TABLE results ALTER id SET DEFAULT NEXTVAL('results_id_seq')",
"INSERT INTO rb_schema (version, date) VALUES (29, '" + now + "')"
],
+
+ 30: [ # Add new table to track diffoscope breake
+ "CREATE TABLE stats_breakage
+ (datum TEXT NOT NULL,
+ diffoscope_timeouts INTEGER,
+ diffoscope_crashes INTEGER,
+ PRIMARY KEY (datum))",
+ "INSERT INTO rb_schema (version, date) VALUES (30, '" + now + "')"
+ ],
}
diff --git a/bin/reproducible_html_breakages.py b/bin/reproducible_html_breakages.py
index a604ac07..90ce256b 100755
--- a/bin/reproducible_html_breakages.py
+++ b/bin/reproducible_html_breakages.py
@@ -287,6 +287,52 @@ def _gen_files_html(header, entries):
html += '</pre></p>\n'
return html
+def create_breakages_graph():
+ png_file = os.path.join(DEBIAN_BASE, 'stats_breakages.png')
+ table = "stats_breakages"
+ columns = ["datum", "diffoscope_timeouts", "diffoscope_crashes"]
+ query = "SELECT {fields} FROM {table} ORDER BY datum".format(
+ fields=", ".join(columns), table=table)
+ result = query_db(query)
+ result_rearranged = [dict(zip(columns, row)) for row in result]
+
+ with create_temp_file(mode='w') as f:
+ csv_tmp_file = f.name
+ csv_writer = csv.DictWriter(f, columns)
+ csv_writer.writeheader()
+ csv_writer.writerows(result_rearranged)
+ f.flush()
+
+ graph_command = os.path.join(BIN_PATH, "make_graph.py")
+ main_label = "source packages causing Diffoscope to timeout and crash"
+ y_label = "Amount (packages)"
+ log.info("Creating graph for stats_breakges.")
+ check_call([graph_command, csv_tmp_file, png_file, '2', main_label,
+ y_label, '1920', '960'])
+
+
+def update_stats_breakage(diffoscope_timeouts, diffoscope_crashes):
+ # we only do stats up until yesterday
+ YESTERDAY = (datetime.now()-timedelta(days=1)).strftime('%Y-%m-%d')
+
+ result = query_db("""
+ SELECT datum, diffoscope_timeouts, diffoscope_crashes
+ FROM stats_breakage
+ WHERE datum = '{date}'
+ """.format(date=YESTERDAY))
+
+ # if there is not a result for this day, add one
+ if not result:
+ insert = "INSERT INTO stats_breakages VALUES ('{date}', " + \
+ "'{diffoscope_timeouts}', '{diffoscope_crashes}')"
+ query_db(insert.format(date=YESTERDAY, diffoscope_timeouts=diffoscope_timeouts, diffoscope_crashes=diffoscope_crashes))
+ log.info("Updating db table stats_breakage on %s with %s timeouts and %s crashes.",
+ YESTERDAY, diffoscope_timeouts, diffoscope_crashes)
+ else:
+ log.debug("Not updating db table stats_breakage as it already has data for %s.",
+ YESTERDAY)
+
+
def gen_html():
html = ''
# files that should not be there (e.g. removed package without cleanup)
@@ -317,7 +363,6 @@ def gen_html():
'diffoscope output does not seem to be an html ' +
'file - so probably diffoscope ran into a ' +
'timeout:', bad_dbd)
- # TODO: graph this
html += str(len(sources_without_dbd))
html += ' source packages on which diffoscope ran into a timeout ('
html += str(count_pkgs(bad_dbd)) + ') or crashed ('
@@ -326,6 +371,11 @@ def gen_html():
# pbuilder-satisfydepends failed
html += _gen_packages_html('failed to satisfy their build-dependencies:',
pbuilder_dep_fail())
+ # gather stats and add graph
+ update_stats_breakage(count_pkgs(bad_dbd), count_pkgs(without_dbd))
+ create_breakages_graph
+ html += '<br> <a href="/debian/stats_breakages.png><img src=/debian/stats_breakages.png" alt="source packages causing Diffoscope to timeout and crash"></a>'
+
return html