#!/usr/bin/python3
|
#
|
# Copyright 2018, The Android Open Source Project
|
#
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
|
"""MLTS benchmark result generator.
|
|
Reads a CSV produced by MLTS benchmark and generates
|
an HTML page with results summary.
|
|
Usage:
|
generate_result [csv input file] [html output file]
|
"""
|
|
import argparse
|
import collections
|
import csv
|
import os
|
import re
|
|
|
class ScoreException(Exception):
|
"""Generator base exception type. """
|
pass
|
|
|
BenchmarkResult = collections.namedtuple(
|
'BenchmarkResult',
|
['name', 'backend_type', 'iterations', 'total_time_sec', 'max_single_error',
|
'testset_size', 'evaluator_keys', 'evaluator_values',
|
'time_freq_start_sec', 'time_freq_step_sec', 'time_freq_sec',
|
'validation_errors'])
|
|
|
ResultsWithBaseline = collections.namedtuple(
|
'ResultsWithBaseline',
|
['baseline', 'other'])
|
|
|
BASELINE_BACKEND = 'TFLite_CPU'
|
KNOWN_GROUPS = [
|
(re.compile('mobilenet_v1.*quant.*'), 'MobileNet v1 Quantized'),
|
(re.compile('mobilenet_v1.*'), 'MobileNet v1 Float'),
|
(re.compile('mobilenet_v2.*quant.*'), 'MobileNet v2 Quantized'),
|
(re.compile('mobilenet_v2.*'), 'MobileNet v2 Float'),
|
(re.compile('tts.*'), 'LSTM Text-to-speech'),
|
(re.compile('asr.*'), 'LSTM Automatic Speech Recognition'),
|
]
|
|
|
def parse_csv_input(input_filename):
|
"""Parse input CSV file, returns: (benchmarkInfo, list of BenchmarkResult)."""
|
with open(input_filename, 'r') as csvfile:
|
csv_reader = csv.reader(filter(lambda row: row[0] != '#', csvfile))
|
|
# First line contain device info
|
benchmark_info = next(csv_reader)
|
|
results = []
|
for row in csv_reader:
|
evaluator_keys_count = int(row[8])
|
time_freq_sec_count = int(row[9])
|
validation_error_count = int(row[10])
|
|
tf_start = 11 + evaluator_keys_count*2
|
time_freq_sec = [float(x) for x in
|
row[tf_start:tf_start + time_freq_sec_count]]
|
ve_start = 11 + evaluator_keys_count*2 + time_freq_sec_count
|
validation_errors = row[ve_start: ve_start + validation_error_count]
|
|
results.append(BenchmarkResult(
|
name=row[0],
|
backend_type=row[1],
|
iterations=int(row[2]),
|
total_time_sec=float(row[3]),
|
max_single_error=float(row[4]),
|
testset_size=int(row[5]),
|
time_freq_start_sec=float(row[6]),
|
time_freq_step_sec=float(row[7]),
|
evaluator_keys=row[11:11 + evaluator_keys_count],
|
evaluator_values=row[
|
11 + evaluator_keys_count: 11 + evaluator_keys_count*2],
|
time_freq_sec=time_freq_sec,
|
validation_errors=validation_errors,
|
))
|
return (benchmark_info, results)
|
|
|
def group_results(results):
|
"""Group list of results by their name/backend, returns list of lists."""
|
# Group by name
|
groupings = collections.defaultdict(list)
|
for result in results:
|
groupings[result.name].append(result)
|
|
# Find baseline for each group, make ResultsWithBaseline for each name
|
groupings_baseline = {}
|
for name, results in groupings.items():
|
baseline = next(filter(lambda x: x.backend_type == BASELINE_BACKEND,
|
results))
|
other = sorted(filter(lambda x: x is not baseline, results),
|
key=lambda x: x.backend_type)
|
groupings_baseline[name] = ResultsWithBaseline(
|
baseline=baseline,
|
other=other)
|
|
# Merge ResultsWithBaseline for known groups
|
known_groupings_baseline = collections.defaultdict(list)
|
for name, results_with_bl in sorted(groupings_baseline.items()):
|
group_name = name
|
for known_group in KNOWN_GROUPS:
|
if known_group[0].match(results_with_bl.baseline.name):
|
group_name = known_group[1]
|
break
|
known_groupings_baseline[group_name].append(results_with_bl)
|
|
# Turn into a list sorted by name
|
groupings_list = []
|
for name, results_wbl in sorted(known_groupings_baseline.items()):
|
groupings_list.append((name, results_wbl))
|
return groupings_list
|
|
|
def get_frequency_graph_min_max(results_with_bl):
|
"""Get min and max times of latencies frequency."""
|
mins = []
|
maxs = []
|
for result in [results_with_bl.baseline] + results_with_bl.other:
|
mins.append(result.time_freq_start_sec)
|
to_add = len(result.time_freq_sec) * result.time_freq_step_sec
|
maxs.append(result.time_freq_start_sec + to_add)
|
return min(mins), max(maxs)
|
|
|
def get_frequency_graph(time_freq_start_sec, time_freq_step_sec, time_freq_sec,
|
start_sec, end_sec):
|
"""Generate input x/y data for latency frequency graph."""
|
left_to_pad = int((time_freq_start_sec - start_sec) / time_freq_step_sec)
|
end_time = time_freq_start_sec + len(time_freq_sec) * time_freq_step_sec
|
right_to_pad = int((end_sec - end_time) / time_freq_step_sec)
|
|
# After pading more that 64 values, graphs start to look messy,
|
# bail out in that case.
|
if (left_to_pad + right_to_pad) < 64:
|
left_pad = (['{:.2f}ms'.format(
|
(start_sec + x * time_freq_step_sec) * 1000.0)
|
for x in range(left_to_pad)], [0] * left_to_pad)
|
|
right_pad = (['{:.2f}ms'.format(
|
(end_time + x * time_freq_step_sec) * 1000.0)
|
for x in range(right_to_pad)], [0] * right_to_pad)
|
else:
|
left_pad = [[], []]
|
right_pad = [[], []]
|
|
data = (['{:.2f}ms'.format(
|
(time_freq_start_sec + x * time_freq_step_sec) * 1000.0)
|
for x in range(len(time_freq_sec))], time_freq_sec)
|
|
return (left_pad[0] + data[0] + right_pad[0],
|
left_pad[1] + data[1] + right_pad[1])
|
|
|
def is_topk_evaluator(evaluator_keys):
|
"""Are these evaluator keys from TopK evaluator?"""
|
return (len(evaluator_keys) == 5 and
|
evaluator_keys[0] == 'top_1' and
|
evaluator_keys[1] == 'top_2' and
|
evaluator_keys[2] == 'top_3' and
|
evaluator_keys[3] == 'top_4' and
|
evaluator_keys[4] == 'top_5')
|
|
|
def is_melceplogf0_evaluator(evaluator_keys):
|
"""Are these evaluator keys from MelCepLogF0 evaluator?"""
|
return (len(evaluator_keys) == 2 and
|
evaluator_keys[0] == 'max_mel_cep_distortion' and
|
evaluator_keys[1] == 'max_log_f0_error')
|
|
|
def is_phone_error_rate_evaluator(evaluator_keys):
|
"""Are these evaluator keys from PhoneErrorRate evaluator?"""
|
return (len(evaluator_keys) == 1 and
|
evaluator_keys[0] == 'max_phone_error_rate')
|
|
|
def generate_accuracy_headers(result):
|
"""Accuracy-related headers for result table."""
|
if is_topk_evaluator(result.evaluator_keys):
|
return ACCURACY_HEADERS_TOPK_TEMPLATE
|
elif is_melceplogf0_evaluator(result.evaluator_keys):
|
return ACCURACY_HEADERS_MELCEPLOGF0_TEMPLATE
|
elif is_phone_error_rate_evaluator(result.evaluator_keys):
|
return ACCURACY_HEADERS_PHONE_ERROR_RATE_TEMPLATE
|
else:
|
return ACCURACY_HEADERS_BASIC_TEMPLATE
|
raise ScoreException('Unknown accuracy headers for: ' + str(result))
|
|
|
def get_diff_span(value, same_delta, positive_is_better):
|
if abs(value) < same_delta:
|
return 'same'
|
if positive_is_better and value > 0 or not positive_is_better and value < 0:
|
return 'better'
|
return 'worse'
|
|
|
def generate_accuracy_values(baseline, result):
|
"""Accuracy-related data for result table."""
|
if is_topk_evaluator(result.evaluator_keys):
|
val = [float(x) * 100.0 for x in result.evaluator_values]
|
if result is baseline:
|
topk = [TOPK_BASELINE_TEMPLATE.format(val=x) for x in val]
|
return ACCURACY_VALUES_TOPK_TEMPLATE.format(
|
top1=topk[0], top2=topk[1], top3=topk[2], top4=topk[3],
|
top5=topk[4]
|
)
|
else:
|
base = [float(x) * 100.0 for x in baseline.evaluator_values]
|
diff = [a - b for a, b in zip(val, base)]
|
topk = [TOPK_DIFF_TEMPLATE.format(
|
val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=True))
|
for v, d in zip(val, diff)]
|
return ACCURACY_VALUES_TOPK_TEMPLATE.format(
|
top1=topk[0], top2=topk[1], top3=topk[2], top4=topk[3],
|
top5=topk[4]
|
)
|
elif is_melceplogf0_evaluator(result.evaluator_keys):
|
val = [float(x) for x in
|
result.evaluator_values + [result.max_single_error]]
|
if result is baseline:
|
return ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE.format(
|
max_log_f0=MELCEPLOGF0_BASELINE_TEMPLATE.format(
|
val=val[0]),
|
max_mel_cep_distortion=MELCEPLOGF0_BASELINE_TEMPLATE.format(
|
val=val[1]),
|
max_single_error=MELCEPLOGF0_BASELINE_TEMPLATE.format(
|
val=val[2]),
|
)
|
else:
|
base = [float(x) for x in
|
baseline.evaluator_values + [baseline.max_single_error]]
|
diff = [a - b for a, b in zip(val, base)]
|
v = [MELCEPLOGF0_DIFF_TEMPLATE.format(
|
val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=False))
|
for v, d in zip(val, diff)]
|
return ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE.format(
|
max_log_f0=v[0],
|
max_mel_cep_distortion=v[1],
|
max_single_error=v[2],
|
)
|
elif is_phone_error_rate_evaluator(result.evaluator_keys):
|
val = [float(x) for x in
|
result.evaluator_values + [result.max_single_error]]
|
if result is baseline:
|
return ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE.format(
|
max_phone_error_rate=PHONE_ERROR_RATE_BASELINE_TEMPLATE.format(
|
val=val[0]),
|
max_single_error=PHONE_ERROR_RATE_BASELINE_TEMPLATE.format(
|
val=val[1]),
|
)
|
else:
|
base = [float(x) for x in
|
baseline.evaluator_values + [baseline.max_single_error]]
|
diff = [a - b for a, b in zip(val, base)]
|
v = [PHONE_ERROR_RATE_DIFF_TEMPLATE.format(
|
val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=False))
|
for v, d in zip(val, diff)]
|
return ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE.format(
|
max_phone_error_rate=v[0],
|
max_single_error=v[1],
|
)
|
else:
|
return ACCURACY_VALUES_BASIC_TEMPLATE.format(
|
max_single_error=result.max_single_error,
|
)
|
raise ScoreException('Unknown accuracy values for: ' + str(result))
|
|
|
def getchartjs_source():
|
return open(os.path.dirname(os.path.abspath(__file__)) + '/' +
|
CHART_JS_FILE).read()
|
|
|
def generate_avg_ms(baseline, result):
|
"""Generate average latency value."""
|
if result is None:
|
result = baseline
|
|
result_avg_ms = (result.total_time_sec / result.iterations)*1000.0
|
if result is baseline:
|
return LATENCY_BASELINE_TEMPLATE.format(val=result_avg_ms)
|
baseline_avg_ms = (baseline.total_time_sec / baseline.iterations)*1000.0
|
diff = (result_avg_ms/baseline_avg_ms - 1.0) * 100.0
|
diff_val = result_avg_ms - baseline_avg_ms
|
return LATENCY_DIFF_TEMPLATE.format(
|
val=result_avg_ms,
|
diff=diff,
|
diff_val=diff_val,
|
span=get_diff_span(diff, same_delta=1.0, positive_is_better=False))
|
|
|
def generate_result_entry(baseline, result):
|
if result is None:
|
result = baseline
|
|
return RESULT_ENTRY_TEMPLATE.format(
|
row_class='failed' if result.validation_errors else 'normal',
|
name=result.name,
|
backend=result.backend_type,
|
iterations=result.iterations,
|
testset_size=result.testset_size,
|
accuracy_values=generate_accuracy_values(baseline, result),
|
avg_ms=generate_avg_ms(baseline, result))
|
|
|
def generate_latency_graph_entry(result, results_with_bl):
|
tmin, tmax = get_frequency_graph_min_max(results_with_bl)
|
return LATENCY_GRAPH_ENTRY_TEMPLATE.format(
|
backend=result.backend_type,
|
i=id(result),
|
freq_data=get_frequency_graph(result.time_freq_start_sec,
|
result.time_freq_step_sec,
|
result.time_freq_sec,
|
tmin, tmax))
|
|
|
def generate_validation_errors(entries_group):
|
"""Generate validation errors table."""
|
errors = []
|
for result_and_bl in entries_group:
|
for result in [result_and_bl.baseline] + result_and_bl.other:
|
for error in result.validation_errors:
|
errors.append((result.name, result.backend_type, error))
|
|
if errors:
|
return VALIDATION_ERRORS_TEMPLATE.format(
|
results=''.join(
|
VALIDATION_ERRORS_ENTRY_TEMPLATE.format(
|
name=name,
|
backend=backend,
|
error=error) for name, backend, error in errors))
|
return ''
|
|
|
def generate_result(benchmark_info, data):
|
"""Turn list of results into HTML."""
|
return MAIN_TEMPLATE.format(
|
jsdeps=getchartjs_source(),
|
device_info=DEVICE_INFO_TEMPLATE.format(
|
benchmark_time=benchmark_info[0],
|
device_info=benchmark_info[1],
|
),
|
results_list=''.join((
|
RESULT_GROUP_TEMPLATE.format(
|
group_name=entries_name,
|
accuracy_headers=generate_accuracy_headers(
|
entries_group[0].baseline),
|
results=''.join(
|
RESULT_ENTRY_WITH_BASELINE_TEMPLATE.format(
|
baseline=generate_result_entry(
|
result_and_bl.baseline, None),
|
other=''.join(
|
generate_result_entry(
|
result_and_bl.baseline, x)
|
for x in result_and_bl.other)
|
) for result_and_bl in entries_group),
|
validation_errors=generate_validation_errors(entries_group),
|
latency_graphs=LATENCY_GRAPHS_TEMPLATE.format(
|
results=''.join(
|
LATENCY_GRAPH_ENTRY_WITH_BL_TEMPLATE.format(
|
name=result_and_bl.baseline.name,
|
baseline=generate_latency_graph_entry(
|
result_and_bl.baseline, result_and_bl),
|
result=''.join(
|
generate_latency_graph_entry(x, result_and_bl)
|
for x in result_and_bl.other)
|
) for result_and_bl in entries_group)
|
)
|
) for entries_name, entries_group in group_results(data))
|
))
|
|
|
def main():
|
parser = argparse.ArgumentParser()
|
parser.add_argument('input', help='input csv filename')
|
parser.add_argument('output', help='output html filename')
|
args = parser.parse_args()
|
|
benchmark_info, data = parse_csv_input(args.input)
|
|
with open(args.output, 'w') as htmlfile:
|
htmlfile.write(generate_result(benchmark_info, data))
|
|
|
# -----------------
|
# Templates below
|
|
MAIN_TEMPLATE = """<!doctype html>
|
<html lang='en-US'>
|
<head>
|
<meta http-equiv='Content-Type' content='text/html; charset=utf-8'>
|
<script src='https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js'></script>
|
<script>{jsdeps}</script>
|
<title>MLTS results</title>
|
<style>
|
.results {{
|
border-collapse: collapse;
|
width: 100%;
|
}}
|
.results td, .results th {{
|
border: 1px solid #ddd;
|
padding: 6px;
|
}}
|
.results tbody.values {{
|
border-bottom: 8px solid #333;
|
}}
|
span.better {{
|
color: #070;
|
}}
|
span.worse {{
|
color: #700;
|
}}
|
span.same {{
|
color: #000;
|
}}
|
.results tr:nth-child(even) {{background-color: #eee;}}
|
.results tr:hover {{background-color: #ddd;}}
|
.results th {{
|
padding: 10px;
|
font-weight: bold;
|
text-align: left;
|
background-color: #333;
|
color: white;
|
}}
|
.results tr.failed {{
|
background-color: #ffc4ca;
|
}}
|
.group {{
|
padding-top: 25px;
|
}}
|
.group_name {{
|
padding-left: 10px;
|
font-size: 140%;
|
font-weight: bold;
|
}}
|
.latency_results {{
|
padding: 10px;
|
border: 1px solid #ddd;
|
overflow: hidden;
|
}}
|
.latency_with_baseline {{
|
padding: 10px;
|
border: 1px solid #ddd;
|
overflow: hidden;
|
}}
|
</style>
|
</head>
|
<body>
|
{device_info}
|
{results_list}
|
</body>
|
</html>"""
|
|
DEVICE_INFO_TEMPLATE = """<div id='device_info'>
|
Benchmark for {device_info}, started at {benchmark_time}
|
</div>"""
|
|
|
RESULT_GROUP_TEMPLATE = """<div class="group">
|
<div class="group_name">{group_name}</div>
|
<table class="results">
|
<tr>
|
<th>Name</th>
|
<th>Backend</th>
|
<th>Iterations</th>
|
<th>Test set size</th>
|
<th>Average latency ms</th>
|
{accuracy_headers}
|
</tr>
|
{results}
|
</table>
|
{validation_errors}
|
{latency_graphs}
|
</div>"""
|
|
|
VALIDATION_ERRORS_TEMPLATE = """
|
<table class="results">
|
<tr>
|
<th>Name</th>
|
<th>Backend</th>
|
<th>Error</th>
|
</tr>
|
{results}
|
</table>"""
|
VALIDATION_ERRORS_ENTRY_TEMPLATE = """
|
<tr class="failed">
|
<td>{name}</td>
|
<td>{backend}</td>
|
<td>{error}</td>
|
</tr>
|
"""
|
|
LATENCY_GRAPHS_TEMPLATE = """
|
<div class="latency_results">
|
{results}
|
</div>
|
<div style="clear: left;"></div>
|
"""
|
|
LATENCY_GRAPH_ENTRY_WITH_BL_TEMPLATE = """
|
<div class="latency_with_baseline" style="float: left;">
|
<b>{name}</b>
|
{baseline}
|
{result}
|
</div>
|
"""
|
|
LATENCY_GRAPH_ENTRY_TEMPLATE = """
|
<div class="latency_result" style='width: 350px;'>
|
{backend}
|
<canvas id='latency_chart{i}' class='latency_chart'></canvas>
|
<script>
|
$(function() {{
|
var freqData = {{
|
labels: {freq_data[0]},
|
datasets: [{{
|
data: {freq_data[1]},
|
backgroundColor: 'rgba(255, 99, 132, 0.6)',
|
borderColor: 'rgba(255, 0, 0, 0.6)',
|
borderWidth: 1,
|
}}]
|
}};
|
var ctx = $('#latency_chart{i}')[0].getContext('2d');
|
window.latency_chart{i} = new Chart(ctx,
|
{{
|
type: 'bar',
|
data: freqData,
|
options: {{
|
responsive: true,
|
title: {{
|
display: false,
|
text: 'Latency frequency'
|
}},
|
legend: {{
|
display: false
|
}},
|
scales: {{
|
xAxes: [ {{
|
barPercentage: 1.0,
|
categoryPercentage: 0.9,
|
}}],
|
yAxes: [{{
|
scaleLabel: {{
|
display: false,
|
labelString: 'Iterations Count'
|
}}
|
}}]
|
}}
|
}}
|
}});
|
}});
|
</script>
|
</div>
|
"""
|
|
|
RESULT_ENTRY_WITH_BASELINE_TEMPLATE = """
|
<tbody class="values">
|
{baseline}
|
{other}
|
</tbody>
|
"""
|
RESULT_ENTRY_TEMPLATE = """
|
<tr class={row_class}>
|
<td>{name}</td>
|
<td>{backend}</td>
|
<td>{iterations:d}</td>
|
<td>{testset_size:d}</td>
|
<td>{avg_ms}</td>
|
{accuracy_values}
|
</tr>"""
|
|
LATENCY_BASELINE_TEMPLATE = """{val:.2f}ms"""
|
LATENCY_DIFF_TEMPLATE = """{val:.2f}ms <span class='{span}'>
|
({diff_val:.2f}ms, {diff:.1f}%)</span>"""
|
|
|
ACCURACY_HEADERS_TOPK_TEMPLATE = """
|
<th>Top 1</th>
|
<th>Top 2</th>
|
<th>Top 3</th>
|
<th>Top 4</th>
|
<th>Top 5</th>
|
"""
|
ACCURACY_VALUES_TOPK_TEMPLATE = """
|
<td>{top1}</td>
|
<td>{top2}</td>
|
<td>{top3}</td>
|
<td>{top4}</td>
|
<td>{top5}</td>
|
"""
|
TOPK_BASELINE_TEMPLATE = """{val:.3f}%"""
|
TOPK_DIFF_TEMPLATE = """{val:.3f}% <span class='{span}'>({diff:.1f}%)</span>"""
|
|
|
ACCURACY_HEADERS_MELCEPLOGF0_TEMPLATE = """
|
<th>Max log(F0) error</th>
|
<th>Max Mel Cep distortion</th>
|
<th>Max scalar error</th>
|
"""
|
|
ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE = """
|
<td>{max_log_f0}</td>
|
<td>{max_mel_cep_distortion}</td>
|
<td>{max_single_error}</td>
|
"""
|
|
MELCEPLOGF0_BASELINE_TEMPLATE = """{val:.2E}"""
|
MELCEPLOGF0_DIFF_TEMPLATE = \
|
"""{val:.2E} <span class='{span}'>({diff:.1f}%)</span>"""
|
|
|
ACCURACY_HEADERS_PHONE_ERROR_RATE_TEMPLATE = """
|
<th>Max phone error rate</th>
|
<th>Max scalar error</th>
|
"""
|
|
ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE = """
|
<td>{max_phone_error_rate}</td>
|
<td>{max_single_error}</td>
|
"""
|
|
PHONE_ERROR_RATE_BASELINE_TEMPLATE = """{val:.3f}"""
|
PHONE_ERROR_RATE_DIFF_TEMPLATE = \
|
"""{val:.3f} <span class='{span}'>({diff:.1f}%)</span>"""
|
|
|
ACCURACY_HEADERS_BASIC_TEMPLATE = """
|
<th>Max single scalar error</th>
|
"""
|
|
|
ACCURACY_VALUES_BASIC_TEMPLATE = """
|
<td>{max_single_error:.2f}</td>
|
"""
|
|
CHART_JS_FILE = 'Chart.bundle.min.js'
|
|
if __name__ == '__main__':
|
main()
|
