You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
213 lines
6.7 KiB
213 lines
6.7 KiB
#!/usr/bin/env python |
|
|
|
import csv |
|
import itertools |
|
import math |
|
import matplotlib |
|
import optparse |
|
import os |
|
import subprocess |
|
import sys |
|
|
|
|
|
class WorkingDirectory: |
|
"Scoped context for changing working directory" |
|
def __init__(self, working_dir): |
|
self.original_dir = os.getcwd() |
|
self.working_dir = working_dir |
|
|
|
def __enter__(self): |
|
sys.stderr.write("Entering directory `%s'\n" % self.working_dir) |
|
os.chdir(self.working_dir) |
|
return self |
|
|
|
def __exit__(self, type, value, traceback): |
|
sys.stderr.write("Leaving directory `%s'\n" % self.working_dir) |
|
os.chdir(self.original_dir) |
|
|
|
|
|
def filename(n): |
|
"Filename for a generated file with n statements" |
|
return 'gen%d.ttl' % n |
|
|
|
|
|
def gen(sp2b_dir, n_min, n_max, step): |
|
"Generate files with n_min ... n_max statements if they are not present" |
|
with WorkingDirectory(sp2b_dir) as dir: |
|
for n in range(n_min, n_max + step, step): |
|
out_path = os.path.join(dir.original_dir, 'build', filename(n)) |
|
if not os.path.exists(out_path): |
|
subprocess.call(['./sp2b_gen', '-t', str(n), out_path]) |
|
|
|
|
|
def write_header(results, progs): |
|
"Write the header line for TSV output" |
|
results.write('n') |
|
for prog in progs: |
|
results.write('\t' + os.path.basename(prog.split()[0])) |
|
results.write('\n') |
|
|
|
|
|
def parse_time(report): |
|
"Return user time and max RSS from a /usr/bin/time -v report" |
|
time = memory = None |
|
for line in report.split('\n'): |
|
if line.startswith('\tUser time'): |
|
time = float(line[line.find(':') + 1:]) |
|
elif line.startswith('\tMaximum resident set'): |
|
memory = float(line[line.find(':') + 1:]) * 1024 |
|
|
|
return (time, memory) |
|
|
|
|
|
def get_dashes(): |
|
"Generator for plot line dash patterns" |
|
dash = 2.0 |
|
space = dot = 0.75 |
|
|
|
yield [] # Solid |
|
yield [dash, space] # Dashed |
|
yield [dot, space] # Dotted |
|
|
|
# Dash-dots, with increasing number of dots for each line |
|
for i in itertools.count(2): |
|
yield [dash, space] + [dot, space] * (i - 1) |
|
|
|
|
|
def plot(in_file, out_filename, x_label, y_label, y_max=None): |
|
"Plot a TSV file as SVG" |
|
|
|
matplotlib.use('agg') |
|
import matplotlib.pyplot as plt |
|
|
|
fig_height = 4.0 |
|
dashes = get_dashes() |
|
markers = itertools.cycle(['o', 's', 'v', 'D', '*', 'p', 'P', 'h', 'X']) |
|
|
|
reader = csv.reader(in_file, delimiter='\t') |
|
header = next(reader) |
|
cols = [x for x in zip(*list(reader))] |
|
|
|
plt.clf() |
|
fig = plt.figure(figsize=(fig_height * math.sqrt(2), fig_height)) |
|
ax = fig.add_subplot(111) |
|
|
|
ax.set_xlabel(x_label) |
|
ax.set_ylabel(y_label) |
|
|
|
if y_max is not None: |
|
ax.set_ylim([0.0, y_max]) |
|
|
|
ax.grid(linewidth=0.25, linestyle=':', color='0', dashes=[0.2, 1.6]) |
|
ax.ticklabel_format(style='sci', scilimits=(4, 0), useMathText=True) |
|
ax.tick_params(axis='both', width=0.75) |
|
|
|
x = list(map(float, cols[0])) |
|
for i, y in enumerate(cols[1::]): |
|
ax.plot(x, |
|
list(map(float, y)), |
|
label=header[i + 1], |
|
marker=next(markers), |
|
dashes=next(dashes), |
|
markersize=3.0, |
|
linewidth=1.0) |
|
|
|
plt.legend() |
|
plt.savefig(out_filename, bbox_inches='tight', pad_inches=0.025) |
|
plt.close() |
|
sys.stderr.write('wrote {}\n'.format(out_filename)) |
|
|
|
|
|
def run(progs, n_min, n_max, step): |
|
"Benchmark each program with n_min ... n_max statements" |
|
with WorkingDirectory('build'): |
|
results = {'time': open('serdi-time.txt', 'w'), |
|
'throughput': open('serdi-throughput.txt', 'w'), |
|
'memory': open('serdi-memory.txt', 'w')} |
|
|
|
# Write TSV header for all output files |
|
for name, f in results.items(): |
|
write_header(f, progs) |
|
|
|
for n in range(n_min, n_max + step, step): |
|
# Add first column (n) to rows |
|
rows = {} |
|
for name, _ in results.items(): |
|
rows[name] = [str(n)] |
|
|
|
# Run each program and fill rows with measurements |
|
for prog in progs: |
|
cmd = '/usr/bin/time -v ' + prog + ' ' + filename(n) |
|
with open(filename(n) + '.out', 'w') as out: |
|
sys.stderr.write(cmd + '\n') |
|
proc = subprocess.Popen( |
|
cmd.split(), stdout=out, stderr=subprocess.PIPE) |
|
|
|
time, memory = parse_time(proc.communicate()[1].decode()) |
|
rows['time'] += ['%.07f' % time] |
|
rows['throughput'] += ['%d' % (n / time)] |
|
rows['memory'] += [str(memory)] |
|
|
|
# Write rows to output files |
|
for name, f in results.items(): |
|
f.write('\t'.join(rows[name]) + '\n') |
|
|
|
for name, f in results.items(): |
|
tsv_filename = 'serdi-%s.txt' % name |
|
sys.stderr.write('wrote %s\n' % tsv_filename) |
|
|
|
|
|
def plot_results(): |
|
"Plot all benchmark results" |
|
with WorkingDirectory('build'): |
|
plot(open('serdi-time.txt', 'r'), 'serdi-time.svg', |
|
'Statements', 'Time (s)') |
|
plot(open('serdi-throughput.txt', 'r'), 'serdi-throughput.svg', |
|
'Statements', 'Statements / s') |
|
plot(open('serdi-memory.txt', 'r'), 'serdi-memory.svg', |
|
'Statements', 'Bytes') |
|
|
|
|
|
if __name__ == "__main__": |
|
class OptParser(optparse.OptionParser): |
|
def format_epilog(self, formatter): |
|
return self.expand_prog_name(self.epilog) |
|
|
|
opt = OptParser( |
|
usage='%prog [OPTION]... SP2B_DIR', |
|
description='Benchmark RDF reading and writing commands\n', |
|
epilog=''' |
|
Example: |
|
%prog --max 100000 \\ |
|
--run 'rapper -i turtle -o turtle' \\ |
|
--run 'riot --output=ttl' \\ |
|
--run 'rdfpipe -i turtle -o turtle' /path/to/sp2b/src/ |
|
''') |
|
|
|
opt.add_option('--max', type='int', default=1000000, |
|
help='maximum triple count') |
|
opt.add_option('--run', type='string', action='append', default=[], |
|
help='additional command to run (input file is appended)') |
|
opt.add_option('--no-generate', action='store_true', |
|
help='do not generate data') |
|
opt.add_option('--no-execute', action='store_true', |
|
help='do not run benchmarks') |
|
opt.add_option('--no-plot', action='store_true', |
|
help='do not plot benchmarks') |
|
|
|
(options, args) = opt.parse_args() |
|
if len(args) != 1: |
|
opt.print_usage() |
|
sys.exit(1) |
|
|
|
progs = ['serdi -b -f -i turtle -o turtle'] + options.run |
|
min_n = int(options.max / 10) |
|
max_n = options.max |
|
step = min_n |
|
|
|
if not options.no_generate: |
|
gen(str(args[0]), min_n, max_n, step) |
|
if not options.no_execute: |
|
run(progs, min_n, max_n, step) |
|
if not options.no_plot: |
|
plot_results()
|
|
|