123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354 |
- #!/usr/bin/python
- """Utility to benchmark the generated source files"""
- # Copyright Abel Sinkovics (abel@sinkovics.hu) 2016.
- # Distributed under the Boost Software License, Version 1.0.
- # (See accompanying file LICENSE_1_0.txt or copy at
- # http://www.boost.org/LICENSE_1_0.txt)
- import argparse
- import os
- import subprocess
- import json
- import math
- import platform
- import matplotlib
- import random
- import re
- import time
- import psutil
- import PIL
- matplotlib.use('Agg')
- import matplotlib.pyplot # pylint:disable=I0011,C0411,C0412,C0413
- def benchmark_command(cmd, progress):
- """Benchmark one command execution"""
- full_cmd = '/usr/bin/time --format="%U %M" {0}'.format(cmd)
- print '{0:6.2f}% Running {1}'.format(100.0 * progress, full_cmd)
- (_, err) = subprocess.Popen(
- ['/bin/sh', '-c', full_cmd],
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- ).communicate('')
- values = err.strip().split(' ')
- if len(values) == 2:
- try:
- return (float(values[0]), float(values[1]))
- except: # pylint:disable=I0011,W0702
- pass # Handled by the code after the "if"
- print err
- raise Exception('Error during benchmarking')
- def benchmark_file(
- filename, compiler, include_dirs, (progress_from, progress_to),
- iter_count, extra_flags = ''):
- """Benchmark one file"""
- time_sum = 0
- mem_sum = 0
- for nth_run in xrange(0, iter_count):
- (time_spent, mem_used) = benchmark_command(
- '{0} -std=c++11 {1} -c {2} {3}'.format(
- compiler,
- ' '.join('-I{0}'.format(i) for i in include_dirs),
- filename,
- extra_flags
- ),
- (
- progress_to * nth_run + progress_from * (iter_count - nth_run)
- ) / iter_count
- )
- os.remove(os.path.splitext(os.path.basename(filename))[0] + '.o')
- time_sum = time_sum + time_spent
- mem_sum = mem_sum + mem_used
- return {
- "time": time_sum / iter_count,
- "memory": mem_sum / (iter_count * 1024)
- }
- def compiler_info(compiler):
- """Determine the name + version of the compiler"""
- (out, err) = subprocess.Popen(
- ['/bin/sh', '-c', '{0} -v'.format(compiler)],
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- ).communicate('')
- gcc_clang = re.compile('(gcc|clang) version ([0-9]+(\\.[0-9]+)*)')
- for line in (out + err).split('\n'):
- mtch = gcc_clang.search(line)
- if mtch:
- return mtch.group(1) + ' ' + mtch.group(2)
- return compiler
- def string_char(char):
- """Turn the character into one that can be part of a filename"""
- return '_' if char in [' ', '~', '(', ')', '/', '\\'] else char
- def make_filename(string):
- """Turn the string into a filename"""
- return ''.join(string_char(c) for c in string)
- def files_in_dir(path, extension):
- """Enumartes the files in path with the given extension"""
- ends = '.{0}'.format(extension)
- return (f for f in os.listdir(path) if f.endswith(ends))
- def format_time(seconds):
- """Format a duration"""
- minute = 60
- hour = minute * 60
- day = hour * 24
- week = day * 7
- result = []
- for name, dur in [
- ('week', week), ('day', day), ('hour', hour),
- ('minute', minute), ('second', 1)
- ]:
- if seconds > dur:
- value = seconds // dur
- result.append(
- '{0} {1}{2}'.format(int(value), name, 's' if value > 1 else '')
- )
- seconds = seconds % dur
- return ' '.join(result)
- def benchmark(src_dir, compiler, include_dirs, iter_count):
- """Do the benchmarking"""
- files = list(files_in_dir(src_dir, 'cpp'))
- random.shuffle(files)
- has_string_templates = True
- string_template_file_cnt = sum(1 for file in files if 'bmp' in file)
- file_count = len(files) + string_template_file_cnt
- started_at = time.time()
- result = {}
- for filename in files:
- progress = len(result)
- result[filename] = benchmark_file(
- os.path.join(src_dir, filename),
- compiler,
- include_dirs,
- (float(progress) / file_count, float(progress + 1) / file_count),
- iter_count
- )
- if 'bmp' in filename and has_string_templates:
- try:
- temp_result = benchmark_file(
- os.path.join(src_dir, filename),
- compiler,
- include_dirs,
- (float(progress + 1) / file_count, float(progress + 2) / file_count),
- iter_count,
- '-Xclang -fstring-literal-templates'
- )
- result[filename.replace('bmp', 'slt')] = temp_result
- except:
- has_string_templates = False
- file_count -= string_template_file_cnt
- print 'Stopping the benchmarking of string literal templates'
- elapsed = time.time() - started_at
- total = float(file_count * elapsed) / len(result)
- print 'Elapsed time: {0}, Remaining time: {1}'.format(
- format_time(elapsed),
- format_time(total - elapsed)
- )
- return result
- def plot(values, mode_names, title, (xlabel, ylabel), out_file):
- """Plot a diagram"""
- matplotlib.pyplot.clf()
- for mode, mode_name in mode_names.iteritems():
- vals = values[mode]
- matplotlib.pyplot.plot(
- [x for x, _ in vals],
- [y for _, y in vals],
- label=mode_name
- )
- matplotlib.pyplot.title(title)
- matplotlib.pyplot.xlabel(xlabel)
- matplotlib.pyplot.ylabel(ylabel)
- if len(mode_names) > 1:
- matplotlib.pyplot.legend()
- matplotlib.pyplot.savefig(out_file)
- def mkdir_p(path):
- """mkdir -p path"""
- try:
- os.makedirs(path)
- except OSError:
- pass
- def configs_in(src_dir):
- """Enumerate all configs in src_dir"""
- for filename in files_in_dir(src_dir, 'json'):
- with open(os.path.join(src_dir, filename), 'rb') as in_f:
- yield json.load(in_f)
- def byte_to_gb(byte):
- """Convert bytes to GB"""
- return byte / (1024.0 * 1024 * 1024)
- def join_images(img_files, out_file):
- """Join the list of images into the out file"""
- images = [PIL.Image.open(f) for f in img_files]
- joined = PIL.Image.new(
- 'RGB',
- (sum(i.size[0] for i in images), max(i.size[1] for i in images))
- )
- left = 0
- for img in images:
- joined.paste(im=img, box=(left, 0))
- left = left + img.size[0]
- joined.save(out_file)
- def plot_temp_diagrams(config, results, temp_dir):
- """Plot temporary diagrams"""
- display_name = {
- 'time': 'Compilation time (s)',
- 'memory': 'Compiler memory usage (MB)',
- }
- files = config['files']
- img_files = []
- if any('slt' in result for result in results) and 'bmp' in files.values()[0]:
- config['modes']['slt'] = 'Using BOOST_METAPARSE_STRING with string literal templates'
- for f in files.values():
- f['slt'] = f['bmp'].replace('bmp', 'slt')
- for measured in ['time', 'memory']:
- mpts = sorted(int(k) for k in files.keys())
- img_files.append(os.path.join(temp_dir, '_{0}.png'.format(measured)))
- plot(
- {
- m: [(x, results[files[str(x)][m]][measured]) for x in mpts]
- for m in config['modes'].keys()
- },
- config['modes'],
- display_name[measured],
- (config['x_axis_label'], display_name[measured]),
- img_files[-1]
- )
- return img_files
- def plot_diagram(config, results, images_dir, out_filename):
- """Plot one diagram"""
- img_files = plot_temp_diagrams(config, results, images_dir)
- join_images(img_files, out_filename)
- for img_file in img_files:
- os.remove(img_file)
- def plot_diagrams(results, configs, compiler, out_dir):
- """Plot all diagrams specified by the configs"""
- compiler_fn = make_filename(compiler)
- total = psutil.virtual_memory().total # pylint:disable=I0011,E1101
- memory = int(math.ceil(byte_to_gb(total)))
- images_dir = os.path.join(out_dir, 'images')
- for config in configs:
- out_prefix = '{0}_{1}'.format(config['name'], compiler_fn)
- plot_diagram(
- config,
- results,
- images_dir,
- os.path.join(images_dir, '{0}.png'.format(out_prefix))
- )
- with open(
- os.path.join(out_dir, '{0}.qbk'.format(out_prefix)),
- 'wb'
- ) as out_f:
- qbk_content = """{0}
- Measured on a {2} host with {3} GB memory. Compiler used: {4}.
- [$images/metaparse/{1}.png [width 100%]]
- """.format(config['desc'], out_prefix, platform.platform(), memory, compiler)
- out_f.write(qbk_content)
- def main():
- """The main function of the script"""
- desc = 'Benchmark the files generated by generate.py'
- parser = argparse.ArgumentParser(description=desc)
- parser.add_argument(
- '--src',
- dest='src_dir',
- default='generated',
- help='The directory containing the sources to benchmark'
- )
- parser.add_argument(
- '--out',
- dest='out_dir',
- default='../../doc',
- help='The output directory'
- )
- parser.add_argument(
- '--include',
- dest='include',
- default='include',
- help='The directory containing the headeres for the benchmark'
- )
- parser.add_argument(
- '--boost_headers',
- dest='boost_headers',
- default='../../../..',
- help='The directory containing the Boost headers (the boost directory)'
- )
- parser.add_argument(
- '--compiler',
- dest='compiler',
- default='g++',
- help='The compiler to do the benchmark with'
- )
- parser.add_argument(
- '--repeat_count',
- dest='repeat_count',
- type=int,
- default=5,
- help='How many times a measurement should be repeated.'
- )
- args = parser.parse_args()
- compiler = compiler_info(args.compiler)
- results = benchmark(
- args.src_dir,
- args.compiler,
- [args.include, args.boost_headers],
- args.repeat_count
- )
- plot_diagrams(results, configs_in(args.src_dir), compiler, args.out_dir)
- if __name__ == '__main__':
- main()
|