diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..34ba4d0 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +benchmark/ \ No newline at end of file diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100755 index 0000000..7600554 --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the the Apache License v2.0 with LLVM Exceptions (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://llvm.org/LICENSE.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run benchmark of the various steps in the pre-merge tests. + +This can be used to tune the build times. +""" + +import argparse +import csv +import datetime +import os +import shutil +import subprocess +import sys +from typing import Optional, Dict, List + + +class Cmd: + """Command to be executed as part of the benchmark. + + If name is not set, the results will not be logged to the result file. + """ + + def __init__(self, cmd: str, title: str = None): + self.cmd = cmd # type: str + self.title = title # type: Optional[str] + self.execution_time = None # type: Optional[datetime.timedelta] + + +# commands for the benchmark +COMMANDS = [ + Cmd('git clone https://github.com/llvm/llvm-project .', 'git clone'), + Cmd('git checkout release/10.x', 'git checkout release'), + Cmd('git checkout {commit}'), + Cmd('git pull', 'git pull'), + Cmd('{pmt_root_path}/scripts/run_cmake.py detect', 'cmake autotdetect'), + Cmd('{pmt_root_path}/scripts/run_ninja.py all', 'ninja all 1st'), + Cmd('{pmt_root_path}/scripts/run_ninja.py all', 'ninja all 2nd'), + Cmd('{pmt_root_path}/scripts/run_ninja.py check-all', 'ninja check-all 1st'), + Cmd('{pmt_root_path}/scripts/run_ninja.py check-all', 'ninja check-all 2nd'), +] + + +def run_benchmark(commit: str, name: str, result_file_path: str, workdir: str, pmt_root_path: str): + """Tun the benchmark, write the results to a file.""" + print('Usingn workdir {}'.format(workdir)) + print('Using scripts from {}'.format(pmt_root_path)) + if os.path.exists(workdir): + shutil.rmtree(workdir) + os.makedirs(workdir) + cmd_parameters = { + 'pmt_root_path': pmt_root_path, + 'commit': commit, + } + for command in COMMANDS: + run_cmd(command, cmd_parameters, workdir) + write_results(COMMANDS, result_file_path, name) + + +def write_results(commands: List[Cmd], result_file_path : str, name: str): + fieldnames = ['name'] + fieldnames.extend([cmd.title for cmd in COMMANDS if cmd.title is not None]) + exists = os.path.exists(result_file_path) + with open(result_file_path, 'a') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames, dialect=csv.excel) + if not exists: + writer.writeheader() + row = { + 'name': name + } + for command in commands: + row[command.title] = command.execution_time.total_seconds() + writer.writerow(row) + print('Benchmark completed.') + + +def run_cmd(command: Cmd, cmd_parameters: Dict[str, str], workdir: str): + """Run a single command.""" + cmdline = command.cmd.format(**cmd_parameters) + print('Running: {}'.format(cmdline)) + start_time = datetime.datetime.now() + proc = subprocess.Popen(cmdline, shell=True, cwd=workdir, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + if proc.poll() != 0: + print(stdout) + print(stderr) + print('Benchmark failed.') + sys.exit(1) + end_time = datetime.datetime.now() + command.execution_time = end_time - start_time + print(' Execution time was: {}'.format(command.execution_time)) + + +if __name__ == '__main__': + pmt_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + parser = argparse.ArgumentParser( + description='Benchmark for LLVM pre-merge tests.') + parser.add_argument('--commit', type=str, default='master', help="LLVM commit to run this benchmark on.") + parser.add_argument('--result-file', type=str, default='pmt-benchmark.csv', + help="path to CSV file where to store the benchmark results") + parser.add_argument('--workdir', type=str, default=os.path.join(os.getcwd(), 'benchmark'), + help='Folder to store the LLVM checkout.') + parser.add_argument('name', type=str, nargs='?', help="name for the benchmark") + args = parser.parse_args() + run_benchmark(args.commit, args.name, args.result_file, args.workdir, pmt_root)