diff --git a/tools/maint/rebaseline_tests.py b/tools/maint/rebaseline_tests.py new file mode 100755 index 0000000000000..8c7ec612fd965 --- /dev/null +++ b/tools/maint/rebaseline_tests.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# Copyright 2024 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import argparse +import json +import os +import subprocess +import statistics +import sys + +script_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.dirname(os.path.dirname(script_dir)) + +sys.path.insert(0, root_dir) +from tools import utils + +TESTS = [ + 'browser.test_small_js_flags', + 'other.test_INCOMING_MODULE_JS_API', + 'other.*code_size*', + 'other.*codesize*' +] + + +def run(cmd, **args): + return subprocess.check_output(cmd, text=True, cwd=root_dir, **args) + + +all_deltas = [] + + +def process_changed_file(filename): + content = open(filename).read() + old_content = run(['git', 'show', f'HEAD:{filename}']) + print(f'processing {filename}') + if len(content.splitlines()) == 1: + size = int(content.strip()) + old_size = int(old_content.strip()) + else: + try: + current_json = json.loads(content) + old_json = json.loads(old_content) + except Exception: + print(f'{filename}: Unable to passe json content. Unsupported file format?') + sys.exit(1) + size = current_json['total'] + old_size = old_json['total'] + + filename = utils.removeprefix(filename, 'test/') + delta = size - old_size + percent_delta = delta * 100 / old_size + all_deltas.append(percent_delta) + return f'{filename}: {old_size} => {size} [{delta:+} bytes / {percent_delta:+.2f}%]\n' + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--skip-tests', action='store_true', help="don't actualy run the tests, just analyize the existing results") + args = parser.parse_args() + + if not args.skip_tests: + if run(['git', 'status', '-uno', '--porcelain']).strip(): + print('tree is not clean') + return 1 + + subprocess.check_call(['test/runner', '--rebaseline', '--browser=0'] + TESTS, cwd=root_dir) + + if not run(['git', 'status', '-uno', '--porcelain']): + print('no updates found') + return 1 + + output = run(['git', 'status', '-uno', '--porcelain']) + filenames = [] + for line in output.splitlines(): + status, filename = line.strip().split(' ', 1) + filenames.append(filename) + + + commit_message = f''' +Automatic rebaseline of codesize expectations. NFC + +This is an automatic change generated by tools/maint/rebaseline_tests.py. + +The following ({len(filenames)}) test expectation files were updated by +running the tests with `--rebaseline`: + +''' + + for file in filenames: + commit_message += process_changed_file(file) + + commit_message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}%\n' + + run(['git', 'checkout', '-b', 'rebaseline_tests']) + run(['git', 'add', '-u', '.']) + run(['git', 'commit', '-F', '-'], input=commit_message) + + print(commit_message) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv))