Add maintenance script to automate test rebaselining. NFC

tools/maint/rebaseline_tests.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# Copyright 2024 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+"""Automatically rebaseline tests that have codesize expectations and create
+a git commit containing the resulting changes along with readable details of
+the generated changes.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import statistics
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+root_dir = os.path.dirname(os.path.dirname(script_dir))
+
+sys.path.insert(0, root_dir)
+from tools import utils
+
+TESTS = [
+  'browser.test_small_js_flags',
+  'other.test_INCOMING_MODULE_JS_API',
+  'other.*code_size*',
+  'other.*codesize*'
+]
+
+
+def run(cmd, **args):
+  return subprocess.check_output(cmd, text=True, cwd=root_dir, **args)
+
+
+all_deltas = []
+
+
+def process_changed_file(filename):
+  content = open(filename).read()
+  old_content = run(['git', 'show', f'HEAD:{filename}'])
+  print(f'processing {filename}')
+  if len(content.splitlines()) == 1:
+    size = int(content.strip())
+    old_size = int(old_content.strip())
+  else:
+    try:
+      current_json = json.loads(content)
+      old_json = json.loads(old_content)
+    except Exception:
+      print(f'{filename}: Unable to parse json content. Unsupported file format?')
+      sys.exit(1)
+    size = current_json['total']
+    old_size = old_json['total']
+
+  filename = utils.removeprefix(filename, 'test/')
+  delta = size - old_size
+  percent_delta = delta * 100 / old_size
+  all_deltas.append(percent_delta)
+  return f'{filename}: {old_size} => {size} [{delta:+} bytes / {percent_delta:+.2f}%]\n'
+
+
+def main(argv):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-s', '--skip-tests', action='store_true', help="don't actually run the tests, just analyze the existing results")
+  args = parser.parse_args()
+
+  if not args.skip_tests:
+    if run(['git', 'status', '-uno', '--porcelain']).strip():
+      print('tree is not clean')
+      return 1
+
+    subprocess.check_call(['test/runner', '--rebaseline', '--browser=0'] + TESTS, cwd=root_dir)
+
+  if not run(['git', 'status', '-uno', '--porcelain']):
+    print('no updates found')
+    return 1
+
+  output = run(['git', 'status', '-uno', '--porcelain'])
+  filenames = []
+  for line in output.splitlines():
+    status, filename = line.strip().split(' ', 1)
+    filenames.append(filename)
+
+
+  commit_message = f'''
+Automatic rebaseline of codesize expectations. NFC
+
+This is an automatic change generated by tools/maint/rebaseline_tests.py.
+
+The following ({len(filenames)}) test expectation files were updated by
+running the tests with `--rebaseline`:
+
+'''
+
+  for file in filenames:
+    commit_message += process_changed_file(file)
+
+  commit_message += f'\nAverage change: {statistics.mean(all_deltas):+.2f}% ({min(all_deltas):+.2f}% - {max(all_deltas):+.2f}%)\n'
+
+  run(['git', 'checkout', '-b', 'rebaseline_tests'])
+  run(['git', 'add', '-u', '.'])
+  run(['git', 'commit', '-F', '-'], input=commit_message)
+
+  print(commit_message)
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))