diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 5bb4c641..e6f4e595 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -526,6 +526,8 @@ jobs: testapp_dir: testapps test_type: "game-loop" test_devices: ${{ matrix.device_detail }} + max_attempts: 3 + validator: ${GITHUB_WORKSPACE}/scripts/gha/read_ftl_test_result.py - name: Read FTL Test Result if: ${{ matrix.device_type == 'real' && !cancelled() }} timeout-minutes: 60 diff --git a/scripts/gha/__init__.py b/scripts/gha/__init__.py new file mode 100644 index 00000000..6bbeaf06 --- /dev/null +++ b/scripts/gha/__init__.py @@ -0,0 +1 @@ +# Copyright 2023 Google diff --git a/scripts/gha/integration_testing/__init__.py b/scripts/gha/integration_testing/__init__.py new file mode 100644 index 00000000..6bbeaf06 --- /dev/null +++ b/scripts/gha/integration_testing/__init__.py @@ -0,0 +1 @@ +# Copyright 2023 Google diff --git a/scripts/gha/read_ftl_test_result.py b/scripts/gha/read_ftl_test_result.py index b1db1796..4c5ddfff 100644 --- a/scripts/gha/read_ftl_test_result.py +++ b/scripts/gha/read_ftl_test_result.py @@ -32,11 +32,13 @@ import attr import subprocess import json +import sys from absl import app from absl import flags from absl import logging +sys.path.append(os.path.dirname(os.path.realpath(__file__))) from integration_testing import test_validation from integration_testing import gcs @@ -59,18 +61,7 @@ def main(argv): test_result = json.loads(FLAGS.test_result) tests = [] for app in test_result.get("apps"): - app_path = app.get("testapp_path") - return_code = app.get("return_code") - logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code)) - if return_code == 0: - gcs_dir = app.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://") - logging.info("gcs_dir: %s" % gcs_dir) - logs = _get_testapp_log_text_from_gcs(gcs_dir) - logging.info("Test result: %s", logs) - tests.append(Test(testapp_path=app_path, logs=logs)) - else: - logging.error("Test failed: %s", app) - tests.append(Test(testapp_path=app_path, logs=None)) + tests.append(_parse_testapp_to_test(app)) (output_dir, file_name) = os.path.split(os.path.abspath(FLAGS.output_path)) return test_validation.summarize_test_results( @@ -80,6 +71,22 @@ def main(argv): file_name=file_name) +def _parse_testapp_to_test(app): + """Parsed the given testapp run into a Test object.""" + app_path = app.get("testapp_path") + return_code = app.get("return_code") + logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code)) + if return_code == 0: + gcs_dir = app.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://") + logging.info("gcs_dir: %s" % gcs_dir) + logs = _get_testapp_log_text_from_gcs(gcs_dir) + logging.info("Test result: %s", logs) + return Test(testapp_path=app_path, logs=logs) + else: + logging.error("Test failed: %s", app) + return Test(testapp_path=app_path, logs=None) + + def _get_testapp_log_text_from_gcs(gcs_path): """Gets the testapp log text generated by game loops.""" try: @@ -130,6 +137,18 @@ def _gcs_read_file(gcs_path): return result.stdout + +def validate(test_summary): + """Validates a given test summary, assuming it is from a Unity testapp.""" + if not test_summary: + logging.error("Nothing to be validate! Please provide test_summary") + return False + + test = _parse_testapp_to_test(test_summary) + result = test_validation.validate_results(test.logs, "unity") + return result.complete and result.fails == 0 + + if __name__ == '__main__': flags.mark_flag_as_required("test_result") flags.mark_flag_as_required("output_path")