Skip to content

Commit

Permalink
Add retries to the FTL integration test runs (#876)
Browse files Browse the repository at this point in the history
* Add retries to the FTL integration test runs

* Fix ModuleNotFound problems

* Make Analytics intentionally fail

* Revert "Make Analytics intentionally fail"

This reverts commit 6969172.
  • Loading branch information
a-maurice authored Oct 11, 2023
1 parent 3061aef commit 4c82830
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 12 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/integration_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -526,6 +526,8 @@ jobs:
testapp_dir: testapps
test_type: "game-loop"
test_devices: ${{ matrix.device_detail }}
max_attempts: 3
validator: ${GITHUB_WORKSPACE}/scripts/gha/read_ftl_test_result.py
- name: Read FTL Test Result
if: ${{ matrix.device_type == 'real' && !cancelled() }}
timeout-minutes: 60
Expand Down
1 change: 1 addition & 0 deletions scripts/gha/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# Copyright 2023 Google
1 change: 1 addition & 0 deletions scripts/gha/integration_testing/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# Copyright 2023 Google
43 changes: 31 additions & 12 deletions scripts/gha/read_ftl_test_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@
import attr
import subprocess
import json
import sys

from absl import app
from absl import flags
from absl import logging

sys.path.append(os.path.dirname(os.path.realpath(__file__)))
from integration_testing import test_validation
from integration_testing import gcs

Expand All @@ -59,18 +61,7 @@ def main(argv):
test_result = json.loads(FLAGS.test_result)
tests = []
for app in test_result.get("apps"):
app_path = app.get("testapp_path")
return_code = app.get("return_code")
logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code))
if return_code == 0:
gcs_dir = app.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://")
logging.info("gcs_dir: %s" % gcs_dir)
logs = _get_testapp_log_text_from_gcs(gcs_dir)
logging.info("Test result: %s", logs)
tests.append(Test(testapp_path=app_path, logs=logs))
else:
logging.error("Test failed: %s", app)
tests.append(Test(testapp_path=app_path, logs=None))
tests.append(_parse_testapp_to_test(app))

(output_dir, file_name) = os.path.split(os.path.abspath(FLAGS.output_path))
return test_validation.summarize_test_results(
Expand All @@ -80,6 +71,22 @@ def main(argv):
file_name=file_name)


def _parse_testapp_to_test(app):
"""Parsed the given testapp run into a Test object."""
app_path = app.get("testapp_path")
return_code = app.get("return_code")
logging.info("testapp: %s\nreturn code: %s" % (app_path, return_code))
if return_code == 0:
gcs_dir = app.get("raw_result_link").replace("https://console.developers.google.com/storage/browser/", "gs://")
logging.info("gcs_dir: %s" % gcs_dir)
logs = _get_testapp_log_text_from_gcs(gcs_dir)
logging.info("Test result: %s", logs)
return Test(testapp_path=app_path, logs=logs)
else:
logging.error("Test failed: %s", app)
return Test(testapp_path=app_path, logs=None)


def _get_testapp_log_text_from_gcs(gcs_path):
"""Gets the testapp log text generated by game loops."""
try:
Expand Down Expand Up @@ -130,6 +137,18 @@ def _gcs_read_file(gcs_path):
return result.stdout



def validate(test_summary):
"""Validates a given test summary, assuming it is from a Unity testapp."""
if not test_summary:
logging.error("Nothing to be validate! Please provide test_summary")
return False

test = _parse_testapp_to_test(test_summary)
result = test_validation.validate_results(test.logs, "unity")
return result.complete and result.fails == 0


if __name__ == '__main__':
flags.mark_flag_as_required("test_result")
flags.mark_flag_as_required("output_path")
Expand Down

0 comments on commit 4c82830

Please sign in to comment.