From 60e8e08929042193698762ffc81b18c18cc83157 Mon Sep 17 00:00:00 2001 From: Yutong Zhang Date: Tue, 21 Jan 2025 10:00:38 +0800 Subject: [PATCH] Roll out T0 and T1 PR checkers --- .../calculate-instance-numbers.yml | 6 +- .../calculate_instance_number.py | 16 ++- .../impacted_area_testing/get_test_scripts.py | 10 +- azure-pipelines.yml | 107 ++---------------- 4 files changed, 24 insertions(+), 115 deletions(-) diff --git a/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml b/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml index c408230b1f..34ed79a4ac 100644 --- a/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml +++ b/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml @@ -7,6 +7,10 @@ parameters: type: string default: "" + - name: PREPARE_TIME + type: number + default: 30 + steps: - script: | set -x @@ -64,7 +68,7 @@ steps: pip install azure-kusto-data pip install azure-kusto-data azure-identity - INSTANCE_NUMBER=$(python ./.azure-pipelines/impacted_area_testing/calculate_instance_number.py --scripts $(SCRIPTS) --topology ${{ parameters.TOPOLOGY }} --branch ${{ parameters.BUILD_BRANCH }}) + INSTANCE_NUMBER=$(python ./.azure-pipelines/impacted_area_testing/calculate_instance_number.py --scripts $(SCRIPTS) --topology ${{ parameters.TOPOLOGY }} --branch ${{ parameters.BUILD_BRANCH }} --prepare_time ${{ parameters.PREPARE_TIME }}) if [[ $? -ne 0 ]]; then echo "##vso[task.complete result=Failed;]Get instances number fails." diff --git a/.azure-pipelines/impacted_area_testing/calculate_instance_number.py b/.azure-pipelines/impacted_area_testing/calculate_instance_number.py index 0df733963c..7e27f8a9a4 100644 --- a/.azure-pipelines/impacted_area_testing/calculate_instance_number.py +++ b/.azure-pipelines/impacted_area_testing/calculate_instance_number.py @@ -64,7 +64,7 @@ def get_access_token(): raise Exception(f"Failed to get token after {MAX_GET_TOKEN_RETRY_TIMES} attempts") -def main(scripts, topology, branch): +def main(scripts, topology, branch, prepare_time): ingest_cluster = os.getenv("TEST_REPORT_QUERY_KUSTO_CLUSTER_BACKUP") access_token = get_access_token() @@ -107,23 +107,25 @@ def main(scripts, topology, branch): # To get the result for a single time, we need to divide by five # If response.primary_results is None, which means where is no historical data in Kusto, # we will use the default 1800s for a script. + running_time = row["TotalRuntime"] actual_count = row["ActualCount"] # There is no relevant records in Kusto - if actual_count == 0: + if running_time == 0: average_running_time = 1800 else: - average_running_time = row["TotalRuntime"] / actual_count + average_running_time = running_time / actual_count total_running_time += average_running_time scripts_running_time[script] = average_running_time logging.info(f"Time for each test script: {scripts_running_time}") logging.info(f"Total running time: {total_running_time}") # Total running time is calculated by seconds, divide by 60 to get minutes - # For one instance, we plan to assign 90 minutes to run test scripts + # Our goal is to limit the whole PR testing into 120 minutes + # As we need some time to prepare testbeds, the prepare time should be subtracted. # Obtain the number of instances by rounding up the calculation. # To prevent unexpected situations, we set the maximum number of instance - print(min(math.ceil(total_running_time / 60 / 90), MAX_INSTANCE_NUMBER)) + print(min(math.ceil(total_running_time / 60 / (120 - prepare_time)), MAX_INSTANCE_NUMBER)) if __name__ == '__main__': @@ -131,9 +133,11 @@ def main(scripts, topology, branch): parser.add_argument("--topology", help="The topology of testplan", type=str, default="") parser.add_argument("--scripts", help="Test scripts to be executed", type=str, default="") parser.add_argument("--branch", help="Test branch", type=str, default="") + parser.add_argument("--prepare_time", help="Time for preparing testbeds", type=int, default=30) args = parser.parse_args() scripts = args.scripts topology = args.topology branch = args.branch - main(scripts, topology, branch) + prepare_time = args.prepare_time + main(scripts, topology, branch, prepare_time) diff --git a/.azure-pipelines/impacted_area_testing/get_test_scripts.py b/.azure-pipelines/impacted_area_testing/get_test_scripts.py index 71d1face8d..d02615689b 100644 --- a/.azure-pipelines/impacted_area_testing/get_test_scripts.py +++ b/.azure-pipelines/impacted_area_testing/get_test_scripts.py @@ -101,15 +101,7 @@ def collect_scripts_by_topology_type(features: str, location: str) -> dict: except Exception as e: raise Exception('Exception occurred while trying to get topology in {}, error {}'.format(s, e)) - test_scripts = {k: v for k, v in test_scripts_per_topology_checker.items() if v} - - # This is just for the first stage of rolling out - # To avoid the overuse of resource, we will ignore the PR which modifies the common part. - if features == "": - test_scripts.pop("t0_checker") - test_scripts.pop("t1_checker") - - return test_scripts + return {k: v for k, v in test_scripts_per_topology_checker.items() if v} def main(features, location): diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dcdabc4fb3..f3def9e549 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -78,19 +78,19 @@ stages: - job: get_impacted_area displayName: "Get impacted area" timeoutInMinutes: 240 - continueOnError: true + continueOnError: false pool: sonic-ubuntu-1c steps: - template: .azure-pipelines/impacted_area_testing/get-impacted-area.yml - job: impacted_area_t0_elastictest - displayName: "impacted-area-kvmtest-t0 by Elastictest - optional" + displayName: "impacted-area-kvmtest-t0 by Elastictest" dependsOn: get_impacted_area condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0_checker') variables: TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] timeoutInMinutes: 240 - continueOnError: true + continueOnError: false pool: sonic-ubuntu-1c steps: - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml @@ -106,7 +106,6 @@ stages: MAX_WORKER: $(INSTANCE_NUMBER) KVM_IMAGE_BRANCH: $(BUILD_BRANCH) MGMT_BRANCH: "master" - STOP_ON_FAILURE: "False" - job: impacted_area_t0_2vlans_elastictest displayName: "impacted-area-kvmtest-t0_2vlans by Elastictest" @@ -134,19 +133,20 @@ stages: MGMT_BRANCH: "master" - job: impacted_area_t1_lag_elastictest - displayName: "impacted-area-kvmtest-t1-lag by Elastictest - optional" + displayName: "impacted-area-kvmtest-t1-lag by Elastictest" dependsOn: get_impacted_area condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1_checker') variables: TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] timeoutInMinutes: 240 - continueOnError: true + continueOnError: false pool: sonic-ubuntu-1c steps: - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml parameters: TOPOLOGY: t1 BUILD_BRANCH: $(BUILD_BRANCH) + PREPARE_TIME: 50 - template: .azure-pipelines/run-test-elastictest-template.yml parameters: @@ -156,7 +156,6 @@ stages: MAX_WORKER: $(INSTANCE_NUMBER) KVM_IMAGE_BRANCH: $(BUILD_BRANCH) MGMT_BRANCH: "master" - STOP_ON_FAILURE: "False" - job: impacted_area_dualtor_elastictest displayName: "impacted-area-kvmtest-dualtor by Elastictest" @@ -172,6 +171,7 @@ stages: parameters: TOPOLOGY: dualtor BUILD_BRANCH: $(BUILD_BRANCH) + PREPARE_TIME: 40 - template: .azure-pipelines/run-test-elastictest-template.yml parameters: @@ -222,6 +222,7 @@ stages: parameters: TOPOLOGY: t0-sonic BUILD_BRANCH: $(BUILD_BRANCH) + PREPARE_TIME: 40 - template: .azure-pipelines/run-test-elastictest-template.yml parameters: @@ -259,68 +260,6 @@ stages: MGMT_BRANCH: "master" # Below is the original PR checkers - - job: t0_elastictest - displayName: "kvmtest-t0 by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0 - MIN_WORKER: $(T0_INSTANCE_NUM) - MAX_WORKER: $(T0_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: "master" - - - job: t1_lag_elastictest - displayName: "kvmtest-t1-lag by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-lag - MIN_WORKER: $(T1_LAG_INSTANCE_NUM) - MAX_WORKER: $(T1_LAG_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: "master" - - - job: onboarding_elastictest_t0 - displayName: "onboarding t0 testcases by Elastictest - optional" - timeoutInMinutes: 240 - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0 - STOP_ON_FAILURE: "False" - RETRY_TIMES: 0 - MIN_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) - MAX_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: "master" - TEST_SET: onboarding_t0 - - - job: onboarding_elastictest_t1 - displayName: "onboarding t1 testcases by Elastictest - optional" - timeoutInMinutes: 240 - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-lag - STOP_ON_FAILURE: "False" - RETRY_TIMES: 0 - MIN_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) - MAX_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: "master" - TEST_SET: onboarding_t1 - - job: onboarding_multi_asic_elastictest_t1 displayName: "onboarding t1 testcases for kvmtest-multi-asic-t1-lag by Elastictest - optional" timeoutInMinutes: 240 @@ -337,33 +276,3 @@ stages: NUM_ASIC: 4 KVM_IMAGE_BRANCH: $(BUILD_BRANCH) MGMT_BRANCH: "master" - -# - job: onboarding_elastictest_dualtor -# displayName: "onboarding dualtor testcases by Elastictest - optional" -# timeoutInMinutes: 240 -# continueOnError: true -# pool: sonic-ubuntu-1c -# steps: -# - template: .azure-pipelines/run-test-elastictest-template.yml -# parameters: -# TOPOLOGY: dualtor -# STOP_ON_FAILURE: "False" -# RETRY_TIMES: 0 -# MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM) -# MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM) -# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) -# MGMT_BRANCH: "master" -# TEST_SET: onboarding_dualtor - -# - job: wan_elastictest -# displayName: "kvmtest-wan by Elastictest" -# timeoutInMinutes: 240 -# continueOnError: false -# pool: sonic-ubuntu-1c -# steps: -# - template: .azure-pipelines/run-test-elastictest-template.yml -# parameters: -# TOPOLOGY: wan-pub -# MIN_WORKER: $(WAN_INSTANCE_NUM) -# MAX_WORKER: $(WAN_INSTANCE_NUM) -# COMMON_EXTRA_PARAMS: "--skip_sanity "