diff --git a/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml b/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml index 4c1e17fecd6..fd31348aa6e 100644 --- a/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml +++ b/.azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml @@ -11,7 +11,7 @@ steps: - script: | set -x - sudo apt-get update && sudo apt-get install -y jq + sudo apt-get -o DPkg::Lock::Timeout=600 update && sudo apt-get -o DPkg::Lock::Timeout=600 -y install jq TEST_SCRIPTS=$(echo '$(TEST_SCRIPTS)' | jq -r -c '."${{ parameters.TOPOLOGY }}"') diff --git a/.azure-pipelines/impacted_area_testing/get-impacted-area.yml b/.azure-pipelines/impacted_area_testing/get-impacted-area.yml index 6bee00aeee6..03ac2725261 100644 --- a/.azure-pipelines/impacted_area_testing/get-impacted-area.yml +++ b/.azure-pipelines/impacted_area_testing/get-impacted-area.yml @@ -21,7 +21,7 @@ steps: pip install PyYAML pip install natsort - sudo apt-get install -y jq + sudo apt-get -o DPkg::Lock::Timeout=600 -y install jq FINAL_FEATURES="" IFS=' ' read -ra FEATURES_LIST <<< "$(DIFF_FOLDERS)" diff --git a/.azure-pipelines/pr_test_scripts.yaml b/.azure-pipelines/pr_test_scripts.yaml index 106bee345da..8bbb27db5e9 100644 --- a/.azure-pipelines/pr_test_scripts.yaml +++ b/.azure-pipelines/pr_test_scripts.yaml @@ -177,7 +177,6 @@ t0: - test_interfaces.py - test_procdockerstatsd.py - testbed_setup/test_populate_fdb.py - - upgrade_path/test_upgrade_path.py - vlan/test_autostate_disabled.py - vlan/test_host_vlan.py - vlan/test_vlan.py @@ -229,9 +228,13 @@ t0: - l2/test_l2_configure.py - srv6/test_srv6_basic_sanity.py - pfcwd/test_pfcwd_all_port_storm.py + - pfcwd/test_pfcwd_cli.py - pfcwd/test_pfcwd_function.py - pfcwd/test_pfcwd_timer_accuracy.py - pfcwd/test_pfcwd_warm_reboot.py + - qos/test_pfc_counters.py + - qos/test_pfc_pause.py + - qos/test_qos_dscp_mapping.py t0-2vlans: - dhcp_relay/test_dhcp_relay.py @@ -444,8 +447,11 @@ t1-lag: - restapi/test_restapi_vxlan_ecmp.py - srv6/test_srv6_basic_sanity.py - pfcwd/test_pfcwd_all_port_storm.py + - pfcwd/test_pfcwd_cli.py - pfcwd/test_pfcwd_function.py - pfcwd/test_pfcwd_timer_accuracy.py + - qos/test_pfc_counters.py + - qos/test_qos_dscp_mapping.py multi-asic-t1-lag: - bgp/test_bgp_bbr.py @@ -488,7 +494,6 @@ onboarding_t0: - gnmi/test_gnmi_smartswitch.py - gnmi/test_gnoi_killprocess.py - onboarding_t1: - pc/test_lag_member_forwarding.py - bgp/test_bgp_bbr_default_state.py diff --git a/.azure-pipelines/pr_test_skip_scripts.yaml b/.azure-pipelines/pr_test_skip_scripts.yaml index cb2969406d3..38467158883 100644 --- a/.azure-pipelines/pr_test_skip_scripts.yaml +++ b/.azure-pipelines/pr_test_skip_scripts.yaml @@ -56,6 +56,10 @@ t0: - platform_tests/mellanox/test_hw_management_service.py - platform_tests/mellanox/test_psu_power_threshold.py - platform_tests/mellanox/test_reboot_cause.py + # Qos buffer traditional test is only supported 201911 branch + - qos/test_buffer_traditional.py + # This test only support on cisco device with cisco sdk-debug mode + - qos/test_ecn_config.py # read_mac test needs specific variables and image urls, currently do not support on KVM and regular nightly test - read_mac/test_read_mac_metadata.py # This script only supported on Mellanox @@ -74,6 +78,9 @@ t0: - mvrf/test_mgmtvrf.py - vrf/test_vrf.py - vrf/test_vrf_attr.py + # Upgrade path test needs base and target image lists, currently do not support on KVM + - upgrade_path/test_upgrade_path.py + - upgrade_path/test_multi_hop_upgrade_path.py t1-lag: # KVM do not support bfd test @@ -129,6 +136,10 @@ t1-lag: - system_health/test_system_health.py # Vrf tests are also skipped in nightly test - mvrf/test_mgmtvrf.py + # This test needs swap syncd support, which is not available on KVM + - qos/test_qos_masic.py + # This test can only run on cisco and mnlx platforms + - vxlan/test_vnet_bgp_route_precedence.py t2: # KVM do not support bfd test @@ -197,6 +208,8 @@ dualtor: - dualtor_io/test_grpc_server_failure.py # This test is only for Nvidia platforms. - dualtor_mgmt/test_egress_drop_nvidia.py + # This test needs some additional SAI attributes, do not support on KVM. + - qos/test_tunnel_qos_remap.py tgen: # Ixia test only support on physical ixia testbed @@ -232,6 +245,7 @@ tgen: - snappi_tests/multidut/bgp/test_bgp_outbound_uplink_po_member_flap.py - snappi_tests/multidut/bgp/test_bgp_outbound_uplink_process_crash.py - snappi_tests/multidut/ecn/test_multidut_dequeue_ecn_with_snappi.py + - snappi_tests/multidut/ecn/test_multidut_ecn_marking_with_pfc_quanta_variance_with_snappi.py - snappi_tests/multidut/ecn/test_multidut_ecn_marking_with_snappi.py - snappi_tests/multidut/ecn/test_multidut_red_accuracy_with_snappi.py - snappi_tests/multidut/pfc/test_lossless_response_to_external_pause_storms.py diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 11cd58d3ceb..a244c5ee7a9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -33,6 +33,7 @@ Fixes # (issue) - [ ] 202305 - [ ] 202311 - [ ] 202405 +- [ ] 202411 ### Approach #### What is the motivation for this PR? diff --git a/ansible/roles/test/files/ptftests/fib_test.py b/ansible/roles/test/files/ptftests/fib_test.py index 03e53c37069..953866259fa 100644 --- a/ansible/roles/test/files/ptftests/fib_test.py +++ b/ansible/roles/test/files/ptftests/fib_test.py @@ -118,6 +118,9 @@ def setUp(self): ''' self.dataplane = ptf.dataplane_instance self.asic_type = self.test_params.get('asic_type') + # if test_params has skip_src_ports then set it otherwise empty + self.skip_src_ports = self.test_params.get('skip_src_ports', []) + if self.asic_type == "marvell": fib.EXCLUDE_IPV4_PREFIXES.append("240.0.0.0/4") @@ -166,6 +169,10 @@ def setUp(self): if not self.src_ports: self.src_ports = [int(port) for port in self.ptf_test_port_map.keys()] + # remove the skip_src_ports + if self.skip_src_ports is not None: + self.src_ports = [port for port in self.src_ports if port not in self.skip_src_ports] + logging.info("Skipping some src ports: new set src_ports: {}".format(self.src_ports)) self.ignore_ttl = self.test_params.get('ignore_ttl', False) self.single_fib = self.test_params.get( diff --git a/ansible/templates/minigraph_link_meta.j2 b/ansible/templates/minigraph_link_meta.j2 index fff5e393cd5..2180faa4618 100644 --- a/ansible/templates/minigraph_link_meta.j2 +++ b/ansible/templates/minigraph_link_meta.j2 @@ -63,7 +63,7 @@ {% else %} {% set autoneg_intf = "Ethernet" ~ if_index ~ "/1" %} {% endif %} -{% if device_conn[inventory_hostname][port_alias_map[autoneg_intf]]['autoneg']|lower == "on" %} +{% if port_alias_map[autoneg_intf] in device_conn[inventory_hostname] and device_conn[inventory_hostname][port_alias_map[autoneg_intf]]['autoneg']|lower == "on" %} diff --git a/tests/arp/test_arp_dualtor.py b/tests/arp/test_arp_dualtor.py index b2abc94d5f2..237f965dbaf 100644 --- a/tests/arp/test_arp_dualtor.py +++ b/tests/arp/test_arp_dualtor.py @@ -148,6 +148,8 @@ def test_arp_update_for_failed_standby_neighbor( 4. Run `arp_update` on the active ToR 5. Verify the incomplete entry is now reachable """ + if ip_address(neighbor_ip).version == 6 and lower_tor_host.facts["asic_type"] == "vs": + pytest.skip("Temporarily skipped to let the sonic-swss submodule be updated.") # We only use ping to trigger an ARP request from the kernel, so exit early to save time ping_cmd = "timeout 0.2 ping -c1 -W1 -i0.2 -n -q {}".format(neighbor_ip) @@ -189,6 +191,8 @@ def test_standby_unsolicited_neigh_learning( 2. Run arp_update on the active ToR 3. Confirm that the standby ToR learned the entry and it is REACHABLE """ + if ip_address(neighbor_ip).version == 6 and lower_tor_host.facts["asic_type"] == "vs": + pytest.skip("Temporarily skipped to let the sonic-swss submodule be updated.") ping_cmd = "timeout 0.2 ping -c1 -W1 -i0.2 -n -q {}".format(neighbor_ip) upper_tor_host.shell(ping_cmd, module_ignore_errors=True) diff --git a/tests/bgp/test_reliable_tsa.py b/tests/bgp/test_reliable_tsa.py index 928e9590d97..06ef29717f5 100644 --- a/tests/bgp/test_reliable_tsa.py +++ b/tests/bgp/test_reliable_tsa.py @@ -1,7 +1,10 @@ import logging +import threading + import pytest from tests.common import reboot, config_reload +from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor from tests.common.reboot import wait_for_startup from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until @@ -24,23 +27,33 @@ CONTAINER_CHECK_INTERVAL_SECS = 2 CONTAINER_STOP_THRESHOLD_SECS = 60 CONTAINER_RESTART_THRESHOLD_SECS = 300 -PROGRAM_STATUS = "RUNNING" BGP_CRIT_PROCESS = "bgpcfgd" supported_tsa_configs = ['false', 'true'] +lock = threading.Lock() -def nbrhosts_to_dut(duthost, nbrhosts): +def nbrhosts_to_dut(duthost, nbrhosts, dut_nbrhosts): """ - @summary: Fetch the neighbor hosts' details for duthost - @returns: dut_nbrhosts dict + @summary: Fetch the neighbor hosts' details for duthost and update the dut_nbrhosts dict """ mg_facts = duthost.minigraph_facts(host=duthost.hostname)['ansible_facts'] - dut_nbrhosts = {} + all_nbrhosts = {} for host in nbrhosts.keys(): if host in mg_facts['minigraph_devices']: new_nbrhost = {host: nbrhosts[host]} - dut_nbrhosts.update(new_nbrhost) - return dut_nbrhosts + all_nbrhosts.update(new_nbrhost) + + with lock: + dut_nbrhosts[duthost] = all_nbrhosts + + +def toggle_bgp_autorestart_state(duthost, current_state, target_state): + feature_list, _ = duthost.get_feature_status() + bgp_autorestart_state = duthost.get_container_autorestart_states()['bgp'] + for feature, status in list(feature_list.items()): + if feature == 'bgp' and status == 'enabled' and bgp_autorestart_state == current_state: + duthost.shell("sudo config feature autorestart {} {}".format(feature, target_state)) + break @pytest.fixture @@ -55,23 +68,26 @@ def enable_disable_bgp_autorestart_state(duthosts): None. """ # Enable autorestart status for bgp feature to overcome pretest changes - for duthost in duthosts.frontend_nodes: - feature_list, _ = duthost.get_feature_status() - bgp_autorestart_state = duthost.get_container_autorestart_states()['bgp'] - for feature, status in list(feature_list.items()): - if feature == 'bgp' and status == 'enabled' and bgp_autorestart_state == 'disabled': - duthost.shell("sudo config feature autorestart {} enabled".format(feature)) - break + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts.frontend_nodes: + executor.submit(toggle_bgp_autorestart_state, duthost, "disabled", "enabled") + yield # Disable autorestart status for bgp feature as in pretest - for duthost in duthosts.frontend_nodes: - feature_list, _ = duthost.get_feature_status() - bgp_autorestart_state = duthost.get_container_autorestart_states()['bgp'] - for feature, status in list(feature_list.items()): - if feature == 'bgp' and status == 'enabled' and bgp_autorestart_state == 'enabled': - duthost.shell("sudo config feature autorestart {} disabled".format(feature)) - break + with SafeThreadPoolExecutor(max_workers=8) as executor: + for duthost in duthosts.frontend_nodes: + executor.submit(toggle_bgp_autorestart_state, duthost, "enabled", "disabled") + + +def run_tsb_on_linecard(linecard): + if verify_dut_configdb_tsa_value(linecard) is not False or get_tsa_chassisdb_config(linecard) != 'false' or \ + get_traffic_shift_state(linecard, cmd='TSC no-stats') != TS_NORMAL: + linecard.shell('TSB') + linecard.shell('sudo config save -y') + # Ensure that the DUT is not in maintenance already before start of the test + pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + "DUT is not in normal state") def set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname): @@ -86,15 +102,10 @@ def set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_host pytest_assert('false' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) - for linecard in duthosts.frontend_nodes: - # Issue TSB on line card before proceeding further - if verify_dut_configdb_tsa_value(linecard) is not False or get_tsa_chassisdb_config(linecard) != 'false' or \ - get_traffic_shift_state(linecard, cmd='TSC no-stats') != TS_NORMAL: - linecard.shell('TSB') - linecard.shell('sudo config save -y') - # Ensure that the DUT is not in maintenance already before start of the test - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT is not in normal state") + # Issue TSB on line card before proceeding further + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsb_on_linecard, linecard) def verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes): @@ -163,9 +174,11 @@ def is_process_running(duthost, container_name, program_name): @summary: Determine whether a process under container is in the expected state (running/not running) @returns: True if its running and false if its not """ - global PROGRAM_STATUS program_status, _ = get_program_info(duthost, container_name, program_name) - PROGRAM_STATUS = program_status + logger.info( + "Program '{}' in container '{}' is in '{}' state".format(program_name, container_name, program_status) + ) + if program_status == "RUNNING": return True elif program_status in ["EXITED", "STOPPED", "STARTING"]: @@ -180,10 +193,9 @@ def restart_bgp(duthost, container_name, service_name, program_name, program_pid @summary: Kill a critical process in a container to verify whether the container is stopped and restarted correctly """ - global PROGRAM_STATUS pytest_assert(wait_until(40, 3, 0, is_process_running, duthost, container_name, program_name), - "Program '{}' in container '{}' is in the '{}' state, expected 'RUNNING'" - .format(program_name, container_name, PROGRAM_STATUS)) + "Program '{}' in container '{}' is not 'RUNNING' state after given time" + .format(program_name, container_name)) kill_process_by_pid(duthost, container_name, program_name, program_pid) logger.info("Waiting until container '{}' is stopped...".format(container_name)) @@ -262,10 +274,13 @@ def test_sup_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_su suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -281,17 +296,22 @@ def test_sup_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_su pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - for linecard in duthosts.frontend_nodes: - # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + def verify_linecard_after_sup_tsa(lc): + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc), "DUT is not in maintenance state when startup_tsa_tsb service is running") - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) + + # Verify DUT is in maintenance state + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsa, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -312,10 +332,13 @@ def test_sup_tsa_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -325,13 +348,19 @@ def test_sup_tsa_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Convert line cards to BGP operational TSA state for the current test as initial config - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT is not in maintenance state") + pytest_assert( + TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), + "DUT is not in maintenance state", + ) + + # Convert line cards to BGP operational TSA state for the current test as initial config + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) # Now Issue TSA from supervisor and make sure it changes from TSB->TSA suphost.shell('TSA') @@ -339,17 +368,22 @@ def test_sup_tsa_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - for linecard in duthosts.frontend_nodes: - # Verify DUT continues to be in maintenance state even with supervisor TSA action - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + def verify_linecard_after_sup_tsa(lc): + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc), "DUT is not in maintenance state with supervisor TSA action") - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) + + # Verify DUT continues to be in maintenance state even with supervisor TSA action + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsa, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -373,12 +407,13 @@ def test_sup_tsb_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -392,11 +427,17 @@ def test_sup_tsb_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Confirm all the line cards are in BGP operational TSA state due to supervisor TSA - for linecard in duthosts.frontend_nodes: + def verify_linecard_after_sup_tsa(lc): # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") + + # Confirm all the line cards are in BGP operational TSA state due to supervisor TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsa, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -409,13 +450,16 @@ def test_sup_tsb_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) # Verify line cards change the state to TSB from TSA after supervisor TSB - for linecard in duthosts.frontend_nodes: + def verify_linecard_after_sup_tsb(lc): # Verify DUT changes to normal state with supervisor TSB action - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard), + pytest_assert(TS_NORMAL == get_traffic_shift_state(lc), "DUT is not in normal state with supervisor TSB action") - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsb, linecard) finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -437,12 +481,13 @@ def test_sup_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -455,36 +500,46 @@ def test_sup_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu suphost.shell('sudo config save -y') pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Similarly keep line cards in TSA mode to start with as part of the test - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") + # Similarly keep line cards in TSA mode to start with as part of the test + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + # Issue TSB on the supervisor suphost.shell('TSB') suphost.shell('sudo config save -y') pytest_assert('false' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + def verify_linecard_after_sup_tsb(lc): + # Verify DUT continues to be in maintenance state even with supervisor TSB action + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), + "DUT is not in maintenance state") + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + # Verify line cards maintains the BGP operational TSA state but with chassisdb tsa-enabled config as 'false' # in sync with supervisor + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsb, linecard) + for linecard in duthosts.frontend_nodes: - # Verify DUT continues to be in maintenance state even with supervisor TSB action - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT is not in maintenance state") - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -506,10 +561,13 @@ def test_dut_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_su suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -519,19 +577,25 @@ def test_dut_tsa_act_when_sup_duts_on_tsb_initially(duthosts, localhost, enum_su orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Issue TSA from line card and verify line cards' BGP operational state changes to TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -561,13 +625,15 @@ def test_dut_tsa_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -581,28 +647,38 @@ def test_dut_tsa_act_when_sup_on_tsa_duts_on_tsb_initially(duthosts, localhost, pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Confirm all the line cards are in BGP operational TSA state due to supervisor TSA - for linecard in duthosts.frontend_nodes: + def verify_linecard_after_sup_tsa(lc): # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Verify line card config TSA enabled is still false - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) - # Issue TSA from line card and verify line cards' BGP operational state continues to be in TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + # Confirm all the line cards are in BGP operational TSA state due to supervisor TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsa, linecard) + + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) + + # Issue TSA from line card and verify line cards' BGP operational state continues to be in TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -634,10 +710,13 @@ def test_dut_tsb_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -647,59 +726,51 @@ def test_dut_tsb_act_when_sup_on_tsb_duts_on_tsa_initially(duthosts, localhost, orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Keep supervisor in TSB mode to start with as part of the test - # And keep the line cards in TSA and verify line cards' BGP operational state changes to TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + + # Keep supervisor in TSB mode to start with as part of the test + # And keep the line cards in TSA and verify line cards' BGP operational state changes to TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - # Issue TSB from line card and verify line cards' BGP operational state changes to TSB - for linecard in duthosts.frontend_nodes: - linecard.shell('TSB') - linecard.shell('sudo config save -y') + def run_tsb_on_linecard_and_verify(lc): + lc.shell('TSB') + lc.shell('sudo config save -y') # Verify line card config changed to tsa_enabled false - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) # Ensure that the DUT is in normal state - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in normal state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) - # Make sure all routes are advertised back to neighbors after TSB on line cards - for linecard in duthosts.frontend_nodes: - # Wait until all routes are announced to neighbors - cur_v4_routes = {} - cur_v6_routes = {} - # Verify that all routes advertised to neighbor at the start of the test - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, - dut_nbrhosts[linecard], - orig_v4_routes[linecard], cur_v4_routes, 4): - if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], - orig_v4_routes[linecard], cur_v4_routes, 4): - pytest.fail("Not all ipv4 routes are announced to neighbors") - - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, - dut_nbrhosts[linecard], - orig_v6_routes[linecard], cur_v6_routes, 6): - if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], - orig_v6_routes[linecard], cur_v6_routes, 6): - pytest.fail("Not all ipv6 routes are announced to neighbors") + # Issue TSB from line card and verify line cards' BGP operational state changes to TSB + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsb_on_linecard_and_verify, linecard) + # Make sure all routes are advertised back to neighbors after TSB on line cards + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) finally: # Bring back the supervisor and line cards to the normal state at the end of test set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -717,13 +788,15 @@ def test_dut_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -737,30 +810,40 @@ def test_dut_tsb_act_when_sup_and_duts_on_tsa_initially(duthosts, localhost, enu pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Similarly keep line cards in TSA mode to start with as part of the test - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") - # Issue TSB from line card and verify line cards' BGP operational state maintained at TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSB') - linecard.shell('sudo config save -y') + # Similarly keep line cards in TSA mode to start with as part of the test + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + + def run_tsb_on_linecard_and_verify(lc): + lc.shell('TSB') + lc.shell('sudo config save -y') # Verify line card config changed to tsa_enabled false - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) + + # Issue TSB from line card and verify line cards' BGP operational state maintained at TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsb_on_linecard_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -792,18 +875,23 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) + tsa_tsb_timer = dict() - orig_v4_routes, orig_v6_routes = dict(), dict() - dut_nbrhosts = dict() - up_bgp_neighbors = dict() int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) int_status_result[linecard] = True crit_process_check[linecard] = True - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + orig_v4_routes, orig_v6_routes = dict(), dict() + up_bgp_neighbors = dict() try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -818,14 +906,18 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - for linecard in duthosts.frontend_nodes: + def verify_linecard_after_sup_tsa(lc): # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc), "DUT is not in maintenance state when startup_tsa_tsb service is running") - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) # Not verifying loopback routes here check since its been checked multiple times with previous test cases + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsa, linecard) + # Get a dut uptime before reboot sup_uptime_before = suphost.get_up_time() # Reboot supervisor and wait for startup_tsa_tsb service to start on line cards @@ -842,63 +934,80 @@ def test_sup_tsa_act_with_sup_reboot(duthosts, localhost, enum_supervisor_dut_ho pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - for linecard in duthosts.frontend_nodes: - wait_for_startup(linecard, localhost, delay=10, timeout=300) + def verify_linecard_after_sup_reboot(lc): + wait_for_startup(lc, localhost, delay=10, timeout=300) # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = linecard.get_up_time() - logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(linecard) + dut_uptime = lc.get_up_time() + logging.info('DUT {} up since {}'.format(lc.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(lc) time_diff = (service_uptime - dut_uptime).total_seconds() pytest_assert(int(time_diff) < 160, "startup_tsa_tsb service started much later than the expected time after dut reboot") # Verify DUT is in the same maintenance state like before supervisor reboot - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state when startup_tsa_tsb service is running") - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) logging.info("Wait until all critical processes are fully started") - crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) - int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) + + crit_process_check_res = wait_until(600, 20, 0, _all_critical_processes_healthy, lc) + int_status_check_res = wait_until(1200, 20, 0, check_interface_status_of_up_ports, lc) + with lock: + crit_process_check[lc] = crit_process_check_res + int_status_result[lc] = int_status_check_res + # verify bgp sessions are established pytest_assert( wait_until( - 300, 10, 0, linecard.check_bgp_session_state_all_asics, up_bgp_neighbors[linecard], "established"), + 300, 10, 0, lc.check_bgp_session_state_all_asics, up_bgp_neighbors[lc], "established"), "All BGP sessions are not up, no point in continuing the test") - # Once all line cards are in maintenance state, proceed further - for linecard in duthosts.frontend_nodes: + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_reboot, linecard) + + def verify_linecard_tsa_tsb(lc): # Verify startup_tsa_tsb service stopped after expected time - pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), + pytest_assert(wait_until(tsa_tsb_timer[lc], 20, 0, get_tsa_tsb_service_status, lc, 'exited'), "startup_tsa_tsb service is not stopped even after configured timer expiry") # Ensure dut comes back to normal state after timer expiry - if not get_tsa_tsb_service_status(linecard, 'running'): + if not get_tsa_tsb_service_status(lc, 'running'): # Verify dut continues to be in TSA even after startup_tsa_tsb service is stopped - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in normal state after startup_tsa_tsb service is stopped") - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) # Verify line card config changed to TSB after startup-tsa-tsb service expiry - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) + + # Once all line cards are in maintenance state, proceed further + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_tsa_tsb, linecard) + for linecard in duthosts.frontend_nodes: pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - for linecard in duthosts.frontend_nodes: - # Make sure linecards are in Normal state, if not do config-reload on the dut - if not (int_status_result[linecard] and crit_process_check[linecard] and - TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + def config_reload_linecard_if_unhealthy(lc): + if not (int_status_result[lc] and crit_process_check[lc] and + TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats')): logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") - config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + config_reload(lc, safe_reload=True, check_intf_up_ports=True) + + # Make sure linecards are in Normal state, if not do config-reload on the dut + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(config_reload_linecard_if_unhealthy, linecard) # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) @@ -918,10 +1027,14 @@ def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - dut_nbrhosts, up_bgp_neighbors = dict(), dict() + + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + up_bgp_neighbors = dict() orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -931,48 +1044,63 @@ def test_sup_tsa_act_when_duts_on_tsa_with_sup_config_reload(duthosts, localhost orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Convert line cards to BGP operational TSA state for the current test as initial config - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") + # Convert line cards to BGP operational TSA state for the current test as initial config + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + # Now Issue TSA from supervisor and make sure it changes from TSB->TSA suphost.shell('TSA') suphost.shell('sudo config save -y') pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - for linecard in duthosts.frontend_nodes: - # Verify DUT continues to be in maintenance state even with supervisor TSA action - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + def verify_tsa_after_sup_tsa(lc): + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc), "DUT is not in maintenance state with supervisor TSA action") - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) - up_bgp_neighbors[linecard] = linecard.get_bgp_neighbors_per_asic("established") + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) + + up_bgp_neighbors_of_linecard = lc.get_bgp_neighbors_per_asic("established") + with lock: + up_bgp_neighbors[lc] = up_bgp_neighbors_of_linecard + + # Verify DUT continues to be in maintenance state even with supervisor TSA action + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_tsa_after_sup_tsa, linecard) # Do config_reload on the supervisor and verify configs are same as before config_reload(suphost, wait=300, safe_reload=True) pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Verify line cards traffic shift states are same as before config_reload - for linecard in duthosts.frontend_nodes: + def verify_line_card_after_sup_config_reload(lc): # Verify DUT is in the same maintenance state like before supervisor config reload - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc), "DUT is not in maintenance state after supervisor config reload") - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled chassisdb config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled chassisdb config is not enabled".format(lc.hostname)) # Before verifying loopback address, make sure IBGP neighbors are in established state - pytest_assert(wait_until(300, 20, 0, linecard.check_bgp_session_state_all_asics, - up_bgp_neighbors[linecard], "established")) + pytest_assert(wait_until(300, 20, 0, lc.check_bgp_session_state_all_asics, + up_bgp_neighbors[lc], "established")) + # Verify line cards traffic shift states are same as before config_reload + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_line_card_after_sup_config_reload, linecard) + + for linecard in duthosts.frontend_nodes: pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -996,16 +1124,21 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + tsa_tsb_timer = dict() - dut_nbrhosts = dict() - up_bgp_neighbors = dict() int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) int_status_result[linecard] = True crit_process_check[linecard] = True + + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + up_bgp_neighbors = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1016,60 +1149,69 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Issue TSA from line card and verify line cards' BGP operational state changes to TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) - # Verify dut reboot scenario for one of the line card to make sure tsa config is in sync - for linecard in duthosts.frontend_nodes: - logger.info("Cold reboot on node: %s", linecard.hostname) - reboot(linecard, localhost, wait=240) + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) - for linecard in duthosts.frontend_nodes: - wait_for_startup(linecard, localhost, delay=10, timeout=300) + def reboot_linecard_and_verify(lc): + logger.info("Cold reboot on node: %s", lc.hostname) + reboot(lc, localhost, wait=240) + + wait_for_startup(lc, localhost, delay=10, timeout=300) # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = linecard.get_up_time() - logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(linecard) + dut_uptime = lc.get_up_time() + logging.info('DUT {} up since {}'.format(lc.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(lc) time_diff = (service_uptime - dut_uptime).total_seconds() pytest_assert(int(time_diff) < 160, "startup_tsa_tsb service started much later than the expected time after dut reboot") # Verify startup_tsa_tsb service is not started and in exited due to manual TSA - pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), + pytest_assert(wait_until(tsa_tsb_timer[lc], 20, 0, get_tsa_tsb_service_status, lc, 'exited'), "startup_tsa_tsb service is in running state after dut reboot which is not expected") # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) # Verify line card config changed is still TSA enabled true after reboot - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) - # Make sure the ports, interfaces are UP and running after reboot - for linecard in duthosts.frontend_nodes: + # Make sure the ports, interfaces are UP and running after reboot logging.info("Wait until all critical processes are fully started") - crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) - int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) + crit_process_check_res = wait_until(600, 20, 0, _all_critical_processes_healthy, lc) + int_status_check_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, lc) + with lock: + crit_process_check[lc] = crit_process_check_res + int_status_result[lc] = int_status_check_result # verify bgp sessions are established pytest_assert( wait_until( - 300, 10, 0, linecard.check_bgp_session_state_all_asics, up_bgp_neighbors[linecard], "established"), + 300, 10, 0, lc.check_bgp_session_state_all_asics, up_bgp_neighbors[lc], "established"), "All BGP sessions are not up, no point in continuing the test") + # Verify dut reboot scenario for one of the line card to make sure tsa config is in sync + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(reboot_linecard_and_verify, linecard) + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced to neighbors when the linecards are in TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( @@ -1084,12 +1226,16 @@ def test_dut_tsa_act_with_reboot_when_sup_dut_on_tsb_init(duthosts, localhost, e # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - for linecard in duthosts.frontend_nodes: - # Make sure linecards are in Normal state, if not do config-reload on the dut to recover - if not (int_status_result[linecard] and crit_process_check[linecard] and - TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + def config_reload_linecard_if_unhealthy(lc): + if not (int_status_result[lc] and crit_process_check[lc] and + TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats')): logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") - config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + config_reload(lc, safe_reload=True, check_intf_up_ports=True) + + # Make sure linecards are in Normal state, if not do config-reload on the dut to recover + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(config_reload_linecard_if_unhealthy, linecard) # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) @@ -1110,10 +1256,13 @@ def test_dut_tsa_with_conf_reload_when_sup_on_tsa_dut_on_tsb_init(duthosts, loca suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1130,30 +1279,33 @@ def test_dut_tsa_with_conf_reload_when_sup_on_tsa_dut_on_tsb_init(duthosts, loca "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) # Verify line cards' BGP operational state changes to TSA - for linecard in duthosts.frontend_nodes: + def verify_line_card_after_sup_tsa(lc): # Verify line card BGP operational state changes to TSA - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled chassisdb config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled chassisdb config is not enabled".format(lc.hostname)) + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_line_card_after_sup_tsa, linecard) # Verify dut config_reload scenario for one of the line card to make sure tsa config is in sync - for linecard in duthosts.frontend_nodes: - linecard.shell('sudo config save -y') - config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + first_linecard = duthosts.frontend_nodes[0] + first_linecard.shell('sudo config save -y') + config_reload(first_linecard, safe_reload=True, check_intf_up_ports=True) - # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT is not in maintenance state after config reload") + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(first_linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state after config reload") - pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( - duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), - "Failed to verify routes on nbr in TSA") - break + pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( + duthosts, first_linecard, dut_nbrhosts[first_linecard], traffic_shift_community), + "Failed to verify routes on nbr in TSA") # Verify supervisor still has tsa_enabled 'true' config pytest_assert('true' == get_tsa_chassisdb_config(suphost), @@ -1183,10 +1335,13 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsa(duthosts, localhost, enum_supe suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1196,19 +1351,23 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsa(duthosts, localhost, enum_supe orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Issue TSA from line card and verify line cards' BGP operational state changes to TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) # Issue TSA from supervisor and verify line cards' BGP operational state continues to be in TSA suphost.shell('TSA') @@ -1216,17 +1375,23 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsa(duthosts, localhost, enum_supe pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Verify line cards' BGP operational state continues in mainternance state - for linecard in duthosts.frontend_nodes: + def verify_linecard_after_sup_tsa(lc): # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) + + # Verify line cards' BGP operational state continues in maintenance state + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsa, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced with TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -1255,10 +1420,13 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsb(duthosts, localhost, enum_supe suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1268,19 +1436,23 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsb(duthosts, localhost, enum_supe orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Issue TSA from line card and verify line cards' BGP operational state changes to TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + + # Issue TSA from line card and verify line cards' BGP operational state changes to TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) # Issue TSB from supervisor and verify line cards' BGP operational state continues to be in TSA suphost.shell('TSB') @@ -1288,17 +1460,23 @@ def test_user_init_tsa_on_dut_followed_by_sup_tsb(duthosts, localhost, enum_supe pytest_assert('false' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) - # Verify line cards' BGP operational state continues in mainternance state - for linecard in duthosts.frontend_nodes: + def verify_linecard_after_sup_tsb(lc): # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + + # Verify line cards' BGP operational state continues in maintenance state + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsb, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced with TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), @@ -1324,16 +1502,21 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() - dut_nbrhosts = dict() + tsa_tsb_timer = dict() - up_bgp_neighbors = dict() int_status_result, crit_process_check = dict(), dict() for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) int_status_result[linecard] = True crit_process_check[linecard] = True + + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + up_bgp_neighbors = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1344,22 +1527,28 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Verify dut reboot scenario for line card to make sure tsa config is in sync - for linecard in duthosts.frontend_nodes: - logger.info("Cold reboot on node: %s", linecard.hostname) - reboot(linecard, localhost, wait=240) - wait_for_startup(linecard, localhost, delay=10, timeout=300) + def reboot_linecard_and_verify(lc): + logger.info("Cold reboot on node: %s", lc.hostname) + reboot(lc, localhost, wait=240) + wait_for_startup(lc, localhost, delay=10, timeout=300) # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = linecard.get_up_time() - logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(linecard) + dut_uptime = lc.get_up_time() + logging.info('DUT {} up since {}'.format(lc.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(lc) time_diff = (service_uptime - dut_uptime).total_seconds() pytest_assert(int(time_diff) < 160, "startup_tsa_tsb service started much later than the expected time after dut reboot") # Verify startup_tsa_tsb service is started and running - pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'running'), + pytest_assert(wait_until(tsa_tsb_timer[lc], 20, 0, get_tsa_tsb_service_status, lc, 'running'), "startup_tsa_tsb service is not in running state after dut reboot") + + # Verify dut reboot scenario for line card to make sure tsa config is in sync + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(reboot_linecard_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Now Issue TSA from supervisor and make sure it changes from TSB->TSA while the service is running if get_tsa_tsb_service_status(linecard, 'running'): # Now Issue TSA from supervisor and make sure it changes from TSB->TSA @@ -1377,31 +1566,40 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) - for linecard in duthosts.frontend_nodes: + def verify_linecard_after_sup_tsa(lc): logging.info("Wait until all critical processes are fully started") - crit_process_check[linecard] = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) - int_status_result[linecard] = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) + crit_process_check_res = wait_until(600, 20, 0, _all_critical_processes_healthy, lc) + int_status_check_res = wait_until(1200, 20, 0, check_interface_status_of_up_ports, lc) + + with lock: + crit_process_check[lc] = crit_process_check_res + int_status_result[lc] = int_status_check_res # verify bgp sessions are established pytest_assert( wait_until( - 300, 10, 0, linecard.check_bgp_session_state_all_asics, up_bgp_neighbors[linecard], "established"), + 300, 10, 0, lc.check_bgp_session_state_all_asics, up_bgp_neighbors[lc], "established"), "All BGP sessions are not up, no point in continuing the test") # Verify startup_tsa_tsb service stopped after expected time - pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), + pytest_assert(wait_until(tsa_tsb_timer[lc], 20, 0, get_tsa_tsb_service_status, lc, 'exited'), "startup_tsa_tsb service is not stopped even after configured timer expiry") # Ensure dut is still in TSA even after startup-tsa-tsb timer expiry - if get_tsa_tsb_service_status(linecard, 'exited'): - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + if get_tsa_tsb_service_status(lc, 'exited'): + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is in normal state after startup_tsa_tsb service is stopped") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) # Verify line card config changed to tsa_enabled false after timer expiry - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(verify_linecard_after_sup_tsa, linecard) + # Verify only loopback routes are announced to neighbors at this state for linecard in duthosts.frontend_nodes: pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( @@ -1411,12 +1609,16 @@ def test_sup_tsa_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - for linecard in duthosts.frontend_nodes: - # Make sure linecards are in Normal state, if not do config-reload on the dut to recover - if not (int_status_result[linecard] and crit_process_check[linecard] and - TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + def config_reload_linecard_if_unhealthy(lc): + if not (int_status_result[lc] and crit_process_check[lc] and + TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats')): logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") - config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + config_reload(lc, safe_reload=True, check_intf_up_ports=True) + + # Make sure linecards are in Normal state, if not do config-reload on the dut to recover + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(config_reload_linecard_if_unhealthy, linecard) # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) @@ -1435,14 +1637,19 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() - dut_nbrhosts = dict() + tsa_tsb_timer = dict() - up_bgp_neighbors = dict() - int_status_result, crit_process_check = True, True for linecard in duthosts.frontend_nodes: tsa_tsb_timer[linecard] = get_startup_tsb_timer(linecard) - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + + int_status_result, crit_process_check = True, True + up_bgp_neighbors = dict() + orig_v4_routes, orig_v6_routes = dict(), dict() # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) try: @@ -1454,69 +1661,73 @@ def test_sup_tsb_when_startup_tsa_tsb_service_running(duthosts, localhost, enum_ orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) # Verify dut reboot scenario for one of the line card to make sure tsa config is in sync - for linecard in duthosts.frontend_nodes: - logger.info("Cold reboot on node: %s", linecard.hostname) - reboot(linecard, localhost, wait=240) - wait_for_startup(linecard, localhost, delay=10, timeout=300) - - # Ensure startup_tsa_tsb service started on expected time since dut rebooted - dut_uptime = linecard.get_up_time() - logging.info('DUT {} up since {}'.format(linecard.hostname, dut_uptime)) - service_uptime = get_tsa_tsb_service_uptime(linecard) - time_diff = (service_uptime - dut_uptime).total_seconds() - pytest_assert(int(time_diff) < 160, - "startup_tsa_tsb service started much later than the expected time after dut reboot") - # Verify startup_tsa_tsb service is started and running - pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'running'), - "startup_tsa_tsb service is not in running state after dut reboot") - # Now Issue TSB from supervisor and make sure it changes from TSA->TSB - if get_tsa_tsb_service_status(linecard, 'running'): - # Now Issue TSB from supervisor - suphost.shell('TSB') - suphost.shell('sudo config save -y') - pytest_assert('false' == get_tsa_chassisdb_config(suphost), - "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) - - # Verify DUT is in maintenance state. - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT is not in maintenance state when startup_tsa_tsb service is running") - - logging.info("Wait until all critical processes are fully started") - crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, linecard) - int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, linecard) - - # verify bgp sessions are established - pytest_assert( - wait_until( - 300, 10, 0, linecard.check_bgp_session_state_all_asics, up_bgp_neighbors[linecard], "established"), - "All BGP sessions are not up, no point in continuing the test") - - # Verify startup_tsa_tsb service stopped after expected time - pytest_assert(wait_until(tsa_tsb_timer[linecard], 20, 0, get_tsa_tsb_service_status, linecard, 'exited'), - "startup_tsa_tsb service is not stopped even after configured timer expiry") - - # Ensure dut gets back to normal state after startup-tsa-tsb timer expiry - if get_tsa_tsb_service_status(linecard, 'exited'): - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), - "DUT is not in normal state after startup_tsa_tsb service is stopped") - # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) - # Verify line card config changed to tsa_enabled false after timer expiry - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) - break + first_linecard = duthosts.frontend_nodes[0] + logger.info("Cold reboot on node: %s", first_linecard.hostname) + reboot(first_linecard, localhost, wait=240) + wait_for_startup(first_linecard, localhost, delay=10, timeout=300) + + # Ensure startup_tsa_tsb service started on expected time since dut rebooted + dut_uptime = first_linecard.get_up_time() + logging.info('DUT {} up since {}'.format(first_linecard.hostname, dut_uptime)) + service_uptime = get_tsa_tsb_service_uptime(first_linecard) + time_diff = (service_uptime - dut_uptime).total_seconds() + pytest_assert(int(time_diff) < 160, + "startup_tsa_tsb service started much later than the expected time after dut reboot") + # Verify startup_tsa_tsb service is started and running + pytest_assert(wait_until(tsa_tsb_timer[first_linecard], 20, 0, + get_tsa_tsb_service_status, first_linecard, 'running'), + "startup_tsa_tsb service is not in running state after dut reboot") + # Now Issue TSB from supervisor and make sure it changes from TSA->TSB + if get_tsa_tsb_service_status(first_linecard, 'running'): + # Now Issue TSB from supervisor + suphost.shell('TSB') + suphost.shell('sudo config save -y') + pytest_assert('false' == get_tsa_chassisdb_config(suphost), + "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) + + # Verify DUT is in maintenance state. + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(first_linecard, cmd='TSC no-stats'), + "DUT is not in maintenance state when startup_tsa_tsb service is running") + logging.info("Wait until all critical processes are fully started") + crit_process_check = wait_until(600, 20, 0, _all_critical_processes_healthy, first_linecard) + int_status_result = wait_until(1200, 20, 0, check_interface_status_of_up_ports, first_linecard) + + # verify bgp sessions are established + pytest_assert( + wait_until( + 300, 10, 0, + first_linecard.check_bgp_session_state_all_asics, up_bgp_neighbors[first_linecard], "established"), + "All BGP sessions are not up, no point in continuing the test") + + # Verify startup_tsa_tsb service stopped after expected time + pytest_assert(wait_until(tsa_tsb_timer[first_linecard], 20, 0, + get_tsa_tsb_service_status, first_linecard, 'exited'), + "startup_tsa_tsb service is not stopped even after configured timer expiry") + + # Ensure dut gets back to normal state after startup-tsa-tsb timer expiry + if get_tsa_tsb_service_status(first_linecard, 'exited'): + pytest_assert(TS_NORMAL == get_traffic_shift_state(first_linecard, cmd='TSC no-stats'), + "DUT is not in normal state after startup_tsa_tsb service is stopped") + # Ensure line card chassisdb config is in sync with supervisor + pytest_assert('false' == get_tsa_chassisdb_config(first_linecard), + "{} tsa_enabled config is enabled".format(first_linecard.hostname)) + # Verify line card config changed to tsa_enabled false after timer expiry + pytest_assert(verify_dut_configdb_tsa_value(first_linecard) is False, + "DUT {} tsa_enabled config is enabled".format(first_linecard.hostname)) finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - for linecard in duthosts.frontend_nodes: - # Make sure linecards are in Normal state, if not do config-reload on the dut to recover + def config_reload_linecard_if_unhealthy(lc): if not (int_status_result and crit_process_check and - TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats')): + TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats')): logging.info("DUT is not in normal state after supervisor cold reboot, doing config-reload") - config_reload(linecard, safe_reload=True, check_intf_up_ports=True) + config_reload(lc, safe_reload=True, check_intf_up_ports=True) + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(config_reload_linecard_if_unhealthy, linecard) # Verify all routes are advertised back to neighbors when duts are in TSB verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) @@ -1537,13 +1748,15 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -1573,26 +1786,28 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_on_tsa_duts_on_tsb( pytest_assert('false' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) - # Restart bgp on the line cards and check the status - for linecard in duthosts.frontend_nodes: - for asic in linecard.asics: + def restart_bgp_on_linecard_and_verify(lc): + # Restart bgp on the line cards and check the status + for asic in lc.asics: service_name = asic.get_service_name("bgp") container_name = asic.get_docker_name("bgp") - logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) - process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + logger.info("Restarting {} container on dut {}".format(container_name, lc.hostname)) + process_status, program_pid = get_program_info(lc, container_name, BGP_CRIT_PROCESS) if process_status == "RUNNING": - restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + restart_bgp(lc, container_name, service_name, BGP_CRIT_PROCESS, program_pid) - # Verify line cards continues to be in TSB state even after bgp restart - for linecard in duthosts.frontend_nodes: + # Verify line cards continues to be in TSB state even after bgp restart # Verify DUT changes to normal state with supervisor TSB action - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in normal state with supervisor TSB action") - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) - + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) + + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(restart_bgp_on_linecard_and_verify, linecard) finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -1615,13 +1830,15 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) + dut_nbrhosts = dict() + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) - try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -1634,46 +1851,55 @@ def test_sup_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, suphost.shell('sudo config save -y') pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Similarly keep line cards in TSA mode to start with as part of the test - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") + # Similarly keep line cards in TSA mode to start with as part of the test + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + # Issue TSB on the supervisor suphost.shell('TSB') suphost.shell('sudo config save -y') pytest_assert('false' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is enabled".format(suphost.hostname)) - # Restart bgp on the line cards and check the status - for linecard in duthosts.frontend_nodes: - for asic in linecard.asics: + def restart_bgp_and_verify(lc): + for asic in lc.asics: service_name = asic.get_service_name("bgp") container_name = asic.get_docker_name("bgp") - logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) - process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + logger.info("Restarting {} container on dut {}".format(container_name, lc.hostname)) + process_status, program_pid = get_program_info(lc, container_name, BGP_CRIT_PROCESS) if process_status == "RUNNING": - restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + restart_bgp(lc, container_name, service_name, BGP_CRIT_PROCESS, program_pid) - # Verify line cards maintains the BGP operational TSA state but with chassisdb tsa-enabled config as 'false' - # in sync with supervisor - for linecard in duthosts.frontend_nodes: + # Verify line cards maintains the BGP operational TSA state but with chassisdb tsa-enabled config as 'false' + # in sync with supervisor # Verify DUT continues to be in maintenance state even with supervisor TSB action - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + + # Restart bgp on the line cards and check the status + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(restart_bgp_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -1699,12 +1925,15 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthost suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) - orig_v4_routes, orig_v6_routes = dict(), dict() + dut_nbrhosts = dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + orig_v4_routes, orig_v6_routes = dict(), dict() try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -1712,80 +1941,75 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_on_tsb_duts_on_tsa(duthost orig_v4_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 4) orig_v6_routes[linecard] = parse_routes_on_neighbors(linecard, dut_nbrhosts[linecard], 6) - # Keep supervisor in the current TSB mode to start with as part of the test - # And keep the line cards in TSA and verify line cards' BGP operational state changes to TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) + + # Keep supervisor in the current TSB mode to start with as part of the test + # And keep the line cards in TSA and verify line cards' BGP operational state changes to TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - # Issue TSB from line card and verify line cards' BGP operational state changes to TSB - for linecard in duthosts.frontend_nodes: - linecard.shell('TSB') - linecard.shell('sudo config save -y') + def run_tsb_on_linecard_and_verify(lc): + lc.shell('TSB') + lc.shell('sudo config save -y') # Verify line card config changed to tsa_enabled false - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) # Ensure that the DUT is in normal state - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in normal state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) - # Restart bgp on the line cards and check the status - for linecard in duthosts.frontend_nodes: - for asic in linecard.asics: + # Issue TSB from line card and verify line cards' BGP operational state changes to TSB + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsb_on_linecard_and_verify, linecard) + + def restart_bgp_and_verify(lc): + for asic in lc.asics: service_name = asic.get_service_name("bgp") container_name = asic.get_docker_name("bgp") - logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) - process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + logger.info("Restarting {} container on dut {}".format(container_name, lc.hostname)) + process_status, program_pid = get_program_info(lc, container_name, BGP_CRIT_PROCESS) if process_status == "RUNNING": - restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + restart_bgp(lc, container_name, service_name, BGP_CRIT_PROCESS, program_pid) - # Verify line cards are in the same state as before docker restart - for linecard in duthosts.frontend_nodes: + # Verify line cards are in the same state as before docker restart # Verify line card config changed to tsa_enabled false - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_NORMAL == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_NORMAL == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in normal state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('false' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert('false' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is enabled".format(lc.hostname)) - for linecard in duthosts.frontend_nodes: - # Wait until all routes are announced to neighbors - cur_v4_routes = {} - cur_v6_routes = {} - # Verify that all routes advertised to neighbor at the start of the test - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, - dut_nbrhosts[linecard], - orig_v4_routes[linecard], cur_v4_routes, 4): - if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], - orig_v4_routes[linecard], cur_v4_routes, 4): - pytest.fail("Not all ipv4 routes are announced to neighbors") - - if not wait_until(300, 3, 0, verify_current_routes_announced_to_neighs, linecard, - dut_nbrhosts[linecard], - orig_v6_routes[linecard], cur_v6_routes, 6): - if not check_and_log_routes_diff(linecard, dut_nbrhosts[linecard], - orig_v6_routes[linecard], cur_v6_routes, 6): - pytest.fail("Not all ipv6 routes are announced to neighbors") + # Restart bgp on the line cards and check the status + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(restart_bgp_and_verify, linecard) + verify_route_on_neighbors_when_duts_on_tsb(duthosts, dut_nbrhosts, orig_v4_routes, orig_v6_routes) finally: # Bring back the supervisor and line cards to the normal state at the end of test set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) @@ -1806,12 +2030,15 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, suphost = duthosts[enum_supervisor_dut_hostname] if get_tsa_chassisdb_config(suphost) not in supported_tsa_configs: pytest.skip("Reliable TSA feature is not supported in this image on dut {}".format(suphost.hostname)) + dut_nbrhosts = dict() - orig_v4_routes, orig_v6_routes = dict(), dict() - for linecard in duthosts.frontend_nodes: - dut_nbrhosts[linecard] = nbrhosts_to_dut(linecard, nbrhosts) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(nbrhosts_to_dut, linecard, nbrhosts, dut_nbrhosts) + # Initially make sure both supervisor and line cards are in BGP operational normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) + orig_v4_routes, orig_v6_routes = dict(), dict() try: # Get the original routes present on the neighbors for each line card for linecard in duthosts.frontend_nodes: @@ -1825,57 +2052,69 @@ def test_dut_tsb_followed_by_dut_bgp_restart_when_sup_and_duts_on_tsa(duthosts, pytest_assert('true' == get_tsa_chassisdb_config(suphost), "Supervisor {} tsa_enabled config is not enabled".format(suphost.hostname)) - # Similarly keep line cards in TSA mode to start with as part of the test - for linecard in duthosts.frontend_nodes: - linecard.shell('TSA') - linecard.shell('sudo config save -y') + def run_tsa_on_linecard_and_verify(lc): + lc.shell('TSA') + lc.shell('sudo config save -y') # Verify line card config changed to TSA enabled true - pytest_assert(verify_dut_configdb_tsa_value(linecard) is True, - "DUT {} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is True, + "DUT {} tsa_enabled config is not enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") - # Issue TSB from line card and verify line cards' BGP operational state maintained at TSA - for linecard in duthosts.frontend_nodes: - linecard.shell('TSB') - linecard.shell('sudo config save -y') + # Similarly keep line cards in TSA mode to start with as part of the test + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsa_on_linecard_and_verify, linecard) + + def run_tsb_on_linecard_and_verify(lc): + lc.shell('TSB') + lc.shell('sudo config save -y') # Verify line card config changed to tsa_enabled false - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) - # Restart bgp on the line cards and check the status - for linecard in duthosts.frontend_nodes: - for asic in linecard.asics: + # Issue TSB from line card and verify line cards' BGP operational state maintained at TSA + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(run_tsb_on_linecard_and_verify, linecard) + + def restart_bgp_and_verify(lc): + for asic in lc.asics: service_name = asic.get_service_name("bgp") container_name = asic.get_docker_name("bgp") - logger.info("Restarting {} container on dut {}".format(container_name, linecard.hostname)) - process_status, program_pid = get_program_info(linecard, container_name, BGP_CRIT_PROCESS) + logger.info("Restarting {} container on dut {}".format(container_name, lc.hostname)) + process_status, program_pid = get_program_info(lc, container_name, BGP_CRIT_PROCESS) if process_status == "RUNNING": - restart_bgp(linecard, container_name, service_name, BGP_CRIT_PROCESS, program_pid) + restart_bgp(lc, container_name, service_name, BGP_CRIT_PROCESS, program_pid) - # Verify line cards are in the same state as before bgp restart - for linecard in duthosts.frontend_nodes: + # Verify line cards are in the same state as before bgp restart # Verify line card config changed to tsa_enabled false - pytest_assert(verify_dut_configdb_tsa_value(linecard) is False, - "DUT {} tsa_enabled config is enabled".format(linecard.hostname)) + pytest_assert(verify_dut_configdb_tsa_value(lc) is False, + "DUT {} tsa_enabled config is enabled".format(lc.hostname)) # Ensure that the DUT is in maintenance state - pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(linecard, cmd='TSC no-stats'), + pytest_assert(TS_MAINTENANCE == get_traffic_shift_state(lc, cmd='TSC no-stats'), "DUT is not in maintenance state") # Ensure line card chassisdb config is in sync with supervisor - pytest_assert('true' == get_tsa_chassisdb_config(linecard), - "{} tsa_enabled config is not enabled".format(linecard.hostname)) + pytest_assert('true' == get_tsa_chassisdb_config(lc), + "{} tsa_enabled config is not enabled".format(lc.hostname)) + + # Restart bgp on the line cards and check the status + with SafeThreadPoolExecutor(max_workers=8) as executor: + for linecard in duthosts.frontend_nodes: + executor.submit(restart_bgp_and_verify, linecard) + + for linecard in duthosts.frontend_nodes: # Verify only loopback routes are announced after TSA pytest_assert(verify_only_loopback_routes_are_announced_to_neighs( duthosts, linecard, dut_nbrhosts[linecard], traffic_shift_community), "Failed to verify routes on nbr in TSA") - finally: # Bring back the supervisor and line cards to the normal state set_tsb_on_sup_duts_before_and_after_test(duthosts, enum_supervisor_dut_hostname) diff --git a/tests/common/fixtures/fib_utils.py b/tests/common/fixtures/fib_utils.py index 36ff543e448..fdb5c927aea 100644 --- a/tests/common/fixtures/fib_utils.py +++ b/tests/common/fixtures/fib_utils.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) -def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts): +def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts, testname=None): """Get parsed FIB information from redis DB for T2 topology. Args: @@ -32,8 +32,12 @@ def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts): ... } """ + timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') fib_info = {} + route_key = 'ROUTE*' + if 'test_ecmp_group_member_flap' in testname: + route_key = 'ROUTE_TABLE:0\.0\.0\.0*' # noqa W605 # Collect system neighbors, inband intf and port channel info to resolve ptf ports # for system neigh or lags. @@ -62,7 +66,7 @@ def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts): asic_index, asic_cfg_facts = asic_cfg_facts_tuple asic = duthost.asic_instance(asic_index) - asic.shell("{} redis-dump -d 0 -k 'ROUTE*' -y > /tmp/fib.{}.txt".format(asic.ns_arg, timestamp)) + asic.shell("{} redis-dump -d 0 -k {} -y > /tmp/fib.{}.txt".format(asic.ns_arg, route_key, timestamp)) duthost.fetch(src="/tmp/fib.{}.txt".format(timestamp), dest="/tmp/fib") po_members = asic_cfg_facts.get('PORTCHANNEL_MEMBER', {}) @@ -158,7 +162,7 @@ def get_t2_fib_info(duthosts, duts_cfg_facts, duts_mg_facts): return fib_info -def get_fib_info(duthost, dut_cfg_facts, duts_mg_facts): +def get_fib_info(duthost, dut_cfg_facts, duts_mg_facts, testname=None): """Get parsed FIB information from redis DB. Args: @@ -180,13 +184,17 @@ def get_fib_info(duthost, dut_cfg_facts, duts_mg_facts): """ timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') fib_info = {} + route_key = 'ROUTE*' + if 'test_ecmp_group_member_flap' in testname: + route_key = r'ROUTE_TABLE:0\.0\.0\.0*' + for list_index, asic_cfg_facts_tuple in enumerate(dut_cfg_facts): asic_index, asic_cfg_facts = asic_cfg_facts_tuple asic = duthost.asic_instance(asic_index) - asic.shell("{} redis-dump -d 0 -k 'ROUTE*' -y > /tmp/fib.{}.txt".format(asic.ns_arg, timestamp)) + asic.shell("{} redis-dump -d 0 -k {} -y > /tmp/fib.{}.txt".format(asic.ns_arg, route_key, timestamp)) duthost.fetch(src="/tmp/fib.{}.txt".format(timestamp), dest="/tmp/fib") po = asic_cfg_facts.get('PORTCHANNEL_MEMBER', {}) @@ -290,7 +298,7 @@ def fib_info_files(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_ if tbinfo['topo']['type'] != "t2": for dut_index, duthost in enumerate(duthosts): fib_info = get_fib_info( - duthost, duts_config_facts[duthost.hostname], duts_minigraph_facts[duthost.hostname] + duthost, duts_config_facts[duthost.hostname], duts_minigraph_facts[duthost.hostname], testname ) if 'test_decap' in testname and 'backend' in tbinfo['topo']['name']: # if it is a storage backend topo and the testcase is test_decap @@ -301,50 +309,7 @@ def fib_info_files(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_ gen_fib_info_file(ptfhost, fib_info, filename) files.append(filename) else: - fib_info = get_t2_fib_info(duthosts, duts_config_facts, duts_minigraph_facts) - filename = '/root/fib_info_all_duts.txt' - gen_fib_info_file(ptfhost, fib_info, filename) - files.append(filename) - - return files - - -@pytest.fixture(scope='function') -def fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, tbinfo, request): - """Get FIB info from database and store to text files on PTF host. - - For T2 topology, generate a single file to /root/fib_info_all_duts.txt to PTF host. - For other topologies, generate one file for each duthost. File name pattern: - /root/fib_info_dut.txt - - Args: - duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts. - ptfhost (PTFHost): Instance of PTFHost for interacting with the PTF host. - duts_running_config_facts (dict): Running config facts of all DUT hosts. - duts_minigraph_facts (dict): Minigraph facts of all DUT hosts. - tbinfo (object): Instance of TestbedInfo. - - Returns: - list: List of FIB info file names on PTF host. - """ - duts_config_facts = duts_running_config_facts - testname = request.node.name - files = [] - if tbinfo['topo']['type'] != "t2": - for dut_index, duthost in enumerate(duthosts): - fib_info = get_fib_info( - duthost, duts_config_facts[duthost.hostname], duts_minigraph_facts[duthost.hostname] - ) - if 'test_basic_fib' in testname and 'backend' in tbinfo['topo']['name']: - # if it is a storage backend topology(bt0 or bt1) and testcase is test_basic_fib - # add a default route as failover in the prefix matching - fib_info['0.0.0.0/0'] = [] - fib_info['::/0'] = [] - filename = '/root/fib_info_dut_{0}_{1}.txt'.format(testname, dut_index) - gen_fib_info_file(ptfhost, fib_info, filename) - files.append(filename) - else: - fib_info = get_t2_fib_info(duthosts, duts_config_facts, duts_minigraph_facts) + fib_info = get_t2_fib_info(duthosts, duts_config_facts, duts_minigraph_facts, testname) filename = '/root/fib_info_all_duts.txt' gen_fib_info_file(ptfhost, fib_info, filename) files.append(filename) diff --git a/tests/common/helpers/tacacs/tacacs_helper.py b/tests/common/helpers/tacacs/tacacs_helper.py index 2c696678260..66856ef82bf 100644 --- a/tests/common/helpers/tacacs/tacacs_helper.py +++ b/tests/common/helpers/tacacs/tacacs_helper.py @@ -177,6 +177,7 @@ def setup_tacacs_server(ptfhost, tacacs_creds, duthost): logger.debug("setup_tacacs_server: duthost options does not contains config for ansible_ssh_user.") ptfhost.host.options['variable_manager'].extra_vars.update(extra_vars) + ptfhost.shell("mkdir -p /etc/tacacs+") ptfhost.template(src="tacacs/tac_plus.conf.j2", dest="/etc/tacacs+/tac_plus.conf") # Find 'python' command symbolic link target, and fix the tac_plus config file diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index e9add868c8d..f08d91a7e08 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -1611,25 +1611,12 @@ qos/test_buffer_traditional.py: conditions: - "release not in ['201911']" - "topo_type in ['m0', 'mx']" - - "asic_type in ['vs']" qos/test_ecn_config.py: skip: - reason: "Temporarily skip in PR testing" - conditions: - - "asic_type in ['vs']" - -qos/test_pfc_counters.py: - skip: - reason: "Temporarily skip in PR testing" - conditions: - - "asic_type in ['vs']" - -qos/test_pfc_pause.py: - skip: - reason: "Temporarily skip in PR testing" + reason: "This test only support on cisco device" conditions: - - "asic_type in ['vs']" + - "asic_type not in ['cisco-8000']" qos/test_pfc_pause.py::test_pfc_pause_lossless: # For this test, we use the fanout connected to the DUT to send PFC pause frames. @@ -1646,10 +1633,6 @@ qos/test_qos_dscp_mapping.py: strict: True conditions: - "asic_type in ['cisco-8000'] and platform in ['x86_64-8122_64eh_o-r0']" - skip: - reason: "Temporarily skip in PR testing" - conditions: - - "asic_type in ['vs']" qos/test_qos_dscp_mapping.py::TestQoSSaiDSCPQueueMapping_IPIP_Base::test_dscp_to_queue_mapping_pipe_mode: skip: @@ -1659,11 +1642,10 @@ qos/test_qos_dscp_mapping.py::TestQoSSaiDSCPQueueMapping_IPIP_Base::test_dscp_to - "asic_type in ['mellanox', 'broadcom', 'cisco-8000']" - https://github.com/sonic-net/sonic-mgmt/issues/12906 - "topo_type in ['m0', 'mx']" - - "asic_type in ['vs']" qos/test_qos_masic.py: skip: - reason: "QoS tests for multi-ASIC only. Supported topos: t1-lag, t1-64-lag, t1-56-lag, t1-backend. / M0/MX topo does not support qos" + reason: "QoS tests for multi-ASIC only. Supported topos: t1-lag, t1-64-lag, t1-56-lag, t1-backend. / M0/MX topo does not support qos. / KVM do not support swap syncd." conditions_logical_operator: or conditions: - "is_multi_asic==False or topo_name not in ['t1-lag', 't1-64-lag', 't1-56-lag', 't1-backend']" @@ -1886,7 +1868,7 @@ qos/test_qos_sai.py::TestQosSai::testQosSaiSharedReservationSize: qos/test_tunnel_qos_remap.py: skip: - reason: "Temporarily skip in PR testing" + reason: "Tunnel qos remap test needs some SAI attributes, which are not supported on KVM." conditions: - "asic_type in ['vs']" @@ -2309,6 +2291,15 @@ test_vs_chassis_setup.py: conditions: - "asic_type not in ['vs']" +####################################### +##### upgrade_path ##### +####################################### +upgrade_path: + skip: + reason: "Upgrade path test needs base and target image lists, currently do not support on KVM." + conditions: + - "asic_type in ['vs']" + ####################################### ##### vlan ##### ####################################### @@ -2355,12 +2346,6 @@ voq/test_fabric_reach.py: - "not any(i in platform for i in ['arista_7800', 'x86_64-nokia_ixr7250e'])" - "(asic_type in ['cisco-8000'])" -voq/test_voq_fabric_capacity.py: - skip: - reason: "Skip test_voq_fabric_capacity on unsupported testbed." - conditions: - - "('t2' not in topo_name) or (asic_subtype not in ['broadcom-dnx']) or (asic_type in ['cisco-8000'])" - voq/test_voq_counter.py::TestVoqCounter::test_voq_queue_counter[multi_dut_shortlink_to_longlink]: skip: reason: "Testcase ignored due to sonic-mgmt issue: https://github.com/sonic-net/sonic-buildimage/issues/21098" @@ -2368,6 +2353,12 @@ voq/test_voq_counter.py::TestVoqCounter::test_voq_queue_counter[multi_dut_shortl - "(switch_type=='voq')" - "('t2' in topo_name)" +voq/test_voq_fabric_capacity.py: + skip: + reason: "Skip test_voq_fabric_capacity on unsupported testbed." + conditions: + - "('t2' not in topo_name) or (asic_subtype not in ['broadcom-dnx']) or (asic_type in ['cisco-8000'])" + voq/test_voq_fabric_isolation.py: skip: reason: "Skip test_voq_fabric_isolation on unsupported testbed." diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml index 12cccb05cd6..b5a28d7c6e2 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_skip_traffic_test.yaml @@ -285,6 +285,15 @@ ipfwd/test_nhop_group.py: conditions: - "asic_type in ['vs']" +####################################### +##### qos ##### +####################################### +qos/test_qos_dscp_mapping.py: + skip_traffic_test: + reason: "Skip traffic test for KVM testbed" + conditions: + - "asic_type in ['vs']" + ####################################### ##### route ##### ####################################### diff --git a/tests/common/plugins/sanity_check/__init__.py b/tests/common/plugins/sanity_check/__init__.py index f9ce9c570af..b62be4ad7bc 100644 --- a/tests/common/plugins/sanity_check/__init__.py +++ b/tests/common/plugins/sanity_check/__init__.py @@ -181,7 +181,7 @@ def prepare_parallel_run(request, parallel_run_context): @pytest.fixture(scope="module") -def sanity_check_full(prepare_parallel_run, localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): +def sanity_check_full(ptfhost, prepare_parallel_run, localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): logger.info("Prepare sanity check") should_skip_sanity = prepare_parallel_run if should_skip_sanity: @@ -299,7 +299,7 @@ def sanity_check_full(prepare_parallel_run, localhost, duthosts, request, fanout pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}" .format(json.dumps(failed_results, indent=4, default=fallback_serializer))) else: - recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, localhost, nbrhosts, + recover_on_sanity_check_failure(ptfhost, duthosts, failed_results, fanouthosts, localhost, nbrhosts, pre_check_items, recover_method, request, tbinfo, STAGE_PRE_TEST) logger.info("Done pre-test sanity check") @@ -325,15 +325,16 @@ def sanity_check_full(prepare_parallel_run, localhost, duthosts, request, fanout pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}" .format(json.dumps(post_failed_results, indent=4, default=fallback_serializer))) else: - recover_on_sanity_check_failure(duthosts, post_failed_results, fanouthosts, localhost, nbrhosts, - post_check_items, recover_method, request, tbinfo, STAGE_POST_TEST) + recover_on_sanity_check_failure(ptfhost, duthosts, post_failed_results, fanouthosts, localhost, + nbrhosts, post_check_items, recover_method, request, tbinfo, + STAGE_POST_TEST) logger.info("Done post-test sanity check") else: logger.info('No post-test sanity check item, skip post-test sanity check.') -def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, localhost, nbrhosts, check_items, +def recover_on_sanity_check_failure(ptfhost, duthosts, failed_results, fanouthosts, localhost, nbrhosts, check_items, recover_method, request, tbinfo, sanity_check_stage: str): cache_key = "pre_sanity_check_failed" recovery_cache_key = "pre_sanity_recovered" @@ -363,7 +364,7 @@ def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, local else: for dut_name, dut_results in list(dut_failed_results.items()): # Attempt to restore DUT state - recover(duthosts[dut_name], localhost, fanouthosts, nbrhosts, tbinfo, dut_results, + recover(ptfhost, duthosts[dut_name], localhost, fanouthosts, nbrhosts, tbinfo, dut_results, recover_method) except BaseException as e: diff --git a/tests/common/plugins/sanity_check/recover.py b/tests/common/plugins/sanity_check/recover.py index 42a4096d51e..bc85a7e6b0b 100644 --- a/tests/common/plugins/sanity_check/recover.py +++ b/tests/common/plugins/sanity_check/recover.py @@ -1,5 +1,6 @@ import json import logging +import time from tests.common import config_reload from tests.common.devices.sonic import SonicHost @@ -7,7 +8,7 @@ from tests.common.platform.device_utils import fanout_switch_port_lookup from tests.common.reboot import REBOOT_TYPE_WARM, REBOOT_TYPE_FAST, REBOOT_TYPE_COLD from tests.common.reboot import reboot -from tests.common.utilities import wait +from tests.common.utilities import wait, wait_until from . import constants from ...helpers.multi_thread_utils import SafeThreadPoolExecutor @@ -147,13 +148,49 @@ def _recover_with_command(dut, cmd, wait_time): wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time)) -def re_announce_routes(localhost, topo_name, ptf_ip): - localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="withdraw", path="../ansible/") - localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="announce", path="../ansible/") +def re_announce_routes(ptfhost, localhost, topo_name, ptf_ip, neighbor_number): + def _check_exabgp(): + # Get pid of exabgp processes + exabgp_pids = [] + output = ptfhost.shell("ps aux | grep exabgp/http_api |grep -v grep | awk '{print $2}'", + module_ignore_errors=True) + if output['rc'] != 0: + logger.warning("cmd to fetch exabgp pid returned with error: {}".format(output["stderr"])) + return False + for line in output["stdout_lines"]: + if len(line.strip()) == 0: + continue + exabgp_pids.append(line.strip()) + # Each bgp neighbor has 2 exabgp process, one is for v4 and another is for v6 + if len(exabgp_pids) != neighbor_number * 2: + logger.info("pids number for exabgp processes is incorrect, expected: {}, actual: {}" + .format(neighbor_number * 2, len(exabgp_pids))) + return False + # Check whether all sockets for exabgp are created + output = ptfhost.shell("ss -nltp | grep -E \"{}\"" + .format("|".join(["pid={}".format(pid) for pid in exabgp_pids])), + module_ignore_errors=True) + return output["rc"] == 0 and len(output["stdout_lines"]) == neighbor_number * 2 + + def _op_routes(action): + try: + localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action=action, path="../ansible/") + time.sleep(5) + except Exception as e: + logger.error("Failed to {} routes with error: {}".format(action, e)) + + ptfhost.shell("supervisorctl restart exabgpv4:*", module_ignore_errors=True) + ptfhost.shell("supervisorctl restart exabgpv6:*", module_ignore_errors=True) + # Wait exabgp to be ready + if not wait_until(120, 5, 0, _check_exabgp): + logger.error("Not all exabgp process are running") + + _op_routes("withdraw") + _op_routes("announce") return None -def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, wait_time): +def adaptive_recover(ptfhost, dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, wait_time): outstanding_action = None for result in check_results: if result['failed']: @@ -169,7 +206,8 @@ def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_result "no_v6_default_route" in result['bgp'] and len(result['bgp']) == 1 or ("no_v4_default_route" in result['bgp'] and "no_v6_default_route" in result['bgp'] and len(result['bgp']) == 2))): - action = re_announce_routes(localhost, tbinfo["topo"]["name"], tbinfo["ptf_ip"]) + action = re_announce_routes(ptfhost, localhost, tbinfo["topo"]["name"], tbinfo["ptf_ip"], + len(nbrhosts)) else: action = neighbor_vm_restore(dut, nbrhosts, tbinfo, result) elif result['check_item'] == "neighbor_macsec_empty": @@ -199,13 +237,13 @@ def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_result _recover_with_command(dut, method['cmd'], wait_time) -def recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, recover_method): +def recover(ptfhost, dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, recover_method): logger.warning("Try to recover %s using method %s" % (dut.hostname, recover_method)) method = constants.RECOVER_METHODS[recover_method] wait_time = method['recover_wait'] if method["adaptive"]: - adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, wait_time) + adaptive_recover(ptfhost, dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, wait_time) elif method["reload"]: config_reload(dut, config_source='running_golden_config', safe_reload=True, check_intf_up_ports=True, wait_for_bgp=True) diff --git a/tests/common/utilities.py b/tests/common/utilities.py index 3c2575601f1..e455621af26 100644 --- a/tests/common/utilities.py +++ b/tests/common/utilities.py @@ -1320,9 +1320,11 @@ def reload_minigraph_with_golden_config(duthost, json_data, safe_reload=True): from tests.common.config_reload import config_reload golden_config = "/etc/sonic/golden_config_db.json" duthost.copy(content=json.dumps(json_data, indent=4), dest=golden_config) - config_reload(duthost, config_source="minigraph", safe_reload=safe_reload, override_config=True) - # Cleanup golden config because some other test or device recover may reload config with golden config - duthost.command('mv {} {}_backup'.format(golden_config, golden_config)) + try: + config_reload(duthost, config_source="minigraph", safe_reload=safe_reload, override_config=True) + finally: + # Cleanup golden config because some other test or device recover may reload config with golden config + duthost.command('mv {} {}_backup'.format(golden_config, golden_config)) def file_exists_on_dut(duthost, filename): diff --git a/tests/fib/test_fib.py b/tests/fib/test_fib.py index 0a4e5fb4bf1..11d1fddb9f5 100644 --- a/tests/fib/test_fib.py +++ b/tests/fib/test_fib.py @@ -1,3 +1,4 @@ +import re import time import logging @@ -20,8 +21,7 @@ from tests.common.dualtor.dual_tor_common import active_active_ports # noqa F401 from tests.common.utilities import is_ipv4_address -from tests.common.fixtures.fib_utils import fib_info_files_per_function # noqa F401 -from tests.common.fixtures.fib_utils import single_fib_for_duts # noqa F401 +from tests.common.fixtures.fib_utils import single_fib_for_duts, get_fib_info, get_t2_fib_info, gen_fib_info_file # noqa F401 from tests.common.utilities import wait from tests.common.helpers.assertions import pytest_require @@ -51,6 +51,162 @@ PTF_TEST_PORT_MAP = '/root/ptf_test_port_map.json' +# Helper Functions +def check_default_route_from_fib_info(ptfhost, file_path): + """ + Check for the default route (0.0.0.0/0) in the FIB information file + and return a list of next hop port indices. + + Args: + ptfhost: The PTF host object. + file_path: The path to the FIB info file. + + Returns: + A list of next hop port indices or an empty list if not found. + """ + + # Attempt to read the FIB info file + result = ptfhost.shell("cat {}".format(file_path)) + if result['rc'] != 0: + logger.error("Failed to read file {} from PTF host.".format(file_path)) + return [] + + lines = result['stdout_lines'] + + # Find the line containing the default route + default_route_line = next((line.strip() for line in lines if '0.0.0.0/0' in line), None) + + if not default_route_line: + logger.info("No default route found. Returning an empty list.") + return [] # Return an empty list if no default route is found + + # Count the number of next hops (each '[]' represents one nexthop) + nexthops_count = len(re.findall(r'\[.*?\]', default_route_line)) + + if nexthops_count <= 1: + logger.info("Number of nexthops is less than or equal to 1. Returning an empty list.") + return [] # Return empty list if only one or no nexthop + + # Extract all numbers inside square brackets and convert them to integers + matches = re.findall(r'\[(\d+(?: \d+)*)\]', default_route_line) + ports = [int(num) for group in matches for num in group.split()] + + return ports + + +def get_all_ptf_port_indices_from_mg_facts(mg_facts): + """ + Retrieve all ptf port indices from the minigraph facts. + + Args: + mg_facts: The minigraph facts containing ASIC information. + + Returns: + A dictionary containing mapping of ptf port indices to dutport(asic_id, port_name). + """ + + all_port_indices = {} # Dictionary to store (port_index, (asic_id, port_name)) + + # Iterate over all ASICs in mg_facts + for asic_id, asic_data in mg_facts: + minigraph_indices = asic_data['minigraph_ptf_indices'] + + # Store (port_index: (asic_id, port_name)) in the dictionary + for port_name, port_index in minigraph_indices.items(): + all_port_indices[port_index] = (asic_id, port_name) + + return all_port_indices + + +def map_ptf_ports_to_dut_port(ptf_ports, all_dut_port_indices): + """ + Map PTF port indices to DUT port information. + + Args: + ptf_ports: List of PTF port indices. + all_dut_port_indices: Dictionary of DUT port indices. + + Returns: + A list of tuples containing (asic_id, port_name) for mapped ports. + """ + + ethernet_ports_with_asic = [ + (asic_id, port_name) + for port in ptf_ports + if (port_info := all_dut_port_indices.get(port)) + for asic_id, port_name in [port_info] + ] + + return ethernet_ports_with_asic + + +def filter_ports(all_port_indices, tbinfo): + """ + Filter PTF ports that need to be skipped while picking up src_port for ptf traffic test. + + Args: + all_port_indices: Dictionary with available port indices. + Keys represent PTF ports; values are port numbers. + tbinfo: Dictionary with testbed information, including topology type. + + Returns: + A list of PTF ports to be skipped as src_port. + """ + + # Note: this filteration is useful for multilinecard DUTs to make sure incoming traffic is + # landing on a different linecard; NA for pizza boxes + + if tbinfo['topo']['type'] != 't2': + return [] + + # Collect all port indices (keys) from all_port_indices + host_ptf_ports_all = list(all_port_indices.keys()) + + return host_ptf_ports_all + + +def fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, tbinfo, request): + """Get FIB info from database and store to text files on PTF host. + + For T2 topology, generate a single file to /root/fib_info_all_duts.txt to PTF host. + For other topologies, generate one file for each duthost. File name pattern: + /root/fib_info_dut.txt + + Args: + duthosts (DutHosts): Instance of DutHosts for interacting with DUT hosts. + ptfhost (PTFHost): Instance of PTFHost for interacting with the PTF host. + duts_running_config_facts (dict): Running config facts of all DUT hosts. + duts_minigraph_facts (dict): Minigraph facts of all DUT hosts. + tbinfo (object): Instance of TestbedInfo. + + Returns: + list: List of FIB info file names on PTF host. + """ + duts_config_facts = duts_running_config_facts + testname = request.node.name + files = [] + if tbinfo['topo']['type'] != "t2": + for dut_index, duthost in enumerate(duthosts): + fib_info = get_fib_info( + duthost, duts_config_facts[duthost.hostname], duts_minigraph_facts[duthost.hostname], testname + ) + if 'test_basic_fib' in testname and 'backend' in tbinfo['topo']['name']: + # if it is a storage backend topology(bt0 or bt1) and testcase is test_basic_fib + # add a default route as failover in the prefix matching + fib_info['0.0.0.0/0'] = [] + fib_info['::/0'] = [] + filename = '/root/fib_info_dut_{0}_{1}.txt'.format(testname, dut_index) + gen_fib_info_file(ptfhost, fib_info, filename) + files.append(filename) + else: + fib_info = get_t2_fib_info(duthosts, duts_config_facts, duts_minigraph_facts, testname) + filename = '/root/fib_info_all_duts.txt' + gen_fib_info_file(ptfhost, fib_info, filename) + files.append(filename) + + return files + + @pytest.fixture(scope="module") def ignore_ttl(duthosts): # on the multi asic devices, the packet can have different ttl based on how the packet is routed @@ -76,18 +232,20 @@ def updated_tbinfo(tbinfo): @pytest.mark.parametrize("ipv4, ipv6, mtu", [pytest.param(True, True, 1514)]) -def test_basic_fib(duthosts, ptfhost, ipv4, ipv6, mtu, +def test_basic_fib(duthosts, ptfhost, tbinfo, ipv4, ipv6, mtu, toggle_all_simulator_ports_to_random_side, # noqa F811 - fib_info_files_per_function, # noqa F401 updated_tbinfo, mux_server_url, # noqa F401 mux_status_from_nic_simulator, ignore_ttl, single_fib_for_duts, # noqa F401 duts_running_config_facts, duts_minigraph_facts, - validate_active_active_dualtor_setup): # noqa F811 + validate_active_active_dualtor_setup, # noqa F401 + request): # noqa F811 if 'dualtor' in updated_tbinfo['topo']['name']: wait(30, 'Wait some time for mux active/standby state to be stable after toggled mux state') + fib_files = fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, + tbinfo, request) timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') switch_type = duthosts[0].facts.get('switch_type') @@ -110,7 +268,7 @@ def test_basic_fib(duthosts, ptfhost, ipv4, ipv6, mtu, platform_dir="ptftests", params={ # Test at most 3 DUTs - "fib_info_files": fib_info_files_per_function[:3], + "fib_info_files": fib_files[:3], "ptf_test_port_map": ptf_test_port_map_active_active( ptfhost, updated_tbinfo, duthosts, mux_server_url, duts_running_config_facts, duts_minigraph_facts, @@ -311,15 +469,18 @@ def setup_active_active_ports( return -def test_hash(add_default_route_to_dut, duthosts, fib_info_files_per_function, setup_vlan, # noqa F811 +def test_hash(add_default_route_to_dut, duthosts, tbinfo, setup_vlan, # noqa F811 hash_keys, ptfhost, ipver, toggle_all_simulator_ports_to_rand_selected_tor_m, # noqa F811 updated_tbinfo, mux_server_url, mux_status_from_nic_simulator, ignore_ttl, # noqa F811 single_fib_for_duts, duts_running_config_facts, duts_minigraph_facts, # noqa F811 - setup_active_active_ports, active_active_ports): # noqa F811 + setup_active_active_ports, active_active_ports, request): # noqa F811 if 'dualtor' in updated_tbinfo['topo']['name']: wait(30, 'Wait some time for mux active/standby state to be stable after toggled mux state') + fib_files = fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, + tbinfo, request) + is_active_active_dualtor = bool(active_active_ports) switch_type = duthosts[0].facts.get('switch_type') timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') @@ -337,7 +498,7 @@ def test_hash(add_default_route_to_dut, duthosts, fib_info_files_per_function, s "hash_test.HashTest", platform_dir="ptftests", params={ - "fib_info_files": fib_info_files_per_function[:3], # Test at most 3 DUTs + "fib_info_files": fib_files[:3], # Test at most 3 DUTs "ptf_test_port_map": ptf_test_port_map_active_active( ptfhost, updated_tbinfo, duthosts, mux_server_url, duts_running_config_facts, duts_minigraph_facts, @@ -362,13 +523,17 @@ def test_hash(add_default_route_to_dut, duthosts, fib_info_files_per_function, s # used as hash keys -def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_per_function, # noqa F811 +def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, # noqa F811 hash_keys, ptfhost, ipver, tbinfo, mux_server_url, # noqa F811 ignore_ttl, single_fib_for_duts, duts_running_config_facts, # noqa F811 - duts_minigraph_facts): # noqa F811 + duts_minigraph_facts, request): # noqa F811 # Skip test on none T1 testbed pytest_require('t1' == tbinfo['topo']['type'], "The test case runs on T1 topology") + + fib_files = fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, + tbinfo, request) + timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') log_file = "/tmp/hash_test.IPinIPHashTest.{}.{}.log".format( ipver, timestamp) @@ -383,7 +548,7 @@ def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files "ptftests", "hash_test.IPinIPHashTest", platform_dir="ptftests", - params={"fib_info_files": fib_info_files_per_function[:3], # Test at most 3 DUTs + params={"fib_info_files": fib_files[:3], # Test at most 3 DUTs "ptf_test_port_map": ptf_test_port_map(ptfhost, tbinfo, duthosts, mux_server_url, duts_running_config_facts, duts_minigraph_facts), "hash_keys": hash_keys, @@ -403,10 +568,13 @@ def test_ipinip_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files # Only inner frame length is tested at this moment -def test_ipinip_hash_negative(add_default_route_to_dut, duthosts, fib_info_files_per_function, # noqa F811 +def test_ipinip_hash_negative(add_default_route_to_dut, duthosts, # noqa F811 ptfhost, ipver, tbinfo, mux_server_url, ignore_ttl, single_fib_for_duts, # noqa F811 - duts_running_config_facts, duts_minigraph_facts, mux_status_from_nic_simulator): + duts_running_config_facts, duts_minigraph_facts, mux_status_from_nic_simulator, + request): # noqa F811 hash_keys = ['inner_length'] + fib_files = fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, + tbinfo, request) timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') log_file = "/tmp/hash_test.IPinIPHashTest.{}.{}.log".format( ipver, timestamp) @@ -421,7 +589,7 @@ def test_ipinip_hash_negative(add_default_route_to_dut, duthosts, fib_info_files "ptftests", "hash_test.IPinIPHashTest", platform_dir="ptftests", - params={"fib_info_files": fib_info_files_per_function[:3], # Test at most 3 DUTs + params={"fib_info_files": fib_files[:3], # Test at most 3 DUTs "ptf_test_port_map": ptf_test_port_map_active_active( ptfhost, tbinfo, duthosts, mux_server_url, duts_running_config_facts, duts_minigraph_facts, @@ -444,10 +612,13 @@ def test_ipinip_hash_negative(add_default_route_to_dut, duthosts, fib_info_files @pytest.fixture(params=["ipv4-ipv4", "ipv4-ipv6", "ipv6-ipv6", "ipv6-ipv4"]) def vxlan_ipver(request): return request.param -def test_vxlan_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_per_function, # noqa F811 +def test_vxlan_hash(add_default_route_to_dut, duthost, duthosts, # noqa F811 hash_keys, ptfhost, vxlan_ipver, tbinfo, mux_server_url, # noqa F811 - ignore_ttl, single_fib_for_duts, duts_running_config_facts, # noqa F811 - duts_minigraph_facts): # noqa F811 + ignore_ttl, single_fib_for_duts, duts_running_config_facts, # noqa F811 + duts_minigraph_facts, request): # noqa F811 + + fib_files = fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, + tbinfo, request) # Query the default VxLAN UDP port from switch's APPL_DB vxlan_dport_check = duthost.shell('redis-cli -n 0 hget "SWITCH_TABLE:switch" "vxlan_port"') if 'stdout' in vxlan_dport_check and vxlan_dport_check['stdout'].isdigit(): @@ -470,7 +641,7 @@ def test_vxlan_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_ "ptftests", "hash_test.VxlanHashTest", platform_dir="ptftests", - params={"fib_info_files": fib_info_files_per_function[:3], # Test at most 3 DUTs + params={"fib_info_files": fib_files[:3], # Test at most 3 DUTs "ptf_test_port_map": ptf_test_port_map(ptfhost, tbinfo, duthosts, mux_server_url, duts_running_config_facts, duts_minigraph_facts), "hash_keys": hash_keys, @@ -491,11 +662,13 @@ def test_vxlan_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_ @pytest.fixture(params=["ipv4-ipv4", "ipv4-ipv6", "ipv6-ipv6", "ipv6-ipv4"]) def nvgre_ipver(request): return request.param -def test_nvgre_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_per_function, # noqa F811 +def test_nvgre_hash(add_default_route_to_dut, duthost, duthosts, # noqa F811 hash_keys, ptfhost, nvgre_ipver, tbinfo, mux_server_url, # noqa F811 - ignore_ttl, single_fib_for_duts, duts_running_config_facts, # noqa F811 - duts_minigraph_facts): # noqa F811 + ignore_ttl, single_fib_for_duts, duts_running_config_facts, # noqa F811 + duts_minigraph_facts, request): # noqa F811 + fib_files = fib_info_files_per_function(duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, + tbinfo, request) # For NVGRE, default hash key is inner 5-tuple. # Due to current limitation, NVGRE hash keys are updated for different vendors. # Hash-key will be updated once we get the full support. @@ -521,7 +694,7 @@ def test_nvgre_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_ "ptftests", "hash_test.NvgreHashTest", platform_dir="ptftests", - params={"fib_info_files": fib_info_files_per_function[:3], # Test at most 3 DUTs + params={"fib_info_files": fib_files[:3], # Test at most 3 DUTs "ptf_test_port_map": ptf_test_port_map(ptfhost, tbinfo, duthosts, mux_server_url, duts_running_config_facts, duts_minigraph_facts), "hash_keys": hash_keys, @@ -536,3 +709,182 @@ def test_nvgre_hash(add_default_route_to_dut, duthost, duthosts, fib_info_files_ qlen=PTF_QLEN, socket_recv_size=16384, is_python3=True) + + +@pytest.mark.parametrize("ipv4, ipv6, mtu", [pytest.param(True, False, 1514)]) +def test_ecmp_group_member_flap( + duthosts, ptfhost, tbinfo, ipv4, ipv6, mtu, + toggle_all_simulator_ports_to_random_side, # noqa F811 + updated_tbinfo, mux_server_url, # noqa F401 + mux_status_from_nic_simulator, ignore_ttl, + single_fib_for_duts, # noqa F401 + duts_running_config_facts, duts_minigraph_facts, + validate_active_active_dualtor_setup, request # noqa F401 +): + """Test ECMP group member flap handling.""" + + if 'dualtor' in updated_tbinfo['topo']['name']: + wait(30, 'Wait some time for mux active/standby state to be stable after toggled mux state') + + # Determine test balancing based on asic type and switch platform + switch_type = duthosts[0].facts.get('switch_type') + + # do not test load balancing for vs platform as kernel 4.9 + # can only do load balance base on L3 + asic_type = duthosts[0].facts['asic_type'] + if asic_type in ["vs"]: + test_balancing = False + else: + test_balancing = True + + # --- Load initial FIB files --- + fib_files = fib_info_files_per_function( + duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, tbinfo, request + ) + + # Verify that the default route has valid nexthops + nh_ptf_ports = check_default_route_from_fib_info(ptfhost, fib_files[0]) + logging.info("nh_ptf_ports: {}".format(nh_ptf_ports)) + if not nh_ptf_ports: + pytest.skip("Skipping test as default route is missing or has fewer than 2 nexthops.") + + # --- Identify the DUT and ports from the minigraph facts --- + upstream_lc = duthosts[0].hostname + logging.info("upstream_lc: {}".format(upstream_lc)) + + all_port_indices = get_all_ptf_port_indices_from_mg_facts(duts_minigraph_facts[upstream_lc]) + nh_dut_ports = map_ptf_ports_to_dut_port(nh_ptf_ports, all_port_indices) + filtered_ports = filter_ports(all_port_indices, tbinfo) + + logging.info("nh_dut_ports: {}".format(nh_dut_ports)) + logging.info("filtered_ports: {}".format(filtered_ports)) + + # --- Prepare logging and timestamps --- + timestamp = datetime.now().strftime('%Y-%m-%d-%H:%M:%S') + log_file = "/tmp/fib_test.ecmp_member_flap.ipv4.{}.ipv6.{}.{}.log".format(ipv4, ipv6, timestamp) + logging.info("PTF log file: {}".format(log_file)) + + # --- Run the initial PTF test to verify default route behavior --- + logging.info("Starting ECMP test for default route.") + ptf_runner( + ptfhost, + "ptftests", + "fib_test.FibTest", + platform_dir="ptftests", + params={ + "fib_info_files": fib_files[:3], # Test up to 3 DUTs + "ptf_test_port_map": ptf_test_port_map_active_active( + ptfhost, updated_tbinfo, duthosts, mux_server_url, + duts_running_config_facts, duts_minigraph_facts, + mux_status_from_nic_simulator() + ), + "ipv4": ipv4, + "ipv6": ipv6, + "testbed_mtu": mtu, + "test_balancing": test_balancing, + "ignore_ttl": ignore_ttl, + "single_fib_for_duts": single_fib_for_duts, + "switch_type": switch_type, + "asic_type": asic_type, + "skip_src_ports": filtered_ports + }, + log_file=log_file, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + # --- Simulate port flap: shutdown one uplink port --- + logging.info("Shutting down one uplink port.") + num_asic = duthosts[0].num_asics() + asic_ns = "" + if num_asic > 1: + asic_ns = "-n asic{}".format(nh_dut_ports[0][0]) + logging.info("Shutting down port {}".format(nh_dut_ports[0][1])) + duthosts[0].shell("sudo config interface {} shutdown {}".format(asic_ns, nh_dut_ports[0][1])) + + time.sleep(10) # Allow time for the state to stabilize + + # --- Re-run the PTF test after member down --- + logging.info("Verifying ECMP behavior after member down.") + new_fib_files1 = fib_info_files_per_function( + duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, tbinfo, request + ) + member_down_log_file = "/tmp/fib_test.ecmp_member_flap.member_down.ipv4.{}.ipv6.{}.{}.log".format( + ipv4, ipv6, timestamp) + logging.info("PTF log file: {}".format(member_down_log_file)) + + ptf_runner( + ptfhost, + "ptftests", + "fib_test.FibTest", + platform_dir="ptftests", + params={ + "fib_info_files": new_fib_files1[:3], + "ptf_test_port_map": ptf_test_port_map_active_active( + ptfhost, updated_tbinfo, duthosts, mux_server_url, + duts_running_config_facts, duts_minigraph_facts, + mux_status_from_nic_simulator() + ), + "ipv4": ipv4, + "ipv6": ipv6, + "testbed_mtu": mtu, + "test_balancing": test_balancing, + "ignore_ttl": ignore_ttl, + "single_fib_for_duts": single_fib_for_duts, + "switch_type": switch_type, + "asic_type": asic_type, + "skip_src_ports": filtered_ports + }, + log_file=member_down_log_file, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + # --- Bring the port back up and verify --- + logging.info("Bringing the uplink port back up.") + + logging.info("Enabling port {}".format(nh_dut_ports[0][1])) + duthosts[0].shell("sudo config interface {} startup {}".format(asic_ns, nh_dut_ports[0][1])) + + time.sleep(60) # Allow time for the state to stabilize + + # --- Re-run the PTF test after member is back up --- + logging.info("Re-verifying ECMP behavior after member up.") + new_fib_files2 = fib_info_files_per_function( + duthosts, ptfhost, duts_running_config_facts, duts_minigraph_facts, tbinfo, request + ) + member_up_log_file = "/tmp/fib_test.ecmp_member_flap.member_up.ipv4.{}.ipv6.{}.{}.log".format( + ipv4, ipv6, timestamp) + logging.info("PTF log file: {}".format(member_up_log_file)) + + ptf_runner( + ptfhost, + "ptftests", + "fib_test.FibTest", + platform_dir="ptftests", + params={ + "fib_info_files": new_fib_files2[:3], + "ptf_test_port_map": ptf_test_port_map_active_active( + ptfhost, updated_tbinfo, duthosts, mux_server_url, + duts_running_config_facts, duts_minigraph_facts, + mux_status_from_nic_simulator() + ), + "ipv4": ipv4, + "ipv6": ipv6, + "testbed_mtu": mtu, + "test_balancing": test_balancing, + "ignore_ttl": ignore_ttl, + "single_fib_for_duts": single_fib_for_duts, + "switch_type": switch_type, + "asic_type": asic_type, + "skip_src_ports": filtered_ports + }, + log_file=member_up_log_file, + qlen=PTF_QLEN, + socket_recv_size=16384, + is_python3=True + ) + + logging.info("All tests passed.") diff --git a/tests/ipfwd/test_nhop_group.py b/tests/ipfwd/test_nhop_group.py index 8f94b836fdb..fc3863e7844 100644 --- a/tests/ipfwd/test_nhop_group.py +++ b/tests/ipfwd/test_nhop_group.py @@ -666,31 +666,31 @@ def built_and_send_tcp_ip_packet(): 45: 'c0:ff:ee:00:00:0c', 46: 'c0:ff:ee:00:00:0d', 47: 'c0:ff:ee:00:00:0b', 48: 'c0:ff:ee:00:00:11', 49: 'c0:ff:ee:00:00:0f'} - td3_asic_flow_map = {0: 'c0:ff:ee:00:00:10', 1: 'c0:ff:ee:00:00:0b', - 2: 'c0:ff:ee:00:00:12', 3: 'c0:ff:ee:00:00:0d', - 4: 'c0:ff:ee:00:00:11', 5: 'c0:ff:ee:00:00:0e', - 6: 'c0:ff:ee:00:00:0f', 7: 'c0:ff:ee:00:00:0c', - 8: 'c0:ff:ee:00:00:0e', 9: 'c0:ff:ee:00:00:11', - 10: 'c0:ff:ee:00:00:0c', 11: 'c0:ff:ee:00:00:0f', - 12: 'c0:ff:ee:00:00:12', 13: 'c0:ff:ee:00:00:0d', - 14: 'c0:ff:ee:00:00:10', 15: 'c0:ff:ee:00:00:0b', - 16: 'c0:ff:ee:00:00:11', 17: 'c0:ff:ee:00:00:0e', - 18: 'c0:ff:ee:00:00:0f', 19: 'c0:ff:ee:00:00:0c', - 20: 'c0:ff:ee:00:00:10', 21: 'c0:ff:ee:00:00:0b', - 22: 'c0:ff:ee:00:00:12', 23: 'c0:ff:ee:00:00:0d', - 24: 'c0:ff:ee:00:00:11', 25: 'c0:ff:ee:00:00:0e', - 26: 'c0:ff:ee:00:00:0f', 27: 'c0:ff:ee:00:00:0c', - 28: 'c0:ff:ee:00:00:0b', 29: 'c0:ff:ee:00:00:10', - 30: 'c0:ff:ee:00:00:0d', 31: 'c0:ff:ee:00:00:12', - 32: 'c0:ff:ee:00:00:0c', 33: 'c0:ff:ee:00:00:0f', - 34: 'c0:ff:ee:00:00:0e', 35: 'c0:ff:ee:00:00:11', - 36: 'c0:ff:ee:00:00:0d', 37: 'c0:ff:ee:00:00:12', - 38: 'c0:ff:ee:00:00:0b', 39: 'c0:ff:ee:00:00:10', - 40: 'c0:ff:ee:00:00:12', 41: 'c0:ff:ee:00:00:0d', - 42: 'c0:ff:ee:00:00:10', 43: 'c0:ff:ee:00:00:0b', - 44: 'c0:ff:ee:00:00:0e', 45: 'c0:ff:ee:00:00:11', - 46: 'c0:ff:ee:00:00:0c', 47: 'c0:ff:ee:00:00:0f', - 48: 'c0:ff:ee:00:00:0d', 49: 'c0:ff:ee:00:00:12'} + td3_asic_flow_map = {0: 'c0:ff:ee:00:00:12', 1: 'c0:ff:ee:00:00:10', + 2: 'c0:ff:ee:00:00:11', 3: 'c0:ff:ee:00:00:0f', + 4: 'c0:ff:ee:00:00:0d', 5: 'c0:ff:ee:00:00:0b', + 6: 'c0:ff:ee:00:00:0e', 7: 'c0:ff:ee:00:00:0c', + 8: 'c0:ff:ee:00:00:0f', 9: 'c0:ff:ee:00:00:11', + 10: 'c0:ff:ee:00:00:10', 11: 'c0:ff:ee:00:00:12', + 12: 'c0:ff:ee:00:00:10', 13: 'c0:ff:ee:00:00:12', + 14: 'c0:ff:ee:00:00:0f', 15: 'c0:ff:ee:00:00:11', + 16: 'c0:ff:ee:00:00:0b', 17: 'c0:ff:ee:00:00:0d', + 18: 'c0:ff:ee:00:00:0c', 19: 'c0:ff:ee:00:00:0e', + 20: 'c0:ff:ee:00:00:10', 21: 'c0:ff:ee:00:00:12', + 22: 'c0:ff:ee:00:00:0f', 23: 'c0:ff:ee:00:00:11', + 24: 'c0:ff:ee:00:00:11', 25: 'c0:ff:ee:00:00:0f', + 26: 'c0:ff:ee:00:00:12', 27: 'c0:ff:ee:00:00:10', + 28: 'c0:ff:ee:00:00:0f', 29: 'c0:ff:ee:00:00:11', + 30: 'c0:ff:ee:00:00:10', 31: 'c0:ff:ee:00:00:12', + 32: 'c0:ff:ee:00:00:0c', 33: 'c0:ff:ee:00:00:0e', + 34: 'c0:ff:ee:00:00:0b', 35: 'c0:ff:ee:00:00:0d', + 36: 'c0:ff:ee:00:00:0f', 37: 'c0:ff:ee:00:00:11', + 38: 'c0:ff:ee:00:00:10', 39: 'c0:ff:ee:00:00:12', + 40: 'c0:ff:ee:00:00:0d', 41: 'c0:ff:ee:00:00:0b', + 42: 'c0:ff:ee:00:00:0e', 43: 'c0:ff:ee:00:00:0c', + 44: 'c0:ff:ee:00:00:0e', 45: 'c0:ff:ee:00:00:0c', + 46: 'c0:ff:ee:00:00:0d', 47: 'c0:ff:ee:00:00:0b', + 48: 'c0:ff:ee:00:00:11', 49: 'c0:ff:ee:00:00:0f'} th2_asic_flow_map = {0: 'c0:ff:ee:00:00:12', 1: 'c0:ff:ee:00:00:10', 2: 'c0:ff:ee:00:00:11', diff --git a/tests/qos/files/qos_params.jr2.yaml b/tests/qos/files/qos_params.jr2.yaml index d9e144b47d3..92649f35731 100644 --- a/tests/qos/files/qos_params.jr2.yaml +++ b/tests/qos/files/qos_params.jr2.yaml @@ -114,15 +114,15 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 415271 - pkts_num_trig_ingr_drp: 418781 + pkts_num_trig_pfc: 369873 + pkts_num_trig_ingr_drp: 373383 pkts_num_margin: 100 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 415271 - pkts_num_trig_ingr_drp: 418781 + pkts_num_trig_pfc: 369873 + pkts_num_trig_ingr_drp: 373383 pkts_num_margin: 100 hdrm_pool_size: dscps: [ 3, 4 ] @@ -131,29 +131,29 @@ qos_params: src_port_ids: [ 0, 2, 4, 6, 8, 10, 12, 14, 16 ] dst_port_id: 18 pgs_num: 18 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 pkts_num_hdrm_full: 362 pkts_num_hdrm_partial: 182 wm_pg_headroom: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 415271 - pkts_num_trig_ingr_drp: 418781 + pkts_num_trig_pfc: 369873 + pkts_num_trig_ingr_drp: 373383 cell_size: 4096 pkts_num_margin: 30 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 pkts_num_dismiss_pfc: 3243 pkts_num_margin: 150 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 pkts_num_dismiss_pfc: 3243 pkts_num_margin: 150 lossy_queue_1: @@ -167,7 +167,7 @@ qos_params: ecn: 1 pg: 3 pkts_num_fill_min: 0 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -185,7 +185,7 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 418781 + pkts_num_trig_ingr_drp: 373383 cell_size: 4096 wm_buf_pool_lossless: dscp: 3 @@ -220,15 +220,15 @@ qos_params: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 415271 - pkts_num_trig_ingr_drp: 593885 + pkts_num_trig_pfc: 369873 + pkts_num_trig_ingr_drp: 548487 pkts_num_margin: 100 xoff_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 415271 - pkts_num_trig_ingr_drp: 593885 + pkts_num_trig_pfc: 369873 + pkts_num_trig_ingr_drp: 548487 pkts_num_margin: 100 hdrm_pool_size: dscps: [ 3, 4 ] @@ -237,29 +237,29 @@ qos_params: src_port_ids: [ 0, 2, 4, 6, 8, 10, 12, 14, 16 ] dst_port_id: 18 pgs_num: 18 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 pkts_num_hdrm_full: 362 pkts_num_hdrm_partial: 182 wm_pg_headroom: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 415271 - pkts_num_trig_ingr_drp: 593885 + pkts_num_trig_pfc: 369873 + pkts_num_trig_ingr_drp: 548487 cell_size: 4096 pkts_num_margin: 30 xon_1: dscp: 3 ecn: 1 pg: 3 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 pkts_num_dismiss_pfc: 3245 pkts_num_margin: 150 xon_2: dscp: 4 ecn: 1 pg: 4 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 pkts_num_dismiss_pfc: 3245 pkts_num_margin: 150 lossy_queue_1: @@ -273,7 +273,7 @@ qos_params: ecn: 1 pg: 3 pkts_num_fill_min: 0 - pkts_num_trig_pfc: 415271 + pkts_num_trig_pfc: 369873 packet_size: 64 cell_size: 4096 pkts_num_margin: 40 @@ -291,7 +291,7 @@ qos_params: ecn: 1 queue: 3 pkts_num_fill_min: 0 - pkts_num_trig_ingr_drp: 593885 + pkts_num_trig_ingr_drp: 548487 cell_size: 4096 wm_buf_pool_lossless: dscp: 3 diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index 03ee343230d..3ae11556906 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -1089,6 +1089,7 @@ def dutConfig( dutPortIps[src_dut_index][src_asic_index] = {} sysPortMap[src_dut_index][src_asic_index] = {} active_ips = src_asic.get_active_ip_interfaces(tbinfo) + src_namespace_prefix = src_asic.namespace + '|' if src_asic.namespace else f'Asic{src_asic.asic_index}|' for iface, addr in active_ips.items(): if iface.startswith("Ethernet") and ("Ethernet-Rec" not in iface): portIndex = src_mgFacts["minigraph_ptf_indices"][iface] @@ -1097,7 +1098,7 @@ def dutConfig( # Map port IDs to system port for dnx chassis if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': - sys_key = src_asic.namespace + '|' + iface if src_asic.namespace else iface + sys_key = src_namespace_prefix + iface if sys_key in src_system_port: system_port = src_system_port[sys_key]['system_port_id'] sysPort = {'port': iface, 'system_port': system_port, 'port_type': iface} @@ -1114,7 +1115,7 @@ def dutConfig( if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': for portName in src_mgFacts["minigraph_portchannels"][iface]["members"]: - sys_key = src_asic.namespace + '|' + portName if src_asic.namespace else portName + sys_key = src_namespace_prefix + portName port_Index = src_mgFacts["minigraph_ptf_indices"][portName] if sys_key in src_system_port: system_port = src_system_port[sys_key]['system_port_id'] @@ -1132,6 +1133,7 @@ def dutConfig( dutPortIps[dst_dut_index] = {} testPortIds[dst_dut_index] = {} sysPortMap[dst_dut_index] = {} + dst_system_port = {} if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': dst_system_port = dst_dut.config_facts(host=dst_dut.hostname, source='running')[ @@ -1141,6 +1143,7 @@ def dutConfig( dst_system_port = src_system_port dutPortIps[dst_dut_index][dst_asic_index] = {} sysPortMap[dst_dut_index][dst_asic_index] = {} + dst_namespace_prefix = dst_asic.namespace + '|' if dst_asic.namespace else f'Asic{dst_asic.asic_index}|' active_ips = dst_asic.get_active_ip_interfaces(tbinfo) for iface, addr in active_ips.items(): if iface.startswith("Ethernet") and ("Ethernet-Rec" not in iface): @@ -1150,7 +1153,7 @@ def dutConfig( # Map port IDs to system port IDs if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': - sys_key = dst_asic.namespace + '|' + iface if dst_asic.namespace else iface + sys_key = dst_namespace_prefix + iface if sys_key in dst_system_port: system_port = dst_system_port[sys_key]['system_port_id'] sysPort = {'port': iface, 'system_port': system_port, 'port_type': iface} @@ -1167,7 +1170,7 @@ def dutConfig( if 'platform_asic' in get_src_dst_asic_and_duts["src_dut"].facts and \ get_src_dst_asic_and_duts["src_dut"].facts['platform_asic'] == 'broadcom-dnx': for portName in dst_mgFacts["minigraph_portchannels"][iface]["members"]: - sys_key = dst_asic.namespace + '|' + portName if dst_asic.namespace else portName + sys_key = dst_namespace_prefix + portName port_Index = dst_mgFacts["minigraph_ptf_indices"][portName] if sys_key in dst_system_port: system_port = dst_system_port[sys_key]['system_port_id'] diff --git a/tests/qos/test_ecn_config.py b/tests/qos/test_ecn_config.py index 7e6b2f12d49..78d4a4139ae 100644 --- a/tests/qos/test_ecn_config.py +++ b/tests/qos/test_ecn_config.py @@ -6,7 +6,6 @@ import logging import time import pytest -from tests.common.cisco_data import is_cisco_device import json @@ -106,8 +105,6 @@ def test_verify_ecn_marking_config(duthosts, rand_one_dut_hostname, request): @summary: Verify output of `show platform npu voq cgm_profile with wred_profile drop probability` """ duthost = duthosts[rand_one_dut_hostname] - if not is_cisco_device(duthost): - pytest.skip("Skipping as not a Cisco device") cmd = "show platform npu rx cgm_global -d" diff --git a/tests/qos/test_pfc_counters.py b/tests/qos/test_pfc_counters.py index a508494d4f0..4a849bf4e5e 100644 --- a/tests/qos/test_pfc_counters.py +++ b/tests/qos/test_pfc_counters.py @@ -81,7 +81,8 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa """ setup_testbed(fanouthosts, duthost, leaf_fanouts) asic = duthost.asic_instance() - conn_facts = conn_graph_facts['device_conn'][duthost.hostname] + asic_type = duthost.facts["asic_type"] + conn_facts = conn_graph_facts['device_conn'].get(duthost.hostname, {}) onyx_pfc_container_name = 'storm' int_status = asic.show_interface(command="status")[ 'ansible_facts']['int_status'] @@ -92,48 +93,49 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa int_status[intf]['oper_state'] == 'up'] only_lossless_rx_counters = "Cisco-8122" in asic.sonichost.facts["hwsku"] no_xon_counters = "Cisco-8122" in asic.sonichost.facts["hwsku"] - if only_lossless_rx_counters: + if only_lossless_rx_counters and asic_type != 'vs': config_facts = asic.config_facts(host=asic.hostname, source='persistent')['ansible_facts'] if not check_continuous_pfc: - """ Generate PFC or FC packets for active physical interfaces """ - for intf in active_phy_intfs: - peer_device = conn_facts[intf]['peerdevice'] - peer_port = conn_facts[intf]['peerport'] - - if peer_device not in fanouthosts: - continue - - peerdev_ans = fanouthosts[peer_device] - fanout_os = peerdev_ans.get_fanout_os() - fanout_hwsku = fanout_graph_facts[peerdev_ans.hostname]["device_info"]["HwSku"] - if fanout_os == "nxos": - peer_port_name = nxos_to_linux_intf(peer_port) - elif fanout_os == "sonic": - peer_port_name = sonic_to_linux_intf(peer_port) - else: - peer_port_name = eos_to_linux_intf( - peer_port, hwsku=fanout_hwsku) + if asic_type != 'vs': + """ Generate PFC or FC packets for active physical interfaces """ + for intf in active_phy_intfs: + peer_device = conn_facts[intf]['peerdevice'] + peer_port = conn_facts[intf]['peerport'] + + if peer_device not in fanouthosts: + continue + + peerdev_ans = fanouthosts[peer_device] + fanout_os = peerdev_ans.get_fanout_os() + fanout_hwsku = fanout_graph_facts[peerdev_ans.hostname]["device_info"]["HwSku"] + if fanout_os == "nxos": + peer_port_name = nxos_to_linux_intf(peer_port) + elif fanout_os == "sonic": + peer_port_name = sonic_to_linux_intf(peer_port) + else: + peer_port_name = eos_to_linux_intf( + peer_port, hwsku=fanout_hwsku) - if is_pfc: - for priority in range(PRIO_COUNT): + if is_pfc: + for priority in range(PRIO_COUNT): + if fanout_hwsku == "MLNX-OS": + cmd = 'docker exec %s "python %s -i %s -p %d -t %d -n %d"' % ( + onyx_pfc_container_name, PFC_GEN_FILE_ABSOLUTE_PATH, + peer_port_name, 2 ** priority, pause_time, PKT_COUNT) + peerdev_ans.host.config(cmd) + else: + cmd = "sudo python %s -i %s -p %d -t %d -n %d" % ( + PFC_GEN_FILE_DEST, peer_port_name, 2 ** priority, pause_time, PKT_COUNT) + peerdev_ans.host.command(cmd) + else: if fanout_hwsku == "MLNX-OS": - cmd = 'docker exec %s "python %s -i %s -p %d -t %d -n %d"' % ( - onyx_pfc_container_name, PFC_GEN_FILE_ABSOLUTE_PATH, - peer_port_name, 2 ** priority, pause_time, PKT_COUNT) + cmd = 'docker exec %s "python %s -i %s -g -t %d -n %d"' % ( + onyx_pfc_container_name, PFC_GEN_FILE_ABSOLUTE_PATH, peer_port_name, pause_time, PKT_COUNT) peerdev_ans.host.config(cmd) else: - cmd = "sudo python %s -i %s -p %d -t %d -n %d" % ( - PFC_GEN_FILE_DEST, peer_port_name, 2 ** priority, pause_time, PKT_COUNT) + cmd = "sudo python %s -i %s -g -t %d -n %d" % ( + PFC_GEN_FILE_DEST, peer_port_name, pause_time, PKT_COUNT) peerdev_ans.host.command(cmd) - else: - if fanout_hwsku == "MLNX-OS": - cmd = 'docker exec %s "python %s -i %s -g -t %d -n %d"' % ( - onyx_pfc_container_name, PFC_GEN_FILE_ABSOLUTE_PATH, peer_port_name, pause_time, PKT_COUNT) - peerdev_ans.host.config(cmd) - else: - cmd = "sudo python %s -i %s -g -t %d -n %d" % ( - PFC_GEN_FILE_DEST, peer_port_name, pause_time, PKT_COUNT) - peerdev_ans.host.command(cmd) """ SONiC takes some time to update counters in database """ time.sleep(5) @@ -141,7 +143,7 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa """ Check results """ counter_facts = duthost.sonic_pfc_counters(method="get")[ 'ansible_facts'] - if only_lossless_rx_counters: + if only_lossless_rx_counters and asic_type != 'vs': pfc_enabled_prios = [int(prio) for prio in config_facts["PORT_QOS_MAP"][intf]['pfc_enable'].split(',')] failures = [] for intf in active_phy_intfs: @@ -157,9 +159,10 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa logger.info("Verifying PFC RX count matches {}".format(expected_prios)) if counter_facts[intf]['Rx'] != expected_prios: failures.append((counter_facts[intf]['Rx'], expected_prios)) - for failure in failures: - logger.error("Got {}, expected {}".format(*failure)) - assert len(failures) == 0, "PFC RX counter increment not matching expected for above logged cases." + if asic_type != 'vs': + for failure in failures: + logger.error("Got {}, expected {}".format(*failure)) + assert len(failures) == 0, "PFC RX counter increment not matching expected for above logged cases." else: for intf in active_phy_intfs: @@ -168,45 +171,47 @@ def run_test(fanouthosts, duthost, conn_graph_facts, fanout_graph_facts, leaf_fa """ Clear PFC counters """ duthost.sonic_pfc_counters(method="clear") - peer_device = conn_facts[intf]['peerdevice'] - peer_port = conn_facts[intf]['peerport'] + if asic_type != 'vs': + peer_device = conn_facts[intf]['peerdevice'] + peer_port = conn_facts[intf]['peerport'] - if peer_device not in fanouthosts: - continue + if peer_device not in fanouthosts: + continue - peerdev_ans = fanouthosts[peer_device] - fanout_os = peerdev_ans.get_fanout_os() - fanout_hwsku = fanout_graph_facts[peerdev_ans.hostname]["device_info"]["HwSku"] - if fanout_os == "nxos": - peer_port_name = nxos_to_linux_intf(peer_port) - elif fanout_os == "sonic": - peer_port_name = sonic_to_linux_intf(peer_port) - else: - peer_port_name = eos_to_linux_intf( - peer_port, hwsku=fanout_hwsku) + peerdev_ans = fanouthosts[peer_device] + fanout_os = peerdev_ans.get_fanout_os() + fanout_hwsku = fanout_graph_facts[peerdev_ans.hostname]["device_info"]["HwSku"] + if fanout_os == "nxos": + peer_port_name = nxos_to_linux_intf(peer_port) + elif fanout_os == "sonic": + peer_port_name = sonic_to_linux_intf(peer_port) + else: + peer_port_name = eos_to_linux_intf( + peer_port, hwsku=fanout_hwsku) - if fanout_hwsku == "MLNX-OS": - cmd = 'docker exec %s "python %s -i %s -p %d -t %d -n %d"' % ( - onyx_pfc_container_name, PFC_GEN_FILE_ABSOLUTE_PATH, - peer_port_name, 2 ** priority, pause_time, PKT_COUNT) - peerdev_ans.host.config(cmd) - else: - cmd = "sudo python %s -i %s -p %d -t %d -n %d" % ( - PFC_GEN_FILE_DEST, peer_port_name, 2 ** priority, pause_time, PKT_COUNT) - peerdev_ans.host.command(cmd) + if fanout_hwsku == "MLNX-OS": + cmd = 'docker exec %s "python %s -i %s -p %d -t %d -n %d"' % ( + onyx_pfc_container_name, PFC_GEN_FILE_ABSOLUTE_PATH, + peer_port_name, 2 ** priority, pause_time, PKT_COUNT) + peerdev_ans.host.config(cmd) + else: + cmd = "sudo python %s -i %s -p %d -t %d -n %d" % ( + PFC_GEN_FILE_DEST, peer_port_name, 2 ** priority, pause_time, PKT_COUNT) + peerdev_ans.host.command(cmd) time.sleep(5) pfc_rx = duthost.sonic_pfc_counters( method="get")['ansible_facts'] - """check pfc Rx frame count on particular priority are increased""" - assert pfc_rx[intf]['Rx'][priority] == str(PKT_COUNT) - """check LHS priorities are 0 count""" - for i in range(priority): - assert pfc_rx[intf]['Rx'][i] == '0' - """check RHS priorities are 0 count""" - for i in range(priority+1, PRIO_COUNT): - assert pfc_rx[intf]['Rx'][i] == '0' + if asic_type != 'vs': + """check pfc Rx frame count on particular priority are increased""" + assert pfc_rx[intf]['Rx'][priority] == str(PKT_COUNT) + """check LHS priorities are 0 count""" + for i in range(priority): + assert pfc_rx[intf]['Rx'][i] == '0' + """check RHS priorities are 0 count""" + for i in range(priority+1, PRIO_COUNT): + assert pfc_rx[intf]['Rx'][i] == '0' def test_pfc_pause(fanouthosts, duthosts, rand_one_tgen_dut_hostname, diff --git a/tests/qos/test_qos_dscp_mapping.py b/tests/qos/test_qos_dscp_mapping.py index 44638aa6b57..1eece8e3d15 100644 --- a/tests/qos/test_qos_dscp_mapping.py +++ b/tests/qos/test_qos_dscp_mapping.py @@ -123,7 +123,12 @@ def send_and_verify_traffic(ptfadapter, testutils.send(ptfadapter, ptf_src_port_id, pkt, count=DEFAULT_PKT_COUNT) try: - port_index, _ = testutils.verify_packet_any_port(ptfadapter, exp_pkt, ports=ptf_dst_port_ids) + result = testutils.verify_packet_any_port(ptfadapter, exp_pkt, ports=ptf_dst_port_ids) + if isinstance(result, bool): + logger.info("Return a dummy value for VS platform") + port_index = 0 + else: + port_index, _ = result logger.info("Received packet(s) on port {}".format(ptf_dst_port_ids[port_index])) global packet_egressed_success packet_egressed_success = True @@ -207,6 +212,7 @@ def _run_test(self, if "backend" in tbinfo["topo"]["type"]: pytest.skip("Dscp-queue mapping is not supported on {}".format(tbinfo["topo"]["type"])) + asic_type = duthost.facts['asic_type'] router_mac = test_params['router_mac'] ptf_src_port_id = test_params['ptf_downlink_port'] ptf_dst_port_ids = test_params['ptf_uplink_ports'] @@ -274,6 +280,9 @@ def _run_test(self, logger.error("{}: Try reducing DEFAULT_PKT_COUNT value".format(str(e))) failed_once = True + if asic_type == 'vs': + logger.info("Skipping queue verification for VS platform") + continue global packet_egressed_success if packet_egressed_success: dut_egress_port = get_dut_pair_port_from_ptf_port(duthost, tbinfo, dst_ptf_port_id) diff --git a/tests/qos/test_tunnel_qos_remap.py b/tests/qos/test_tunnel_qos_remap.py index 74cc8dc5b4e..70d57558ba3 100644 --- a/tests/qos/test_tunnel_qos_remap.py +++ b/tests/qos/test_tunnel_qos_remap.py @@ -36,7 +36,7 @@ pytestmark = [ pytest.mark.enable_active_active, - pytest.mark.topology('t0') + pytest.mark.topology('dualtor') ] logger = logging.getLogger(__name__) diff --git a/tests/saitests/py3/sai_qos_tests.py b/tests/saitests/py3/sai_qos_tests.py index bfd06eb47c5..4291f0bc74e 100755 --- a/tests/saitests/py3/sai_qos_tests.py +++ b/tests/saitests/py3/sai_qos_tests.py @@ -4018,10 +4018,9 @@ def runTest(self): self.dst_client, asic_type, port_list['dst'][dst_port_id]) # for t2 chassis if platform_asic and platform_asic == "broadcom-dnx": - if dst_port_id in dst_sys_port_ids: - for port_id, sysport in dst_sys_port_ids.items(): - if dst_port_id == port_id: - dst_sys_port_id = int(sysport) + assert dst_port_id in dst_sys_port_ids, \ + "dst_port_id does not have a sys port id configured" + dst_sys_port_id = int(dst_sys_port_ids[dst_port_id]) log_message("actual dst_sys_port_id: {}".format(dst_sys_port_id), to_stderr=True) voq_list = sai_thrift_get_voq_port_id(self.src_client, dst_sys_port_id) voq_queue_counters_base = sai_thrift_read_port_voq_counters(self.src_client, voq_list) @@ -4539,7 +4538,7 @@ def runTest(self): time.sleep(8) if pg_min_pkts_num > 0 and check_leackout_compensation_support(asic_type, hwsku): - dynamically_compensate_leakout(self.src_client, asic_type, sai_thrift_read_port_counters, + dynamically_compensate_leakout(self.dst_client, asic_type, sai_thrift_read_port_counters, port_list['dst'][dst_port_id], TRANSMITTED_PKTS, xmit_counters_history, self, src_port_id, pkt, 40) @@ -4606,7 +4605,7 @@ def runTest(self): and (pkts_num <= 1 + margin) and check_leackout_compensation_support(asic_type, hwsku) ): - dynamically_compensate_leakout(self.src_client, asic_type, sai_thrift_read_port_counters, + dynamically_compensate_leakout(self.dst_client, asic_type, sai_thrift_read_port_counters, port_list['dst'][dst_port_id], TRANSMITTED_PKTS, xmit_counters_history, self, src_port_id, pkt, 40) diff --git a/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py b/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py index bc38364a935..4ad39ee47c3 100755 --- a/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py +++ b/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py @@ -4,6 +4,7 @@ import json import time import math +import os import pexpect from ixnetwork_restpy import SessionAssistant from ixnetwork_restpy.testplatform.testplatform import TestPlatform @@ -1870,3 +1871,165 @@ def get_convergence_for_blackout(duthosts, total_routes, mean(avg_pld)], [test_name+' (Link Up)', iteration, traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns, tablefmt="psql")) + + +def send_kernel_panic_command(duthost, creds): + username = creds.get('sonicadmin_user') + password = creds.get('sonicadmin_password') + ip = duthost.mgmt_ip + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, port=22, username=username, password=password) + command = 'echo c | sudo tee /proc/sysrq-trigger' + stdin, stdout, stderr = ssh.exec_command(command) + + +def ping_device(duthost, timeout): + response = os.system(f"ping -c 1 {duthost.mgmt_ip}") + start_time = time.time() + while True: + response = os.system(f"ping -c 1 {duthost.mgmt_ip}") + if response == 0: + logger.info('PASS:PING SUCCESSFUL for {}'.format(duthost.hostname)) + break + logger.info('Polling for {} to come UP.....'.format(duthost.hostname)) + elapsed_time = time.time() - start_time + pytest_assert(elapsed_time < timeout, "Unable to ping for {}".format(timeout)) + time.sleep(1) + + +def get_convergence_for_ungraceful_restart(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + device_name, + route_range, + test_name, + creds, + is_supervisor): + """ + Args: + duthost (pytest fixture): duthost fixture + api (pytest fixture): Snappi API + snappi_bgp_config: __snappi_bgp_config + flap_details: contains device name and port / services that needs to be flapped + traffic_type : IPv4 / IPv6 traffic type + iteration : Number of iterations + device_name: Device in which restart needs to be performed + route_range: V4 and v6 routes + test_name: Name of the test + """ + api.set_config(snappi_bgp_config) + avg_pld = [] + avg_pld2 = [] + + test_platform = TestPlatform(api._address) + test_platform.Authenticate(api._username, api._password) + session = SessionAssistant(IpAddress=api._address, UserName=api._username, + SessionId=test_platform.Sessions.find()[-1].Id, Password=api._password) + ixnetwork = session.Ixnetwork + for index, topology in enumerate(ixnetwork.Topology.find()): + try: + topology.DeviceGroup.find()[0].RouterData.find().RouterId.Single(router_ids[index]) + logger.info('Setting Router id {} for {}'.format(router_ids[index], topology.DeviceGroup.find()[0].Name)) + except Exception: + logger.info('Skipping Router id for {}, Since bgp is not configured'. + format(topology.DeviceGroup.find()[0].Name)) + continue + logger.info('\n') + logger.info('Testing with Route Range: {}'.format(route_range)) + logger.info('\n') + for i in range(0, iteration): + logger.info( + '|--------------------------- Iteration : {} -----------------------|'.format(i+1)) + logger.info("Starting all protocols ...") + ps = api.protocol_state() + ps.state = ps.START + api.set_protocol_state(ps) + wait(SNAPPI_TRIGGER, "For Protocols To start") + logger.info('Verifying protocol sessions state') + protocolsSummary = StatViewAssistant(ixnetwork, 'Protocols Summary') + protocolsSummary.CheckCondition('Sessions Down', StatViewAssistant.EQUAL, 0) + logger.info('Starting Traffic') + ts = api.transmit_state() + ts.state = ts.START + api.set_transmit_state(ts) + wait(SNAPPI_TRIGGER, "For Traffic To start") + + flow_stats = get_flow_stats(api) + port_stats = get_port_stats(api) + logger.info('\n') + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + logger.info('\n') + for i in range(0, len(traffic_type)): + logger.info('{} Loss %: {}'.format(flow_stats[i].name, int(flow_stats[i].loss))) + pytest_assert(int(flow_stats[i].loss) == 0, f'Loss Observed in {flow_stats[i].name}') + + # Getting rx rate on uplink ports + sum_t2_rx_frame_rate = 0 + for port_stat in port_stats: + if 'Snappi_Uplink' in port_stat.name: + sum_t2_rx_frame_rate = sum_t2_rx_frame_rate + int(port_stat.frames_rx_rate) + + logger.info('Issuing Ungraceful restart') + for duthost in duthosts: + if duthost.hostname == device_name: + send_kernel_panic_command(duthost, creds) + wait(DUT_TRIGGER, "Issued ungraceful restart on {}".format(device_name)) + for i in range(0, len(traffic_type)): + pytest_assert(float((int(flow_stats[i].frames_tx_rate) - int(flow_stats[i].frames_rx_rate)) / + int(flow_stats[i].frames_tx_rate)) < 0.005, + 'Traffic has not converged after issuing kernel panic') + logger.info('Traffic has converged after issuing kernel panic command in {}'.format(device_name)) + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After Device is DOWN (ms): {}'.format(pkt_loss_duration)) + avg_pld.append(pkt_loss_duration) + + logger.info('Clearing Stats') + ixnetwork.ClearStats() + for duthost in duthosts: + ping_device(duthost, timeout=180) + wait(DUT_TRIGGER, "Contaniers on the DUT to stabalize after restart") + + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After device is UP (ms): {}'.format(pkt_loss_duration)) + avg_pld2.append(pkt_loss_duration) + + for duthost in duthosts: + if duthost.hostname == device_name: + if is_supervisor is True: + exec_tsa_tsb_cmd_on_linecard(duthost, creds, "sudo TSB") + else: + duthost.command('sudo TSB') + logger.info('Stopping Traffic') + ts = api.transmit_state() + ts.state = ts.STOP + api.set_transmit_state(ts) + + logger.info("Stopping all protocols ...") + ps = api.protocol_state() + ps.state = ps.STOP + api.set_protocol_state(ps) + logger.info('\n') + + columns = ['Test Name', 'Iterations', 'Traffic Type', 'Uplink ECMP Paths', 'Route Count', + 'Avg Calculated Packet Loss Duration (ms)'] + logger.info("\n%s" % tabulate([[test_name+' (DOWN))', iteration, traffic_type, portchannel_count, + total_routes, mean(avg_pld)], [test_name+' (UP)', iteration, + traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns, + tablefmt="psql")) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_ungraceful_restart.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_ungraceful_restart.py new file mode 100644 index 00000000000..858763d9b7c --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_ungraceful_restart.py @@ -0,0 +1,266 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + run_bgp_outbound_ungraceful_restart, get_hw_platform, run_dut_configuration) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 500], + ['200.1.1.1', 24, 500] + ], + 'IPv6': [ + ['5000::1', 64, 500], + ['4000::1', 64, 500] + ], + }, + { + 'IPv4': [ + ['100.1.1.1', 24, 2500], + ['200.1.1.1', 24, 2500] + ], + 'IPv6': [ + ['5000::1', 64, 2500], + ['4000::1', 64, 2500] + ], + }] + + +def test_dut_configuration(multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts): # noqa: F811 + """ + Configures BGP in T1, T2 Uplink and T2 Downlink + + Args: + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.test_name = "Dut Configuration" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unknown HW Platform") + logger.info("HW Platform: {}".format(hw_platform)) + + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + run_dut_configuration(snappi_extra_params) + + +def test_bgp_outbound_uplink_ungraceful_restart(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + + """ + Gets the packet loss duration on issuing ungraceful restart in uplink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Uplink Ungraceful Re-start" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unknown HW Platform") + logger.info("HW Platform: {}".format(hw_platform)) + + snappi_extra_params.device_name = t1_t2_device_hostnames[hw_platform][1] + + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[hw_platform][3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + run_bgp_outbound_ungraceful_restart(api=snappi_api, + creds=creds, + is_supervisor=False, + snappi_extra_params=snappi_extra_params) + + +def test_bgp_outbound_downlink_ungraceful_restart(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + + """ + Gets the packet loss duration on issuing ungraceful restart in downlink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Downlink Ungraceful Re-start" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unknown HW Platform") + logger.info("HW Platform: {}".format(hw_platform)) + + snappi_extra_params.device_name = t1_t2_device_hostnames[hw_platform][2] + + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[hw_platform][3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + run_bgp_outbound_ungraceful_restart(api=snappi_api, + creds=creds, + is_supervisor=False, + snappi_extra_params=snappi_extra_params) + + +def test_bgp_outbound_supervisor_ungraceful_restart(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + + """ + Gets the packet loss duration on issuing ungraceful restart in supervisor + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Supervisor Ungraceful Re-start" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unknown HW Platform") + logger.info("HW Platform: {}".format(hw_platform)) + + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + snappi_extra_params.device_name = t1_t2_device_hostnames[hw_platform][3] + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[hw_platform][3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + run_bgp_outbound_ungraceful_restart(api=snappi_api, + creds=creds, + is_supervisor=True, + snappi_extra_params=snappi_extra_params) diff --git a/tests/tacacs/test_authorization.py b/tests/tacacs/test_authorization.py index c0559a0e3df..3f48c0d2eae 100644 --- a/tests/tacacs/test_authorization.py +++ b/tests/tacacs/test_authorization.py @@ -650,17 +650,18 @@ def test_fallback_to_local_authorization_with_config_reload( tacacs_server_ip: {"priority": "60", "tcp_port": "59", "timeout": "2"} } } - reload_minigraph_with_golden_config(duthost, override_config) - - # Shutdown tacacs server to simulate network unreachable because BGP shutdown - stop_tacacs_server(ptfhost) + try: + reload_minigraph_with_golden_config(duthost, override_config) - # Test "sudo config save -y" can success after reload minigraph - exit_code, stdout, stderr = ssh_run_command(remote_rw_user_client, "sudo config save -y") - pytest_assert(exit_code == 0) + # Shutdown tacacs server to simulate network unreachable because BGP shutdown + stop_tacacs_server(ptfhost) - # Cleanup UT. - start_tacacs_server(ptfhost) + # Test "sudo config save -y" can success after reload minigraph + exit_code, stdout, stderr = ssh_run_command(remote_rw_user_client, "sudo config save -y") + pytest_assert(exit_code == 0) - # Restore config after test finish - restore_config(duthost, CONFIG_DB, CONFIG_DB_BACKUP) + # Cleanup UT. + start_tacacs_server(ptfhost) + finally: + # Restore config after test finish + restore_config(duthost, CONFIG_DB, CONFIG_DB_BACKUP)