From 68abae3be93a863d5c962a3571f1fe2486058d0f Mon Sep 17 00:00:00 2001 From: deepak-singhal0408 Date: Wed, 22 Jan 2025 07:55:15 +0000 Subject: [PATCH] T2-Snappi-route-conv: Add Ungraceful restart testcase --- .../multidut/bgp/files/bgp_outbound_helper.py | 208 ++++++++++++++ .../test_bgp_outbound_ungraceful_restart.py | 257 ++++++++++++++++++ 2 files changed, 465 insertions(+) create mode 100644 tests/snappi_tests/multidut/bgp/test_bgp_outbound_ungraceful_restart.py diff --git a/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py b/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py index e5166fed09..38337e3987 100755 --- a/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py +++ b/tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py @@ -4,6 +4,7 @@ import json import time import math +import os from ixnetwork_restpy import SessionAssistant from ixnetwork_restpy.testplatform.testplatform import TestPlatform from ixnetwork_restpy.assistants.statistics.statviewassistant import StatViewAssistant @@ -167,6 +168,55 @@ def run_bgp_outbound_tsa_tsb_test(api, is_supervisor) +def run_bgp_outbound_ungraceful_restart(api, + creds, + is_supervisor, + snappi_extra_params): + """ + Run outbound test with ungraceful restart on the dut + Args: + api (pytest fixture): snappi API + snappi_extra_params (SnappiTestParams obj): additional parameters for Snappi traffic + """ + if snappi_extra_params is None: + snappi_extra_params = SnappiTestParams() # noqa F821 + + duthost1 = snappi_extra_params.multi_dut_params.duthost1 + duthost2 = snappi_extra_params.multi_dut_params.duthost2 + duthost3 = snappi_extra_params.multi_dut_params.duthost3 + duthost4 = snappi_extra_params.multi_dut_params.duthost4 + duthosts = [duthost1, duthost2, duthost3, duthost4] + hw_platform = snappi_extra_params.multi_dut_params.hw_platform + route_ranges = snappi_extra_params.ROUTE_RANGES + snappi_ports = snappi_extra_params.multi_dut_params.multi_dut_ports + device_name = snappi_extra_params.device_name + iteration = snappi_extra_params.iteration + test_name = snappi_extra_params.test_name + + """ Create snappi config """ + for route_range in route_ranges: + traffic_type = [] + for key, value in route_range.items(): + traffic_type.append(key) + snappi_bgp_config = __snappi_bgp_config(api, + duthosts, + hw_platform, + snappi_ports, + traffic_type, + route_range) + + get_convergence_for_ungraceful_restart(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + device_name, + route_range, + test_name, + creds, + is_supervisor) + + def run_bgp_outbound_process_restart_test(api, creds, snappi_extra_params): @@ -1831,3 +1881,161 @@ def get_convergence_for_blackout(duthosts, total_routes, mean(avg_pld)], [test_name+' (Link Up)', iteration, traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns, tablefmt="psql")) + + +def send_kernel_panic_command(duthost, creds): + username = creds.get('sonicadmin_user') + password = creds.get('sonicadmin_password') + ip = duthost.mgmt_ip + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, port=22, username=username, password=password) + command = 'echo c | sudo tee /proc/sysrq-trigger' + stdin, stdout, stderr = ssh.exec_command(command) + + +def ping_device(duthost, timeout): + response = os.system(f"ping -c 1 {duthost.mgmt_ip}") + start_time = time.time() + while True: + response = os.system(f"ping -c 1 {duthost.mgmt_ip}") + if response == 0: + logger.info('PASS:PING SUCCESSFUL for {}'.format(duthost.hostname)) + break + logger.info('Polling for {} to come UP.....'.format(duthost.hostname)) + elapsed_time = time.time() - start_time + pytest_assert(elapsed_time < timeout, "Unable to ping for {}".format(timeout)) + time.sleep(1) + + +def get_convergence_for_ungraceful_restart(duthosts, + api, + snappi_bgp_config, + traffic_type, + iteration, + device_name, + route_range, + test_name, + creds, + is_supervisor): + """ + Args: + duthost (pytest fixture): duthost fixture + api (pytest fixture): Snappi API + snappi_bgp_config: __snappi_bgp_config + flap_details: contains device name and port / services that needs to be flapped + traffic_type : IPv4 / IPv6 traffic type + iteration : Number of iterations + device_name: Device in which restart needs to be performed + route_range: V4 and v6 routes + test_name: Name of the test + """ + api.set_config(snappi_bgp_config) + avg_pld = [] + avg_pld2 = [] + + test_platform = TestPlatform(api._address) + test_platform.Authenticate(api._username, api._password) + session = SessionAssistant(IpAddress=api._address, UserName=api._username, + SessionId=test_platform.Sessions.find()[-1].Id, Password=api._password) + ixnetwork = session.Ixnetwork + for index, topology in enumerate(ixnetwork.Topology.find()): + try: + topology.DeviceGroup.find()[0].RouterData.find().RouterId.Single(router_ids[index]) + logger.info('Setting Router id {} for {}'.format(router_ids[index], topology.DeviceGroup.find()[0].Name)) + except Exception: + logger.info('Skipping Router id for {}, Since bgp is not configured'. + format(topology.DeviceGroup.find()[0].Name)) + continue + logger.info('\n') + logger.info('Testing with Route Range: {}'.format(route_range)) + logger.info('\n') + for i in range(0, iteration): + logger.info( + '|--------------------------- Iteration : {} -----------------------|'.format(i+1)) + logger.info("Starting all protocols ...") + ps = api.protocol_state() + ps.state = ps.START + api.set_protocol_state(ps) + wait(SNAPPI_TRIGGER, "For Protocols To start") + logger.info('Verifying protocol sessions state') + protocolsSummary = StatViewAssistant(ixnetwork, 'Protocols Summary') + protocolsSummary.CheckCondition('Sessions Down', StatViewAssistant.EQUAL, 0) + logger.info('Starting Traffic') + ts = api.transmit_state() + ts.state = ts.START + api.set_transmit_state(ts) + wait(SNAPPI_TRIGGER, "For Traffic To start") + + flow_stats = get_flow_stats(api) + port_stats = get_port_stats(api) + logger.info('\n') + logger.info('Rx Snappi Port Name : Rx Frame Rate') + for port_stat in port_stats: + if 'Snappi_Tx_Port' not in port_stat.name: + logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate)) + pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name)) + logger.info('\n') + for i in range(0, len(traffic_type)): + logger.info('{} Loss %: {}'.format(flow_stats[i].name, int(flow_stats[i].loss))) + pytest_assert(int(flow_stats[i].loss) == 0, f'Loss Observed in {flow_stats[i].name}') + + # Getting rx rate on uplink ports + sum_t2_rx_frame_rate = 0 + for port_stat in port_stats: + if 'Snappi_Uplink' in port_stat.name: + sum_t2_rx_frame_rate = sum_t2_rx_frame_rate + int(port_stat.frames_rx_rate) + logger.info('Issuing Ungraceful restart') + for duthost in duthosts: + if duthost.hostname == device_name: + send_kernel_panic_command(duthost, creds) + wait(DUT_TRIGGER, "Issued ungraceful restart on {}".format(device_name)) + for i in range(0, len(traffic_type)): + pytest_assert(float((int(flow_stats[i].frames_tx_rate) - int(flow_stats[i].frames_rx_rate)) / + int(flow_stats[i].frames_tx_rate)) < 0.005, + 'Traffic has not converged after issuing kernel panic') + logger.info('Traffic has converged after issuing kernel panic command in {}'.format(device_name)) + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After Device is DOWN (ms): {}'.format(pkt_loss_duration)) + avg_pld.append(pkt_loss_duration) + + logger.info('Clearing Stats') + ixnetwork.ClearStats() + for duthost in duthosts: + ping_device(duthost, timeout=180) + wait(DUT_TRIGGER, "Contaniers on the DUT to stabalize after restart") + + flow_stats = get_flow_stats(api) + delta_frames = 0 + for i in range(0, len(traffic_type)): + delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx + pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate) + logger.info('Delta Frames : {}'.format(delta_frames)) + logger.info('PACKET LOSS DURATION After device is UP (ms): {}'.format(pkt_loss_duration)) + avg_pld2.append(pkt_loss_duration) + + for duthost in duthosts: + if duthost.hostname == device_name: + duthost.command("sudo TSB") + logger.info('Stopping Traffic') + ts = api.transmit_state() + ts.state = ts.STOP + api.set_transmit_state(ts) + + logger.info("Stopping all protocols ...") + ps = api.protocol_state() + ps.state = ps.STOP + api.set_protocol_state(ps) + logger.info('\n') + + columns = ['Test Name', 'Iterations', 'Traffic Type', 'Uplink ECMP Paths', 'Route Count', + 'Avg Calculated Packet Loss Duration (ms)'] + logger.info("\n%s" % tabulate([[test_name+' (DOWN))', iteration, traffic_type, portchannel_count, + total_routes, mean(avg_pld)], [test_name+' (UP)', iteration, + traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns, + tablefmt="psql")) diff --git a/tests/snappi_tests/multidut/bgp/test_bgp_outbound_ungraceful_restart.py b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_ungraceful_restart.py new file mode 100644 index 0000000000..307e73f856 --- /dev/null +++ b/tests/snappi_tests/multidut/bgp/test_bgp_outbound_ungraceful_restart.py @@ -0,0 +1,257 @@ +import pytest +import logging +from tests.common.helpers.assertions import pytest_require, pytest_assert # noqa: F401 +from tests.common.fixtures.conn_graph_facts import conn_graph_facts, \ + fanout_graph_facts_multidut # noqa: F401 +from tests.common.snappi_tests.snappi_fixtures import snappi_api_serv_ip, snappi_api_serv_port, \ + snappi_api, multidut_snappi_ports_for_bgp # noqa: F401 +from tests.snappi_tests.variables import t1_t2_device_hostnames # noqa: F401 +from tests.snappi_tests.multidut.bgp.files.bgp_outbound_helper import ( + get_hw_platform, run_bgp_outbound_ungraceful_restart, run_dut_configuration) # noqa: F401 +from tests.common.snappi_tests.snappi_test_params import SnappiTestParams # noqa: F401 + +logger = logging.getLogger(__name__) + +pytestmark = [pytest.mark.topology('multidut-tgen')] + +ITERATION = 1 +ROUTE_RANGES = [{ + 'IPv4': [ + ['100.1.1.1', 24, 15000], + ['200.1.1.1', 24, 15000] + ], + 'IPv6': [ + ['5000::1', 64, 15000], + ['4000::1', 64, 15000] + ], + }] + + +def test_dut_configuration(multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts): # noqa: F811 + """ + Configures BGP in T1, T2 Uplink and T2 Downlink + + Args: + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.test_name = "Dut Configuration" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unable to get the hardware platform") + + logger.info("hw_platform: {}".format(hw_platform)) + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + else: + continue + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + run_dut_configuration(snappi_extra_params) + + +def test_bgp_outbound_uplink_ungraceful_restart(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + + """ + Gets the packet loss duration on issuing ungraceful restart in uplink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Uplink Ungraceful Re-start" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unknown HW Platform") + logger.info("HW Platform: {}".format(hw_platform)) + + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[hw_platform][3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.device_name = t1_t2_device_hostnames[hw_platform][1] + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + + run_bgp_outbound_ungraceful_restart(api=snappi_api, + creds=creds, + is_supervisor=False, + snappi_extra_params=snappi_extra_params) + + +def test_bgp_outbound_downlink_ungraceful_restart(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + + """ + Gets the packet loss duration on issuing ungraceful restart in downlink + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Downlink Ungraceful Re-start" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unknown HW Platform") + logger.info("HW Platform: {}".format(hw_platform)) + + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[hw_platform][3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.device_name = t1_t2_device_hostnames[hw_platform][2] + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + + run_bgp_outbound_ungraceful_restart(api=snappi_api, + creds=creds, + is_supervisor=False, + snappi_extra_params=snappi_extra_params) + + +def test_bgp_outbound_supervisor_ungraceful_restart(snappi_api, # noqa: F811 + multidut_snappi_ports_for_bgp, # noqa: F811 + conn_graph_facts, # noqa: F811 + fanout_graph_facts_multidut, # noqa: F811 + duthosts, + creds): # noqa: F811 + + """ + Gets the packet loss duration on issuing ungraceful restart in supervisor + + Args: + snappi_api (pytest fixture): SNAPPI session + multidut_snappi_ports_for_bgp (pytest fixture): Port mapping info on multidut testbed + conn_graph_facts (pytest fixture): connection graph + fanout_graph_facts_multidut (pytest fixture): fanout graph + duthosts (pytest fixture): list of DUTs + Returns: + N/A + """ + snappi_extra_params = SnappiTestParams() + snappi_extra_params.ROUTE_RANGES = ROUTE_RANGES + snappi_extra_params.iteration = ITERATION + snappi_extra_params.test_name = "Supervisor Ungraceful Re-start" + + ansible_dut_hostnames = [] + for duthost in duthosts: + ansible_dut_hostnames.append(duthost.hostname) + + hw_platform = get_hw_platform(ansible_dut_hostnames) + if hw_platform is None: + pytest_require(False, "Unknown HW Platform") + logger.info("HW Platform: {}".format(hw_platform)) + + for device_hostname in t1_t2_device_hostnames[hw_platform]: + if device_hostname not in ansible_dut_hostnames: + logger.info('!!!!! Attention: {} not in : {} derived from ansible dut hostnames'. + format(device_hostname, ansible_dut_hostnames)) + pytest_assert(False, "Mismatch between the dut hostnames in ansible and in variables.py files") + + for duthost in duthosts: + if t1_t2_device_hostnames[hw_platform][0] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost1 = duthost + elif t1_t2_device_hostnames[hw_platform][1] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost2 = duthost + elif t1_t2_device_hostnames[hw_platform][2] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost3 = duthost + elif t1_t2_device_hostnames[hw_platform][3] in duthost.hostname: + snappi_extra_params.multi_dut_params.duthost4 = duthost + else: + continue + + snappi_extra_params.device_name = t1_t2_device_hostnames[hw_platform][3] + snappi_extra_params.multi_dut_params.hw_platform = hw_platform + snappi_extra_params.multi_dut_params.multi_dut_ports = multidut_snappi_ports_for_bgp + + run_bgp_outbound_ungraceful_restart(api=snappi_api, + creds=creds, + is_supervisor=True, + snappi_extra_params=snappi_extra_params)