[Snappi] Adding Ungraceful Restart script for BGP Outbound cases

tests/snappi_tests/multidut/bgp/files/bgp_outbound_helper.py

@@ -4,6 +4,7 @@
 import json
 import time
 import math
+import os
 import pexpect
 from ixnetwork_restpy import SessionAssistant
 from ixnetwork_restpy.testplatform.testplatform import TestPlatform
@@ -20,8 +21,7 @@
      snappi_portchannel_ipv4_list, snappi_portchannel_ipv6_list, AS_PATHS, \
      BGP_TYPE, t1_side_interconnected_port, t2_side_interconnected_port, router_ids, \
      snappi_community_for_t1, snappi_community_for_t1_drop, snappi_community_for_t2, num_regionalhubs,  \
-     SNAPPI_TRIGGER, DUT_TRIGGER, fanout_presence, t2_uplink_fanout_info  # noqa: F401
-from tests.common.snappi_tests.variables import v6_prefix_length
+     SNAPPI_TRIGGER, DUT_TRIGGER, fanout_presence, t2_uplink_fanout_info, v6_prefix_length  # noqa: F401
 
 logger = logging.getLogger(__name__)
 total_routes = 0
@@ -1870,3 +1870,165 @@ def get_convergence_for_blackout(duthosts,
                                   total_routes, mean(avg_pld)], [test_name+' (Link Up)', iteration,
                                   traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns,
                                   tablefmt="psql"))
+
+
+def send_kernel_panic_command(duthost, creds):
+    username = creds.get('sonicadmin_user')
+    password = creds.get('sonicadmin_password')
+    ip = duthost.mgmt_ip
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(ip, port=22, username=username, password=password)
+    command = 'echo c | sudo tee /proc/sysrq-trigger'
+    stdin, stdout, stderr = ssh.exec_command(command)
+
+
+def ping_device(duthost, timeout):
+    response = os.system(f"ping -c 1 {duthost.mgmt_ip}")
+    start_time = time.time()
+    while True:
+        response = os.system(f"ping -c 1 {duthost.mgmt_ip}")
+        if response == 0:
+            logger.info('PASS:PING SUCCESSFUL for {}'.format(duthost.hostname))
+            break
+        logger.info('Polling for {} to come UP.....'.format(duthost.hostname))
+        elapsed_time = time.time() - start_time
+        pytest_assert(elapsed_time < timeout, "Unable to ping for {}".format(timeout))
+        time.sleep(1)
+
+
+def get_convergence_for_ungraceful_restart(duthosts,
+                                           api,
+                                           snappi_bgp_config,
+                                           traffic_type,
+                                           iteration,
+                                           device_name,
+                                           route_range,
+                                           test_name,
+                                           creds,
+                                           is_supervisor):
+    """
+    Args:
+        duthost (pytest fixture): duthost fixture
+        api (pytest fixture): Snappi API
+        snappi_bgp_config: __snappi_bgp_config
+        flap_details: contains device name and port / services that needs to be flapped
+        traffic_type : IPv4 / IPv6 traffic type
+        iteration : Number of iterations
+        device_name: Device in which restart needs to be performed
+        route_range: V4 and v6 routes
+        test_name: Name of the test
+    """
+    api.set_config(snappi_bgp_config)
+    avg_pld = []
+    avg_pld2 = []
+
+    test_platform = TestPlatform(api._address)
+    test_platform.Authenticate(api._username, api._password)
+    session = SessionAssistant(IpAddress=api._address, UserName=api._username,
+                               SessionId=test_platform.Sessions.find()[-1].Id, Password=api._password)
+    ixnetwork = session.Ixnetwork
+    for index, topology in enumerate(ixnetwork.Topology.find()):
+        try:
+            topology.DeviceGroup.find()[0].RouterData.find().RouterId.Single(router_ids[index])
+            logger.info('Setting Router id {} for {}'.format(router_ids[index], topology.DeviceGroup.find()[0].Name))
+        except Exception:
+            logger.info('Skipping Router id for {}, Since bgp is not configured'.
+                        format(topology.DeviceGroup.find()[0].Name))
+            continue
+    logger.info('\n')
+    logger.info('Testing with Route Range: {}'.format(route_range))
+    logger.info('\n')
+    for i in range(0, iteration):
+        logger.info(
+            '|--------------------------- Iteration : {} -----------------------|'.format(i+1))
+        logger.info("Starting all protocols ...")
+        ps = api.protocol_state()
+        ps.state = ps.START
+        api.set_protocol_state(ps)
+        wait(SNAPPI_TRIGGER, "For Protocols To start")
+        logger.info('Verifying protocol sessions state')
+        protocolsSummary = StatViewAssistant(ixnetwork, 'Protocols Summary')
+        protocolsSummary.CheckCondition('Sessions Down', StatViewAssistant.EQUAL, 0)
+        logger.info('Starting Traffic')
+        ts = api.transmit_state()
+        ts.state = ts.START
+        api.set_transmit_state(ts)
+        wait(SNAPPI_TRIGGER, "For Traffic To start")
+
+        flow_stats = get_flow_stats(api)
+        port_stats = get_port_stats(api)
+        logger.info('\n')
+        logger.info('Rx Snappi Port Name : Rx Frame Rate')
+        for port_stat in port_stats:
+            if 'Snappi_Tx_Port' not in port_stat.name:
+                logger.info('{} : {}'.format(port_stat.name, port_stat.frames_rx_rate))
+                pytest_assert(port_stat.frames_rx_rate > 0, '{} is not receiving any packet'.format(port_stat.name))
+        logger.info('\n')
+        for i in range(0, len(traffic_type)):
+            logger.info('{} Loss %: {}'.format(flow_stats[i].name, int(flow_stats[i].loss)))
+            pytest_assert(int(flow_stats[i].loss) == 0, f'Loss Observed in {flow_stats[i].name}')
+
+        # Getting rx rate on uplink ports
+        sum_t2_rx_frame_rate = 0
+        for port_stat in port_stats:
+            if 'Snappi_Uplink' in port_stat.name:
+                sum_t2_rx_frame_rate = sum_t2_rx_frame_rate + int(port_stat.frames_rx_rate)
+
+        logger.info('Issuing Ungraceful restart')
+        for duthost in duthosts:
+            if duthost.hostname == device_name:
+                send_kernel_panic_command(duthost, creds)
+        wait(DUT_TRIGGER, "Issued ungraceful restart on {}".format(device_name))
+        for i in range(0, len(traffic_type)):
+            pytest_assert(float((int(flow_stats[i].frames_tx_rate) - int(flow_stats[i].frames_rx_rate)) /
+                          int(flow_stats[i].frames_tx_rate)) < 0.005,
+                          'Traffic has not converged after issuing kernel panic')
+        logger.info('Traffic has converged after issuing kernel panic command in {}'.format(device_name))
+        flow_stats = get_flow_stats(api)
+        delta_frames = 0
+        for i in range(0, len(traffic_type)):
+            delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx
+        pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate)
+        logger.info('Delta Frames : {}'.format(delta_frames))
+        logger.info('PACKET LOSS DURATION  After Device is DOWN (ms): {}'.format(pkt_loss_duration))
+        avg_pld.append(pkt_loss_duration)
+
+        logger.info('Clearing Stats')
+        ixnetwork.ClearStats()
+        for duthost in duthosts:
+            ping_device(duthost, timeout=180)
+        wait(DUT_TRIGGER, "Contaniers on the DUT to stabalize after restart")
+
+        flow_stats = get_flow_stats(api)
+        delta_frames = 0
+        for i in range(0, len(traffic_type)):
+            delta_frames = delta_frames + flow_stats[i].frames_tx - flow_stats[i].frames_rx
+        pkt_loss_duration = 1000 * (delta_frames / sum_t2_rx_frame_rate)
+        logger.info('Delta Frames : {}'.format(delta_frames))
+        logger.info('PACKET LOSS DURATION  After device is UP (ms): {}'.format(pkt_loss_duration))
+        avg_pld2.append(pkt_loss_duration)
+
+        for duthost in duthosts:
+            if duthost.hostname == device_name:
+                if is_supervisor is True:
+                    exec_tsa_tsb_cmd_on_linecard(duthost, creds, "sudo TSB")
+                else:
+                    duthost.command('sudo TSB')
+        logger.info('Stopping Traffic')
+        ts = api.transmit_state()
+        ts.state = ts.STOP
+        api.set_transmit_state(ts)
+
+        logger.info("Stopping all protocols ...")
+        ps = api.protocol_state()
+        ps.state = ps.STOP
+        api.set_protocol_state(ps)
+        logger.info('\n')
+
+    columns = ['Test Name', 'Iterations', 'Traffic Type', 'Uplink ECMP Paths', 'Route Count',
+               'Avg Calculated Packet Loss Duration (ms)']
+    logger.info("\n%s" % tabulate([[test_name+' (DOWN))', iteration, traffic_type, portchannel_count,
+                                  total_routes, mean(avg_pld)], [test_name+' (UP)', iteration,
+                                  traffic_type, portchannel_count, total_routes, mean(avg_pld2)]], headers=columns,
+                                  tablefmt="psql"))