update rsyslog fixture to workaround /var/log size limitation (sonic-…

…net#14331) (sonic-net#14380)

What is the motivation for this PR?
The case failed sometimes due to cannot found the log.
It should be related to the small storage, since the var/log size limitation, logrotate would remove the previous syslogs during the test, then cause loganalyzer cannot find the previous log and teardown failure.

How did you do it?
It is caused by storage resource limitation, could not fix the issue, this PR is a workaround.
Since set the syslog to error lever and update the logroate to compress log immediately during the test still failed sometimes. Create a thread to do the logrotate every 60s during the test.

How did you verify/test it?
https://elastictest.org/scheduler/testplan/66d1ebb7f90c6cd947f4dd3e

tests/conftest.py

@@ -10,6 +10,9 @@
 import yaml
 import jinja2
 import copy
+import time
+import subprocess
+import threading
 
 from datetime import datetime
 from ipaddress import ip_interface, IPv4Interface
@@ -51,6 +54,7 @@
 from tests.common.config_reload import config_reload
 from tests.common.connections.console_host import ConsoleHost
 from tests.common.helpers.assertions import pytest_assert as pt_assert
+from tests.common.utilities import InterruptableThread
 
 try:
     from tests.macsec import MacsecPlugin
@@ -2313,3 +2317,39 @@ def format_failure(port, failure):
 # HACK: We are using set_do_not_care_scapy but it will be deprecated.
 if not hasattr(Mask, "set_do_not_care_scapy"):
     Mask.set_do_not_care_scapy = Mask.set_do_not_care_packet
+
+
+def run_logrotate(duthost, stop_event):
+    logger.info("Start rotate_syslog on {}".format(duthost))
+    while not stop_event.is_set():
+        try:
+            # Run logrotate for rsyslog
+            duthost.shell("logrotate -f /etc/logrotate.conf", module_ignore_errors=True)
+        except subprocess.CalledProcessError as e:
+            logger.error("Error: {}".format(str(e)))
+        # Wait for 60 seconds before the next rotation
+        time.sleep(60)
+
+
+@pytest.fixture(scope="function")
+def rotate_syslog(duthosts, enum_rand_one_per_hwsku_frontend_hostname):
+    duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
+
+    stop_event = threading.Event()
+    thread = InterruptableThread(
+        target=run_logrotate,
+        args=(duthost, stop_event,)
+    )
+    thread.daemon = True
+    thread.start()
+
+    yield
+    stop_event.set()
+    try:
+        if thread.is_alive():
+            thread.join(timeout=30)
+            logger.info("thread {} joined".format(thread))
+    except Exception as e:
+        logger.debug("Exception occurred in thread {}".format(str(e)))
+
+    logger.info("rotate_syslog exit {}".format(thread))

tests/fdb/test_fdb_mac_move.py

@@ -27,50 +27,6 @@
 ]
 
 
-def modify_rsyslog_severity_level(dut):
-    logger.info('Modify rsyslog severity level to error on dut: {}'.format(dut.hostname))
-    dut.shell("sudo mv /etc/rsyslog.d /etc/rsyslog.d.bak")
-    dut.shell("sudo mkdir /etc/rsyslog.d")
-    dut.shell("sudo echo '*.err    /var/log/syslog' > /etc/rsyslog.d/50_debug.conf")
-    dut.shell("sudo systemctl restart rsyslog")
-    time.sleep(5)
-
-
-def revert_rsyslog_severity_level(dut):
-    logger.info('Revert rsyslog severity level to error on dut: {}'.format(dut.hostname))
-    dut.shell("sudo rm -rf /etc/rsyslog.d")
-    dut.shell("sudo mv /etc/rsyslog.d.bak /etc/rsyslog.d")
-    dut.shell("sudo systemctl restart rsyslog")
-    time.sleep(5)
-
-
-@pytest.fixture(scope="function")
-def fixture_rsyslog_conf_setup_teardown(duthosts, rand_one_dut_hostname):
-    duthost = duthosts[rand_one_dut_hostname]
-
-    # workaround for small disk devices which also has limitation on /var/log size
-    cmd = "df -m"
-    cmd_response_lines = duthost.shell(cmd, module_ignore_errors=True).get('stdout_lines', [])
-    logger.debug("cmd {} rsp {}".format(cmd, cmd_response_lines))
-
-    available_size = 0
-    for line in cmd_response_lines:
-        if "/var/log" in line:
-            available_size = int(line.split()[3])
-            break
-
-    if available_size < 400:
-        modify_rsyslog_severity_level(duthost)
-        rsyslog_severity_level_modified = True
-    else:
-        rsyslog_severity_level_modified = False
-
-    yield
-
-    if rsyslog_severity_level_modified:
-        revert_rsyslog_severity_level(duthost)
-
-
 def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count):
     """
     :param ptfadapter: PTF adapter object
@@ -101,7 +57,7 @@ def get_fdb_dict(ptfadapter, vlan_table, dummay_mac_count):
 
 
 def test_fdb_mac_move(ptfadapter, duthosts, rand_one_dut_hostname, ptfhost, get_function_conpleteness_level,
-                      fixture_rsyslog_conf_setup_teardown):
+                      rotate_syslog):
     # Perform FDB clean up before each test
     fdb_cleanup(duthosts, rand_one_dut_hostname)
 

tests/stress/test_stress_routes.py

@@ -39,7 +39,8 @@ def announce_withdraw_routes(duthost, localhost, ptf_ip, topo_name):
 
 def test_announce_withdraw_route(duthosts, localhost, tbinfo, get_function_conpleteness_level,
                                  withdraw_and_announce_existing_routes, loganalyzer,
-                                 enum_rand_one_per_hwsku_frontend_hostname):
+                                 enum_rand_one_per_hwsku_frontend_hostname,
+                                 rotate_syslog):
     ptf_ip = tbinfo["ptf_ip"]
     topo_name = tbinfo["topo"]["name"]
     duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]