Fixes for SNMP Tests Flakiness

tests/cacl/test_cacl_function.py

@@ -34,7 +34,8 @@ def test_cacl_function(duthosts, enum_rand_one_per_hwsku_hostname, localhost, cr
         logging.warning("Will not check NTP connection. ntplib is not installed.")
 
     # Ensure we can gather basic SNMP facts from the device. Should fail on timeout
-    get_snmp_facts(localhost,
+    get_snmp_facts(duthost,
+                   localhost,
                    host=dut_mgmt_ip,
                    version="v2c",
                    community=creds['snmp_rocommunity'],
@@ -83,7 +84,7 @@ def test_cacl_function(duthosts, enum_rand_one_per_hwsku_hostname, localhost, cr
         pytest_assert(res.is_failed, "SSH did not timeout when expected. {}".format(res.get('msg', '')))
 
         # Ensure we CANNOT gather basic SNMP facts from the device
-        res = get_snmp_facts(localhost, host=dut_mgmt_ip, version='v2c', community=creds['snmp_rocommunity'],
+        res = get_snmp_facts(duthost, localhost, host=dut_mgmt_ip, version='v2c', community=creds['snmp_rocommunity'],
                              module_ignore_errors=True)
 
         pytest_assert('ansible_facts' not in res and "No SNMP response received before timeout" in res.get('msg', ''))
@@ -114,7 +115,8 @@ def test_cacl_function(duthosts, enum_rand_one_per_hwsku_hostname, localhost, cr
         duthost.file(path="/tmp/config_service_acls.sh", state="absent")
 
         # Ensure we can gather basic SNMP facts from the device once again. Should fail on timeout
-        get_snmp_facts(localhost,
+        get_snmp_facts(duthost,
+                       localhost,
                        host=dut_mgmt_ip,
                        version="v2c",
                        community=creds['snmp_rocommunity'],

tests/common/helpers/snmp_helpers.py

@@ -14,35 +14,46 @@
 global_snmp_facts = {}
 
 
+def is_snmp_subagent_running(duthost):
+    cmd = "docker exec snmp supervisorctl status snmp-subagent"
+    output = duthost.shell(cmd)
+    if "RUNNING" in output["stdout"]:
+        logger.info("SNMP Sub-Agent is Running")
+        return True
+    logger.info("SNMP Sub-Agent is Not Running")
+    return False
+
+
 def _get_snmp_facts(localhost, host, version, community, is_dell, include_swap, module_ignore_errors):
     snmp_facts = localhost.snmp_facts(host=host, version=version, community=community, is_dell=is_dell,
                                       module_ignore_errors=module_ignore_errors, include_swap=include_swap)
     return snmp_facts
 
 
-def _update_snmp_facts(localhost, host, version, community, is_dell, include_swap):
+def _update_snmp_facts(localhost, host, version, community, is_dell, include_swap, duthost):
     global global_snmp_facts
 
     try:
+        snmp_subagent_running = is_snmp_subagent_running(duthost)
         global_snmp_facts = _get_snmp_facts(localhost, host, version, community, is_dell, include_swap,
                                             module_ignore_errors=False)
     except RunAnsibleModuleFail as e:
         logger.info("encountered error when getting snmp facts: {}".format(e))
         global_snmp_facts = {}
         return False
 
-    return True
+    return snmp_subagent_running and True
 
 
-def get_snmp_facts(localhost, host, version, community, is_dell=False, module_ignore_errors=False,
+def get_snmp_facts(duthost, localhost, host, version, community, is_dell=False, module_ignore_errors=False,
                    wait=False, include_swap=False, timeout=DEF_WAIT_TIMEOUT, interval=DEF_CHECK_INTERVAL):
     if not wait:
         return _get_snmp_facts(localhost, host, version, community, is_dell, include_swap, module_ignore_errors)
 
     global global_snmp_facts
 
     pytest_assert(wait_until(timeout, interval, 0, _update_snmp_facts, localhost, host, version,
-                             community, is_dell, include_swap), "Timeout waiting for SNMP facts")
+                             community, is_dell, include_swap, duthost), "Timeout waiting for SNMP facts")
     return global_snmp_facts
 
 

tests/mvrf/test_mgmtvrf.py

@@ -176,7 +176,7 @@ def test_ping(self, duthost):
         duthost.ping()
 
     def test_snmp_fact(self, localhost, duthost, creds):
-        get_snmp_facts(localhost, host=duthost.mgmt_ip, version="v2c", community=creds['snmp_rocommunity'])
+        get_snmp_facts(duthost, localhost, host=duthost.mgmt_ip, version="v2c", community=creds['snmp_rocommunity'])
 
 
 class TestMvrfOutbound():

tests/snmp/test_snmp_cpu.py

@@ -39,7 +39,7 @@ def test_snmp_cpu(duthosts, enum_rand_one_per_hwsku_hostname, localhost, creds_a
 
     # Gather facts with SNMP version 2
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], is_dell=True, wait=True)['ansible_facts']
 
     assert int(snmp_facts['ansible_ChStackUnitCpuUtil5sec'])
@@ -53,7 +53,7 @@ def test_snmp_cpu(duthosts, enum_rand_one_per_hwsku_hostname, localhost, creds_a
 
         # Gather facts with SNMP version 2
         snmp_facts = get_snmp_facts(
-            localhost, host=hostip, version="v2c",
+            duthost, localhost, host=hostip, version="v2c",
             community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], is_dell=True, wait=True)['ansible_facts']
 
         # Pull CPU utilization via shell

tests/snmp/test_snmp_default_route.py

@@ -21,7 +21,7 @@ def test_snmp_default_route(duthosts, enum_rand_one_per_hwsku_frontend_hostname,
     hostip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     dut_result = duthost.shell(r'show ip route 0.0.0.0/0 | grep "\*"')
 

tests/snmp/test_snmp_fdb.py

@@ -130,7 +130,7 @@ def test_snmp_fdb_send_tagged(ptfadapter, duthosts, rand_one_dut_hostname,
     hostip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     assert 'snmp_fdb' in snmp_facts
     assert 'snmp_interfaces' in snmp_facts

tests/snmp/test_snmp_interfaces.py

@@ -168,7 +168,7 @@ def test_snmp_interfaces(localhost, creds_all_duts, duthosts, enum_rand_one_per_
     config_facts = duthost.config_facts(
         host=duthost.hostname, source="persistent", namespace=namespace)['ansible_facts']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
 
     snmp_ifnames = [v['name']
@@ -193,7 +193,7 @@ def test_snmp_mgmt_interface(localhost, creds_all_duts, duthosts, enum_rand_one_
         duthost.hostname).vars['ansible_host']
 
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     config_facts = duthost.config_facts(
         host=duthost.hostname, source="persistent")['ansible_facts']
@@ -227,7 +227,7 @@ def test_snmp_interfaces_mibs(duthosts, enum_rand_one_per_hwsku_hostname, localh
     hostip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     config_facts = duthost.config_facts(
         host=duthost.hostname, source="persistent", namespace=namespace)['ansible_facts']

tests/snmp/test_snmp_link_local.py

@@ -31,7 +31,7 @@ def test_snmp_link_local_ip(duthosts,
     hostip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"],
         wait=True)['ansible_facts']
     # Get link local IP of mamangement interface

tests/snmp/test_snmp_lldp.py

@@ -43,7 +43,7 @@ def test_snmp_lldp(duthosts, enum_rand_one_per_hwsku_hostname, localhost, creds_
         duthost.hostname).vars['ansible_host']
 
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     mg_facts = {}
     for asic_id in duthost.get_asic_ids():

tests/snmp/test_snmp_loopback.py

@@ -23,7 +23,7 @@ def test_snmp_loopback(duthosts, enum_rand_one_per_hwsku_frontend_hostname,
     hostip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     config_facts = duthost.config_facts(
         host=duthost.hostname, source="persistent")['ansible_facts']

tests/snmp/test_snmp_memory.py

@@ -88,7 +88,7 @@ def test_snmp_memory(duthosts, enum_rand_one_per_hwsku_hostname, localhost, cred
     # Allow the test to retry a few times before claiming failure.
     for _ in range(3):
         snmp_facts = get_snmp_facts(
-            localhost, host=host_ip, version="v2c",
+            duthost, localhost, host=host_ip, version="v2c",
             community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
         facts = collect_memory(duthost)
         # net-snmp calculate cached memory as cached + sreclaimable
@@ -175,7 +175,7 @@ def test_snmp_swap(duthosts, enum_rand_one_per_hwsku_hostname, localhost, creds_
             "Swap is not on for this device, snmp does not support swap related queries when swap isn't on")
 
     snmp_facts = get_snmp_facts(
-        localhost, host=host_ip, version="v2c", include_swap=True,
+        duthost, localhost, host=host_ip, version="v2c", include_swap=True,
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     snmp_total_swap = snmp_facts['ansible_sysTotalSwap']
     snmp_free_swap = snmp_facts['ansible_sysTotalFreeSwap']

tests/snmp/test_snmp_pfc_counters.py

@@ -14,7 +14,7 @@ def test_snmp_pfc_counters(duthosts, enum_rand_one_per_hwsku_frontend_hostname,
         duthost.hostname).vars['ansible_host']
 
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
 
     # Check PFC counters

tests/snmp/test_snmp_phy_entity.py

@@ -209,7 +209,7 @@ def get_entity_and_sensor_mib(duthost, localhost, creds_all_duts):
     hostip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     entity_mib = {}
     sensor_mib = {}

tests/snmp/test_snmp_psu.py

@@ -20,7 +20,7 @@ def test_snmp_numpsu(duthosts, enum_supervisor_dut_hostname, localhost, creds_al
         duthost.hostname).vars['ansible_host']
 
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     res = duthost.shell("psuutil numpsus", module_ignore_errors=True)
 
@@ -41,7 +41,7 @@ def test_snmp_psu_status(duthosts, enum_supervisor_dut_hostname, localhost, cred
     hostip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=hostip, version="v2c",
+        duthost, localhost, host=hostip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
 
     psus_on = 0

tests/snmp/test_snmp_queue.py

@@ -58,7 +58,7 @@ def test_snmp_queues(duthosts, enum_rand_one_per_hwsku_hostname, localhost, cred
                 q_interfaces[intf[intf_idx]] = set()
             q_interfaces[intf[intf_idx]].add(intf[queue_idx])
 
-    snmp_facts = get_snmp_facts(localhost, host=hostip, version="v2c",
+    snmp_facts = get_snmp_facts(duthost, localhost, host=hostip, version="v2c",
                                 community=creds_all_duts[duthost.hostname]["snmp_rocommunity"],
                                 wait=True)['ansible_facts']
 

tests/snmp/test_snmp_v2mib.py

@@ -19,7 +19,7 @@ def test_snmp_v2mib(duthosts, enum_rand_one_per_hwsku_hostname, localhost, creds
     host_ip = duthost.host.options['inventory_manager'].get_host(
         duthost.hostname).vars['ansible_host']
     snmp_facts = get_snmp_facts(
-        localhost, host=host_ip, version="v2c",
+        duthost, localhost, host=host_ip, version="v2c",
         community=creds_all_duts[duthost.hostname]["snmp_rocommunity"], wait=True)['ansible_facts']
     dut_facts = duthost.setup()['ansible_facts']
     debian_ver = duthost.shell('cat /etc/debian_version')['stdout']