Fix: hidden nodes test and related node-counters for CCA (#60)

* [pylibs] testWithOTNS unit test expanded: assertions added.

* [pylibs][script] Added test_topologies unit test

* [pylibs] Fix for 'go' Python command parameter and added unit test for Python go command.

* [ot-rfsim] bump ot-rfsim with fix radio CCA -> busy response.

* [radiomodel] MIDisc fixed to perform the CCA only within the disc limit. Now more consistent as a physical model.

* [pylibs] hidden_nodes_test updated to use MIDisc model.

* [pylibs] MLEID/service connectivity stress tests commented and updated. Using less End Devices for the moment since child capacity of router is limited to 10 unlike in OT-NS v1.

* [pylibs] decrease ping interval in connectivity stress tests to have more chances to get ping across after a node-move. Optimized code for 'ns.time'.

.github/workflows/stress.yml

@@ -64,6 +64,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Stress Test
         env:
-          STRESS_LEVEL: 10
+          STRESS_LEVEL: 2
         run: |
           ./script/test stress-tests ${{ matrix.suite }}

pylibs/examples/hidden_nodes_test.py

@@ -32,7 +32,7 @@
 TEST_SCENARIO = 0
 
 SPEED = 1e6
-RADIO_RANGE = 250
+RADIO_RANGE = 360
 PING_DATA_SIZE = 64
 NUM_PINGS = 100
 PING_INTERVAL = 1
@@ -55,8 +55,9 @@ def main():
     ns = OTNS(otns_args=["-log", "info", "-watch", "trace"])
     ns.speed = SPEED
     ns.web()
-    # The test setups are only valid for this radio model.
-    ns.radiomodel = 'MutualInterference'
+    # The test setups are only valid for this radio model. Disc model is used to enforce a hidden
+    # node situation.
+    ns.radiomodel = 'MIDisc'
 
     if TEST_SCENARIO == 0:
         print('\n ==============Scenario A (Static): Interferer out of Source range ======================')
@@ -71,17 +72,13 @@ def main():
         ns.go(10)
         dst = ns.add("router", x=700, y=300, radio_range=RADIO_RANGE)
         intf = ns.add("router", x=525, y=450, radio_range=RADIO_RANGE)
-        ns.node_cmd(dst, "state router") # due to very low RSSI, a node may stay End Device. This forces it to Router.
-        ns.node_cmd(intf, "state router")
 
     elif TEST_SCENARIO == 2:
         print('\n ==============Scenario C (Static): Interferer out of Destination range ======================')
         src = ns.add("router", x=350, y=300, radio_range=RADIO_RANGE)
         ns.go(10)
         dst = ns.add("router", x=700, y=300, radio_range=RADIO_RANGE)
         intf = ns.add("router", x=175, y=300, radio_range=RADIO_RANGE)
-        ns.node_cmd(dst, "state router") # due to very low RSSI, a node may stay End Device. This forces it to Router.
-        ns.node_cmd(intf, "state router")
 
     else:
         return

pylibs/otns/cli/OTNS.py

@@ -342,6 +342,24 @@ def watch(self, *nodeids: int) -> None:
         cmd = f'watch {" ".join(map(str, nodeids))}'
         self._do_command(cmd)
 
+    def watch_all(self, level: str) -> None:
+        """
+        Set watch log level on all current nodes.
+
+        :param level: watch log level 'trace', 'debug', 'info', 'warn', 'crit', or 'off'.
+        """
+        cmd = f'watch all {level}'
+        self._do_command(cmd)
+
+    def watch_default(self, level: str) -> None:
+        """
+        Set default watch log level for newly to be created nodes.
+
+        :param level: watch log level 'trace', 'debug', 'info', 'warn', 'crit', or 'off'.
+        """
+        cmd = f'watch default {level}'
+        self._do_command(cmd)
+
     def watched(self) -> List[int]:
         """
         Get the nodes currently being watched.

pylibs/stress_tests/BaseStressTest.py

@@ -32,10 +32,10 @@
 import time
 import traceback
 from functools import wraps
-from otns.cli import OTNS
-from otns.cli.errors import UnexpectedError
 from typing import Collection
 
+from otns.cli import OTNS
+from otns.cli.errors import UnexpectedError
 from StressTestResult import StressTestResult
 from errors import UnexpectedNodeAddr, UnexpectedNodeState
 

pylibs/stress_tests/mleid_connectivity.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (c) 2020, The OTNS Authors.
+# Copyright (c) 2020-2023, The OTNS Authors.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -29,47 +29,50 @@
 # ML-EID Connectivity Stress Test:
 #   Nodes pings the BR by it's MLEID and measure the connectivity
 # Topology:
-#   Router x20
-#   FED x10
-#   MED x10
-#   SED x10
+#   Router xROUTER_COUNT
+#   FED xFED_COUNT
+#   MED xMED_COUNT
+#   SED xSED_COUNT
 # Fault Injections:
 #   Nodes are constantly moving
-#   Nodes fail for 60s in every 600s
-#   Packet Loss Ratio set to 0.2
+#   Nodes fail for 30s in every 600s
+#   Packet Loss Ratio set to 0.1
 # Pass Criteria:
-#   Max Delay (of all nodes) <= 3600s
+#   Max Delay (over all nodes) <= MAX_DELAY_TIME s
+#   Avg Delay (over all nodes) <= MAX_AVG_DELAY_TIME s
 #
+
 import logging
 import os
 import random
 
 from BaseStressTest import BaseStressTest
 
-ROUTER_COUNT = 20
-
-FED_COUNT = 10
-MED_COUNT = 10
-SED_COUNT = 10
+ROUTER_COUNT = 35
+FED_COUNT = 5
+MED_COUNT = 5
+SED_COUNT = 5
 TOTAL_NODE_COUNT = ROUTER_COUNT + FED_COUNT + MED_COUNT + SED_COUNT
 
 RADIO_RANGE = 400
-
 XMAX = 1000
 YMAX = 1000
 
+# Note: the "delay time" for a node is the time period since contact with the BR by means of ping was
+# last successful.
 TOTAL_SIMULATION_TIME = 3600 * int(os.getenv("STRESS_LEVEL", 1))
-MOVE_INTERVAL = 60
-PING_INTERVAL = 60
-PING_DATA_SIZE = 32
-
-FAIL_DURATION = 60
-FAIL_INTERVAL = 600
-MOVE_COUNT = 5
+MAX_DELAY_TIME = 1800 # seconds - the max allowable delay time for any node.
+MAX_AVG_DELAY_TIME = 1000 # seconds - the max allowable average of all nodes' delay times.
+MOVE_INTERVAL = 60 # seconds
+PING_INTERVAL = 10 # seconds
+PING_DATA_SIZE = 32 # bytes
+FAIL_DURATION = 30 # seconds (of failure during one FAIL_INTERVAL)
+FAIL_INTERVAL = 600 # seconds
+MOVE_COUNT = 3 # number of nodes moved per move-interval
 
 BR = None  # the Border Router
 
-SED_PULL_PERIOD = 1
+SED_PULL_PERIOD = 1 # seconds
 
 
 class MleidConnectivityStressTest(BaseStressTest):
@@ -79,17 +82,18 @@ def __init__(self):
         super(MleidConnectivityStressTest, self).__init__("ML-EID Connectivity Test",
                                                           ["Simulation Time", "Max Delay", "Min Delay", "Avg Delay"])
         self._last_ping_succ_time = {}
-        self._cur_time = 0
         self._ping_fail_count = 0
         self._ping_succ_count = 0
 
     def run(self):
         ns = self.ns
         ns.packet_loss_ratio = 0.1
+        ns.radiomodel = 'MIDisc'
+        #ns.watch_default('warn') # enable OT node warnings or higher to be printed.
         ns.config_visualization(broadcast_message=False)
 
         assert ROUTER_COUNT >= 1
-        BR = ns.add("router", x=random.randint(0, XMAX), y=random.randint(0, YMAX))
+        BR = ns.add("router", x=random.randint(0, XMAX), y=random.randint(0, YMAX), radio_range=RADIO_RANGE)
         ns.radio_set_fail_time(BR, fail_time=(FAIL_DURATION, FAIL_INTERVAL))
 
         BR_ADDR = self.expect_node_mleid(BR, 10)
@@ -114,39 +118,37 @@ def run(self):
         for nodeid in range(1, TOTAL_NODE_COUNT + 1):
             ns.ping(nodeid, BR_ADDR, datasize=PING_DATA_SIZE, count=TOTAL_SIMULATION_TIME // PING_INTERVAL,
                     interval=PING_INTERVAL)
+            ns.go(PING_INTERVAL/TOTAL_NODE_COUNT) # spread out pings over time within interval.
 
         for _ in range(TOTAL_SIMULATION_TIME // MOVE_INTERVAL):
             nodeids = list(range(1, TOTAL_NODE_COUNT + 1))
             for nodeid in random.sample(nodeids, min(MOVE_COUNT, len(nodeids))):
                 ns.move(nodeid, random.randint(0, XMAX), random.randint(0, YMAX))
-
             ns.go(MOVE_INTERVAL)
-            self._collect_pings()
-
-            self._cur_time += MOVE_INTERVAL
-
-        ns.go(100)
-        self._collect_pings()
+            self._collect_pings(MOVE_INTERVAL)
 
-        self._cur_time += 100
+        ns.go(MOVE_INTERVAL)
+        self._collect_pings(PING_INTERVAL)
 
         delays = [TOTAL_SIMULATION_TIME - self._last_ping_succ_time.get(nodeid, 0) for nodeid in
                   range(1, TOTAL_NODE_COUNT + 1)]
         logging.debug("_last_ping_succ_time %s delays %s", self._last_ping_succ_time, delays)
         avg_delay = sum(delays) / TOTAL_NODE_COUNT
         self.result.append_row("%dh" % (TOTAL_SIMULATION_TIME // 3600),
                                '%ds' % max(delays), '%ds' % min(delays), '%ds' % avg_delay)
-        self.result.fail_if(max(delays) > 3600, "Max Delay (%ds)> 3600s" % max(delays))
+        self.result.fail_if(avg_delay > MAX_AVG_DELAY_TIME, "Avg Delay (%ds)> %ds" % (avg_delay, MAX_AVG_DELAY_TIME))
+        self.result.fail_if(max(delays) > MAX_DELAY_TIME, "Max Delay (%ds)> %ds" % (max(delays), MAX_DELAY_TIME))
 
-    def _collect_pings(self):
+    def _collect_pings(self, lastIntervalSec):
+        current_sim_time = self.ns.time/1e6
         for srcid, dstaddr, _, delay in self.ns.pings():
             if delay >= 10000:
                 # ignore failed pings
                 self._ping_fail_count += 1
                 continue
 
             self._ping_succ_count += 1
-            self._last_ping_succ_time[srcid] = self._cur_time
+            self._last_ping_succ_time[srcid] = current_sim_time - lastIntervalSec
 
 
 if __name__ == '__main__':

pylibs/stress_tests/service_connectivity.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (c) 2020, The OTNS Authors.
+# Copyright (c) 2020-2023, The OTNS Authors.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -29,42 +29,46 @@
 # ML-EID Connectivity Stress Test:
 #   Nodes pings the BR by it's service ALOC and measure the connectivity
 # Topology:
-#   Router x20
-#   FED x10
-#   MED x10
-#   SED x10
+#   Router xROUTER_COUNT
+#   FED xFED_COUNT
+#   MED xMED_COUNT
+#   SED xSED_COUNT
 # Fault Injections:
 #   Nodes are constantly moving
 #   Nodes fail for 30s in every 600s
 #   Packet Loss Ratio set to 0.1
 # Pass Criteria:
 #   Max Delay (over all nodes) <= MAX_DELAY_TIME s
+#   Avg Delay (over all nodes) <= MAX_AVG_DELAY_TIME s
 #
+
 import logging
 import os
 import random
 
 from BaseStressTest import BaseStressTest
 
-ROUTER_COUNT = 20
-FED_COUNT = 10
-MED_COUNT = 10
-SED_COUNT = 10
+ROUTER_COUNT = 35
+FED_COUNT = 5
+MED_COUNT = 5
+SED_COUNT = 5
 TOTAL_NODE_COUNT = ROUTER_COUNT + FED_COUNT + MED_COUNT + SED_COUNT
 
 RADIO_RANGE = 400
 XMAX = 1000
 YMAX = 1000
 
+# Note: the "delay time" for a node is the time period since contact with the BR by means of ping was
+# last successful.
 TOTAL_SIMULATION_TIME = 3600 * int(os.getenv("STRESS_LEVEL", "1")) # seconds
-MAX_DELAY_TIME = 1800 # seconds - TODO what is this?
+MAX_DELAY_TIME = 1800 # seconds - the max allowable delay time for any node.
+MAX_AVG_DELAY_TIME = 1000 # seconds - the max allowable average of all nodes' delay times.
 MOVE_INTERVAL = 60 # seconds
-PING_INTERVAL = 60 # seconds
+PING_INTERVAL = 10 # seconds
 PING_DATA_SIZE = 32 # bytes
-
 FAIL_DURATION = 30 # seconds (of failure during one FAIL_INTERVAL)
 FAIL_INTERVAL = 600 # seconds
-MOVE_COUNT = 3
+MOVE_COUNT = 3 # number of nodes moved per move-interval
 
 BR = None  # the Border Router
 SVR1, SVR1_DATA = "112233", "aabbcc"
@@ -73,23 +77,25 @@
 SED_PULL_PERIOD = 1
 
 
-class StressTest(BaseStressTest):
+class ServiceConnectivityStressTest(BaseStressTest):
     SUITE = 'connectivity'
 
     def __init__(self):
-        super(StressTest, self).__init__("Service Connectivity Test",
+        super(ServiceConnectivityStressTest, self).__init__("Service Connectivity Test",
                                          ["Simulation Time", "Max Delay", "Min Delay", "Avg Delay"])
         self._last_ping_succ_time = {}
-        self._cur_time = 0
         self._ping_fail_count = 0
         self._ping_succ_count = 0
 
     def run(self):
         ns = self.ns
         ns.packet_loss_ratio = 0.1
+        ns.radiomodel = 'MIDisc'
+        #ns.watch_default('warn') # enable OT node warnings or higher to be printed.
+        ns.config_visualization(broadcast_message=False)
 
         assert ROUTER_COUNT >= 1
-        BR = ns.add("router", x=random.randint(0, XMAX), y=random.randint(0, YMAX))
+        BR = ns.add("router", x=random.randint(0, XMAX), y=random.randint(0, YMAX), radio_range=RADIO_RANGE)
         ns.radio_set_fail_time(BR, fail_time=(FAIL_DURATION, FAIL_INTERVAL))
         ns.node_cmd(BR, "prefix add 2001:dead:beef:cafe::/64 paros med")
         ns.node_cmd(BR, f"service add 44970 {SVR1} {SVR1_DATA}")
@@ -117,40 +123,38 @@ def run(self):
         for nodeid in range(1, TOTAL_NODE_COUNT + 1):
             ns.ping(nodeid, BR_ADDR, datasize=PING_DATA_SIZE, count=TOTAL_SIMULATION_TIME // PING_INTERVAL,
                     interval=PING_INTERVAL)
+            ns.go(PING_INTERVAL/TOTAL_NODE_COUNT) # spread out pings over time within interval.
 
         for _ in range(TOTAL_SIMULATION_TIME // MOVE_INTERVAL):
             nodeids = list(range(1, TOTAL_NODE_COUNT + 1))
             for nodeid in random.sample(nodeids, min(MOVE_COUNT, len(nodeids))):
                 ns.move(nodeid, random.randint(0, XMAX), random.randint(0, YMAX))
-
             ns.go(MOVE_INTERVAL)
-            self._collect_pings()
-
-            self._cur_time += MOVE_INTERVAL
-
-        ns.go(100)
-        self._collect_pings()
+            self._collect_pings(MOVE_INTERVAL)
 
-        self._cur_time += 100
+        ns.go(MOVE_INTERVAL)
+        self._collect_pings(PING_INTERVAL)
 
         delays = [TOTAL_SIMULATION_TIME - self._last_ping_succ_time.get(nodeid, 0) for nodeid in
                   range(1, TOTAL_NODE_COUNT + 1)]
         logging.debug("_last_ping_succ_time %s delays %s", self._last_ping_succ_time, delays)
         avg_delay = sum(delays) / TOTAL_NODE_COUNT
         self.result.append_row("%dh" % (TOTAL_SIMULATION_TIME // 3600),
                                '%ds' % max(delays), '%ds' % min(delays), '%ds' % avg_delay)
+        self.result.fail_if(avg_delay > MAX_AVG_DELAY_TIME, "Avg Delay (%ds)> %ds" % (avg_delay, MAX_AVG_DELAY_TIME))
         self.result.fail_if(max(delays) > MAX_DELAY_TIME, "Max Delay (%ds)> %ds" % (max(delays), MAX_DELAY_TIME))
 
-    def _collect_pings(self):
+    def _collect_pings(self, lastIntervalSec):
+        current_sim_time = self.ns.time/1e6
         for srcid, dstaddr, _, delay in self.ns.pings():
             if delay >= 10000:
                 # ignore failed pings
                 self._ping_fail_count += 1
                 continue
 
             self._ping_succ_count += 1
-            self._last_ping_succ_time[srcid] = self._cur_time
+            self._last_ping_succ_time[srcid] = current_sim_time - lastIntervalSec
 
 
 if __name__ == '__main__':
-    StressTest().run()
+    ServiceConnectivityStressTest().run()