Prepare future inclusion of Thread CCM nodes ; Improved node build sy…

…stem ; Make-pretty for all code types (#165)

* [pylibs] Add CCM-commissioning unit test; with Commissioning unit test to include multi-hop commissioning.
* [pylibs] test  to include normal and CCM joiner in one test.
* [ot-rfsim] fixed filter on outgoing (off mesh) packets
* [ot-rfsim] fix build for FTD
* [event] bugfix in event data len; tests added.
* [pylibs][script] aiocoap install for unit-tests only
* [dispatcher][pylibs] added HostEvents counter
* [pcap] insert a dummy 802.15.4 frame at start of every PCAP file, to create a t=0 simulation time reference. This allows any PCAP entry to be associated to a corresponding radio (trace) log entry with exact (to the microsecond) precision.
* [pylibs] added further comments in partial_dataset case study.
* [pcap] fix unit test, related to introduction of t=0 time-reference frame.
* [ot-rfsim] fix macos build error (due to warn)
* [ot-rfsim] add C lang definitions for macOS build cpp in ot-rfsim project.
* [ot-rfsim] adding 'a' in front of function call parameters; and minor format fixes in C code; adding comments; removing FIXME items.
* [pylibs] added more documentation to test_sim_hosts and fixed testing-for-coaps in one test.
* [ot-rfsim] fix build for v1.1, v1.2 for using ot....() platform logging functions.
* [pylibs] fixes to test_commissioning - test sometimes failed.
* [pylibs] narrow test topology to avoid spurious test fails.
* [pylibs] Added  CCM unit test for one CCM joiner
* [ot-rfsim][cli] add version 'ccm' to place CCM nodes; adding branch pr-ccm repo of openthread for building CCM nodes; .gitignore cleanup.
* [script] add make-pretty exclusion for new submodule
* [ccm] added OT Registrar use in unit tests.
* [ot-rfsim] change ccm build script style to match latest 'main' branch merged.
* bumped openthread  to latest code; build and API updates
* [ot-rfsim] radio: compile-fix for otMacFrameGetSequence function signature change.
* [ot-rfsim] script: build only needed targets; store all build results under ./build; introduced main() in all build scripts; separate build dirs for BR/non-BR builds.
* [pylibs] 'joiner startccm' command added.
* [.golangci.yml] new syntax to replace deprecated one
* [.gitmodules] add commented OT CCM submodule (for future)
* [.github][script] updated make-pretty script to support all languages and checks; removed check-pretty.
* [ot-rfsim] improved build system, can now build/update all nodes quickly
* [script] make-pretty script updated; added clang config files from OpenThread.

.clang-format

@@ -0,0 +1,113 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+AccessModifierOffset: -4
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: true
+AlignConsecutiveDeclarations: true
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: true
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: false
+BinPackArguments: true
+BinPackParameters: false
+BraceWrapping:
+  AfterCaseLabel:  true
+  AfterClass:      true
+  AfterControlStatement: true
+  AfterEnum:       true
+  AfterFunction:   true
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     true
+  AfterUnion:      true
+  BeforeCatch:     false
+  BeforeElse:      true
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Custom
+BreakBeforeInheritanceComma: false
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeComma
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeCategories:
+  - Regex:           '^<openthread/.*/'
+    Priority:        4
+  - Regex:           '^<openthread/'
+    Priority:        3
+  - Regex:           '^<'
+    Priority:        2
+  - Regex:           '^".*/'
+    Priority:        5
+  - Regex:           '^"'
+    Priority:        1
+IncludeIsMainRegex: '(Test)?$'
+IndentCaseLabels: false
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 1000
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    true
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        4
+UseTab:          Never
+...
+

.clang-tidy

@@ -0,0 +1,23 @@
+---
+Checks: >
+  -*,
+  bugprone-argument-comment,
+  bugprone-too-small-loop-variable,
+  google-explicit-constructor,
+  google-readability-casting,
+  misc-unused-using-decls,
+  modernize-loop-convert,
+  modernize-use-bool-literals,
+  modernize-use-equals-default,
+  modernize-use-equals-delete,
+  modernize-use-nullptr,
+  readability-avoid-const-params-in-decls,
+  readability-else-after-return,
+  readability-inconsistent-declaration-parameter-name,
+  readability-make-member-function-const,
+  readability-redundant-control-flow,
+  readability-redundant-member-init,
+  readability-simplify-boolean-expr,
+  readability-static-accessed-through-instance
+WarningsAsErrors: '*'
+HeaderFilterRegex: '(examples|include|src).*(?<!third_party.*repo)'

.github/workflows/lint.yml

@@ -49,4 +49,5 @@ jobs:
       - uses: actions/checkout@v3
       - name: Check pretty
         run: |
-          ./script/check-pretty
+          ./script/install
+          ./script/make-pretty check

.gitignore

@@ -10,42 +10,51 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+# IDE files
 .idea/
+
+# Go generated files
 go.sum
-tmp/
-*.json
+go.mod.orig
+
+# ignore OTNS Go executables
+/cmd/otns/otns
+
+# OT-NS output files
+tmp*/
+?_kpi*.json
 *.log
 *.rep
-ot-cli-ftd
-ot-cli-mtd
-ot-cli-ftd_*
-ot-cli-mtd_*
-ot-rcp
+*.replay
+
+# Python/NodeJS generated
 *.pyc
 __pycache__
 node_modules/
 coverage.txt
 
-# Ignore any pcap
+# Ignore any pcap (generated)
 *.pcap
 
-# Ignore OTNS replay files
-*.replay
-
-# ignore OTNS executables
-/cmd/otns/otns
-
-# ignore generated files
+# ignore generated files (gRPC/web)
 /visualize/grpc/pb/*.pb.go
 /web/site/js/proto/*_pb.js
 /web/site/static/js/*.js
 
 # ot-rfsim build items
 /ot-rfsim/build*/
 /ot-rfsim/cmake-build-debug
+ot-cli-ftd*
+ot-cli-mtd*
+ot-rcp
 
 # pylibs installation files
 /pylibs/build/
 /pylibs/dist/
 /pylibs/pyOTNS.egg-info/
 /.venv-otns
+
+# lock files
+package-lock.json
+

.gitmodules

@@ -16,3 +16,7 @@
 	url = https://github.com/EskoDijk/openthread.git
 	branch = otns-v13
 	shallow = true
+#[submodule "openthread-ccm"]
+#	path = openthread-ccm
+#	url = https://github.com/EskoDijk/openthread-priv.git
+#	branch = pr-ccm

.golangci.yml

@@ -1,3 +1,3 @@
-run:
-  skip-files:
+issues:
+  exclude-files:
     - "web/site/bindata.go"

cli/ast.go

@@ -332,7 +332,7 @@ type RestoreFlag struct {
 
 // noinspection GoVetStructTag
 type ThreadVersion struct {
-	Val string `@("v11"|"v12"|"v13"|"v14")` //nolint
+	Val string `@("v11"|"v12"|"v13"|"v14"|"ccm")` //nolint
 }
 
 // noinspection GoVetStructTag

dispatcher/Node.go

@@ -185,7 +185,7 @@ func (node *Node) sendEvent(evt *Event) {
 	node.CurTime += evt.Delay
 	logger.AssertTrue(node.CurTime == node.D.CurTime)
 
-	// re-evaluate the FailutreCtrl when node time advances.
+	// re-evaluate the FailureCtrl when node time advances.
 	if evt.Timestamp > oldTime {
 		reEvaluateTime, isUpdated := node.failureCtrl.OnTimeAdvanced(oldTime)
 		if isUpdated {

dispatcher/dispatcher.go

@@ -406,7 +406,7 @@ func (d *Dispatcher) handleRecvEvent(evt *Event) {
 	switch evt.Type {
 	case EventTypeAlarmFired:
 		d.Counters.AlarmEvents += 1
-		if evt.MsgId == node.msgId { // if OT-node has seen my last sent event (so is done processing)
+		if evt.MsgId == node.msgId { // if OT-node has seen my last sent event (so it is done processing)
 			d.setSleeping(node.Id)
 		}
 		d.alarmMgr.SetTimestamp(nodeid, d.CurTime+delay) // schedule future wake-up of node
@@ -443,8 +443,9 @@ func (d *Dispatcher) handleRecvEvent(evt *Event) {
 	case EventTypeUdpFromHost,
 		EventTypeIp6FromHost:
 		d.Counters.HostEvents += 1
-		evt.MustDispatch = true // asap resend again to the target (BR) node.
-		d.eventQueue.Add(evt)
+		evt2 := evt.Copy()
+		evt2.MustDispatch = true // asap resend again to the target (BR) node.
+		d.eventQueue.Add(&evt2)
 	default:
 		d.Counters.OtherEvents += 1
 		d.cbHandler.OnRfSimEvent(node.Id, evt)
@@ -909,7 +910,7 @@ func (d *Dispatcher) syncAliveNodes() {
 		return
 	}
 
-	logger.Warnf("syncing %d alive nodes: %v", len(d.aliveNodes), d.aliveNodes)
+	logger.Warnf("syncing %d non-responding (alive) nodes: %v at simtime=%v", len(d.aliveNodes), d.aliveNodes, d.CurTime)
 	for nodeid := range d.aliveNodes {
 		d.advanceNodeTime(d.nodes[nodeid], d.CurTime, true)
 	}

event/event.go

@@ -28,12 +28,10 @@ package event
 
 import (
 	"encoding/binary"
+	"encoding/hex"
 	"fmt"
 	"math"
 	"net"
-	"strings"
-	"unicode"
-
 	"net/netip"
 
 	"github.com/openthread/ot-ns/logger"
@@ -240,7 +238,7 @@ func (e *Event) Deserialize(data []byte) int {
 	}
 
 	data2 := make([]byte, datalen-payloadOffset)
-	copy(data2, e.Data[payloadOffset:payloadOffset+datalen])
+	copy(data2, e.Data[payloadOffset:datalen])
 	e.Data = data2
 
 	// e.Timestamp is not in the event, so set to invalid initially.
@@ -316,12 +314,13 @@ func (e *Event) Copy() Event {
 func (e *Event) String() string {
 	paylStr := ""
 	if len(e.Data) > 0 {
-		paylStr = fmt.Sprintf(",payl=%s", keepPrintableChars(string(e.Data)))
+		paylStr = fmt.Sprintf(",payl=%s", hex.EncodeToString(e.Data))
 	}
 	s := fmt.Sprintf("Ev{%2d,nid=%d,mid=%d,dly=%v%s}", e.Type, e.NodeId, e.MsgId, e.Delay, paylStr)
 	return s
 }
 
+/*
 func keepPrintableChars(s string) string {
 	return strings.Map(func(r rune) rune {
 		if unicode.IsPrint(r) {
@@ -330,3 +329,4 @@ func keepPrintableChars(s string) string {
 		return -1
 	}, s)
 }
+*/

event/event_test.go

@@ -28,9 +28,8 @@ package event
 
 import (
 	"encoding/hex"
-	"testing"
-
 	"net/netip"
+	"testing"
 
 	"github.com/openthread/ot-ns/types"
 	"github.com/stretchr/testify/assert"
@@ -304,3 +303,26 @@ func TestEventCopy(t *testing.T) {
 	assert.Equal(t, uint8(types.OT_ERROR_FCS), evCopy.RadioCommData.Error)
 	assert.Equal(t, uint64(11234), evCopy.MsgId)
 }
+
+func TestDeserializeEventFromLargerData(t *testing.T) {
+	data, _ := hex.DecodeString("0403020100000000060c0d0e0f00000000100002b01140e201000000000002102030400809070605040302010000")
+	ev := Event{}
+	n := ev.Deserialize(data)
+	assert.Equal(t, 35, n)
+	assert.Equal(t, 35-eventMsgHeaderLen-radioCommEventDataHeaderLen, len(ev.Data))
+}
+
+func TestDeserializeEventFromTooLittleData(t *testing.T) {
+	// header intact, but incomplete data (i.e. datalen too large)
+	data, _ := hex.DecodeString("0403020100000000060c0d0e0f00000000100002b01140e201000000000002102030")
+	ev := Event{}
+	n := ev.Deserialize(data)
+	assert.Equal(t, 0, n)
+	assert.Equal(t, 0, len(ev.Data))
+
+	data, _ = hex.DecodeString("0403020100000000060c0d0e0f00000000")
+	ev = Event{}
+	n = ev.Deserialize(data)
+	assert.Equal(t, 0, n)
+	assert.Equal(t, 0, len(ev.Data))
+}

ot-rfsim/CMakeLists.txt

@@ -41,8 +41,16 @@ if(DEFINED ENV{OT_DIR})
 else()
     set(OT_DIR "${PROJECT_SOURCE_DIR}/../openthread")
 endif()
+if(DEFINED ENV{OTNS_NODE_TYPE})
+    set(OTNS_NODE_TYPE "$ENV{OTNS_NODE_TYPE}")
+    message( "-- [ot-rfsim] Caller has set OTNS_NODE_TYPE to: ${OTNS_NODE_TYPE}")
+else()
+    set(OTNS_NODE_TYPE "custom")
+endif()
 get_filename_component(OT_DIR "${OT_DIR}" REALPATH)
-message( "-- [ot-rfsim] Using openthread directory for build: ${OT_DIR}" )
+set(BUILD_DIR "build/ot-rfsim-${OTNS_NODE_TYPE}")
+message( "-- [ot-rfsim] Using openthread stack for build       : ${OT_DIR}" )
+message( "-- [ot-rfsim] Using openthread build output directory: ${BUILD_DIR}" )
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
@@ -52,7 +60,7 @@ add_definitions( -DRFSIM_SW_VERSION="${CMAKE_PROJECT_VERSION}" )
 
 # an external openthread repo may be defined - its build files are stored in there, so that coverage info
 # collection will work.
-add_subdirectory(${OT_DIR} ${OT_DIR}/build/ot-rfsim)
+add_subdirectory(${OT_DIR} ${OT_DIR}/${BUILD_DIR})
 
 target_compile_definitions(ot-config INTERFACE
     OPENTHREAD_CONFIG_FILE="openthread-core-rfsim-config.h"