diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eaf5b1fe..fdba112f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,7 @@ jobs: steps: - uses: actions/setup-go@v5 with: - go-version: '1.22' + go-version: '1.18' - uses: actions/checkout@v3 - name: Test run: | @@ -117,7 +117,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.10'] + python-version: ['3.9'] os: [ubuntu-22.04, macos-13] env: HOMEBREW_NO_AUTO_UPDATE: 1 diff --git a/GUIDE.md b/GUIDE.md index dc50ac20..6028ff5c 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -1,44 +1,57 @@ # OTNS Guide -This guide covers the installation of Go, installation of OTNS, use of the OTNS Web UI and the OTNS CLI. +This guide covers the installation of OTNS, installation of OpenThread nodes used in simulations, use of the OTNS Web UI and the OTNS CLI. + +## Prerequisites / OS + +OTNS supports Linux and MacOS. Windows is supported only through the Windows Subsystem for Linux (WSL2) running Ubuntu 22.04, though this +is tested less frequently. If Ubuntu Linux is used, the minimum version and tested version is 22.04. +For MacOS, currently version 13 is best supported. Version 14 is supported but currently without the Thread legacy +node versions (Thread 1.1, 1.2, 1.3). MacOS also requires the Homebrew package manager (`brew` command) already installed. + +OTNS requires Go 1.18 or higher and Python 3.9 or higher. These will be installed (if needed) by following the rest of this guide. +Also Git is required to get the code and for building OpenThread nodes. If not already available, this needs to be installed using e.g. +a package manager (`sudo apt install git`). ## Install Go -OTNS requires [Go 1.18 or higher](https://golang.org/dl/) to build: +OTNS requires [Go 1.18 or higher](https://golang.org/dl/) to build. You can check your installed Go version using +the `go version` command. If Go is not found, it needs to be installed in one of these ways: + + 1. Install Go from https://golang.org/dl/ and select your version or + 2. Install Go via a package manager (in this case check the Go version is high enough). + 3. Install Go automatically (uses APT or Brew package manager) by the installation script called later in this guide. In + Ubuntu 22.04, this approach should work to get Go 1.18. - - Install Go from https://golang.org/dl/ or via a package manager (in this case check the Go version is high enough). - - Add `$(go env GOPATH)/bin` (normally `$HOME/go/bin`) to `$PATH`. +After installation, manually check that the directory `$(go env GOPATH)/bin` (normally `$HOME/go/bin`) is in the user's `$PATH`. This +directory is where the `otns` binary will be installed into. If needed, add this directory to the user's `$PATH` by configuring the +user's `~/.bashrc` file. -## Get OTNS code +## Get OTNS code using Git ```bash git clone https://github.com/EskoDijk/ot-ns.git ./otns cd otns ``` -## Automated installation +## Installation of OTNS An automated way to install dependencies, OTNS and all OT nodes, and test the result, is the following command: ```bash -./script/test go-tests +./script/bootstrap ``` -Alternatively the more extensive Python unit tests can be run also with the following command: - -```bash -./script/test py-unittests -``` - -However, this can take a long time (5-10 minutes). Running any Python (`py-*`) tests will set up a Python 3 virtual +This will set up (if not yet done) a Python 3 virtual environment locally in the `.venv-otns` directory. This virtual environment must also be active when manually running -any OTNS Python scripts. +any OTNS Python scripts, as shown later in this guide. -## Manual step-by-step installation +### Manual step-by-step installation of OTNS (Alternative) As alternative to above automated installation, scripts can also be run for the individual phases of the installation. This is shown in the below subsections. Here, also some more explanation is given of the result of each phase. +This approach is also useful for troubleshooting. -### Install Dependencies +#### Install Dependencies ```bash ./script/install-deps @@ -46,7 +59,7 @@ This is shown in the below subsections. Here, also some more explanation is give Running this script will also set up a Python 3 virtual environment (venv) in `.venv-otns` in the project directory. -### Install OTNS +#### Install OTNS ```bash ./script/install @@ -54,9 +67,9 @@ Running this script will also set up a Python 3 virtual environment (venv) in `. This installs `otns` in the Go binary directory of the user (typically `~/go/bin`) and makes the command available in the path. Also, it installs the pyOTNS library in the local Python virtual environment `.venv-otns`. -The OT nodes required for running a simulation are not yet installed at this point. +The OT nodes required for running a simulation are not yet installed at this point: this is the next step. -### Install OT nodes +#### Install OT Nodes ```bash ./script/install-nodes @@ -69,88 +82,33 @@ order to access older OpenThread codebases. In case the build stops unexpectedly the case that an older OT commit is checked out in the `./openthread` subdirectory. This can be manually restored again by running `git submodule update`. -These nodes of specific versions can be added to a simulation using specific flags in the `add` command that adds -a node. Type `help add` in OTNS to see this. - -## Manually Build OpenThread for OTNS (Optional) - -This fork of OTNS uses POSIX simulation to simulate Thread nodes, with a specific platform `ot-rfsim` located in the -`ot-rfsim` directory. -The simulator uses node executables such as `ot-cli-ftd`. By default, the `install-nodes` script will build -a common set of OpenThread nodes of different version (v1.1, v1.2, v1.3.0, and "latest" v1.4) that -are used in the various examples and unit-tests of OTNS. - -To build or rebuild yourself an executable with platform `ot-rfsim` for OTNS, see the example build below. -It shows a build with default settings that builds an OpenThread node of version "latest". This version is the -latest one that is bundled with the current checked out commit of OTNS. It gets bumped occassionally to the -latest OpenThread main branch after verifying that it works. - -NOTE: the `bootstrap` step only has to be executed only once, to install the required dev tools. - -```bash -$ cd ot-rfsim -$ ./script/bootstrap -$ ./script/build_latest -$ cd .. -``` - -To build earlier OpenThread code versions for inclusion in the simulation, such as 1.1, 1.2 or 1.3, the following -commands can be used: - -```bash -$ cd ot-rfsim -$ ./script/build_v11 -$ ./script/build_v12 -$ ./script/build_v13 -$ ./script/build_br -$ ./script/build_latest -$ cd .. -``` - -NOTE: all of the above version-specific build scripts will check if the submodule 'openthread' is at the right specific -commit that is expected. The `./script/build_all` script provides the automated Git commit checkout and building for -all versions. - -The generic build script can be invoked as shown below. This will build whatever code is currently -residing in the 'openthread' submodule without switching to a specific OT version and without clearing any previous -build files. So, it can be used for (faster) incremental builds while developing code. - -```bash -$ cd ot-rfsim -$ ./script/build -$ cd .. -``` - -Finally, the generic build script allows setting or overriding any OT build arguments. See an example below: +NOTE: the mechanism of using a single repo and checking out different commits from it will be replaced in an upcoming +new version of OTNS. There's a better method that can be used. -```bash -$ cd ot-rfsim -$ ./script/build -DOT_FULL_LOGS=OFF -DOT_COAP_OBSERVE=ON -DOT_TCP=ON -$ cd .. -``` - -In this example a node is built with debug logs off (for speed in simulation), CoAP-observe enabled, and TCP enabled. +These nodes of specific versions can be added to a simulation using specific flags in the `add` command that adds +a node. Type `help add` later on in the OTNS CLI to see this. ## Run OTNS Interactively Preferably run OTNS from the working directory (i.e. the root of this repo): ```bash -cd ~/otns -otns +$ cd ~/otns +$ otns ``` Running from this directory ensures that OTNS can find the standard binaries (version latest, v11, v12, v13, etc - these are stored in `./ot-rfsim/ot-versions`). OTNS can be run also from any directory in which the node executable(s) such as `ot-cli-ftd` (and optionally `ot-cli-mtd`) are placed. In this case, it will use the executable from the current directory if it can find the -right one. +right executable type/name there. If started successfully, OTNS by default opens a web browser for network visualization and management. -To see what command-line parameters are supported, use `-h`: +To see what command-line parameters are supported for OTNS, use `-h`: ```bash -otns -h +$ otns -h +... ``` ## Use OTNS-Web @@ -166,6 +124,8 @@ Use a web browser to manage the simulated Thread network: * See some logged events * See nodes' energy usage (Beta feature - pending validation) +A detailed CodeLab guide for this is still under development. + ## Use OTNS CLI See [OTNS CLI Reference](cli/README.md). @@ -173,8 +133,7 @@ See [OTNS CLI Reference](cli/README.md). ## OTNS Python Scripting [pyOTNS](pylibs/otns) library provides utilities to create and manage simulations through OTNS CLI. It is installed in a -Python 3 virtual environment `.venv-otns` by the `./script/install` script. The `./script/test` script also calls this -install script when needed as part of setup. +Python 3 virtual environment `.venv-otns` by the installation process. ### Python Scripting Documentation @@ -206,11 +165,84 @@ To ensure that the `pyOTNS` library can be found, and the virtual environment is ```bash $ source .venv-otns/bin/activate +(.venv-otns) $ ``` -Then run a Python script: +The prompt will change to reflect the virtual environment has been entered. Then run a Python script: ```bash (.venv-otns) $ ./pylibs/examples/farm.py ... ``` + +## Manually Build OpenThread Nodes for OTNS (Optional, for Advanced Use Only) + +OTNS uses POSIX simulation to simulate Thread nodes, with a custom OpenThread platform `ot-rfsim` located in the +`ot-rfsim` directory. +The simulator uses node executables such as `ot-cli-ftd`. By default, the `install-nodes` script will build +a common set of OpenThread nodes of different version (v1.1, v1.2, v1.3.0, and "latest" v1.4) that +are used in the various examples and unit-tests of OTNS. + +To build or rebuild yourself an executable with platform `ot-rfsim` for OTNS, see the example build below. +It shows a build with default settings that builds an OpenThread node of version "latest". This version is the +latest one that is bundled with the current checked out commit of OTNS. It gets bumped occassionally to the +latest OpenThread main branch after verifying that it works. + +```bash +$ cd ot-rfsim +$ ./script/build_latest +... +$ cd .. +``` + +After the build, the executable gets placed in `./ot-rfsim/ot-versions`. This is the default location where OTNS will +look for OpenThread node executables. (Other locations can be configured/selected also in OTNS.) + +To build earlier OpenThread code versions for inclusion in the simulation, such as 1.1, 1.2 or 1.3, the following +commands can be used: + +```bash +$ cd ot-rfsim +$ ./script/build_v11 +... +$ ./script/build_v12 +... +$ ./script/build_v13 +... +$ ./script/build_br +... +$ cd .. +``` + +NOTE: all of the above version-specific build scripts will check if the submodule 'openthread' is at the right specific +commit that is expected. The `./script/build_all` script provides the automated Git commit checkout and building for +all versions. + +The generic build script can be invoked as shown below. This will build whatever code is currently +residing in the 'openthread' submodule without switching to a specific OT version and without clearing any previous +build files. So, it can be used for (faster) incremental builds while developing code. This can also be used to test code +for a particular 'openthread' PR/branch/fix in a simulation. + +```bash +$ cd ot-rfsim +$ ./script/build +$ cd .. +``` + +The result of this build is in `./ot-rfsim/build/bin` and is not copied to any other location. If the user wants to use the +executable for simulations, it's easiest to copy the build result file (`ot-cli-ftd`) into `./ot-rfsim/ot-versions` so that the new-built node +will be used in simulations as the default node. Be careful: this overwrites the existing default node executable. If you want to use +both the default node executable and the new-built node executable in simulations, then the new-built executable will need to get a new name such +as `ot-cli-ftd_myfix` or so. With this new name, the OTNS `add` CLI command can be used with the `exe` argument to provide the name of the new +executable. + +Finally, the generic build script allows setting or overriding any OT build arguments. See an example below: + +```bash +$ cd ot-rfsim +$ ./script/build -DOT_FULL_LOGS=OFF -DOT_COAP_OBSERVE=ON -DOT_TCP=ON +$ cd .. +``` + +In this example a node is built with debug logs off (for more speed in simulation), CoAP-observe enabled, and TCP enabled. + diff --git a/go.mod b/go.mod index e2ebbdf2..64fd5df1 100644 --- a/go.mod +++ b/go.mod @@ -34,21 +34,21 @@ require ( github.com/mitchellh/go-wordwrap v1.0.1 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.9.0 - go.uber.org/zap v1.27.0 + go.uber.org/zap v1.24.0 golang.org/x/net v0.22.0 golang.org/x/term v0.21.0 - google.golang.org/grpc v1.64.0 + google.golang.org/grpc v1.56.0 google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - go.uber.org/multierr v1.10.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect ) diff --git a/otnstester/OtnsTest.go b/otnstester/OtnsTest.go index c2204017..107a3473 100644 --- a/otnstester/OtnsTest.go +++ b/otnstester/OtnsTest.go @@ -429,7 +429,7 @@ func NewOtnsTest(t *testing.T) *OtnsTest { }) }() - grpcConn, err := grpc.NewClient("localhost:8999", grpc.WithTransportCredentials(insecure.NewCredentials())) + grpcConn, err := grpc.Dial("localhost:8999", grpc.WithTransportCredentials(insecure.NewCredentials())) ot.ExpectNoError(err) grpcClient := visualize_grpc_pb.NewVisualizeGrpcServiceClient(grpcConn) diff --git a/pylibs/setup.py b/pylibs/setup.py index 6d8e90da..5c9b6369 100755 --- a/pylibs/setup.py +++ b/pylibs/setup.py @@ -39,6 +39,6 @@ "License :: OSI Approved :: BSD 3-Clause License", "Operating System :: OS Independent", ], - python_requires='>=3.10', + python_requires='>=3.9', install_requires=['PyYAML'], ) diff --git a/pylibs/unittests/requirements.txt b/pylibs/unittests/requirements.txt index 768234ae..2f8206f6 100644 --- a/pylibs/unittests/requirements.txt +++ b/pylibs/unittests/requirements.txt @@ -2,4 +2,4 @@ grpcio-tools==1.53.0 grpcio==1.53.0 protobuf==4.21.6 wheel==0.42.0 -aiocoap==0.4.9 \ No newline at end of file +aiocoap==0.4.7 \ No newline at end of file diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 00000000..c0f36765 --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) 2024, The OTNS Authors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# This script bootstraps a complete install of OTNS, nodes, dependencies. + +set -euox pipefail + +if [[ "$(uname)" == "Darwin" ]]; then + export readonly Darwin=1 + export readonly Linux=0 +elif [[ "$(uname)" == "Linux" ]]; then + export readonly Darwin=0 + export readonly Linux=1 +else + die "Unsupported OS: $(uname)" +fi + +# shellcheck source=script/utils.sh +. "$(dirname "$0")"/utils.sh + +# Some basic items missing on Ubuntu Docker, assumed present on MacOS. +if [[ $Linux == 1 ]]; then + if ! installed sudo; then + apt install sudo + fi + install_package xdg-open --apt xdg-utils || true +fi + +# Install Go if needed +install_package go --apt golang --brew golang || true + +# Do the rest via the test script +# TODO: once MacOS supports OT versions, remove the below if. +if [[ $Linux == 1 ]]; then + ./script/test py-ver-unittests +else + ./script/test py-unittests +fi diff --git a/script/common.sh b/script/common.sh index 01e40065..efea0bbf 100644 --- a/script/common.sh +++ b/script/common.sh @@ -34,16 +34,16 @@ elif [[ "$(uname)" == "Linux" ]]; then export readonly Darwin=0 export readonly Linux=1 else - die "Unknown OS: $(uname)" + die "Unsupported OS: $(uname)" fi # shellcheck source=script/utils.sh . "$(dirname "$0")"/utils.sh export readonly SCRIPTDIR -SCRIPTDIR=$(realpath "$(dirname "$0")") +SCRIPTDIR=$(realpathf "$(dirname "$0")") export readonly OTNSDIR -OTNSDIR=$(realpath "$SCRIPTDIR"/..) +OTNSDIR=$(realpathf "$SCRIPTDIR"/..) export readonly GOPATH GOPATH=$(go env GOPATH) export PATH=$PATH:"$GOPATH"/bin diff --git a/script/install-deps b/script/install-deps index 70127e14..a4367805 100755 --- a/script/install-deps +++ b/script/install-deps @@ -34,8 +34,14 @@ apt_update_once=0 install_packages() { + install_package git --apt git --brew git || true + install_package cmake --apt cmake --brew cmake || true + install_package g++ --apt build-essential --brew gcc || true install_package wget --apt wget --brew wget || true - install_package python3-venv --apt wget --brew wget || true + install_package unzip --apt unzip --brew unzip || true + install_package python3 --apt python3 --brew python3 || true + # FIXME: on Mac, this brew package doesn't exist - seems not needed(?) + install_package virtualenv --apt python3-venv --brew python3-venv || true } install_grpcwebproxy() diff --git a/script/utils.sh b/script/utils.sh index 2de0ebf2..52373b5a 100644 --- a/script/utils.sh +++ b/script/utils.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2022, The OTNS Authors. +# Copyright (c) 2020-2024, The OTNS Authors. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -25,17 +25,16 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -readonly Darwin - function die() { echo "fatal: $1" false } -function realpath() +function realpathf() { - python3 -c "import os; print(os.path.realpath('$1'))" + # the Python3 method is a backup. + realpath -s "$1" || python3 -c "import os; print(os.path.realpath('$1'))" } function installed() @@ -120,13 +119,6 @@ function install_pretty_tools() install_openthread_buildtools() { - if installed "ninja"; then - return 0 - fi - - if [[ $Darwin == 1 ]]; then - brew install ninja - else - sudo apt-get install ninja-build - fi + install_package cmake --apt cmake --brew cmake + install_package ninja --apt ninja-build --brew ninja }