Skip to content

Commit

Permalink
Add manual script to enable p1 networking
Browse files Browse the repository at this point in the history
Once the fxp-net_linux-networking.pkg package has been loaded, we need
to enable primary networking by designating enp0s1f0d5 as the primary
networking interface on the acc.

By default when the ACC first comes up, this package will not be loaded
(we will use the default package which enables networking on P0/P1 on
the d1/d2 interfaces respectively.

However after the first time we install the vsp, this will become
required.

Signed-off-by: Salvatore Daniele <[email protected]>
  • Loading branch information
SalDaniele committed Nov 13, 2024
1 parent a4f2eb3 commit de8cd57
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 4 deletions.
18 changes: 15 additions & 3 deletions extraConfigDpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,11 +151,15 @@ def wait_vsp_ds_running(client: K8sClient) -> None:

def ensure_p4_pod_running(lh: host.Host, acc: host.Host, imgReg: ImageRegistry) -> None:
lh.run_or_die(f"podman pull --tls-verify=false {P4_IMG}")
lh.run_or_die(f"podman tag {P4_IMG} {imgReg.url()}/kubecon-aarch64")
lh.run_or_die(f"podman push {imgReg.url()}/kubecon-aarch64")
local_img = f"{imgReg.url()}/kubecon-aarch64:latest"
lh.run_or_die(f"podman tag {P4_IMG} {local_img}")
lh.run_or_die(f"podman push {local_img}")
uname = acc.run("uname -r").out.strip()
# If p4 pod already exists from previous run, kill this first.
acc.run(f"podman ps --filter ancestor={local_img} --format '{{{{.ID}}}}' | xargs -r podman kill")
logger.info("Manually starting P4 container")
cmd = f"podman run -d --privileged -v /lib/modules/{uname}:/lib/modules/{uname} -v /opt/p4/p4-cp-nws/var/run:/opt/p4/p4-cp-nws/var/run -v /sys:/sys -p 9559:9559 {imgReg.url()}/kubecon-aarch64"
acc.run_or_die("mkdir -p /opt/p4/p4-cp-nws/var/run/openvswitch") # WA https://issues.redhat.com/browse/IIC-421
cmd = f"podman run -d --privileged -v /lib/modules/{uname}:/lib/modules/{uname} -v /opt/p4/p4-cp-nws/var/run:/opt/p4/p4-cp-nws/var/run -v /sys:/sys -p 9559:9559 {local_img}"
acc.run_or_die(cmd)
# Occasionally the P4 pod fails to start
while True:
Expand All @@ -165,6 +169,14 @@ def ensure_p4_pod_running(lh: host.Host, acc: host.Host, imgReg: ImageRegistry)
logger.info("Failed to start p4 container, retrying")
acc.run_or_die(cmd)

logger.info("Waiting for P4 container to finish initialization")
container_id = acc.run_or_die(f"podman ps --filter ancestor={local_img} --format '{{{{.ID}}}}'").out.strip()
while True:
logs = acc.run_or_die(f"podman logs {container_id} 2>&1").out
if "Attempting P4RT communication" in logs:
break
time.sleep(5)


def ExtraConfigDpu(cc: ClustersConfig, cfg: ExtraConfigArgs, futures: dict[str, Future[Optional[host.Result]]]) -> None:
[f.result() for (_, f) in futures.items()]
Expand Down
10 changes: 9 additions & 1 deletion ipu.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,11 @@ def _prepare_imc(self, server_with_key: str) -> None:
nohup sh -c '
while true; do
sleep 30
python /usr/bin/scripts/cfg_acc_apf_x2.py
if [ -f /work/scripts/ipu_p1_setup.sh ]; then
/work/scripts/ipu_p1_setup.sh
else
break
fi
ping -c 1 192.168.0.2
if [ $? -eq 0 ]; then
break
Expand All @@ -184,6 +188,10 @@ def _prepare_imc(self, server_with_key: str) -> None:
imc.run("cp /etc/imc-redfish-configuration.json /work/redfish/")
imc.run(f"echo {self.password} | bash /usr/bin/ipu-redfish-generate-password-hash.sh")

# WA: We need to manually install this file to enable networking with the fxp-net_linux-networking.pkg
imc.copy_to("./manifests/dpu/ipu_port1_setup.sh", "/work/scripts/ipu_port1_setup.sh")
imc.run("chmod +x /work/scripts/ipu_port1_set.sh")

imc.run("reboot")
time.sleep(10)
imc.wait_ping()
Expand Down
6 changes: 6 additions & 0 deletions manifests/dpu/ipu_port1_setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# This script needs to be run on reboot each time in order for P1 to be used as the primary interface via enp0s1f0d5
echo "Configure enp0s1f0d5 as designated primary network interface to Microshift OVS"
echo "#Add to VSI Group 1 : enp0s1f0d5 [vsi: 0x0C]"
devmem 0x20292002a0 64 0x800005000000000c
devmem 0x2029200388 64 0x1
devmem 0x20292002a0 64 0xa00005000000000c

0 comments on commit de8cd57

Please sign in to comment.