diff --git a/packages/daemons/src/bind/ensureBindContainerIpAndRunning.ts b/packages/daemons/src/bind/ensureBindContainerIpAndRunning.ts index 40b7f73e3..0fa5e37d4 100644 --- a/packages/daemons/src/bind/ensureBindContainerIpAndRunning.ts +++ b/packages/daemons/src/bind/ensureBindContainerIpAndRunning.ts @@ -31,41 +31,60 @@ async function disconnectConflictingContainerAndStartBind(): Promise { } export async function ensureBindContainerIpAndRunning(): Promise { - const isBindRunning = ( - await docker.getContainer(params.bindContainerName).inspect() - ).State.Running; + try { + const isBindRunning = ( + await docker.getContainer(params.bindContainerName).inspect() + ).State.Running; - // check if the bind container is running - if (!isBindRunning) { - logs.info(`${params.bindContainerName} container is not running`); - await disconnectConflictingContainerAndStartBind(); - } else { - // check is connected to dncore_network - const isConnectedToNetwork = Object.values( - ( - (await docker - .getNetwork(params.DOCKER_PRIVATE_NETWORK_NAME) - .inspect()) as Dockerode.NetworkInspectInfo - ).Containers ?? [] - ).some((container) => container.Name === params.bindContainerName); - - if (!isConnectedToNetwork) { - logs.info( - `${params.bindContainerName} container is not connected to ${params.DOCKER_PRIVATE_NETWORK_NAME} network` - ); + // check if the bind container is running + if (!isBindRunning) { + logs.info(`${params.bindContainerName} container is not running`); await disconnectConflictingContainerAndStartBind(); } else { - // check it has the right IP - const hasRightIp = - (await docker.getContainer(params.bindContainerName).inspect()) - .NetworkSettings.Networks[params.DOCKER_PRIVATE_NETWORK_NAME] - .IPAddress === params.BIND_IP; - if (hasRightIp) { - logs.info(`${params.bindContainerName} container has right IP`); - } else { - logs.info(`${params.bindContainerName} container has wrong IP`); + // check is connected to dncore_network + const isConnectedToNetwork = Object.values( + ( + (await docker + .getNetwork(params.DOCKER_PRIVATE_NETWORK_NAME) + .inspect()) as Dockerode.NetworkInspectInfo + ).Containers ?? [] + ).some((container) => container.Name === params.bindContainerName); + + if (!isConnectedToNetwork) { + logs.info( + `${params.bindContainerName} container is not connected to ${params.DOCKER_PRIVATE_NETWORK_NAME} network` + ); await disconnectConflictingContainerAndStartBind(); + } else { + // check it has the right IP + const hasRightIp = + (await docker.getContainer(params.bindContainerName).inspect()) + .NetworkSettings.Networks[params.DOCKER_PRIVATE_NETWORK_NAME] + .IPAddress === params.BIND_IP; + if (hasRightIp) { + logs.info(`${params.bindContainerName} container has right IP`); + } else { + logs.info(`${params.bindContainerName} container has wrong IP`); + await disconnectConflictingContainerAndStartBind(); + } } } + } catch (e) { + // check if container does not exist 404 + if (e.statusCode === 404) { + logs.warn( + `container ${params.bindContainerName} not found and it might be in an intermedium state` + ); + // the container might be in intermedium state with different name + // TODO: what if there is a docker container already using this IP. + // This would be extremley dangerous once the migration to the private ip range is done + // and less ips are available. + logs.info( + `recreating container ${params.bindContainerName} with compose up` + ); + await dockerComposeUp(getDockerComposePath(params.bindDnpName, true), { + forceRecreate: true, + }); + } else throw e; } } diff --git a/packages/dockerCompose/src/setDappnodeComposeDefaults.ts b/packages/dockerCompose/src/setDappnodeComposeDefaults.ts index 950bae02c..0ad7d235c 100644 --- a/packages/dockerCompose/src/setDappnodeComposeDefaults.ts +++ b/packages/dockerCompose/src/setDappnodeComposeDefaults.ts @@ -117,9 +117,11 @@ function setServiceNetworksWithAliases( serviceNetworks = parseServiceNetworks(serviceNetworks); const dncoreServiceNetwork = serviceNetworks[params.DOCKER_PRIVATE_NETWORK_NAME] || {}; - // do not allow to set hardcoded IPs - // TODO: allow bind in the future, after 0.3.0 - if (dncoreServiceNetwork.ipv4_address) + // do not allow to set hardcoded IPs except for bind + if ( + dncoreServiceNetwork.ipv4_address && + service.dnpName !== params.bindDnpName + ) delete dncoreServiceNetwork.ipv4_address; return { ...serviceNetworks, diff --git a/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainerWithIp.ts b/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainerWithIp.ts index 39014f08e..2c1ad28ed 100644 --- a/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainerWithIp.ts +++ b/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainerWithIp.ts @@ -1,11 +1,12 @@ import { disconnectConflictingContainerIfAny, docker, + dockerComposeUp, findContainerByIP, } from "@dappnode/dockerapi"; import { logs } from "@dappnode/logger"; import { params } from "@dappnode/params"; -import { removeCidrSuffix } from "@dappnode/utils"; +import { getDockerComposePath, removeCidrSuffix } from "@dappnode/utils"; import Dockerode from "dockerode"; /** @@ -55,34 +56,53 @@ export async function connectContainerWithIp({ // check target container is running, otherwise the docker network connect might not take effect const targetContainer = docker.getContainer(containerName); - const containerInfo = await targetContainer.inspect(); - if ( - !containerInfo.State.Running && - containerName !== params.dappmanagerContainerName - ) { - logs.warn(`container ${containerName} is not running, restarting it`); - await targetContainer.restart(); - } + try { + // There has been edge cases where docker containers are in an intermedium state with a different docker container name + // i.e b7903a289091_DAppNodeCore-bind.dnp.dappnode.eth instead of DAppNodeCore-bind.dnp.dappnode.eth + const containerInfo = await targetContainer.inspect(); + if ( + !containerInfo.State.Running && + containerName !== params.dappmanagerContainerName + ) { + logs.warn(`container ${containerName} is not running, restarting it`); + await targetContainer.restart(); + } - const hasContainerRightIp = - removeCidrSuffix(aliasesIpsMap.get(containerName)?.ip || "") === - containerIp; + const hasContainerRightIp = + removeCidrSuffix(aliasesIpsMap.get(containerName)?.ip || "") === + containerIp; - if (hasContainerRightIp) - logs.info( - `container ${containerName} has right IP and is connected to docker network` - ); - else { - logs.info( - `container ${containerName} does not have right IP and/or is not connected to docker network` - ); - await connectContainerRetryOnIpUsed({ - network, - containerName, - maxAttempts: 20, - ip: containerIp, - aliasesIpsMap, - }); + if (hasContainerRightIp) + logs.info( + `container ${containerName} has right IP and is connected to docker network` + ); + else { + logs.info( + `container ${containerName} does not have right IP and/or is not connected to docker network` + ); + await connectContainerRetryOnIpUsed({ + network, + containerName, + maxAttempts: 20, + ip: containerIp, + aliasesIpsMap, + }); + } + } catch (e) { + // check if container does not exist 404 + if (containerName === params.bindContainerName && e.statusCode === 404) { + logs.warn( + `container ${params.bindContainerName} not found and it might be in an intermedium state` + ); + // the container might be in intermedium state with different name + // TODO: what if there is a docker container already using this IP. + // This would be extremley dangerous once the migration to the private ip range is done + // and less ips are available. + logs.info(`recreating container ${containerName} with compose up`); + await dockerComposeUp(getDockerComposePath(params.bindDnpName, true), { + forceRecreate: true, + }); + } else throw e; } } diff --git a/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainersToNetworkWithPrio.ts b/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainersToNetworkWithPrio.ts index aec05e80b..7229a9d5e 100644 --- a/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainersToNetworkWithPrio.ts +++ b/packages/migrations/src/migrateDockerNetworkIpRange/connectContainersToNetworkWithPrio/connectContainersToNetworkWithPrio.ts @@ -49,7 +49,11 @@ export async function connectContainersToNetworkWithPrio({ containerName: dappmanagerContainer.name, containerIp: dappmanagerContainer.ip, aliasesIpsMap, - }); + }).catch((e) => + logs.error( + `Failed to connect container ${dappmanagerContainer.name} to network ${network.id}: ${e}` + ) + ); // 2. Connect bind container await connectContainerWithIp({ @@ -57,12 +61,18 @@ export async function connectContainersToNetworkWithPrio({ containerName: bindContainer.name, containerIp: bindContainer.ip, aliasesIpsMap, - }); + }).catch((e) => + logs.error( + `Failed to connect container ${bindContainer.name} to network ${network.id}: ${e}` + ) + ); await restoreContainersToNetworkNotThrow({ containersToRestart, network, aliasesIpsMap, containersToRecreate, - }); + }).catch((e) => + logs.error(`Failed to restore containers to network ${network.id}: ${e}`) + ); }