From 0f51f337ff4f0a55f7a567031bb072003164f578 Mon Sep 17 00:00:00 2001 From: apk <13803120+apkd@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:46:03 +0100 Subject: [PATCH] Retry unanswered root handshakes sooner Unanswered root requests and expired paths trigger earlier retries, reducing reconnection delays after sleep or network changes. Reply tracking distinguishes delayed replies from replies to newer requests. Recovery completes after accepting the reply path. --- node/IncomingPacket.cpp | 3 + node/Node.cpp | 12 ++-- node/Peer.cpp | 2 + node/Peer.hpp | 34 +++++++++ selftest.cpp | 153 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 4 deletions(-) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index cf55537a51..ebc8526809 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -662,6 +662,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment* RR, void* tPtr, const Share } peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_OK, inRePacketId, inReVerb, false, networkId, ZT_QOS_NO_FLOW); + if (inReVerb == Packet::VERB_HELLO && ! hops()) { + peer->acknowledgeFullHello(_path, at(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)); + } return true; } diff --git a/node/Node.cpp b/node/Node.cpp index c6b46fb713..5db468d01b 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -260,15 +260,19 @@ class _PingPeersThatNeedPing { // Contact upstream peers as infrequently as possible int roleBasedTimerScale = (role == ZT_PEER_ROLE_LEAF) ? 2 : 16; - // Unless we don't any have paths to the roots, then we shouldn't wait a long time to contact them - bool hasPaths = p->paths(RR->node->now()).size() > 0; - roleBasedTimerScale = (role != ZT_PEER_ROLE_LEAF && ! hasPaths) ? 0 : roleBasedTimerScale; + // Retry roots promptly when paths expire or a full HELLO goes unanswered. + const bool recoverRoot = (role != ZT_PEER_ROLE_LEAF) && p->needsPathRecovery(_now); + roleBasedTimerScale = recoverRoot ? 0 : roleBasedTimerScale; if ((RR->node->now() - p->lastSentFullHello()) <= (ZT_PATH_HEARTBEAT_PERIOD * roleBasedTimerScale)) { return; } - const unsigned int sent = p->doPingAndKeepalive(_tPtr, _now); + unsigned int sent = p->doPingAndKeepalive(_tPtr, _now); + if (recoverRoot) { + // Sending on a saved path does not prove that it still reaches the root. + sent = 0; + } bool contacted = (sent != 0); if ((sent & 0x1) == 0) { // bit 0x1 == IPv4 sent diff --git a/node/Peer.cpp b/node/Peer.cpp index 65199badbc..3e922df908 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -38,6 +38,7 @@ Peer::Peer(const RuntimeEnvironment* renv, const Identity& myIdentity, const Ide , _lastTrustEstablishedPacketReceived(0) , _lastSentFullHello(0) , _lastEchoCheck(0) + , _fullHelloPending(false) , _freeRandomByte((unsigned char)((uintptr_t)this >> 4) ^ ++s_freeRandomByteCounter) , _vProto(0) , _vMajor(0) @@ -549,6 +550,7 @@ unsigned int Peer::doPingAndKeepalive(void* tPtr, int64_t now) const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD); if (sendFullHello) { _lastSentFullHello = now; + _fullHelloPending = true; } // Right now we only keep pinging links that have the maximum priority. The diff --git a/node/Peer.hpp b/node/Peer.hpp index 8bd36bd3df..0a6f307338 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -319,6 +319,39 @@ class Peer { return _lastSentFullHello; } + /** + * Acknowledge a scheduled full HELLO after its direct reply path has been accepted. + */ + inline void acknowledgeFullHello(const SharedPtr& path, int64_t timestamp) + { + Mutex::Lock _l(_paths_m); + if (_fullHelloPending && timestamp >= _lastSentFullHello) { + for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS && _paths[i].p; ++i) { + if (_paths[i].p == path) { + _fullHelloPending = false; + break; + } + } + } + } + + /** + * @return True if paths have expired or a full HELLO has had time to receive a reply but remains unanswered + */ + inline bool needsPathRecovery(int64_t now) const + { + Mutex::Lock _l(_paths_m); + if (_fullHelloPending && ((now - _lastSentFullHello) >= ZT_PATH_HEARTBEAT_PERIOD)) { + return true; + } + for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS && _paths[i].p; ++i) { + if ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) { + return false; + } + } + return true; + } + /** * @return Latency in milliseconds of best/aggregate path or 0xffff if unknown / no paths */ @@ -691,6 +724,7 @@ class Peer { int64_t _lastSentFullHello; int64_t _lastEchoCheck; + bool _fullHelloPending; unsigned char _freeRandomByte; uint16_t _vProto; diff --git a/selftest.cpp b/selftest.cpp index 60e26dc060..c7fc4fcd0f 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -24,13 +24,18 @@ #include "node/RuntimeEnvironment.hpp" #include "node/SHA512.hpp" #include "node/Salsa20.hpp" +#include "node/Topology.hpp" #include "node/Utils.hpp" +#include "node/World.hpp" #include "osdep/OSUtils.hpp" #include "osdep/Phy.hpp" #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" +#include #include +#include +#include #include #include #include @@ -1192,6 +1197,153 @@ static int testPacket() return 0; } +static int testRootRecovery() +{ + std::cout << "[root recovery] Testing a lost wake probe and delayed reply... " << std::flush; + struct Endpoint { + InetAddress address; + std::map state; + std::deque incoming; + Endpoint* remote; + int64_t socket = 1; + unsigned int hellos = 0; + bool connected = true; + std::unique_ptr node; + }; + const auto require = [](bool passed, const char* message) { + if (! passed) { + throw std::runtime_error(message); + } + }; + try { + int64_t now = 1000000; + Endpoint root { InetAddress("198.51.100.1/9993") }, client { InetAddress("192.0.2.1/9993") }; + root.remote = &client; + client.remote = &root; + root.state[ZT_STATE_OBJECT_IDENTITY_SECRET] = KNOWN_GOOD_IDENTITY; + const ECC::Pair key = ECC::generate(); + World::Root worldRoot { Identity(KNOWN_GOOD_IDENTITY), { root.address } }; + Buffer planet; + World::make(World::TYPE_PLANET, 1, now, key.pub, { worldRoot }, key).serialize(planet); + ZT_Node_Callbacks callbacks {}; + callbacks.stateGetFunction = [](ZT_Node*, void* user, void*, ZT_StateObjectType type, const uint64_t[2], + void* buffer, unsigned int capacity) { + const auto& state = static_cast(user)->state; + const auto value = state.find(type); + if (value == state.end() || value->second.size() > capacity) { + return -1; + } + memcpy(buffer, value->second.data(), value->second.size()); + return (int)value->second.size(); + }; + callbacks.statePutFunction = [](ZT_Node*, void*, void*, ZT_StateObjectType, const uint64_t[2], const void*, + int) {}; + callbacks.wirePacketSendFunction = [](ZT_Node*, void* user, void*, int64_t localSocket, + const sockaddr_storage* address, const void* bytes, unsigned int length, + unsigned int) { + Endpoint& sender = *static_cast(user); + Packet packet(bytes, length); + if (! packet.isEncrypted() && packet.verb() == Packet::VERB_HELLO) { + ++sender.hellos; + } + if (sender.connected && (localSocket == -1 || localSocket == sender.socket)) { + if (sender.remote->address == *reinterpret_cast(address)) { + sender.remote->incoming.push_back(packet); + } + } + return 0; + }; + callbacks.virtualNetworkFrameFunction = [](ZT_Node*, void*, void*, uint64_t, void**, uint64_t, uint64_t, + unsigned int, unsigned int, const void*, unsigned int) {}; + callbacks.virtualNetworkConfigFunction = [](ZT_Node*, void*, void*, uint64_t, void**, + ZT_VirtualNetworkConfigOperation, + const ZT_VirtualNetworkConfig*) { return 0; }; + callbacks.eventCallback = [](ZT_Node*, void*, void*, ZT_Event, const void*) {}; + ZT_Node_Config config {}; + for (Endpoint* endpoint : { &root, &client }) { + endpoint->state[ZT_STATE_OBJECT_PLANET] = std::string((const char*)planet.data(), planet.size()); + endpoint->node.reset(new Node(endpoint, nullptr, &config, &callbacks, now)); + } + volatile int64_t deadline; + const auto background = [&] { + require( + client.node->processBackgroundTasks(nullptr, now, &deadline) == ZT_RESULT_OK, + "Background tasks failed"); + }; + const auto deliver = [&](Endpoint& endpoint) { + require(! endpoint.incoming.empty(), "Expected a queued packet"); + const Packet packet = endpoint.incoming.front(); + endpoint.incoming.pop_front(); + require( + endpoint.node->processWirePacket( + nullptr, now, endpoint.socket, &endpoint.remote->address, packet.data(), packet.size(), &deadline) + == ZT_RESULT_OK, + "Packet delivery failed"); + }; + const auto drain = [&] { + unsigned int delivered = 0; + while (! root.incoming.empty() || ! client.incoming.empty()) { + require(++delivered < 1000, "Packet exchange did not settle"); + for (Endpoint* endpoint : { &root, &client }) { + if (! endpoint->incoming.empty()) { + deliver(*endpoint); + } + } + } + }; + const auto tick = [&] { + now += ZT_PING_CHECK_INTERVAL; + background(); + drain(); + }; + const auto checkQuiet = [&] { + const unsigned int hellos = client.hellos; + const int64_t until = now + ZT_PEER_PING_PERIOD * 2; + while (now < until) { + tick(); + } + require(client.hellos == hellos, "A healthy idle root received extra HELLOs"); + }; + background(); + drain(); + const SharedPtr peer(client.node->RR->topology->getPeerNoCache(root.node->identity().address())); + require(peer && peer->lastReceive() == now, "Initial root connection failed"); + checkQuiet(); + + // Hold a real reply across sleep, then deliver it after the wake probe has been lost. + now += ZT_PEER_PING_PERIOD * 4; + background(); + deliver(root); + require(! client.incoming.empty(), "No root reply to delay across sleep"); + now += 6 * 60 * 60 * 1000; + client.connected = false; + const unsigned int beforeWake = client.hellos; + background(); + require(client.hellos > beforeWake, "No wake probe was lost"); + drain(); + require(peer->lastReceive() == now, "The delayed reply did not arrive"); + + ++client.socket; + client.address = InetAddress("192.0.2.2/9993"); + client.connected = true; + const int64_t restoredAt = now; + const unsigned int hellos = client.hellos; + now += ZT_PATH_HEARTBEAT_PERIOD; + background(); + drain(); + require( + peer->lastReceive() > restoredAt && client.hellos > hellos, + "Root recovery waited for the normal contact interval"); + checkQuiet(); + } + catch (const std::exception& error) { + std::cout << "FAILED: " << error.what() << std::endl; + return -1; + } + std::cout << "OK" << std::endl; + return 0; +} + static int testOther() { char buf[1024]; @@ -1673,6 +1825,7 @@ int main(int argc, char** argv) r |= testPacket(); r |= testIdentity(); r |= testCertificate(); + r |= testRootRecovery(); r |= testPhy(); //*/