diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index cf55537a5..ebc852680 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -662,6 +662,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment* RR, void* tPtr, const Share } peer->received(tPtr, _path, hops(), packetId(), payloadLength(), Packet::VERB_OK, inRePacketId, inReVerb, false, networkId, ZT_QOS_NO_FLOW); + if (inReVerb == Packet::VERB_HELLO && ! hops()) { + peer->acknowledgeFullHello(_path, at(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)); + } return true; } diff --git a/node/Node.cpp b/node/Node.cpp index c6b46fb71..5db468d01 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -260,15 +260,19 @@ class _PingPeersThatNeedPing { // Contact upstream peers as infrequently as possible int roleBasedTimerScale = (role == ZT_PEER_ROLE_LEAF) ? 2 : 16; - // Unless we don't any have paths to the roots, then we shouldn't wait a long time to contact them - bool hasPaths = p->paths(RR->node->now()).size() > 0; - roleBasedTimerScale = (role != ZT_PEER_ROLE_LEAF && ! hasPaths) ? 0 : roleBasedTimerScale; + // Retry roots promptly when paths expire or a full HELLO goes unanswered. + const bool recoverRoot = (role != ZT_PEER_ROLE_LEAF) && p->needsPathRecovery(_now); + roleBasedTimerScale = recoverRoot ? 0 : roleBasedTimerScale; if ((RR->node->now() - p->lastSentFullHello()) <= (ZT_PATH_HEARTBEAT_PERIOD * roleBasedTimerScale)) { return; } - const unsigned int sent = p->doPingAndKeepalive(_tPtr, _now); + unsigned int sent = p->doPingAndKeepalive(_tPtr, _now); + if (recoverRoot) { + // Sending on a saved path does not prove that it still reaches the root. + sent = 0; + } bool contacted = (sent != 0); if ((sent & 0x1) == 0) { // bit 0x1 == IPv4 sent diff --git a/node/Peer.cpp b/node/Peer.cpp index 65199badb..3e922df90 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -38,6 +38,7 @@ Peer::Peer(const RuntimeEnvironment* renv, const Identity& myIdentity, const Ide , _lastTrustEstablishedPacketReceived(0) , _lastSentFullHello(0) , _lastEchoCheck(0) + , _fullHelloPending(false) , _freeRandomByte((unsigned char)((uintptr_t)this >> 4) ^ ++s_freeRandomByteCounter) , _vProto(0) , _vMajor(0) @@ -549,6 +550,7 @@ unsigned int Peer::doPingAndKeepalive(void* tPtr, int64_t now) const bool sendFullHello = ((now - _lastSentFullHello) >= ZT_PEER_PING_PERIOD); if (sendFullHello) { _lastSentFullHello = now; + _fullHelloPending = true; } // Right now we only keep pinging links that have the maximum priority. The diff --git a/node/Peer.hpp b/node/Peer.hpp index 8bd36bd3d..0a6f30733 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -319,6 +319,39 @@ class Peer { return _lastSentFullHello; } + /** + * Acknowledge a scheduled full HELLO after its direct reply path has been accepted. + */ + inline void acknowledgeFullHello(const SharedPtr& path, int64_t timestamp) + { + Mutex::Lock _l(_paths_m); + if (_fullHelloPending && timestamp >= _lastSentFullHello) { + for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS && _paths[i].p; ++i) { + if (_paths[i].p == path) { + _fullHelloPending = false; + break; + } + } + } + } + + /** + * @return True if paths have expired or a full HELLO has had time to receive a reply but remains unanswered + */ + inline bool needsPathRecovery(int64_t now) const + { + Mutex::Lock _l(_paths_m); + if (_fullHelloPending && ((now - _lastSentFullHello) >= ZT_PATH_HEARTBEAT_PERIOD)) { + return true; + } + for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS && _paths[i].p; ++i) { + if ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) { + return false; + } + } + return true; + } + /** * @return Latency in milliseconds of best/aggregate path or 0xffff if unknown / no paths */ @@ -691,6 +724,7 @@ class Peer { int64_t _lastSentFullHello; int64_t _lastEchoCheck; + bool _fullHelloPending; unsigned char _freeRandomByte; uint16_t _vProto; diff --git a/selftest.cpp b/selftest.cpp index 60e26dc06..c7fc4fcd0 100644 --- a/selftest.cpp +++ b/selftest.cpp @@ -24,13 +24,18 @@ #include "node/RuntimeEnvironment.hpp" #include "node/SHA512.hpp" #include "node/Salsa20.hpp" +#include "node/Topology.hpp" #include "node/Utils.hpp" +#include "node/World.hpp" #include "osdep/OSUtils.hpp" #include "osdep/Phy.hpp" #include "osdep/PortMapper.hpp" #include "osdep/Thread.hpp" +#include #include +#include +#include #include #include #include @@ -1192,6 +1197,153 @@ static int testPacket() return 0; } +static int testRootRecovery() +{ + std::cout << "[root recovery] Testing a lost wake probe and delayed reply... " << std::flush; + struct Endpoint { + InetAddress address; + std::map state; + std::deque incoming; + Endpoint* remote; + int64_t socket = 1; + unsigned int hellos = 0; + bool connected = true; + std::unique_ptr node; + }; + const auto require = [](bool passed, const char* message) { + if (! passed) { + throw std::runtime_error(message); + } + }; + try { + int64_t now = 1000000; + Endpoint root { InetAddress("198.51.100.1/9993") }, client { InetAddress("192.0.2.1/9993") }; + root.remote = &client; + client.remote = &root; + root.state[ZT_STATE_OBJECT_IDENTITY_SECRET] = KNOWN_GOOD_IDENTITY; + const ECC::Pair key = ECC::generate(); + World::Root worldRoot { Identity(KNOWN_GOOD_IDENTITY), { root.address } }; + Buffer planet; + World::make(World::TYPE_PLANET, 1, now, key.pub, { worldRoot }, key).serialize(planet); + ZT_Node_Callbacks callbacks {}; + callbacks.stateGetFunction = [](ZT_Node*, void* user, void*, ZT_StateObjectType type, const uint64_t[2], + void* buffer, unsigned int capacity) { + const auto& state = static_cast(user)->state; + const auto value = state.find(type); + if (value == state.end() || value->second.size() > capacity) { + return -1; + } + memcpy(buffer, value->second.data(), value->second.size()); + return (int)value->second.size(); + }; + callbacks.statePutFunction = [](ZT_Node*, void*, void*, ZT_StateObjectType, const uint64_t[2], const void*, + int) {}; + callbacks.wirePacketSendFunction = [](ZT_Node*, void* user, void*, int64_t localSocket, + const sockaddr_storage* address, const void* bytes, unsigned int length, + unsigned int) { + Endpoint& sender = *static_cast(user); + Packet packet(bytes, length); + if (! packet.isEncrypted() && packet.verb() == Packet::VERB_HELLO) { + ++sender.hellos; + } + if (sender.connected && (localSocket == -1 || localSocket == sender.socket)) { + if (sender.remote->address == *reinterpret_cast(address)) { + sender.remote->incoming.push_back(packet); + } + } + return 0; + }; + callbacks.virtualNetworkFrameFunction = [](ZT_Node*, void*, void*, uint64_t, void**, uint64_t, uint64_t, + unsigned int, unsigned int, const void*, unsigned int) {}; + callbacks.virtualNetworkConfigFunction = [](ZT_Node*, void*, void*, uint64_t, void**, + ZT_VirtualNetworkConfigOperation, + const ZT_VirtualNetworkConfig*) { return 0; }; + callbacks.eventCallback = [](ZT_Node*, void*, void*, ZT_Event, const void*) {}; + ZT_Node_Config config {}; + for (Endpoint* endpoint : { &root, &client }) { + endpoint->state[ZT_STATE_OBJECT_PLANET] = std::string((const char*)planet.data(), planet.size()); + endpoint->node.reset(new Node(endpoint, nullptr, &config, &callbacks, now)); + } + volatile int64_t deadline; + const auto background = [&] { + require( + client.node->processBackgroundTasks(nullptr, now, &deadline) == ZT_RESULT_OK, + "Background tasks failed"); + }; + const auto deliver = [&](Endpoint& endpoint) { + require(! endpoint.incoming.empty(), "Expected a queued packet"); + const Packet packet = endpoint.incoming.front(); + endpoint.incoming.pop_front(); + require( + endpoint.node->processWirePacket( + nullptr, now, endpoint.socket, &endpoint.remote->address, packet.data(), packet.size(), &deadline) + == ZT_RESULT_OK, + "Packet delivery failed"); + }; + const auto drain = [&] { + unsigned int delivered = 0; + while (! root.incoming.empty() || ! client.incoming.empty()) { + require(++delivered < 1000, "Packet exchange did not settle"); + for (Endpoint* endpoint : { &root, &client }) { + if (! endpoint->incoming.empty()) { + deliver(*endpoint); + } + } + } + }; + const auto tick = [&] { + now += ZT_PING_CHECK_INTERVAL; + background(); + drain(); + }; + const auto checkQuiet = [&] { + const unsigned int hellos = client.hellos; + const int64_t until = now + ZT_PEER_PING_PERIOD * 2; + while (now < until) { + tick(); + } + require(client.hellos == hellos, "A healthy idle root received extra HELLOs"); + }; + background(); + drain(); + const SharedPtr peer(client.node->RR->topology->getPeerNoCache(root.node->identity().address())); + require(peer && peer->lastReceive() == now, "Initial root connection failed"); + checkQuiet(); + + // Hold a real reply across sleep, then deliver it after the wake probe has been lost. + now += ZT_PEER_PING_PERIOD * 4; + background(); + deliver(root); + require(! client.incoming.empty(), "No root reply to delay across sleep"); + now += 6 * 60 * 60 * 1000; + client.connected = false; + const unsigned int beforeWake = client.hellos; + background(); + require(client.hellos > beforeWake, "No wake probe was lost"); + drain(); + require(peer->lastReceive() == now, "The delayed reply did not arrive"); + + ++client.socket; + client.address = InetAddress("192.0.2.2/9993"); + client.connected = true; + const int64_t restoredAt = now; + const unsigned int hellos = client.hellos; + now += ZT_PATH_HEARTBEAT_PERIOD; + background(); + drain(); + require( + peer->lastReceive() > restoredAt && client.hellos > hellos, + "Root recovery waited for the normal contact interval"); + checkQuiet(); + } + catch (const std::exception& error) { + std::cout << "FAILED: " << error.what() << std::endl; + return -1; + } + std::cout << "OK" << std::endl; + return 0; +} + static int testOther() { char buf[1024]; @@ -1673,6 +1825,7 @@ int main(int argc, char** argv) r |= testPacket(); r |= testIdentity(); r |= testCertificate(); + r |= testRootRecovery(); r |= testPhy(); //*/