From 088d7f40f9fd0ad49f63bc5a3171a990d7788661 Mon Sep 17 00:00:00 2001 From: Jared Dohrman Date: Thu, 13 Aug 2026 19:23:12 +1000 Subject: [PATCH] feat(observer-wifi): gateway watchdog and wifi reconnect command Both Clyde North observers sat WL_CONNECTED on an AP that kept beaconing after its bridge to the wired LAN died: association up, good RSSI, stale DHCP lease, but ARP-invisible and both MQTT brokers stuck in backoff. The firmware equated "associated" with "online", so neither node ever re-scanned and the outage held until a manual reassociation. Add a connectivity watchdog to NetworkService: while connected, ARP-probe the gateway every 30s (posted to the lwIP tcpip thread, safe on both the IDF4 and IDF5 cores). If the gateway stays silent for 3 minutes, clear the channel hint and force a full disconnect/rescan so the node can roam to a healthy AP, backing off exponentially (up to 48min) while the outage persists. Nodes without a gateway skip the probe. Add a `wifi reconnect` repeater command that triggers the same forced reassociation on demand, and report `gw:ok|lost wd:` in `get wifi.status`. Teach the web panel parser the new fields (also fixes the IP metric rendering as "x.x.x.x channel:n") and show Gateway and Channel in the Wi-Fi card. Co-Authored-By: Claude Fable 5 --- eastmesh-docs/custom-cli.md | 7 +- examples/simple_repeater/MyMesh.cpp | 3 + src/helpers/NetworkService.cpp | 102 +++++++++++++++++++++++++++- src/helpers/NetworkService.h | 17 +++++ src/helpers/web/WebPanelServer.cpp | 11 ++- 5 files changed, 134 insertions(+), 6 deletions(-) diff --git a/eastmesh-docs/custom-cli.md b/eastmesh-docs/custom-cli.md index 45bb77e8..cc2d5e6d 100644 --- a/eastmesh-docs/custom-cli.md +++ b/eastmesh-docs/custom-cli.md @@ -109,12 +109,15 @@ Legacy dotted aliases are also accepted: ### Wi-Fi Settings For Observers -- `get wifi.status`: shows SSID, connection state, raw Wi-Fi status code, IP, channel, and signal when connected. +- `get wifi.status`: shows SSID, connection state, raw Wi-Fi status code, IP, channel, and signal when connected, plus gateway health (`gw:ok|lost`) and the watchdog reconnect count (`wd:`). - `get wifi.ssid`: shows the configured Wi-Fi SSID. - `set wifi.ssid `: sets the Wi-Fi SSID. - `set wifi.pwd `: sets the Wi-Fi password. - `get wifi.powersaving`: shows the current Wi-Fi power save mode. - `set wifi.powersaving none|min|max`: sets Wi-Fi power saving mode. +- `wifi reconnect`: drops the current association and rejoins from a full channel scan. Use when a node reports connected but is unreachable over the network. + +Observers also run a connectivity watchdog: while Wi-Fi reports connected, the node ARP-probes its gateway every 30 seconds. If the gateway stays silent for 3 minutes (for example, an AP that keeps beaconing after losing its wired uplink), the node forces a full reconnect on its own, backing off up to 48 minutes between attempts while the outage persists. `wd:` in `get wifi.status` counts these forced reconnects since boot. ### NTP Settings For Observers @@ -152,7 +155,7 @@ Example: ```text > get wifi.status -> ssid:EastMesh-IoT status:connected code:3 state:connected ip:192.168.1.50 channel:6 rssi:-61 quality:78% signal:good +> ssid:EastMesh-IoT status:connected code:3 state:connected ip:192.168.1.50 channel:6 rssi:-61 quality:78% signal:good gw:ok wd:0 > get bridge.channel > 1 > set bridge.channel 6 diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 010231d0..838da63f 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -2393,6 +2393,9 @@ void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply } else { strcpy(reply, "Err - use none|min|max"); } + } else if (strcmp(command, "wifi reconnect") == 0) { + network.forceReconnect(); + strcpy(reply, "OK - wifi reconnecting"); } else if (memcmp(command, "set ntp.server1 ", 16) == 0) { if (network.setNtpServer(1, &command[16])) { strcpy(reply, "OK"); diff --git a/src/helpers/NetworkService.cpp b/src/helpers/NetworkService.cpp index bc43d309..8bf53ac6 100644 --- a/src/helpers/NetworkService.cpp +++ b/src/helpers/NetworkService.cpp @@ -6,7 +6,12 @@ #if defined(ESP_PLATFORM) #include + #include + #include #include + #include + #include + #include #endif namespace { @@ -15,6 +20,9 @@ namespace { constexpr unsigned long kWifiRetryMillis = 15000; constexpr unsigned long kWifiConnectTimeoutMillis = 45000; constexpr unsigned long kWifiChannelHintTimeoutMillis = 7000; +constexpr unsigned long kWatchdogProbeMillis = 30000; +constexpr unsigned long kWatchdogTimeoutMillis = 180000; +constexpr uint8_t kWatchdogMaxBackoffShift = 4; // 180s .. 48min between forced reconnects constexpr time_t kMinSaneEpoch = 1735689600; // 2025-01-01T00:00:00Z constexpr size_t kNtpServerMaxLen = 64; @@ -50,6 +58,16 @@ const char* getWifiQualityLabel(int rssi_dbm) { NetworkService::NetworkService() : _fs(nullptr), _prefs{}, _wifi_started(false), _sntp_started(false), _have_time_sync(false), _last_wifi_status(-1), _last_wifi_attempt(0) { +#if defined(ESP_PLATFORM) + _wd_gateway_seen = false; + _wd_probe_pending = false; + _wd_gateway_ip = 0; + _wd_was_connected = false; + _wd_last_gateway_ok = 0; + _wd_last_probe = 0; + _wd_backoff_shift = 0; + _wd_reconnect_count = 0; +#endif NetworkPrefsStore::setDefaults(_prefs); } @@ -85,6 +103,7 @@ void NetworkService::loop(bool network_required) { #if defined(ESP_PLATFORM) ensureWifi(network_required); updateTimeSync(); + updateConnectivityWatchdog(); #else (void)network_required; #endif @@ -280,10 +299,12 @@ void NetworkService::formatWifiStatusReply(char* reply, size_t reply_size) const if (wifi_status == WL_CONNECTED) { const int rssi_dbm = WiFi.RSSI(); + const unsigned long gateway_silence_ms = millis() - _wd_last_gateway_ok; snprintf(reply, reply_size, - "> ssid:%s status:%s code:%d state:%s ip:%s channel:%d rssi:%d quality:%d%% signal:%s", + "> ssid:%s status:%s code:%d state:%s ip:%s channel:%d rssi:%d quality:%d%% signal:%s gw:%s wd:%u", _prefs.wifi_ssid, status, static_cast(wifi_status), state, WiFi.localIP().toString().c_str(), - WiFi.channel(), rssi_dbm, getWifiQualityPercent(rssi_dbm), getWifiQualityLabel(rssi_dbm)); + WiFi.channel(), rssi_dbm, getWifiQualityPercent(rssi_dbm), getWifiQualityLabel(rssi_dbm), + gateway_silence_ms < (kWatchdogProbeMillis * 3) ? "ok" : "lost", _wd_reconnect_count); } else { snprintf(reply, reply_size, "> ssid:%s status:%s code:%d state:%s", _prefs.wifi_ssid[0] ? _prefs.wifi_ssid : "-", status, static_cast(wifi_status), state); @@ -299,6 +320,7 @@ void NetworkService::reconnectWifi() { WiFi.disconnect(true, true); WiFi.mode(WIFI_OFF); } + _wd_was_connected = false; #endif _wifi_started = false; _sntp_started = false; @@ -306,6 +328,15 @@ void NetworkService::reconnectWifi() { _last_wifi_attempt = 0; } +void NetworkService::forceReconnect() { +#if defined(ESP_PLATFORM) + // Clear the channel hint (RAM only) so the retry does a full scan and can land + // on a different AP; the hint is re-learned and persisted on the next connect. + _prefs.wifi_channel = 0; +#endif + reconnectWifi(); +} + void NetworkService::restartTimeSync() { _sntp_started = false; _have_time_sync = false; @@ -451,4 +482,71 @@ void NetworkService::updateTimeSync() { bool sync_ready = sync_status == SNTP_SYNC_STATUS_COMPLETED || sync_status == SNTP_SYNC_STATUS_IN_PROGRESS; _have_time_sync = sane_time && (sync_ready || prev_have_time_sync); } + +// Runs in the lwIP tcpip thread (posted via tcpip_callback), where raw etharp +// calls are safe without core locking. +void NetworkService::watchdogProbeCallback(void* arg) { + NetworkService* self = static_cast(arg); + esp_netif_t* esp_nif = esp_netif_get_handle_from_ifkey("WIFI_STA_DEF"); + struct netif* nif = esp_nif ? static_cast(esp_netif_get_netif_impl(esp_nif)) : nullptr; + if (nif != nullptr && netif_is_up(nif)) { + ip4_addr_t gw; + gw.addr = self->_wd_gateway_ip; + struct eth_addr* eth_ret = nullptr; + const ip4_addr_t* ip_ret = nullptr; + if (etharp_find_addr(nif, &gw, ð_ret, &ip_ret) >= 0) { + self->_wd_gateway_seen = true; + } + etharp_request(nif, &gw); + } + self->_wd_probe_pending = false; +} + +void NetworkService::updateConnectivityWatchdog() { + if (!_wifi_started || WiFi.status() != WL_CONNECTED) { + _wd_was_connected = false; + return; + } + + const unsigned long now_ms = millis(); + if (!_wd_was_connected) { + // Fresh association: give the gateway a full window before judging it. + _wd_was_connected = true; + _wd_last_gateway_ok = now_ms; + _wd_last_probe = 0; + _wd_gateway_seen = false; + } + + if (_wd_gateway_seen.exchange(false)) { + _wd_last_gateway_ok = now_ms; + _wd_backoff_shift = 0; + } + + const uint32_t gateway_ip = static_cast(WiFi.gatewayIP()); + if (gateway_ip == 0) { + // No gateway (e.g. static IP without one) - nothing meaningful to probe. + _wd_last_gateway_ok = now_ms; + return; + } + + if (now_ms - _wd_last_probe >= kWatchdogProbeMillis && !_wd_probe_pending.load()) { + _wd_last_probe = now_ms; + _wd_gateway_ip = gateway_ip; + _wd_probe_pending = true; + if (tcpip_callback(watchdogProbeCallback, this) != ERR_OK) { + _wd_probe_pending = false; + } + } + + const unsigned long timeout = kWatchdogTimeoutMillis << _wd_backoff_shift; + if (now_ms - _wd_last_gateway_ok >= timeout) { + _wd_reconnect_count++; + if (_wd_backoff_shift < kWatchdogMaxBackoffShift) { + _wd_backoff_shift++; + } + Serial.printf("[WDOG] gateway unreachable for %lus, forcing wifi reconnect (count=%u)\n", + (now_ms - _wd_last_gateway_ok) / 1000, _wd_reconnect_count); + forceReconnect(); + } +} #endif diff --git a/src/helpers/NetworkService.h b/src/helpers/NetworkService.h index ef911f62..505ccde1 100644 --- a/src/helpers/NetworkService.h +++ b/src/helpers/NetworkService.h @@ -6,6 +6,8 @@ #if defined(ESP_PLATFORM) #include + + #include #endif #include "NetworkPrefs.h" @@ -30,6 +32,7 @@ public: const char* getNtpServer(uint8_t index) const; void formatWifiStatusReply(char* reply, size_t reply_size) const; void reconnectWifi(); + void forceReconnect(); bool isWifiConnected() const override; bool hasTimeSync() const override { return _have_time_sync; } @@ -41,6 +44,8 @@ private: void ensureWifi(bool network_required); void updateTimeSync(); void restartTimeSync(); + void updateConnectivityWatchdog(); + static void watchdogProbeCallback(void* arg); #endif static bool isValidNtpServer(const char* server); bool savePrefs(); @@ -52,4 +57,16 @@ private: bool _have_time_sync; int _last_wifi_status; unsigned long _last_wifi_attempt; +#if defined(ESP_PLATFORM) + // Connectivity watchdog: _wd_gateway_seen and _wd_probe_pending are shared with + // the lwIP tcpip thread; _wd_gateway_ip is only written while no probe is pending. + std::atomic _wd_gateway_seen; + std::atomic _wd_probe_pending; + uint32_t _wd_gateway_ip; + bool _wd_was_connected; + unsigned long _wd_last_gateway_ok; + unsigned long _wd_last_probe; + uint8_t _wd_backoff_shift; + uint16_t _wd_reconnect_count; +#endif }; diff --git a/src/helpers/web/WebPanelServer.cpp b/src/helpers/web/WebPanelServer.cpp index 7694e718..69431dd0 100644 --- a/src/helpers/web/WebPanelServer.cpp +++ b/src/helpers/web/WebPanelServer.cpp @@ -1906,20 +1906,25 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( return parsed; } function parseWifiStatusReply(text) { - const parsed = parseKeyedReply(text, ["ssid", "status", "code", "state", "ip", "rssi", "quality", "signal"]); + const parsed = parseKeyedReply(text, ["ssid", "status", "code", "state", "ip", "channel", "rssi", "quality", "signal", "gw", "wd"]); if (!parsed) return null; const rssi = Number.parseInt(parsed.rssi, 10); const quality = Number.parseInt(String(parsed.quality || "").replace("%", ""), 10); const code = Number.parseInt(parsed.code, 10); + const channel = Number.parseInt(parsed.channel, 10); + const watchdogCount = Number.parseInt(parsed.wd, 10); return { ssid: parsed.ssid || "-", status: parsed.status || "unknown", code: Number.isFinite(code) ? code : null, state: parsed.state || "unknown", ip: parsed.ip || "--", + channel: Number.isFinite(channel) ? channel : null, rssi: Number.isFinite(rssi) ? rssi : null, quality: Number.isFinite(quality) ? quality : null, - signal: parsed.signal || "--" + signal: parsed.signal || "--", + gateway: parsed.gw || null, + watchdog_count: Number.isFinite(watchdogCount) ? watchdogCount : null }; } function renderCoreCard(core) { @@ -1966,6 +1971,8 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( ${renderMetric("State", wifi.state || "--")} ${renderMetric("SSID", wifi.ssid || "-")} ${renderMetric("IP", wifi.ip || "--")} + ${renderMetric("Channel", wifi.channel == null ? "--" : wifi.channel)} + ${renderMetric("Gateway", wifi.gateway ? wifi.gateway + (wifi.watchdog_count ? " (wd " + wifi.watchdog_count + ")" : "") : "--")} ${renderMetric("Power Save", powersave || "--")} ${renderMetric("Code", wifi.code == null ? "--" : wifi.code)}