feat(observer-wifi): gateway watchdog and wifi reconnect command

Both Clyde North observers sat WL_CONNECTED on an AP that kept beaconing
after its bridge to the wired LAN died: association up, good RSSI, stale
DHCP lease, but ARP-invisible and both MQTT brokers stuck in backoff.
The firmware equated "associated" with "online", so neither node ever
re-scanned and the outage held until a manual reassociation.

Add a connectivity watchdog to NetworkService: while connected, ARP-probe
the gateway every 30s (posted to the lwIP tcpip thread, safe on both the
IDF4 and IDF5 cores). If the gateway stays silent for 3 minutes, clear
the channel hint and force a full disconnect/rescan so the node can roam
to a healthy AP, backing off exponentially (up to 48min) while the
outage persists. Nodes without a gateway skip the probe.

Add a `wifi reconnect` repeater command that triggers the same forced
reassociation on demand, and report `gw:ok|lost wd:<count>` in
`get wifi.status`. Teach the web panel parser the new fields (also fixes
the IP metric rendering as "x.x.x.x channel:n") and show Gateway and
Channel in the Wi-Fi card.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

eastmesh-docs/custom-cli.md

@@ -109,12 +109,15 @@ Legacy dotted aliases are also accepted:
 
 ### Wi-Fi Settings For Observers
 
-- `get wifi.status`: shows SSID, connection state, raw Wi-Fi status code, IP, channel, and signal when connected.
+- `get wifi.status`: shows SSID, connection state, raw Wi-Fi status code, IP, channel, and signal when connected, plus gateway health (`gw:ok|lost`) and the watchdog reconnect count (`wd:<n>`).
 - `get wifi.ssid`: shows the configured Wi-Fi SSID.
 - `set wifi.ssid <ssid>`: sets the Wi-Fi SSID.
 - `set wifi.pwd <password>`: sets the Wi-Fi password.
 - `get wifi.powersaving`: shows the current Wi-Fi power save mode.
 - `set wifi.powersaving none|min|max`: sets Wi-Fi power saving mode.
+- `wifi reconnect`: drops the current association and rejoins from a full channel scan. Use when a node reports connected but is unreachable over the network.
+
+Observers also run a connectivity watchdog: while Wi-Fi reports connected, the node ARP-probes its gateway every 30 seconds. If the gateway stays silent for 3 minutes (for example, an AP that keeps beaconing after losing its wired uplink), the node forces a full reconnect on its own, backing off up to 48 minutes between attempts while the outage persists. `wd:<n>` in `get wifi.status` counts these forced reconnects since boot.
 
 ### NTP Settings For Observers
 
@@ -152,7 +155,7 @@ Example:
 
 ```text
 > get wifi.status
-> ssid:EastMesh-IoT status:connected code:3 state:connected ip:192.168.1.50 channel:6 rssi:-61 quality:78% signal:good
+> ssid:EastMesh-IoT status:connected code:3 state:connected ip:192.168.1.50 channel:6 rssi:-61 quality:78% signal:good gw:ok wd:0
 > get bridge.channel
 > 1
 > set bridge.channel 6

examples/simple_repeater/MyMesh.cpp

@@ -2393,6 +2393,9 @@ void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply
     } else {
       strcpy(reply, "Err - use none|min|max");
     }
+  } else if (strcmp(command, "wifi reconnect") == 0) {
+    network.forceReconnect();
+    strcpy(reply, "OK - wifi reconnecting");
   } else if (memcmp(command, "set ntp.server1 ", 16) == 0) {
     if (network.setNtpServer(1, &command[16])) {
       strcpy(reply, "OK");

src/helpers/NetworkService.cpp

@@ -6,7 +6,12 @@
 
 #if defined(ESP_PLATFORM)
   #include <WiFi.h>
+  #include <esp_netif.h>
+  #include <esp_netif_net_stack.h>
   #include <esp_sntp.h>
+  #include <lwip/etharp.h>
+  #include <lwip/netif.h>
+  #include <lwip/tcpip.h>
 #endif
 
 namespace {
@@ -15,6 +20,9 @@ namespace {
 constexpr unsigned long kWifiRetryMillis = 15000;
 constexpr unsigned long kWifiConnectTimeoutMillis = 45000;
 constexpr unsigned long kWifiChannelHintTimeoutMillis = 7000;
+constexpr unsigned long kWatchdogProbeMillis = 30000;
+constexpr unsigned long kWatchdogTimeoutMillis = 180000;
+constexpr uint8_t kWatchdogMaxBackoffShift = 4;  // 180s .. 48min between forced reconnects
 constexpr time_t kMinSaneEpoch = 1735689600;  // 2025-01-01T00:00:00Z
 constexpr size_t kNtpServerMaxLen = 64;
 
@@ -50,6 +58,16 @@ const char* getWifiQualityLabel(int rssi_dbm) {
 
 NetworkService::NetworkService()
     : _fs(nullptr), _prefs{}, _wifi_started(false), _sntp_started(false), _have_time_sync(false), _last_wifi_status(-1), _last_wifi_attempt(0) {
+#if defined(ESP_PLATFORM)
+  _wd_gateway_seen = false;
+  _wd_probe_pending = false;
+  _wd_gateway_ip = 0;
+  _wd_was_connected = false;
+  _wd_last_gateway_ok = 0;
+  _wd_last_probe = 0;
+  _wd_backoff_shift = 0;
+  _wd_reconnect_count = 0;
+#endif
   NetworkPrefsStore::setDefaults(_prefs);
 }
 
@@ -85,6 +103,7 @@ void NetworkService::loop(bool network_required) {
 #if defined(ESP_PLATFORM)
   ensureWifi(network_required);
   updateTimeSync();
+  updateConnectivityWatchdog();
 #else
   (void)network_required;
 #endif
@@ -280,10 +299,12 @@ void NetworkService::formatWifiStatusReply(char* reply, size_t reply_size) const
 
   if (wifi_status == WL_CONNECTED) {
     const int rssi_dbm = WiFi.RSSI();
+    const unsigned long gateway_silence_ms = millis() - _wd_last_gateway_ok;
     snprintf(reply, reply_size,
-             "> ssid:%s status:%s code:%d state:%s ip:%s channel:%d rssi:%d quality:%d%% signal:%s",
+             "> ssid:%s status:%s code:%d state:%s ip:%s channel:%d rssi:%d quality:%d%% signal:%s gw:%s wd:%u",
              _prefs.wifi_ssid, status, static_cast<int>(wifi_status), state, WiFi.localIP().toString().c_str(),
-             WiFi.channel(), rssi_dbm, getWifiQualityPercent(rssi_dbm), getWifiQualityLabel(rssi_dbm));
+             WiFi.channel(), rssi_dbm, getWifiQualityPercent(rssi_dbm), getWifiQualityLabel(rssi_dbm),
+             gateway_silence_ms < (kWatchdogProbeMillis * 3) ? "ok" : "lost", _wd_reconnect_count);
   } else {
     snprintf(reply, reply_size, "> ssid:%s status:%s code:%d state:%s", _prefs.wifi_ssid[0] ? _prefs.wifi_ssid : "-",
              status, static_cast<int>(wifi_status), state);
@@ -299,6 +320,7 @@ void NetworkService::reconnectWifi() {
     WiFi.disconnect(true, true);
     WiFi.mode(WIFI_OFF);
   }
+  _wd_was_connected = false;
 #endif
   _wifi_started = false;
   _sntp_started = false;
@@ -306,6 +328,15 @@ void NetworkService::reconnectWifi() {
   _last_wifi_attempt = 0;
 }
 
+void NetworkService::forceReconnect() {
+#if defined(ESP_PLATFORM)
+  // Clear the channel hint (RAM only) so the retry does a full scan and can land
+  // on a different AP; the hint is re-learned and persisted on the next connect.
+  _prefs.wifi_channel = 0;
+#endif
+  reconnectWifi();
+}
+
 void NetworkService::restartTimeSync() {
   _sntp_started = false;
   _have_time_sync = false;
@@ -451,4 +482,71 @@ void NetworkService::updateTimeSync() {
   bool sync_ready = sync_status == SNTP_SYNC_STATUS_COMPLETED || sync_status == SNTP_SYNC_STATUS_IN_PROGRESS;
   _have_time_sync = sane_time && (sync_ready || prev_have_time_sync);
 }
+
+// Runs in the lwIP tcpip thread (posted via tcpip_callback), where raw etharp
+// calls are safe without core locking.
+void NetworkService::watchdogProbeCallback(void* arg) {
+  NetworkService* self = static_cast<NetworkService*>(arg);
+  esp_netif_t* esp_nif = esp_netif_get_handle_from_ifkey("WIFI_STA_DEF");
+  struct netif* nif = esp_nif ? static_cast<struct netif*>(esp_netif_get_netif_impl(esp_nif)) : nullptr;
+  if (nif != nullptr && netif_is_up(nif)) {
+    ip4_addr_t gw;
+    gw.addr = self->_wd_gateway_ip;
+    struct eth_addr* eth_ret = nullptr;
+    const ip4_addr_t* ip_ret = nullptr;
+    if (etharp_find_addr(nif, &gw, &eth_ret, &ip_ret) >= 0) {
+      self->_wd_gateway_seen = true;
+    }
+    etharp_request(nif, &gw);
+  }
+  self->_wd_probe_pending = false;
+}
+
+void NetworkService::updateConnectivityWatchdog() {
+  if (!_wifi_started || WiFi.status() != WL_CONNECTED) {
+    _wd_was_connected = false;
+    return;
+  }
+
+  const unsigned long now_ms = millis();
+  if (!_wd_was_connected) {
+    // Fresh association: give the gateway a full window before judging it.
+    _wd_was_connected = true;
+    _wd_last_gateway_ok = now_ms;
+    _wd_last_probe = 0;
+    _wd_gateway_seen = false;
+  }
+
+  if (_wd_gateway_seen.exchange(false)) {
+    _wd_last_gateway_ok = now_ms;
+    _wd_backoff_shift = 0;
+  }
+
+  const uint32_t gateway_ip = static_cast<uint32_t>(WiFi.gatewayIP());
+  if (gateway_ip == 0) {
+    // No gateway (e.g. static IP without one) - nothing meaningful to probe.
+    _wd_last_gateway_ok = now_ms;
+    return;
+  }
+
+  if (now_ms - _wd_last_probe >= kWatchdogProbeMillis && !_wd_probe_pending.load()) {
+    _wd_last_probe = now_ms;
+    _wd_gateway_ip = gateway_ip;
+    _wd_probe_pending = true;
+    if (tcpip_callback(watchdogProbeCallback, this) != ERR_OK) {
+      _wd_probe_pending = false;
+    }
+  }
+
+  const unsigned long timeout = kWatchdogTimeoutMillis << _wd_backoff_shift;
+  if (now_ms - _wd_last_gateway_ok >= timeout) {
+    _wd_reconnect_count++;
+    if (_wd_backoff_shift < kWatchdogMaxBackoffShift) {
+      _wd_backoff_shift++;
+    }
+    Serial.printf("[WDOG] gateway unreachable for %lus, forcing wifi reconnect (count=%u)\n",
+                  (now_ms - _wd_last_gateway_ok) / 1000, _wd_reconnect_count);
+    forceReconnect();
+  }
+}
 #endif

src/helpers/NetworkService.h

@@ -6,6 +6,8 @@
 
 #if defined(ESP_PLATFORM)
   #include <WiFi.h>
+
+  #include <atomic>
 #endif
 
 #include "NetworkPrefs.h"
@@ -30,6 +32,7 @@ public:
   const char* getNtpServer(uint8_t index) const;
   void formatWifiStatusReply(char* reply, size_t reply_size) const;
   void reconnectWifi();
+  void forceReconnect();
 
   bool isWifiConnected() const override;
   bool hasTimeSync() const override { return _have_time_sync; }
@@ -41,6 +44,8 @@ private:
   void ensureWifi(bool network_required);
   void updateTimeSync();
   void restartTimeSync();
+  void updateConnectivityWatchdog();
+  static void watchdogProbeCallback(void* arg);
 #endif
   static bool isValidNtpServer(const char* server);
   bool savePrefs();
@@ -52,4 +57,16 @@ private:
   bool _have_time_sync;
   int _last_wifi_status;
   unsigned long _last_wifi_attempt;
+#if defined(ESP_PLATFORM)
+  // Connectivity watchdog: _wd_gateway_seen and _wd_probe_pending are shared with
+  // the lwIP tcpip thread; _wd_gateway_ip is only written while no probe is pending.
+  std::atomic<bool> _wd_gateway_seen;
+  std::atomic<bool> _wd_probe_pending;
+  uint32_t _wd_gateway_ip;
+  bool _wd_was_connected;
+  unsigned long _wd_last_gateway_ok;
+  unsigned long _wd_last_probe;
+  uint8_t _wd_backoff_shift;
+  uint16_t _wd_reconnect_count;
+#endif
 };

src/helpers/web/WebPanelServer.cpp

@@ -1906,20 +1906,25 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
       return parsed;
     }
     function parseWifiStatusReply(text) {
-      const parsed = parseKeyedReply(text, ["ssid", "status", "code", "state", "ip", "rssi", "quality", "signal"]);
+      const parsed = parseKeyedReply(text, ["ssid", "status", "code", "state", "ip", "channel", "rssi", "quality", "signal", "gw", "wd"]);
       if (!parsed) return null;
       const rssi = Number.parseInt(parsed.rssi, 10);
       const quality = Number.parseInt(String(parsed.quality || "").replace("%", ""), 10);
       const code = Number.parseInt(parsed.code, 10);
+      const channel = Number.parseInt(parsed.channel, 10);
+      const watchdogCount = Number.parseInt(parsed.wd, 10);
       return {
         ssid: parsed.ssid || "-",
         status: parsed.status || "unknown",
         code: Number.isFinite(code) ? code : null,
         state: parsed.state || "unknown",
         ip: parsed.ip || "--",
+        channel: Number.isFinite(channel) ? channel : null,
         rssi: Number.isFinite(rssi) ? rssi : null,
         quality: Number.isFinite(quality) ? quality : null,
-        signal: parsed.signal || "--"
+        signal: parsed.signal || "--",
+        gateway: parsed.gw || null,
+        watchdog_count: Number.isFinite(watchdogCount) ? watchdogCount : null
       };
     }
     function renderCoreCard(core) {
@@ -1966,6 +1971,8 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
           ${renderMetric("State", wifi.state || "--")}
           ${renderMetric("SSID", wifi.ssid || "-")}
           ${renderMetric("IP", wifi.ip || "--")}
+          ${renderMetric("Channel", wifi.channel == null ? "--" : wifi.channel)}
+          ${renderMetric("Gateway", wifi.gateway ? wifi.gateway + (wifi.watchdog_count ? " (wd " + wifi.watchdog_count + ")" : "") : "--")}
           ${renderMetric("Power Save", powersave || "--")}
           ${renderMetric("Code", wifi.code == null ? "--" : wifi.code)}
         </div>