fix(web): stop HTTPS panel heap fragmentation, add self-heal and heap counters

Both Clyde North observers stopped answering over WiFi after days of
uptime while LoRa and MQTT kept working. `memory` showed heap_min ~1KB
and a largest internal block of 19KB: the HTTPS listener was alive but
no mbedTLS handshake (~40KB contiguous) could be allocated.

The trigger was the Aug 9 lru_purge change: with a 2-socket pool, every
browser page-load burst evicted a live session and forced a fresh TLS
handshake, and the repeated 40KB alloc/free cycles fragmented internal
RAM. Before that change the pool simply jammed, so nothing churned.

- Send `Connection: close` and trigger a session close on one-shot
  responses (pages, favicon, redirect, login, 401s) so page-load bursts
  release their sockets immediately. The authenticated /api/* polling
  connection keeps keep-alive so it does not pay a handshake per poll.
- Gate WebPanelServer::start() on internal heap headroom (56KB free /
  32KB largest), retrying every 15s instead of every loop tick.
- Self-heal: when the panel is idle and the largest internal block drops
  below 24KB, stop and re-create the server to return its pools.
- Count the MQTT teardown path that deliberately abandons a client on a
  heap-integrity failure.
- Expose the above as `heals:`/`deferred:` in `get web.status` and
  `leaked:` in `get mqtt.status`.
- Fix the stats page Channel/Gateway cells showing `--`: the /api/stats
  summary JSON never carried channel, gateway health, or the watchdog
  count (only the `get wifi.status` string did).
- Add eastmesh-tools/web-heap-check.sh to read heap/service health over
  the API and optionally stress the panel with browser-style bursts.
- Release notes 2026.8.3 for both observer tracks.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jared Dohrman
2026-08-23 10:53:53 +00:00
co-authored by Claude Fable 5
parent 4445fb9f4d
commit 866498299e
12 changed files with 343 additions and 30 deletions
+17 -2
View File
@@ -299,12 +299,11 @@ void NetworkService::formatWifiStatusReply(char* reply, size_t reply_size) const
if (wifi_status == WL_CONNECTED) {
const int rssi_dbm = WiFi.RSSI();
const unsigned long gateway_silence_ms = millis() - _wd_last_gateway_ok;
snprintf(reply, reply_size,
"> ssid:%s status:%s code:%d state:%s ip:%s channel:%d rssi:%d quality:%d%% signal:%s gw:%s wd:%u",
_prefs.wifi_ssid, status, static_cast<int>(wifi_status), state, WiFi.localIP().toString().c_str(),
WiFi.channel(), rssi_dbm, getWifiQualityPercent(rssi_dbm), getWifiQualityLabel(rssi_dbm),
gateway_silence_ms < (kWatchdogProbeMillis * 3) ? "ok" : "lost", _wd_reconnect_count);
isGatewayReachable() ? "ok" : "lost", _wd_reconnect_count);
} else {
snprintf(reply, reply_size, "> ssid:%s status:%s code:%d state:%s", _prefs.wifi_ssid[0] ? _prefs.wifi_ssid : "-",
status, static_cast<int>(wifi_status), state);
@@ -328,6 +327,22 @@ void NetworkService::reconnectWifi() {
_last_wifi_attempt = 0;
}
bool NetworkService::isGatewayReachable() const {
#if defined(ESP_PLATFORM)
return millis() - _wd_last_gateway_ok < (kWatchdogProbeMillis * 3);
#else
return false;
#endif
}
uint16_t NetworkService::getWatchdogReconnectCount() const {
#if defined(ESP_PLATFORM)
return _wd_reconnect_count;
#else
return 0;
#endif
}
void NetworkService::forceReconnect() {
#if defined(ESP_PLATFORM)
// Clear the channel hint (RAM only) so the retry does a full scan and can land
+3
View File
@@ -33,6 +33,9 @@ public:
void formatWifiStatusReply(char* reply, size_t reply_size) const;
void reconnectWifi();
void forceReconnect();
// Gateway watchdog state, as reported by `get wifi.status` (gw:ok|lost wd:<n>).
bool isGatewayReachable() const;
uint16_t getWatchdogReconnectCount() const;
bool isWifiConnected() const override;
bool hasTimeSync() const override { return _have_time_sync; }
+6 -4
View File
@@ -142,7 +142,7 @@ const MQTTUplink::BrokerSpec MQTTUplink::kBrokerSpecs[kBrokerCount] = {
MQTTUplink::MQTTUplink(mesh::RTCClock& rtc, mesh::LocalIdentity& identity)
: _fs(nullptr), _rtc(&rtc), _identity(&identity), _running(false), _last_status_publish(0),
_token_refresh_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr)
_token_refresh_count(0), _abandoned_client_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr)
{
memset(_device_id, 0, sizeof(_device_id));
MQTTPrefsStore::setDefaults(_prefs);
@@ -615,6 +615,7 @@ void MQTTUplink::destroyBroker(BrokerState& broker, bool reset_retry_state) {
MQTT_LOG("%s destroy broker client rc=0x%x", broker.spec->label, destroy_rc);
} else {
// Avoid freeing through esp-mqtt after the IDF 4.4 WSS transport has already poisoned the heap.
_abandoned_client_count++;
MQTT_LOG("%s abandon stopped broker client: heap corrupt", broker.spec->label);
}
} else {
@@ -623,6 +624,7 @@ void MQTTUplink::destroyBroker(BrokerState& broker, bool reset_retry_state) {
esp_err_t destroy_rc = esp_mqtt_client_destroy(broker.client);
MQTT_LOG("%s destroy broker client rc=0x%x", broker.spec->label, destroy_rc);
} else {
_abandoned_client_count++;
MQTT_LOG("%s abandon broker client: heap corrupt", broker.spec->label);
}
}
@@ -1249,11 +1251,11 @@ void MQTTUplink::formatStatusReply(char* reply, size_t reply_size) const {
format_slot(secondary, secondary_slot, sizeof(secondary_slot));
snprintf(reply, reply_size,
"> wifi:%s ntp:%s iata:%s p:%s s:%s status:%s tx:%s",
"> wifi:%s ntp:%s iata:%s p:%s s:%s status:%s tx:%s leaked:%lu",
(_network != nullptr && _network->isWifiConnected()) ? "up" : "down",
(_network != nullptr && _network->hasTimeSync()) ? "up" : "wait",
_prefs.iata, primary_slot, secondary_slot, _prefs.status_enabled ? "on" : "off",
_prefs.tx_enabled ? "on" : "off");
_prefs.tx_enabled ? "on" : "off", static_cast<unsigned long>(_abandoned_client_count));
}
bool MQTTUplink::setEndpointEnabled(uint8_t bit, bool enabled) {
@@ -1521,7 +1523,7 @@ const char* MQTTUplink::getAggregateBrokerState() const {
MQTTUplink::MQTTUplink(mesh::RTCClock&, mesh::LocalIdentity&)
: _fs(nullptr), _rtc(nullptr), _identity(nullptr), _running(false), _last_status_publish(0),
_token_refresh_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr) {
_token_refresh_count(0), _abandoned_client_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr) {
MQTTPrefsStore::setDefaults(_prefs);
}
+4
View File
@@ -80,6 +80,7 @@ public:
const char* getAggregateBrokerState() const;
void setNetworkStateProvider(NetworkStateProvider* network) { _network = network; }
uint32_t getTokenRefreshCount() const { return _token_refresh_count; }
uint32_t getAbandonedClientCount() const { return _abandoned_client_count; }
bool isTokenRefreshInProgress() const;
private:
@@ -124,6 +125,9 @@ private:
bool _running;
unsigned long _last_status_publish;
uint32_t _token_refresh_count;
// Clients never freed because heap_caps_check_integrity_all() failed at teardown.
// Each one leaks its TLS context and task stack (~45KB of internal RAM).
uint32_t _abandoned_client_count;
unsigned long _token_refresh_active_until_ms;
MQTTStatusSnapshot _last_status;
char _device_id[65];
+88 -16
View File
@@ -54,6 +54,33 @@ void freeScratchBuffer(void* ptr) {
}
}
// Internal (DMA-capable) heap headroom needed before we start an HTTPS listener.
// A single mbedTLS session needs ~40KB of internal RAM in one piece; starting the
// server when that cannot be satisfied just leaves a listener that resets every
// handshake. Mirrors the dual-broker gate in MQTTUplink.
constexpr size_t kWebMinFreeHeap = 56U * 1024U;
constexpr size_t kWebMinLargestHeap = 32U * 1024U;
// Below this largest-block size a TLS handshake can no longer be satisfied; the
// listener is alive but every connect resets. Tear it down and re-create it so
// its pools are returned and coalesced.
constexpr size_t kWebHealLargestHeap = 24U * 1024U;
// Mark a one-shot response so the socket is released as soon as it is sent.
// Page loads open several parallel connections; with a 2-socket pool and LRU
// purge each one otherwise evicts a live session and forces a fresh TLS
// handshake. The authenticated polling connection keeps keep-alive.
void markCloseAfterSend(httpd_req_t* req) {
httpd_resp_set_hdr(req, "Connection", "close");
}
esp_err_t finishAndClose(httpd_req_t* req, esp_err_t rc) {
int fd = httpd_req_to_sockfd(req);
if (fd >= 0) {
httpd_sess_trigger_close(req->handle, fd);
}
return rc;
}
void rebootAfterFirmwareUpdateTask(void*) {
vTaskDelay(pdMS_TO_TICKS(1200));
esp_restart();
@@ -1971,7 +1998,7 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
${renderMetric("State", wifi.state || "--")}
${renderMetric("SSID", wifi.ssid || "-")}
${renderMetric("IP", wifi.ip || "--")}
${renderMetric("Channel", wifi.channel == null ? "--" : wifi.channel)}
${renderMetric("Channel", wifi.channel ? wifi.channel : "--")}
${renderMetric("Gateway", wifi.gateway ? wifi.gateway + (wifi.watchdog_count ? " (wd " + wifi.watchdog_count + ")" : "") : "--")}
${renderMetric("Power Save", powersave || "--")}
${renderMetric("Code", wifi.code == null ? "--" : wifi.code)}
@@ -3299,7 +3326,8 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
} // namespace
WebPanelServer::WebPanelServer()
: _runner(nullptr), _server(nullptr), _redirect_server(nullptr), _token{0}, _last_activity_ms(0), _route_context{this} {
: _runner(nullptr), _server(nullptr), _redirect_server(nullptr), _token{0}, _last_activity_ms(0),
_start_deferred_count(0), _restart_count(0), _route_context{this} {
}
void WebPanelServer::setCommandRunner(WebPanelCommandRunner* runner) {
@@ -3311,6 +3339,16 @@ bool WebPanelServer::start() {
return _server != nullptr;
}
const size_t free_heap = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
const size_t largest_heap = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
if (free_heap < kWebMinFreeHeap || largest_heap < kWebMinLargestHeap) {
_start_deferred_count++;
WEB_PANEL_LOG("server start deferred: heap_free=%u heap_max=%u need=%u/%u", static_cast<unsigned>(free_heap),
static_cast<unsigned>(largest_heap), static_cast<unsigned>(kWebMinFreeHeap),
static_cast<unsigned>(kWebMinLargestHeap));
return false;
}
noteActivity();
httpd_ssl_config_t config = HTTPD_SSL_CONFIG_DEFAULT();
@@ -3412,6 +3450,18 @@ bool WebPanelServer::hasSessionToken() const {
return _token[0] != 0;
}
bool WebPanelServer::isIdle(unsigned long now_ms, unsigned long quiet_ms) const {
return _last_activity_ms == 0 || now_ms - _last_activity_ms >= quiet_ms;
}
bool WebPanelServer::isHeapStarved() {
return heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT) < kWebHealLargestHeap;
}
void WebPanelServer::noteRestart() {
_restart_count++;
}
void WebPanelServer::stopRedirectServer() {
if (_redirect_server != nullptr) {
httpd_handle_t redirect_server = _redirect_server;
@@ -3441,15 +3491,17 @@ esp_err_t WebPanelServer::handleIndex(httpd_req_t* req) {
ctx->self->noteActivity();
httpd_resp_set_type(req, "text/html; charset=utf-8");
httpd_resp_set_hdr(req, "Cache-Control", "no-store");
return sendProgmemChunked(req, kWebPanelLoginHtml);
markCloseAfterSend(req);
return finishAndClose(req, sendProgmemChunked(req, kWebPanelLoginHtml));
}
esp_err_t WebPanelServer::handleFavicon(httpd_req_t* req) {
httpd_resp_set_type(req, "image/png");
httpd_resp_set_hdr(req, "Cache-Control", "max-age=86400");
return httpd_resp_send(req,
reinterpret_cast<const char*>(eastmesh_web_assets::kFaviconPng),
eastmesh_web_assets::kFaviconPngLen);
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_send(req,
reinterpret_cast<const char*>(eastmesh_web_assets::kFaviconPng),
eastmesh_web_assets::kFaviconPngLen));
}
esp_err_t WebPanelServer::handleHttpRedirect(httpd_req_t* req) {
@@ -3464,7 +3516,8 @@ esp_err_t WebPanelServer::handleHttpRedirect(httpd_req_t* req) {
httpd_resp_set_status(req, "302 Found");
httpd_resp_set_hdr(req, "Location", location);
httpd_resp_set_hdr(req, "Cache-Control", "no-store");
return httpd_resp_send(req, "", 0);
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_send(req, "", 0));
}
esp_err_t WebPanelServer::handleApp(httpd_req_t* req) {
@@ -3475,7 +3528,8 @@ esp_err_t WebPanelServer::handleApp(httpd_req_t* req) {
ctx->self->noteActivity();
httpd_resp_set_type(req, "text/html; charset=utf-8");
httpd_resp_set_hdr(req, "Cache-Control", "no-store");
return sendProgmemChunked(req, kWebPanelAppHtml);
markCloseAfterSend(req);
return finishAndClose(req, sendProgmemChunked(req, kWebPanelAppHtml));
}
esp_err_t WebPanelServer::handleStatsPage(httpd_req_t* req) {
@@ -3486,10 +3540,11 @@ esp_err_t WebPanelServer::handleStatsPage(httpd_req_t* req) {
ctx->self->noteActivity();
httpd_resp_set_type(req, "text/html; charset=utf-8");
httpd_resp_set_hdr(req, "Cache-Control", "no-store");
markCloseAfterSend(req);
if (ctx->self->_runner != nullptr && !ctx->self->_runner->isWebStatsEnabled()) {
return sendProgmemChunked(req, kWebPanelStatsDisabledHtml);
return finishAndClose(req, sendProgmemChunked(req, kWebPanelStatsDisabledHtml));
}
return sendProgmemChunked(req, kWebPanelAppHtml);
return finishAndClose(req, sendProgmemChunked(req, kWebPanelAppHtml));
}
esp_err_t WebPanelServer::handleLogin(httpd_req_t* req) {
@@ -3511,7 +3566,8 @@ esp_err_t WebPanelServer::handleLogin(httpd_req_t* req) {
if (strcmp(password, ctx->self->_runner->getWebAdminPassword()) != 0) {
freeScratchBuffer(password);
WEB_PANEL_LOG("login denied");
return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Bad password");
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Bad password"));
}
freeScratchBuffer(password);
@@ -3520,7 +3576,8 @@ esp_err_t WebPanelServer::handleLogin(httpd_req_t* req) {
WEB_PANEL_LOG("login accepted");
httpd_resp_set_type(req, "text/plain; charset=utf-8");
httpd_resp_set_hdr(req, "Cache-Control", "no-store");
return httpd_resp_sendstr(req, ctx->self->_token);
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_sendstr(req, ctx->self->_token));
}
esp_err_t WebPanelServer::handleSession(httpd_req_t* req) {
@@ -3529,7 +3586,8 @@ esp_err_t WebPanelServer::handleSession(httpd_req_t* req) {
return httpd_resp_send_500(req);
}
if (!ctx->self->isAuthorized(req)) {
return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized");
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"));
}
ctx->self->noteActivity();
@@ -3544,7 +3602,8 @@ esp_err_t WebPanelServer::handleCommand(httpd_req_t* req) {
return httpd_resp_send_500(req);
}
if (!ctx->self->isAuthorized(req)) {
return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized");
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"));
}
char* command = allocScratchBuffer(kWebCommandBufferSize);
@@ -3578,7 +3637,8 @@ esp_err_t WebPanelServer::handleFirmwareUpdate(httpd_req_t* req) {
return httpd_resp_send_500(req);
}
if (!ctx->self->isAuthorized(req)) {
return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized");
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"));
}
if (req->content_len <= 0) {
return httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "Missing firmware body");
@@ -3647,7 +3707,8 @@ esp_err_t WebPanelServer::handleStats(httpd_req_t* req) {
return httpd_resp_send_500(req);
}
if (!ctx->self->isAuthorized(req)) {
return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized");
markCloseAfterSend(req);
return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"));
}
ctx->self->noteActivity();
@@ -3766,4 +3827,15 @@ bool WebPanelServer::shouldAutoLock(unsigned long) const {
void WebPanelServer::lockSession() {
}
bool WebPanelServer::isIdle(unsigned long, unsigned long) const {
return true;
}
bool WebPanelServer::isHeapStarved() {
return false;
}
void WebPanelServer::noteRestart() {
}
#endif
+14
View File
@@ -46,6 +46,18 @@ public:
bool hasSessionToken() const;
bool shouldAutoLock(unsigned long now_ms) const;
void lockSession();
// True when no request has touched the panel for quiet_ms (or ever).
bool isIdle(unsigned long now_ms, unsigned long quiet_ms) const;
// True when the largest internal heap block can no longer fit a TLS handshake.
static bool isHeapStarved();
void noteRestart();
#if defined(ESP_PLATFORM) && WITH_WEB_PANEL
unsigned int startDeferredCount() const { return _start_deferred_count; }
unsigned int restartCount() const { return _restart_count; }
#else
unsigned int startDeferredCount() const { return 0; }
unsigned int restartCount() const { return 0; }
#endif
private:
#if defined(ESP_PLATFORM) && WITH_WEB_PANEL
@@ -58,6 +70,8 @@ private:
httpd_handle_t _redirect_server;
char _token[33];
unsigned long _last_activity_ms;
unsigned int _start_deferred_count;
unsigned int _restart_count;
RouteContext _route_context;
static esp_err_t handleIndex(httpd_req_t* req);
+45 -5
View File
@@ -4,7 +4,19 @@
#include <WiFi.h>
#endif
WebService::WebService() : _fs(nullptr), _prefs{}, _runner(nullptr), _network(nullptr), _suspended_for_ota(false) {
namespace {
// Retry a heap-deferred start at this cadence rather than every loop tick.
constexpr unsigned long kWebStartRetryMillis = 15000;
// Self-heal: only restart a starved server when nobody has used it for this long,
// and no more often than this.
constexpr unsigned long kWebHealQuietMillis = 60000;
constexpr unsigned long kWebHealMinIntervalMillis = 5UL * 60UL * 1000UL;
constexpr unsigned long kWebHealCheckMillis = 10000;
} // namespace
WebService::WebService()
: _fs(nullptr), _prefs{}, _runner(nullptr), _network(nullptr), _suspended_for_ota(false), _last_start_attempt_ms(0),
_last_heal_ms(0), _last_heal_check_ms(0) {
WebPrefsStore::setDefaults(_prefs);
}
@@ -36,9 +48,11 @@ void WebService::prepareForOTAStart() {
void WebService::loop() {
#if defined(ESP_PLATFORM) && WITH_WEB_PANEL
ensureWebServer();
if (_panel.isRunning() && _panel.shouldAutoLock(millis())) {
const unsigned long now_ms = millis();
if (_panel.isRunning() && _panel.shouldAutoLock(now_ms)) {
_panel.lockSession();
}
healIfStarved(now_ms);
#endif
}
@@ -59,6 +73,7 @@ bool WebService::setWebEnabled(bool enabled) {
bool ok = savePrefs();
#if defined(ESP_PLATFORM) && WITH_WEB_PANEL
if (_prefs.web_enabled != 0 && !_suspended_for_ota) {
_last_start_attempt_ms = 0; // manual enable bypasses the retry backoff
ensureWebServer();
} else {
_panel.stop();
@@ -85,12 +100,13 @@ void WebService::formatWebStatusReply(char* reply, size_t reply_size) const {
}
if (!_panel.isRunning() || _network == nullptr || !_network->isWifiConnected()) {
snprintf(reply, reply_size, "> web:down");
snprintf(reply, reply_size, "> web:down heals:%u deferred:%u", _panel.restartCount(), _panel.startDeferredCount());
return;
}
snprintf(reply, reply_size, "> web:up url:https://%s/ auth:%s", WiFi.localIP().toString().c_str(),
_panel.hasSessionToken() ? "unlocked" : "locked");
snprintf(reply, reply_size, "> web:up url:https://%s/ auth:%s heals:%u deferred:%u",
WiFi.localIP().toString().c_str(), _panel.hasSessionToken() ? "unlocked" : "locked",
_panel.restartCount(), _panel.startDeferredCount());
#else
snprintf(reply, reply_size, "> web:unsupported");
#endif
@@ -115,6 +131,30 @@ void WebService::ensureWebServer() {
if (_panel.isRunning()) {
return;
}
const unsigned long now_ms = millis();
if (_last_start_attempt_ms != 0 && now_ms - _last_start_attempt_ms < kWebStartRetryMillis) {
return;
}
_last_start_attempt_ms = now_ms;
_panel.start();
}
void WebService::healIfStarved(unsigned long now_ms) {
if (!_panel.isRunning() || now_ms - _last_heal_check_ms < kWebHealCheckMillis) {
return;
}
_last_heal_check_ms = now_ms;
if (!_panel.isIdle(now_ms, kWebHealQuietMillis) || !WebPanelServer::isHeapStarved()) {
return;
}
if (_last_heal_ms != 0 && now_ms - _last_heal_ms < kWebHealMinIntervalMillis) {
return;
}
_last_heal_ms = now_ms;
_panel.noteRestart();
Serial.printf("[WEB] heap starved (largest internal block too small for TLS), restarting web panel (count=%u)\n",
_panel.restartCount());
_panel.stop(false);
_last_start_attempt_ms = 0; // let ensureWebServer() retry immediately (subject to the heap gate)
}
#endif
+4
View File
@@ -32,6 +32,7 @@ public:
private:
#if defined(ESP_PLATFORM) && WITH_WEB_PANEL
void ensureWebServer();
void healIfStarved(unsigned long now_ms);
#endif
bool savePrefs();
@@ -41,4 +42,7 @@ private:
NetworkStateProvider* _network;
WebPanelServer _panel;
bool _suspended_for_ota;
unsigned long _last_start_attempt_ms;
unsigned long _last_heal_ms;
unsigned long _last_heal_check_ms;
};