From 866498299e133ffdec07b1fb0a1ccac87678ed2e Mon Sep 17 00:00:00 2001 From: Jared Dohrman Date: Sun, 23 Aug 2026 10:53:53 +0000 Subject: [PATCH] fix(web): stop HTTPS panel heap fragmentation, add self-heal and heap counters Both Clyde North observers stopped answering over WiFi after days of uptime while LoRa and MQTT kept working. `memory` showed heap_min ~1KB and a largest internal block of 19KB: the HTTPS listener was alive but no mbedTLS handshake (~40KB contiguous) could be allocated. The trigger was the Aug 9 lru_purge change: with a 2-socket pool, every browser page-load burst evicted a live session and forced a fresh TLS handshake, and the repeated 40KB alloc/free cycles fragmented internal RAM. Before that change the pool simply jammed, so nothing churned. - Send `Connection: close` and trigger a session close on one-shot responses (pages, favicon, redirect, login, 401s) so page-load bursts release their sockets immediately. The authenticated /api/* polling connection keeps keep-alive so it does not pay a handshake per poll. - Gate WebPanelServer::start() on internal heap headroom (56KB free / 32KB largest), retrying every 15s instead of every loop tick. - Self-heal: when the panel is idle and the largest internal block drops below 24KB, stop and re-create the server to return its pools. - Count the MQTT teardown path that deliberately abandons a client on a heap-integrity failure. - Expose the above as `heals:`/`deferred:` in `get web.status` and `leaked:` in `get mqtt.status`. - Fix the stats page Channel/Gateway cells showing `--`: the /api/stats summary JSON never carried channel, gateway health, or the watchdog count (only the `get wifi.status` string did). - Add eastmesh-tools/web-heap-check.sh to read heap/service health over the API and optionally stress the panel with browser-style bursts. - Release notes 2026.8.3 for both observer tracks. Co-Authored-By: Claude Fable 5 --- eastmesh-docs/custom-cli.md | 8 ++- eastmesh-tools/web-heap-check.sh | 97 ++++++++++++++++++++++++++ examples/simple_repeater/MyMesh.cpp | 12 +++- release-notes.yml | 48 +++++++++++++ src/helpers/NetworkService.cpp | 19 ++++- src/helpers/NetworkService.h | 3 + src/helpers/mqtt/MQTTUplink.cpp | 10 +-- src/helpers/mqtt/MQTTUplink.h | 4 ++ src/helpers/web/WebPanelServer.cpp | 104 +++++++++++++++++++++++----- src/helpers/web/WebPanelServer.h | 14 ++++ src/helpers/web/WebService.cpp | 50 +++++++++++-- src/helpers/web/WebService.h | 4 ++ 12 files changed, 343 insertions(+), 30 deletions(-) create mode 100755 eastmesh-tools/web-heap-check.sh diff --git a/eastmesh-docs/custom-cli.md b/eastmesh-docs/custom-cli.md index cc2d5e6d..8b226e79 100644 --- a/eastmesh-docs/custom-cli.md +++ b/eastmesh-docs/custom-cli.md @@ -34,7 +34,7 @@ No-argument `get` commands must be entered exactly as shown. ### MQTT Status And Routing -- `get mqtt.status`: shows Wi-Fi, NTP, IATA, endpoint status, custom endpoint transport, status publishing state, and TX state. +- `get mqtt.status`: shows Wi-Fi, NTP, IATA, endpoint status, custom endpoint transport, status publishing state, TX state, and `leaked:` (MQTT clients intentionally left unfreed after a heap-integrity failure at teardown; should stay `0`). - `get mqtt.statuscfg`: shows whether periodic status messages are enabled as a simple `on` or `off` value. Most users can just use `get mqtt.status`. - `get mqtt.client_version`: shows the MQTT `client_version` string published by the repeater. - `get mqtt.client_env`: shows the PlatformIO env used to build the repeater firmware. @@ -165,7 +165,7 @@ OK ### Web Panel Controls - `get web` -- `get web.status`: shows whether the local HTTPS panel is available. +- `get web.status`: shows whether the local HTTPS panel is available, plus `heals:` (times the panel was restarted because internal heap was too fragmented for a TLS handshake) and `deferred:` (start attempts postponed until heap headroom recovered). - `get web.stats.status`: shows whether the dedicated stats page and history subsystem are enabled, whether recent history is active, whether PSRAM-backed history is available, and whether the SD-backed archive is mounted. When enabled, the history capture now covers supported environment telemetry too, not just the original battery/radio series. GPS-active boards also record per-minute satellites samples for the `/stats` history view. If archive access drops while stats remain enabled, the repeater retries the SD mount periodically. - At boot, archive-backed stats restore uses bounded reads of the latest summary, events, and neighbour snapshots so malformed or unexpectedly large archive files do not delay MQTT or web startup. - `set web on|off` @@ -320,3 +320,7 @@ Notes for roaming use: network first), and stays up for as long as that network is unreachable - back in range of its configured network, the device joins it and shuts the AP down automatically — reconnect the app via the LAN address instead + +## Heap Health Check Script + +`eastmesh-tools/web-heap-check.sh [--stress] [--rounds N]` reads `memory`, `get web.status`, `get mqtt.status` and `get wifi.status` over the HTTPS API and, with `--stress`, simulates browser page-load bursts before re-reading `memory`. Authenticate with `REPEATER_PASSWORD` or `REPEATER_TOKEN` in the environment (or enter the password at the prompt). Watch `heap_max` (largest internal block; TLS needs ~40KB), `heap_min`, and the `heals`/`deferred`/`leaked` counters. diff --git a/eastmesh-tools/web-heap-check.sh b/eastmesh-tools/web-heap-check.sh new file mode 100755 index 00000000..d16a3f59 --- /dev/null +++ b/eastmesh-tools/web-heap-check.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Web panel heap-health check for *_repeater_observer nodes. +# +# Reads memory / web / mqtt / wifi status over the HTTPS API, then (optionally) +# hammers the panel the way a browser page-load does and re-reads memory to see +# whether the internal heap fragments. Written while chasing the Aug 2026 +# "responds over LoRa but not WiFi" issue (TLS handshakes failing once the +# largest internal heap block dropped below ~40KB). +# +# What to look at: +# heap_max largest contiguous internal block. Must stay well above ~40KB. +# heap_min low-water mark since boot. Near 0 = the heap ran dry at some point. +# heals web panel self-restarts because heap_max fell below 24KB. +# deferred web panel starts postponed for lack of heap headroom. +# leaked MQTT clients abandoned after a heap-integrity failure. +# +# Usage: +# eastmesh-tools/web-heap-check.sh [--stress] [--rounds N] +# +# Auth: set REPEATER_PASSWORD (the admin password; the script logs in itself) or +# REPEATER_TOKEN (an existing session token). If neither is set you are prompted +# for the password. Tokens are never written to disk. +# +# Example: +# REPEATER_PASSWORD=... eastmesh-tools/web-heap-check.sh 10.33.135.208 --stress +set -euo pipefail + +host="${1:-}" +stress=0 +rounds=5 +shift || true +while [ $# -gt 0 ]; do + case "$1" in + --stress) stress=1 ;; + --rounds) rounds="$2"; shift ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac + shift +done +if [ -z "$host" ]; then + sed -n '2,24p' "$0"; exit 2 +fi + +base="https://$host" +curlq() { curl -sk -m 20 "$@"; } + +token="${REPEATER_TOKEN:-}" +if [ -z "$token" ]; then + pw="${REPEATER_PASSWORD:-}" + if [ -z "$pw" ]; then + read -r -s -p "admin password for $host: " pw; echo + fi + token=$(curlq -X POST "$base/login" --data "$pw") || true + if [ ${#token} -ne 32 ]; then + echo "login failed: $token" >&2; exit 1 + fi +fi + +cmd() { curlq "$base/api/command" -H "X-Auth-Token: $token" --data "$1"; echo; } + +echo "== $host $(date '+%Y-%m-%d %H:%M:%S')" +for c in ver clock "get wifi.status" "get web.status" "get mqtt.status"; do + printf '%-16s %s\n' "$c" "$(cmd "$c")" +done +before=$(cmd memory) +printf '%-16s %s\n' "memory" "$before" + +if [ "$stress" = 1 ]; then + echo "-- stress: $rounds rounds x (8 parallel page loads + 6 API polls)" + for r in $(seq 1 "$rounds"); do + for i in 1 2 3 4; do + curlq -o /dev/null "$base/" & curlq -o /dev/null "$base/app" & + done + wait + for i in 1 2 3; do + curlq -o /dev/null "$base/api/stats" -H "X-Auth-Token: $token" + curlq -o /dev/null "$base/api/session" -H "X-Auth-Token: $token" + done + done + after=$(cmd memory) + sleep 20 + settled=$(cmd memory) + printf '%-16s %s\n' "memory before" "$before" + printf '%-16s %s\n' "memory after" "$after" + printf '%-16s %s\n' "memory +20s" "$settled" + printf '%-16s %s\n' "web.status" "$(cmd 'get web.status')" +fi + +# One-line verdict on the numbers that matter (post-stress sample if we have one). +final="${settled:-$before}" +hm=$(echo "$final" | sed -n 's/.*"heap_max":\([0-9]*\).*/\1/p') +hmin=$(echo "$final" | sed -n 's/.*"heap_min":\([0-9]*\).*/\1/p') +if [ "${hm:-0}" -lt 40000 ] || [ "${hmin:-0}" -lt 16000 ]; then + echo "VERDICT: heap starved or fragmented (heap_max=$hm heap_min=$hmin) - TLS handshakes at risk" +else + echo "VERDICT: ok (heap_max=$hm heap_min=$hmin)" +fi diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 2782a1d6..cdc0f7a9 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -2644,6 +2644,9 @@ bool MyMesh::formatWebStatsSummaryJson(char* reply, size_t reply_size) { int wifi_rssi = 0; int wifi_quality = 0; int wifi_code = 0; + int wifi_channel = 0; + const char* wifi_gateway = "--"; + unsigned wifi_watchdog_count = 0; #if defined(ESP32) if (network.getWifiSSID()[0] == 0) { @@ -2664,6 +2667,9 @@ bool MyMesh::formatWebStatsSummaryJson(char* reply, size_t reply_size) { escapeJsonString(ip.c_str(), wifi_ip, sizeof(wifi_ip)); wifi_rssi = WiFi.RSSI(); wifi_code = static_cast(WiFi.status()); + wifi_channel = WiFi.channel(); + wifi_gateway = network.isGatewayReachable() ? "ok" : "lost"; + wifi_watchdog_count = network.getWatchdogReconnectCount(); if (wifi_rssi <= -100) { wifi_quality = 0; strncpy(wifi_signal, "poor", sizeof(wifi_signal) - 1); @@ -2778,7 +2784,8 @@ bool MyMesh::formatWebStatsSummaryJson(char* reply, size_t reply_size) { "\"packets\":{\"recv\":%u,\"sent\":%u,\"flood_tx\":%u,\"direct_tx\":%u,\"flood_rx\":%u,\"direct_rx\":%u," "\"recv_errors\":%u,\"direct_dups\":%u,\"flood_dups\":%u,\"neighbors\":%u}," "\"memory\":{\"heap_free\":%u,\"heap_min\":%u,\"heap_max\":%u,\"psram_free\":%u,\"psram_min\":%u,\"psram_max\":%u}," - "\"wifi\":{\"ssid\":\"%s\",\"status\":\"%s\",\"connected\":%s,\"state\":\"%s\",\"code\":%d,\"ip\":\"%s\",\"rssi\":%d,\"quality\":%d,\"signal\":\"%s\",\"powersave\":\"%s\"}," + "\"wifi\":{\"ssid\":\"%s\",\"status\":\"%s\",\"connected\":%s,\"state\":\"%s\",\"code\":%d,\"ip\":\"%s\",\"rssi\":%d,\"quality\":%d,\"signal\":\"%s\",\"powersave\":\"%s\"," + "\"channel\":%d,\"gateway\":\"%s\",\"watchdog_count\":%u}," "\"services\":{\"mqtt_connected\":%s,\"mqtt_state\":\"%s\",\"web_enabled\":%s,\"web_panel_up\":%s,\"web_auth\":\"%s\"," "\"archive_available\":%s}", (_stats_history.isEnabled() && _stats_history.isRecentHistoryAvailable()) ? "true" : "false", @@ -2841,6 +2848,9 @@ bool MyMesh::formatWebStatsSummaryJson(char* reply, size_t reply_size) { wifi_quality, wifi_signal, wifi_powersave, + wifi_channel, + wifi_gateway, + wifi_watchdog_count, mqtt_connected ? "true" : "false", mqtt_state, web.isWebEnabled() ? "true" : "false", diff --git a/release-notes.yml b/release-notes.yml index ebbb581e..41143ced 100644 --- a/release-notes.yml +++ b/release-notes.yml @@ -458,6 +458,30 @@ releases: text: "Adopted upstream's SH1106 display address fallback (0x3C/0x3D) so board revisions with the other SA0 strap setting still get a working display, while keeping the EastMesh T-Beam S3 Supreme bring-up order and headless fallback." breaking_changes: [] + - track: observer-eastmesh + version: "2026.8.3" + tag: "observer-eastmesh-v2026.8.3" + date: "2026-08-23" + previous_version: "2026.8.2" + summary: "Fixes the web panel going unreachable after long uptimes due to internal heap fragmentation, adds web panel self-healing and heap-health CLI counters, and fixes the stats page Channel/Gateway cells." + changes: + - type: fixed + area: web + text: "Fixed the web panel becoming unreachable after days of uptime while LoRa and MQTT kept working: browser page-load bursts were evicting live HTTPS sessions and forcing repeated TLS handshakes, fragmenting internal RAM until no handshake could be allocated. One-shot responses (pages, favicon, login, redirect, 401s) now release their socket immediately, while the authenticated API connection keeps keep-alive." + - type: added + area: web + text: "Added web panel self-healing: the HTTPS server now refuses to start without enough internal heap headroom (retrying every 15 seconds), and restarts itself when idle if the largest free block shrinks below what a TLS handshake needs." + - type: changed + area: cli + text: "Extended `get web.status` with `heals:` and `deferred:`, and `get mqtt.status` with `leaked:` (MQTT clients abandoned after a heap-integrity failure), so heap-related failures are visible over LoRa." + - type: fixed + area: web + text: "Fixed the Channel and Gateway cells on the web panel stats page always showing `--`; the stats summary now carries the Wi-Fi channel, gateway health, and watchdog reconnect count." + - type: added + area: tools + text: "Added `eastmesh-tools/web-heap-check.sh` to read heap and service health over the HTTPS API and optionally stress the web panel with browser-style bursts." + breaking_changes: [] + - track: observer-eastmesh-bridge-espnow version: "2026.5.1" tag: "observer-eastmesh-bridge-espnow-v2026.5.1" @@ -729,3 +753,27 @@ releases: area: board text: "Adopted upstream's SH1106 display address fallback (0x3C/0x3D) so board revisions with the other SA0 strap setting still get a working display, while keeping the EastMesh T-Beam S3 Supreme bring-up order and headless fallback." breaking_changes: [] + - track: observer-eastmesh-bridge-espnow + version: "2026.8.3" + tag: "observer-eastmesh-bridge-espnow-v2026.8.3" + date: "2026-08-23" + previous_version: "2026.8.2" + summary: "Fixes the web panel going unreachable after long uptimes due to internal heap fragmentation, adds web panel self-healing and heap-health CLI counters, and fixes the stats page Channel/Gateway cells." + changes: + - type: fixed + area: web + text: "Fixed the web panel becoming unreachable after days of uptime while LoRa and MQTT kept working: browser page-load bursts were evicting live HTTPS sessions and forcing repeated TLS handshakes, fragmenting internal RAM until no handshake could be allocated. One-shot responses (pages, favicon, login, redirect, 401s) now release their socket immediately, while the authenticated API connection keeps keep-alive." + - type: added + area: web + text: "Added web panel self-healing: the HTTPS server now refuses to start without enough internal heap headroom (retrying every 15 seconds), and restarts itself when idle if the largest free block shrinks below what a TLS handshake needs." + - type: changed + area: cli + text: "Extended `get web.status` with `heals:` and `deferred:`, and `get mqtt.status` with `leaked:` (MQTT clients abandoned after a heap-integrity failure), so heap-related failures are visible over LoRa." + - type: fixed + area: web + text: "Fixed the Channel and Gateway cells on the web panel stats page always showing `--`; the stats summary now carries the Wi-Fi channel, gateway health, and watchdog reconnect count." + - type: added + area: tools + text: "Added `eastmesh-tools/web-heap-check.sh` to read heap and service health over the HTTPS API and optionally stress the web panel with browser-style bursts." + breaking_changes: [] + diff --git a/src/helpers/NetworkService.cpp b/src/helpers/NetworkService.cpp index 8bf53ac6..76109b03 100644 --- a/src/helpers/NetworkService.cpp +++ b/src/helpers/NetworkService.cpp @@ -299,12 +299,11 @@ void NetworkService::formatWifiStatusReply(char* reply, size_t reply_size) const if (wifi_status == WL_CONNECTED) { const int rssi_dbm = WiFi.RSSI(); - const unsigned long gateway_silence_ms = millis() - _wd_last_gateway_ok; snprintf(reply, reply_size, "> ssid:%s status:%s code:%d state:%s ip:%s channel:%d rssi:%d quality:%d%% signal:%s gw:%s wd:%u", _prefs.wifi_ssid, status, static_cast(wifi_status), state, WiFi.localIP().toString().c_str(), WiFi.channel(), rssi_dbm, getWifiQualityPercent(rssi_dbm), getWifiQualityLabel(rssi_dbm), - gateway_silence_ms < (kWatchdogProbeMillis * 3) ? "ok" : "lost", _wd_reconnect_count); + isGatewayReachable() ? "ok" : "lost", _wd_reconnect_count); } else { snprintf(reply, reply_size, "> ssid:%s status:%s code:%d state:%s", _prefs.wifi_ssid[0] ? _prefs.wifi_ssid : "-", status, static_cast(wifi_status), state); @@ -328,6 +327,22 @@ void NetworkService::reconnectWifi() { _last_wifi_attempt = 0; } +bool NetworkService::isGatewayReachable() const { +#if defined(ESP_PLATFORM) + return millis() - _wd_last_gateway_ok < (kWatchdogProbeMillis * 3); +#else + return false; +#endif +} + +uint16_t NetworkService::getWatchdogReconnectCount() const { +#if defined(ESP_PLATFORM) + return _wd_reconnect_count; +#else + return 0; +#endif +} + void NetworkService::forceReconnect() { #if defined(ESP_PLATFORM) // Clear the channel hint (RAM only) so the retry does a full scan and can land diff --git a/src/helpers/NetworkService.h b/src/helpers/NetworkService.h index 505ccde1..e59db894 100644 --- a/src/helpers/NetworkService.h +++ b/src/helpers/NetworkService.h @@ -33,6 +33,9 @@ public: void formatWifiStatusReply(char* reply, size_t reply_size) const; void reconnectWifi(); void forceReconnect(); + // Gateway watchdog state, as reported by `get wifi.status` (gw:ok|lost wd:). + bool isGatewayReachable() const; + uint16_t getWatchdogReconnectCount() const; bool isWifiConnected() const override; bool hasTimeSync() const override { return _have_time_sync; } diff --git a/src/helpers/mqtt/MQTTUplink.cpp b/src/helpers/mqtt/MQTTUplink.cpp index 26868856..8d609a7c 100644 --- a/src/helpers/mqtt/MQTTUplink.cpp +++ b/src/helpers/mqtt/MQTTUplink.cpp @@ -142,7 +142,7 @@ const MQTTUplink::BrokerSpec MQTTUplink::kBrokerSpecs[kBrokerCount] = { MQTTUplink::MQTTUplink(mesh::RTCClock& rtc, mesh::LocalIdentity& identity) : _fs(nullptr), _rtc(&rtc), _identity(&identity), _running(false), _last_status_publish(0), - _token_refresh_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr) + _token_refresh_count(0), _abandoned_client_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr) { memset(_device_id, 0, sizeof(_device_id)); MQTTPrefsStore::setDefaults(_prefs); @@ -615,6 +615,7 @@ void MQTTUplink::destroyBroker(BrokerState& broker, bool reset_retry_state) { MQTT_LOG("%s destroy broker client rc=0x%x", broker.spec->label, destroy_rc); } else { // Avoid freeing through esp-mqtt after the IDF 4.4 WSS transport has already poisoned the heap. + _abandoned_client_count++; MQTT_LOG("%s abandon stopped broker client: heap corrupt", broker.spec->label); } } else { @@ -623,6 +624,7 @@ void MQTTUplink::destroyBroker(BrokerState& broker, bool reset_retry_state) { esp_err_t destroy_rc = esp_mqtt_client_destroy(broker.client); MQTT_LOG("%s destroy broker client rc=0x%x", broker.spec->label, destroy_rc); } else { + _abandoned_client_count++; MQTT_LOG("%s abandon broker client: heap corrupt", broker.spec->label); } } @@ -1249,11 +1251,11 @@ void MQTTUplink::formatStatusReply(char* reply, size_t reply_size) const { format_slot(secondary, secondary_slot, sizeof(secondary_slot)); snprintf(reply, reply_size, - "> wifi:%s ntp:%s iata:%s p:%s s:%s status:%s tx:%s", + "> wifi:%s ntp:%s iata:%s p:%s s:%s status:%s tx:%s leaked:%lu", (_network != nullptr && _network->isWifiConnected()) ? "up" : "down", (_network != nullptr && _network->hasTimeSync()) ? "up" : "wait", _prefs.iata, primary_slot, secondary_slot, _prefs.status_enabled ? "on" : "off", - _prefs.tx_enabled ? "on" : "off"); + _prefs.tx_enabled ? "on" : "off", static_cast(_abandoned_client_count)); } bool MQTTUplink::setEndpointEnabled(uint8_t bit, bool enabled) { @@ -1521,7 +1523,7 @@ const char* MQTTUplink::getAggregateBrokerState() const { MQTTUplink::MQTTUplink(mesh::RTCClock&, mesh::LocalIdentity&) : _fs(nullptr), _rtc(nullptr), _identity(nullptr), _running(false), _last_status_publish(0), - _token_refresh_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr) { + _token_refresh_count(0), _abandoned_client_count(0), _token_refresh_active_until_ms(0), _last_status{}, _node_name(nullptr), _network(nullptr) { MQTTPrefsStore::setDefaults(_prefs); } diff --git a/src/helpers/mqtt/MQTTUplink.h b/src/helpers/mqtt/MQTTUplink.h index 960d71e3..4ae80e2e 100644 --- a/src/helpers/mqtt/MQTTUplink.h +++ b/src/helpers/mqtt/MQTTUplink.h @@ -80,6 +80,7 @@ public: const char* getAggregateBrokerState() const; void setNetworkStateProvider(NetworkStateProvider* network) { _network = network; } uint32_t getTokenRefreshCount() const { return _token_refresh_count; } + uint32_t getAbandonedClientCount() const { return _abandoned_client_count; } bool isTokenRefreshInProgress() const; private: @@ -124,6 +125,9 @@ private: bool _running; unsigned long _last_status_publish; uint32_t _token_refresh_count; + // Clients never freed because heap_caps_check_integrity_all() failed at teardown. + // Each one leaks its TLS context and task stack (~45KB of internal RAM). + uint32_t _abandoned_client_count; unsigned long _token_refresh_active_until_ms; MQTTStatusSnapshot _last_status; char _device_id[65]; diff --git a/src/helpers/web/WebPanelServer.cpp b/src/helpers/web/WebPanelServer.cpp index 69431dd0..8036c899 100644 --- a/src/helpers/web/WebPanelServer.cpp +++ b/src/helpers/web/WebPanelServer.cpp @@ -54,6 +54,33 @@ void freeScratchBuffer(void* ptr) { } } +// Internal (DMA-capable) heap headroom needed before we start an HTTPS listener. +// A single mbedTLS session needs ~40KB of internal RAM in one piece; starting the +// server when that cannot be satisfied just leaves a listener that resets every +// handshake. Mirrors the dual-broker gate in MQTTUplink. +constexpr size_t kWebMinFreeHeap = 56U * 1024U; +constexpr size_t kWebMinLargestHeap = 32U * 1024U; +// Below this largest-block size a TLS handshake can no longer be satisfied; the +// listener is alive but every connect resets. Tear it down and re-create it so +// its pools are returned and coalesced. +constexpr size_t kWebHealLargestHeap = 24U * 1024U; + +// Mark a one-shot response so the socket is released as soon as it is sent. +// Page loads open several parallel connections; with a 2-socket pool and LRU +// purge each one otherwise evicts a live session and forces a fresh TLS +// handshake. The authenticated polling connection keeps keep-alive. +void markCloseAfterSend(httpd_req_t* req) { + httpd_resp_set_hdr(req, "Connection", "close"); +} + +esp_err_t finishAndClose(httpd_req_t* req, esp_err_t rc) { + int fd = httpd_req_to_sockfd(req); + if (fd >= 0) { + httpd_sess_trigger_close(req->handle, fd); + } + return rc; +} + void rebootAfterFirmwareUpdateTask(void*) { vTaskDelay(pdMS_TO_TICKS(1200)); esp_restart(); @@ -1971,7 +1998,7 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( ${renderMetric("State", wifi.state || "--")} ${renderMetric("SSID", wifi.ssid || "-")} ${renderMetric("IP", wifi.ip || "--")} - ${renderMetric("Channel", wifi.channel == null ? "--" : wifi.channel)} + ${renderMetric("Channel", wifi.channel ? wifi.channel : "--")} ${renderMetric("Gateway", wifi.gateway ? wifi.gateway + (wifi.watchdog_count ? " (wd " + wifi.watchdog_count + ")" : "") : "--")} ${renderMetric("Power Save", powersave || "--")} ${renderMetric("Code", wifi.code == null ? "--" : wifi.code)} @@ -3299,7 +3326,8 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( } // namespace WebPanelServer::WebPanelServer() - : _runner(nullptr), _server(nullptr), _redirect_server(nullptr), _token{0}, _last_activity_ms(0), _route_context{this} { + : _runner(nullptr), _server(nullptr), _redirect_server(nullptr), _token{0}, _last_activity_ms(0), + _start_deferred_count(0), _restart_count(0), _route_context{this} { } void WebPanelServer::setCommandRunner(WebPanelCommandRunner* runner) { @@ -3311,6 +3339,16 @@ bool WebPanelServer::start() { return _server != nullptr; } + const size_t free_heap = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + const size_t largest_heap = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + if (free_heap < kWebMinFreeHeap || largest_heap < kWebMinLargestHeap) { + _start_deferred_count++; + WEB_PANEL_LOG("server start deferred: heap_free=%u heap_max=%u need=%u/%u", static_cast(free_heap), + static_cast(largest_heap), static_cast(kWebMinFreeHeap), + static_cast(kWebMinLargestHeap)); + return false; + } + noteActivity(); httpd_ssl_config_t config = HTTPD_SSL_CONFIG_DEFAULT(); @@ -3412,6 +3450,18 @@ bool WebPanelServer::hasSessionToken() const { return _token[0] != 0; } +bool WebPanelServer::isIdle(unsigned long now_ms, unsigned long quiet_ms) const { + return _last_activity_ms == 0 || now_ms - _last_activity_ms >= quiet_ms; +} + +bool WebPanelServer::isHeapStarved() { + return heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT) < kWebHealLargestHeap; +} + +void WebPanelServer::noteRestart() { + _restart_count++; +} + void WebPanelServer::stopRedirectServer() { if (_redirect_server != nullptr) { httpd_handle_t redirect_server = _redirect_server; @@ -3441,15 +3491,17 @@ esp_err_t WebPanelServer::handleIndex(httpd_req_t* req) { ctx->self->noteActivity(); httpd_resp_set_type(req, "text/html; charset=utf-8"); httpd_resp_set_hdr(req, "Cache-Control", "no-store"); - return sendProgmemChunked(req, kWebPanelLoginHtml); + markCloseAfterSend(req); + return finishAndClose(req, sendProgmemChunked(req, kWebPanelLoginHtml)); } esp_err_t WebPanelServer::handleFavicon(httpd_req_t* req) { httpd_resp_set_type(req, "image/png"); httpd_resp_set_hdr(req, "Cache-Control", "max-age=86400"); - return httpd_resp_send(req, - reinterpret_cast(eastmesh_web_assets::kFaviconPng), - eastmesh_web_assets::kFaviconPngLen); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_send(req, + reinterpret_cast(eastmesh_web_assets::kFaviconPng), + eastmesh_web_assets::kFaviconPngLen)); } esp_err_t WebPanelServer::handleHttpRedirect(httpd_req_t* req) { @@ -3464,7 +3516,8 @@ esp_err_t WebPanelServer::handleHttpRedirect(httpd_req_t* req) { httpd_resp_set_status(req, "302 Found"); httpd_resp_set_hdr(req, "Location", location); httpd_resp_set_hdr(req, "Cache-Control", "no-store"); - return httpd_resp_send(req, "", 0); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_send(req, "", 0)); } esp_err_t WebPanelServer::handleApp(httpd_req_t* req) { @@ -3475,7 +3528,8 @@ esp_err_t WebPanelServer::handleApp(httpd_req_t* req) { ctx->self->noteActivity(); httpd_resp_set_type(req, "text/html; charset=utf-8"); httpd_resp_set_hdr(req, "Cache-Control", "no-store"); - return sendProgmemChunked(req, kWebPanelAppHtml); + markCloseAfterSend(req); + return finishAndClose(req, sendProgmemChunked(req, kWebPanelAppHtml)); } esp_err_t WebPanelServer::handleStatsPage(httpd_req_t* req) { @@ -3486,10 +3540,11 @@ esp_err_t WebPanelServer::handleStatsPage(httpd_req_t* req) { ctx->self->noteActivity(); httpd_resp_set_type(req, "text/html; charset=utf-8"); httpd_resp_set_hdr(req, "Cache-Control", "no-store"); + markCloseAfterSend(req); if (ctx->self->_runner != nullptr && !ctx->self->_runner->isWebStatsEnabled()) { - return sendProgmemChunked(req, kWebPanelStatsDisabledHtml); + return finishAndClose(req, sendProgmemChunked(req, kWebPanelStatsDisabledHtml)); } - return sendProgmemChunked(req, kWebPanelAppHtml); + return finishAndClose(req, sendProgmemChunked(req, kWebPanelAppHtml)); } esp_err_t WebPanelServer::handleLogin(httpd_req_t* req) { @@ -3511,7 +3566,8 @@ esp_err_t WebPanelServer::handleLogin(httpd_req_t* req) { if (strcmp(password, ctx->self->_runner->getWebAdminPassword()) != 0) { freeScratchBuffer(password); WEB_PANEL_LOG("login denied"); - return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Bad password"); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Bad password")); } freeScratchBuffer(password); @@ -3520,7 +3576,8 @@ esp_err_t WebPanelServer::handleLogin(httpd_req_t* req) { WEB_PANEL_LOG("login accepted"); httpd_resp_set_type(req, "text/plain; charset=utf-8"); httpd_resp_set_hdr(req, "Cache-Control", "no-store"); - return httpd_resp_sendstr(req, ctx->self->_token); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_sendstr(req, ctx->self->_token)); } esp_err_t WebPanelServer::handleSession(httpd_req_t* req) { @@ -3529,7 +3586,8 @@ esp_err_t WebPanelServer::handleSession(httpd_req_t* req) { return httpd_resp_send_500(req); } if (!ctx->self->isAuthorized(req)) { - return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized")); } ctx->self->noteActivity(); @@ -3544,7 +3602,8 @@ esp_err_t WebPanelServer::handleCommand(httpd_req_t* req) { return httpd_resp_send_500(req); } if (!ctx->self->isAuthorized(req)) { - return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized")); } char* command = allocScratchBuffer(kWebCommandBufferSize); @@ -3578,7 +3637,8 @@ esp_err_t WebPanelServer::handleFirmwareUpdate(httpd_req_t* req) { return httpd_resp_send_500(req); } if (!ctx->self->isAuthorized(req)) { - return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized")); } if (req->content_len <= 0) { return httpd_resp_send_err(req, HTTPD_400_BAD_REQUEST, "Missing firmware body"); @@ -3647,7 +3707,8 @@ esp_err_t WebPanelServer::handleStats(httpd_req_t* req) { return httpd_resp_send_500(req); } if (!ctx->self->isAuthorized(req)) { - return httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized"); + markCloseAfterSend(req); + return finishAndClose(req, httpd_resp_send_err(req, HTTPD_401_UNAUTHORIZED, "Unauthorized")); } ctx->self->noteActivity(); @@ -3766,4 +3827,15 @@ bool WebPanelServer::shouldAutoLock(unsigned long) const { void WebPanelServer::lockSession() { } +bool WebPanelServer::isIdle(unsigned long, unsigned long) const { + return true; +} + +bool WebPanelServer::isHeapStarved() { + return false; +} + +void WebPanelServer::noteRestart() { +} + #endif diff --git a/src/helpers/web/WebPanelServer.h b/src/helpers/web/WebPanelServer.h index e8263975..c26c6fc3 100644 --- a/src/helpers/web/WebPanelServer.h +++ b/src/helpers/web/WebPanelServer.h @@ -46,6 +46,18 @@ public: bool hasSessionToken() const; bool shouldAutoLock(unsigned long now_ms) const; void lockSession(); + // True when no request has touched the panel for quiet_ms (or ever). + bool isIdle(unsigned long now_ms, unsigned long quiet_ms) const; + // True when the largest internal heap block can no longer fit a TLS handshake. + static bool isHeapStarved(); + void noteRestart(); +#if defined(ESP_PLATFORM) && WITH_WEB_PANEL + unsigned int startDeferredCount() const { return _start_deferred_count; } + unsigned int restartCount() const { return _restart_count; } +#else + unsigned int startDeferredCount() const { return 0; } + unsigned int restartCount() const { return 0; } +#endif private: #if defined(ESP_PLATFORM) && WITH_WEB_PANEL @@ -58,6 +70,8 @@ private: httpd_handle_t _redirect_server; char _token[33]; unsigned long _last_activity_ms; + unsigned int _start_deferred_count; + unsigned int _restart_count; RouteContext _route_context; static esp_err_t handleIndex(httpd_req_t* req); diff --git a/src/helpers/web/WebService.cpp b/src/helpers/web/WebService.cpp index 6a8b8856..05dc4176 100644 --- a/src/helpers/web/WebService.cpp +++ b/src/helpers/web/WebService.cpp @@ -4,7 +4,19 @@ #include #endif -WebService::WebService() : _fs(nullptr), _prefs{}, _runner(nullptr), _network(nullptr), _suspended_for_ota(false) { +namespace { +// Retry a heap-deferred start at this cadence rather than every loop tick. +constexpr unsigned long kWebStartRetryMillis = 15000; +// Self-heal: only restart a starved server when nobody has used it for this long, +// and no more often than this. +constexpr unsigned long kWebHealQuietMillis = 60000; +constexpr unsigned long kWebHealMinIntervalMillis = 5UL * 60UL * 1000UL; +constexpr unsigned long kWebHealCheckMillis = 10000; +} // namespace + +WebService::WebService() + : _fs(nullptr), _prefs{}, _runner(nullptr), _network(nullptr), _suspended_for_ota(false), _last_start_attempt_ms(0), + _last_heal_ms(0), _last_heal_check_ms(0) { WebPrefsStore::setDefaults(_prefs); } @@ -36,9 +48,11 @@ void WebService::prepareForOTAStart() { void WebService::loop() { #if defined(ESP_PLATFORM) && WITH_WEB_PANEL ensureWebServer(); - if (_panel.isRunning() && _panel.shouldAutoLock(millis())) { + const unsigned long now_ms = millis(); + if (_panel.isRunning() && _panel.shouldAutoLock(now_ms)) { _panel.lockSession(); } + healIfStarved(now_ms); #endif } @@ -59,6 +73,7 @@ bool WebService::setWebEnabled(bool enabled) { bool ok = savePrefs(); #if defined(ESP_PLATFORM) && WITH_WEB_PANEL if (_prefs.web_enabled != 0 && !_suspended_for_ota) { + _last_start_attempt_ms = 0; // manual enable bypasses the retry backoff ensureWebServer(); } else { _panel.stop(); @@ -85,12 +100,13 @@ void WebService::formatWebStatusReply(char* reply, size_t reply_size) const { } if (!_panel.isRunning() || _network == nullptr || !_network->isWifiConnected()) { - snprintf(reply, reply_size, "> web:down"); + snprintf(reply, reply_size, "> web:down heals:%u deferred:%u", _panel.restartCount(), _panel.startDeferredCount()); return; } - snprintf(reply, reply_size, "> web:up url:https://%s/ auth:%s", WiFi.localIP().toString().c_str(), - _panel.hasSessionToken() ? "unlocked" : "locked"); + snprintf(reply, reply_size, "> web:up url:https://%s/ auth:%s heals:%u deferred:%u", + WiFi.localIP().toString().c_str(), _panel.hasSessionToken() ? "unlocked" : "locked", + _panel.restartCount(), _panel.startDeferredCount()); #else snprintf(reply, reply_size, "> web:unsupported"); #endif @@ -115,6 +131,30 @@ void WebService::ensureWebServer() { if (_panel.isRunning()) { return; } + const unsigned long now_ms = millis(); + if (_last_start_attempt_ms != 0 && now_ms - _last_start_attempt_ms < kWebStartRetryMillis) { + return; + } + _last_start_attempt_ms = now_ms; _panel.start(); } + +void WebService::healIfStarved(unsigned long now_ms) { + if (!_panel.isRunning() || now_ms - _last_heal_check_ms < kWebHealCheckMillis) { + return; + } + _last_heal_check_ms = now_ms; + if (!_panel.isIdle(now_ms, kWebHealQuietMillis) || !WebPanelServer::isHeapStarved()) { + return; + } + if (_last_heal_ms != 0 && now_ms - _last_heal_ms < kWebHealMinIntervalMillis) { + return; + } + _last_heal_ms = now_ms; + _panel.noteRestart(); + Serial.printf("[WEB] heap starved (largest internal block too small for TLS), restarting web panel (count=%u)\n", + _panel.restartCount()); + _panel.stop(false); + _last_start_attempt_ms = 0; // let ensureWebServer() retry immediately (subject to the heap gate) +} #endif diff --git a/src/helpers/web/WebService.h b/src/helpers/web/WebService.h index be6aa43e..3b4b7cd4 100644 --- a/src/helpers/web/WebService.h +++ b/src/helpers/web/WebService.h @@ -32,6 +32,7 @@ public: private: #if defined(ESP_PLATFORM) && WITH_WEB_PANEL void ensureWebServer(); + void healIfStarved(unsigned long now_ms); #endif bool savePrefs(); @@ -41,4 +42,7 @@ private: NetworkStateProvider* _network; WebPanelServer _panel; bool _suspended_for_ota; + unsigned long _last_start_attempt_ms; + unsigned long _last_heal_ms; + unsigned long _last_heal_check_ms; };