diff --git a/docs/custom-cli.md b/docs/custom-cli.md index 1cac81bb..1c16403b 100644 --- a/docs/custom-cli.md +++ b/docs/custom-cli.md @@ -69,7 +69,7 @@ Legacy dotted aliases are also accepted: - `get web` - `get web.status`: shows whether the local HTTPS panel is available. After `start ota`, this reports `web:suspended ota` until the repeater reboots. -- `get web.stats.status`: shows whether the dedicated stats page and history subsystem are enabled, whether recent history is active, whether PSRAM-backed history is available, and whether the SD-backed archive is mounted. When enabled, the history capture now covers supported environment telemetry too, not just the original battery/radio series. +- `get web.stats.status`: shows whether the dedicated stats page and history subsystem are enabled, whether recent history is active, whether PSRAM-backed history is available, and whether the SD-backed archive is mounted. When enabled, the history capture now covers supported environment telemetry too, not just the original battery/radio series. GPS-enabled boards also record per-minute satellites samples for the `/stats` history view. - `set web on|off` - `set.web on|off`: enables or disables the local HTTPS panel. - `set web.stats on|off` diff --git a/docs/web-panel.md b/docs/web-panel.md index 2e59cd58..f1268be6 100644 --- a/docs/web-panel.md +++ b/docs/web-panel.md @@ -192,7 +192,7 @@ The `/stats` page currently shows: - `Services`: MQTT, web, archive, neighbour count, and, when mounted, card and archive capacity - optional full-width `Environment` summary card on boards that report GPS or environmental telemetry -- `Trends`: battery, heap free, packet activity, signal, noise floor, and supported environment history such as voltage, temperatures, humidity, barometer, altitude, and, when GPS is enabled, satellites +- `Trends`: battery, heap free, packet activity, signal, noise floor, and, when GPS is enabled, satellites - `Neighbours`: current neighbour table with ID, SNR, heard age, and advert age - `Events`: current boot/session events @@ -207,11 +207,11 @@ The trend graphs load sequentially rather than as one large payload: 3. memory 4. packet activity 5. signal -6. remaining supported environment series +6. satellites when GPS is enabled This keeps browser-side and device-side memory use lower than the previous in-page stats view. -If `web.stats` is enabled and an SD archive is mounted, trends can restore archived summary points after reboot. Recent live points are still added from in-memory history. This now includes the supported environment series as well as the original core/radio stats. +If `web.stats` is enabled and an SD archive is mounted, trends can restore archived summary points after reboot from the latest SD snapshot. Recent live points are still added from in-memory history. ### Stats History Capacity @@ -230,7 +230,7 @@ On boards with roughly `2 MB` PSRAM or more, stats history starts capturing from Archive-backed restore requires `web.stats` enabled plus a mounted SD card on boards that support the EastMesh archive path. -The main purpose of the SD card is to let the repeater retain and restore stats history for `/stats`. As a secondary option, the archive files can also be removed and inspected on a computer for deeper manual review. +The main purpose of the SD card is to let the repeater retain and restore stats history for `/stats`. The archive keeps fast `.latest` snapshot files for quick restore and UTC-dated daily `.log` files for longer-term history. As a secondary option, those files can also be removed and inspected on a computer for deeper manual review. On no-PSRAM boards, `/stats` can still show recent graphs while the stats view is active, but the history is smaller and does not provide the same archive-backed behaviour as PSRAM-capable boards. diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 90e1b6ac..1ca1e9fc 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -79,9 +79,9 @@ WebSensorSnapshot collectWebSensorSnapshot(mesh::MainBoard& board, SensorManager snapshot.gps_enabled = location->isEnabled(); snapshot.gps_fix = location->isValid(); const long satellites = location->satellitesCount(); - if (satellites > 0) { + if (snapshot.gps_enabled || satellites > 0) { snapshot.has_satellites = true; - snapshot.satellites = satellites; + snapshot.satellites = max(satellites, 0); } if (snapshot.gps_fix) { snapshot.has_gps_lat = true; @@ -207,6 +207,20 @@ bool appendJsonFloatField(char* reply, size_t reply_size, size_t& offset, bool& constexpr unsigned long kArchiveNeighboursFlushIntervalMs = 60UL * 1000UL; constexpr const char* kArchiveNeighboursSnapshotPath = "/stats/neighbours.snapshot"; +constexpr const char* kArchiveNeighboursLatestPath = "/stats/neighbours.latest"; + +bool buildUtcDailyArchivePath(const char* prefix, uint32_t epoch_secs, char* path, size_t path_size) { + if (prefix == nullptr || path == nullptr || path_size == 0) { + return false; + } + if (epoch_secs == 0) { + snprintf(path, path_size, "/stats/%s-unknown.log", prefix); + return true; + } + const DateTime dt(epoch_secs); + snprintf(path, path_size, "/stats/%s-%04d-%02d-%02d.log", prefix, dt.year(), dt.month(), dt.day()); + return true; +} File openArchiveWrite(FILESYSTEM* fs, const char* filename) { #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) @@ -247,6 +261,33 @@ File openArchiveWriteWithRecovery(ArchiveStorage* archive, const char* filename) return fs != nullptr ? openArchiveWrite(fs, filename) : File(); } +File openArchiveAppend(FILESYSTEM* fs, const char* filename) { +#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) || defined(RP2040_PLATFORM) + return fs->open(filename, "a"); +#else + return fs->open(filename, FILE_APPEND, true); +#endif +} + +File openArchiveAppendWithRecovery(ArchiveStorage* archive, const char* filename) { + if (archive == nullptr) { + return File(); + } + FILESYSTEM* fs = archive->getFS(); + if (fs == nullptr) { + return File(); + } + File file = openArchiveAppend(fs, filename); + if (file) { + return file; + } + if (!archive->recover()) { + return File(); + } + fs = archive->getFS(); + return fs != nullptr ? openArchiveAppend(fs, filename) : File(); +} + File openArchiveReadWithRecovery(ArchiveStorage* archive, const char* filename) { if (archive == nullptr) { return File(); @@ -1549,13 +1590,22 @@ bool MyMesh::restoreArchiveNeighbours() { } FILESYSTEM* fs = _archive->getFS(); - if (fs == nullptr || !fs->exists(kArchiveNeighboursSnapshotPath)) { + if (fs == nullptr) { return false; } - File file = openArchiveReadWithRecovery(_archive, kArchiveNeighboursSnapshotPath); + const char* restore_path = nullptr; + if (fs->exists(kArchiveNeighboursLatestPath)) { + restore_path = kArchiveNeighboursLatestPath; + } else if (fs->exists(kArchiveNeighboursSnapshotPath)) { + restore_path = kArchiveNeighboursSnapshotPath; + } else { + return false; + } + + File file = openArchiveReadWithRecovery(_archive, restore_path); if (!file) { - ARCHIVE_LOG("neighbours restore open failed path=%s", kArchiveNeighboursSnapshotPath); + ARCHIVE_LOG("neighbours restore open failed path=%s", restore_path); return false; } @@ -1651,9 +1701,39 @@ void MyMesh::flushArchiveNeighbours() { return a->heard_timestamp > b->heard_timestamp; }); - File file = openArchiveWriteWithRecovery(_archive, kArchiveNeighboursSnapshotPath); + char latest_block[1536]; + size_t latest_len = 0; + uint32_t latest_epoch_secs = 0; + for (int i = 0; i < neighbours_count; ++i) { + char full_hex[65]; + mesh::Utils::toHex(full_hex, sorted_neighbours[i]->id.pub_key, PUB_KEY_SIZE); + latest_len += snprintf(&latest_block[latest_len], sizeof(latest_block) - latest_len, + "%s,%lu,%lu,%d\n", + full_hex, + static_cast(sorted_neighbours[i]->advert_timestamp), + static_cast(sorted_neighbours[i]->heard_timestamp), + static_cast(sorted_neighbours[i]->snr)); + latest_epoch_secs = max(latest_epoch_secs, sorted_neighbours[i]->heard_timestamp); + if (latest_len >= sizeof(latest_block)) { + latest_len = sizeof(latest_block) - 1; + break; + } + } + + File latest_file = openArchiveWriteWithRecovery(_archive, kArchiveNeighboursLatestPath); + if (!latest_file) { + ARCHIVE_LOG("neighbours open failed path=%s", kArchiveNeighboursLatestPath); + return; + } + const size_t latest_written = latest_file.print(latest_block); + latest_file.flush(); + latest_file.close(); + + char daily_path[52]; + buildUtcDailyArchivePath("neighbours", latest_epoch_secs, daily_path, sizeof(daily_path)); + File file = openArchiveAppendWithRecovery(_archive, daily_path); if (!file) { - ARCHIVE_LOG("neighbours open failed path=%s", kArchiveNeighboursSnapshotPath); + ARCHIVE_LOG("neighbours open failed path=%s", daily_path); return; } @@ -1669,8 +1749,10 @@ void MyMesh::flushArchiveNeighbours() { } file.flush(); file.close(); - ARCHIVE_LOG("neighbours flushed path=%s bytes=%u count=%d", - kArchiveNeighboursSnapshotPath, + ARCHIVE_LOG("neighbours flushed latest=%s bytes=%u log=%s log_bytes=%u count=%d", + kArchiveNeighboursLatestPath, + static_cast(latest_written), + daily_path, static_cast(total_written), static_cast(neighbours_count)); _archive_neighbours_dirty = false; diff --git a/src/helpers/StatsHistory.cpp b/src/helpers/StatsHistory.cpp index d97f5378..24e79e28 100644 --- a/src/helpers/StatsHistory.cpp +++ b/src/helpers/StatsHistory.cpp @@ -2,6 +2,7 @@ #include #include +#include #if defined(ESP32) #include @@ -32,6 +33,11 @@ constexpr uint32_t kBootAutoCapturePsramMinBytes = 2000000UL; constexpr size_t kLiveOnlySampleCapacity = 24; constexpr size_t kLiveOnlyEventCapacity = 8; +constexpr const char* kSummaryLatestPath = "/stats/summary.latest"; +constexpr const char* kSummaryLegacyLogPath = "/stats/summary.log"; +constexpr const char* kEventsLatestPath = "/stats/events.latest"; +constexpr const char* kEventsLegacyLogPath = "/stats/events.log"; + struct HistoryCapacityBucket { size_t sample_capacity; size_t event_capacity; @@ -187,6 +193,19 @@ File openArchiveWriteWithRecovery(ArchiveStorage* archive, const char* filename) return fs != nullptr ? openArchiveWrite(fs, filename) : File(); } +bool buildUtcDailyLogPath(const char* prefix, uint32_t epoch_secs, char* path, size_t path_size) { + if (prefix == nullptr || path == nullptr || path_size == 0) { + return false; + } + if (epoch_secs == 0) { + snprintf(path, path_size, "/stats/%s-unknown.log", prefix); + return true; + } + const DateTime dt(epoch_secs); + snprintf(path, path_size, "/stats/%s-%04d-%02d-%02d.log", prefix, dt.year(), dt.month(), dt.day()); + return true; +} + bool buildPointValue(const HistorySample& sample, const HistorySample* previous, const char* series, int& value) { if (strcmp(series, "battery") == 0) { value = static_cast(sample.battery_mv); @@ -752,17 +771,59 @@ bool StatsHistory::restoreSummaryLog() { } FILESYSTEM* fs = _archive->getFS(); - if (fs == nullptr || !fs->exists("/stats/summary.log")) { - return false; - } - - File file = openArchiveReadWithRecovery(_archive, "/stats/summary.log"); - if (!file) { - ARCHIVE_LOG("summary restore open failed path=%s", "/stats/summary.log"); + if (fs == nullptr) { return false; } _restored_sample_count = 0; + + if (fs->exists(kSummaryLatestPath)) { + File latest_file = openArchiveReadWithRecovery(_archive, kSummaryLatestPath); + if (latest_file) { + char line[384]; + size_t line_len = 0; + while (latest_file.available()) { + const int raw = latest_file.read(); + if (raw < 0) { + break; + } + const char ch = static_cast(raw); + if (ch == '\r') { + continue; + } + if (ch == '\n') { + break; + } + if (line_len + 1 < sizeof(line)) { + line[line_len++] = ch; + } + } + line[line_len] = 0; + latest_file.close(); + HistorySample latest_sample{}; + if (line_len > 0 && parseSummaryLine(line, latest_sample)) { + storeSample(latest_sample, false); + _restored_sample_count = 1; + return true; + } + } else { + ARCHIVE_LOG("summary restore open failed path=%s", kSummaryLatestPath); + } + } + + const char* summary_restore_path = nullptr; + if (fs->exists(kSummaryLegacyLogPath)) { + summary_restore_path = kSummaryLegacyLogPath; + } else { + return false; + } + + File file = openArchiveReadWithRecovery(_archive, summary_restore_path); + if (!file) { + ARCHIVE_LOG("summary restore open failed path=%s", summary_restore_path); + return false; + } + const size_t size = static_cast(file.size()); const size_t start = (size > kSummaryRestoreWindowBytes) ? (size - kSummaryRestoreWindowBytes) : 0; if (start > 0 && !file.seek(start)) { @@ -851,17 +912,28 @@ bool StatsHistory::restoreEventsLog() { } FILESYSTEM* fs = _archive->getFS(); - if (fs == nullptr || !fs->exists("/stats/events.log")) { + if (fs == nullptr) { return false; } - File file = openArchiveRead(fs, "/stats/events.log"); + const char* events_restore_path = nullptr; + if (fs->exists(kEventsLatestPath)) { + events_restore_path = kEventsLatestPath; + } else if (fs->exists(kEventsLegacyLogPath)) { + events_restore_path = kEventsLegacyLogPath; + } else { + return false; + } + + File file = openArchiveReadWithRecovery(_archive, events_restore_path); if (!file) { return false; } const size_t size = static_cast(file.size()); - const size_t start = (size > kEventsRestoreWindowBytes) ? (size - kEventsRestoreWindowBytes) : 0; + const size_t start = (strcmp(events_restore_path, kEventsLatestPath) == 0) + ? 0 + : ((size > kEventsRestoreWindowBytes) ? (size - kEventsRestoreWindowBytes) : 0); if (start > 0 && !file.seek(start)) { file.close(); return false; @@ -994,12 +1066,6 @@ void StatsHistory::flushSummaryLog() { return; } - File file = openArchiveAppendWithRecovery(_archive, "/stats/summary.log"); - if (!file) { - ARCHIVE_LOG("summary open failed path=%s", "/stats/summary.log"); - return; - } - char line[384]; snprintf(line, sizeof(line), "%lu,%lu,%u,%u,%d,%d,%d,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%d,%d,%u,%u,%d,%d,%ld,%ld,%u,%u\n", @@ -1031,11 +1097,31 @@ void StatsHistory::flushSummaryLog() { static_cast(latest.gps_lon_e6), static_cast(latest.sensor_flags), static_cast(latest.gps_satellites)); + + File latest_file = openArchiveWriteWithRecovery(_archive, kSummaryLatestPath); + if (!latest_file) { + ARCHIVE_LOG("summary latest open failed path=%s", kSummaryLatestPath); + return; + } + const size_t latest_written = latest_file.print(line); + latest_file.flush(); + latest_file.close(); + + char daily_path[48]; + buildUtcDailyLogPath("summary", latest.epoch_secs, daily_path, sizeof(daily_path)); + File file = openArchiveAppendWithRecovery(_archive, daily_path); + if (!file) { + ARCHIVE_LOG("summary open failed path=%s", daily_path); + return; + } + const size_t written = file.print(line); file.flush(); file.close(); - ARCHIVE_LOG("summary flushed path=%s bytes=%u sample_count=%u", - "/stats/summary.log", + ARCHIVE_LOG("summary flushed latest=%s bytes=%u log=%s log_bytes=%u sample_count=%u", + kSummaryLatestPath, + static_cast(latest_written), + daily_path, static_cast(written), static_cast(_sample_count)); writeMetaFile(); @@ -1051,13 +1137,24 @@ void StatsHistory::flushEventsLog() { return; } - File file = openArchiveAppendWithRecovery(_archive, "/stats/events.log"); + File latest_file = openArchiveWriteWithRecovery(_archive, kEventsLatestPath); + if (!latest_file) { + ARCHIVE_LOG("events latest open failed path=%s", kEventsLatestPath); + return; + } + + char latest_epoch_path[48]; + uint32_t latest_epoch_secs = _pending_events[_pending_event_count - 1].epoch_secs; + buildUtcDailyLogPath("events", latest_epoch_secs, latest_epoch_path, sizeof(latest_epoch_path)); + File file = openArchiveAppendWithRecovery(_archive, latest_epoch_path); if (!file) { - ARCHIVE_LOG("events open failed path=%s", "/stats/events.log"); + latest_file.close(); + ARCHIVE_LOG("events open failed path=%s", latest_epoch_path); return; } size_t total_written = 0; + size_t latest_written = 0; for (size_t i = 0; i < _pending_event_count; ++i) { const HistoryEvent& event = _pending_events[i]; char line[160]; @@ -1068,11 +1165,16 @@ void StatsHistory::flushEventsLog() { getEventTypeName(event.type), static_cast(event.value)); total_written += file.print(line); + latest_written += latest_file.print(line); } + latest_file.flush(); + latest_file.close(); file.flush(); file.close(); - ARCHIVE_LOG("events flushed path=%s bytes=%u count=%u", - "/stats/events.log", + ARCHIVE_LOG("events flushed latest=%s bytes=%u log=%s log_bytes=%u count=%u", + kEventsLatestPath, + static_cast(latest_written), + latest_epoch_path, static_cast(total_written), static_cast(_pending_event_count)); _pending_event_count = 0; diff --git a/src/helpers/web/WebPanelServer.cpp b/src/helpers/web/WebPanelServer.cpp index c5617160..41e73836 100644 --- a/src/helpers/web/WebPanelServer.cpp +++ b/src/helpers/web/WebPanelServer.cpp @@ -1780,6 +1780,7 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( } function sparkStrokeColor(key, points) { if (key === "packets") return "#d97706"; + if (key === "gps_satellites") return "#2f8f4e"; if (key === "signal") return "#3b82f6"; if (key === "noise_floor") return "#94a3b8"; if (key === "memory") { @@ -1791,6 +1792,11 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( } return "#2f8f4e"; } + function sparkHoverColor(key, baseColor) { + if (key === "packets") return "#f59e0b"; + if (key === "gps_satellites") return "#48b267"; + return baseColor || "#2f8f4e"; + } function sparkValueRange(key, values) { if (key === "signal") { return { min:(-125 * 4), max:(-30 * 4) }; @@ -1880,14 +1886,15 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( y: scaleY(point[1]) })); const strokeColor = sparkStrokeColor(key, points); - if (key === "packets") { + if (key === "packets" || key === "gps_satellites") { const slotWidth = (plotRight - plotLeft) / Math.max(points.length, 1); const barWidth = Math.max(3, Math.min(18, slotWidth * 0.68)); + const hoverColor = sparkHoverColor(key, strokeColor); coords.forEach((point, index) => { const left = clamp(point.x - (barWidth / 2), plotLeft, plotRight - barWidth); const top = point.y; const barHeight = Math.max(1, plotBottom - top); - ctx.fillStyle = Number.isInteger(hoverIndex) && hoverIndex === index ? "#f59e0b" : strokeColor; + ctx.fillStyle = Number.isInteger(hoverIndex) && hoverIndex === index ? hoverColor : strokeColor; ctx.fillRect(left, top, barWidth, barHeight); }); return; @@ -1993,7 +2000,7 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML( if (gpsEnabled) { order.push("gps_satellites"); } - return order.concat(["voltage", "sensor_temp", "humidity", "pressure", "pressure_altitude", "mcu_temp", "gps_altitude"]); + return order; } function initTrendCards(seriesOrder) { const trendsEl = document.getElementById("statsTrends"); diff --git a/variants/lilygo_tbeam_supreme_SX1262/target.cpp b/variants/lilygo_tbeam_supreme_SX1262/target.cpp index 23983bd3..589a4348 100644 --- a/variants/lilygo_tbeam_supreme_SX1262/target.cpp +++ b/variants/lilygo_tbeam_supreme_SX1262/target.cpp @@ -27,6 +27,7 @@ AutoDiscoverRTCClock rtc_clock(fallback_clock); bool radio_init() { fallback_clock.begin(); + Wire1.begin(PIN_BOARD_SDA1, PIN_BOARD_SCL1); rtc_clock.begin(Wire1); return radio.std_init(&spi); }