Merge pull request #43 from xJARiD/develop

fix: improve stats archive rotation and satellites trend handling

docs/custom-cli.md

@@ -69,7 +69,7 @@ Legacy dotted aliases are also accepted:
 
 - `get web`
 - `get web.status`: shows whether the local HTTPS panel is available. After `start ota`, this reports `web:suspended ota` until the repeater reboots.
-- `get web.stats.status`: shows whether the dedicated stats page and history subsystem are enabled, whether recent history is active, whether PSRAM-backed history is available, and whether the SD-backed archive is mounted. When enabled, the history capture now covers supported environment telemetry too, not just the original battery/radio series.
+- `get web.stats.status`: shows whether the dedicated stats page and history subsystem are enabled, whether recent history is active, whether PSRAM-backed history is available, and whether the SD-backed archive is mounted. When enabled, the history capture now covers supported environment telemetry too, not just the original battery/radio series. GPS-enabled boards also record per-minute satellites samples for the `/stats` history view.
 - `set web on|off`
 - `set.web on|off`: enables or disables the local HTTPS panel.
 - `set web.stats on|off`

docs/web-panel.md

@@ -192,7 +192,7 @@ The `/stats` page currently shows:
 
 - `Services`: MQTT, web, archive, neighbour count, and, when mounted, card and archive capacity
 - optional full-width `Environment` summary card on boards that report GPS or environmental telemetry
-- `Trends`: battery, heap free, packet activity, signal, noise floor, and supported environment history such as voltage, temperatures, humidity, barometer, altitude, and, when GPS is enabled, satellites
+- `Trends`: battery, heap free, packet activity, signal, noise floor, and, when GPS is enabled, satellites
 - `Neighbours`: current neighbour table with ID, SNR, heard age, and advert age
 - `Events`: current boot/session events
 
@@ -207,11 +207,11 @@ The trend graphs load sequentially rather than as one large payload:
 3. memory
 4. packet activity
 5. signal
-6. remaining supported environment series
+6. satellites when GPS is enabled
 
 This keeps browser-side and device-side memory use lower than the previous in-page stats view.
 
-If `web.stats` is enabled and an SD archive is mounted, trends can restore archived summary points after reboot. Recent live points are still added from in-memory history. This now includes the supported environment series as well as the original core/radio stats.
+If `web.stats` is enabled and an SD archive is mounted, trends can restore archived summary points after reboot from the latest SD snapshot. Recent live points are still added from in-memory history.
 
 ### Stats History Capacity
 
@@ -230,7 +230,7 @@ On boards with roughly `2 MB` PSRAM or more, stats history starts capturing from
 
 Archive-backed restore requires `web.stats` enabled plus a mounted SD card on boards that support the EastMesh archive path.
 
-The main purpose of the SD card is to let the repeater retain and restore stats history for `/stats`. As a secondary option, the archive files can also be removed and inspected on a computer for deeper manual review.
+The main purpose of the SD card is to let the repeater retain and restore stats history for `/stats`. The archive keeps fast `.latest` snapshot files for quick restore and UTC-dated daily `.log` files for longer-term history. As a secondary option, those files can also be removed and inspected on a computer for deeper manual review.
 
 On no-PSRAM boards, `/stats` can still show recent graphs while the stats view is active, but the history is smaller and does not provide the same archive-backed behaviour as PSRAM-capable boards.
 

examples/simple_repeater/MyMesh.cpp

@@ -79,9 +79,9 @@ WebSensorSnapshot collectWebSensorSnapshot(mesh::MainBoard& board, SensorManager
     snapshot.gps_enabled = location->isEnabled();
     snapshot.gps_fix = location->isValid();
     const long satellites = location->satellitesCount();
-    if (satellites > 0) {
+    if (snapshot.gps_enabled || satellites > 0) {
       snapshot.has_satellites = true;
-      snapshot.satellites = satellites;
+      snapshot.satellites = max<long>(satellites, 0);
     }
     if (snapshot.gps_fix) {
       snapshot.has_gps_lat = true;
@@ -207,6 +207,20 @@ bool appendJsonFloatField(char* reply, size_t reply_size, size_t& offset, bool&
 
 constexpr unsigned long kArchiveNeighboursFlushIntervalMs = 60UL * 1000UL;
 constexpr const char* kArchiveNeighboursSnapshotPath = "/stats/neighbours.snapshot";
+constexpr const char* kArchiveNeighboursLatestPath = "/stats/neighbours.latest";
+
+bool buildUtcDailyArchivePath(const char* prefix, uint32_t epoch_secs, char* path, size_t path_size) {
+  if (prefix == nullptr || path == nullptr || path_size == 0) {
+    return false;
+  }
+  if (epoch_secs == 0) {
+    snprintf(path, path_size, "/stats/%s-unknown.log", prefix);
+    return true;
+  }
+  const DateTime dt(epoch_secs);
+  snprintf(path, path_size, "/stats/%s-%04d-%02d-%02d.log", prefix, dt.year(), dt.month(), dt.day());
+  return true;
+}
 
 File openArchiveWrite(FILESYSTEM* fs, const char* filename) {
 #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
@@ -247,6 +261,33 @@ File openArchiveWriteWithRecovery(ArchiveStorage* archive, const char* filename)
   return fs != nullptr ? openArchiveWrite(fs, filename) : File();
 }
 
+File openArchiveAppend(FILESYSTEM* fs, const char* filename) {
+#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) || defined(RP2040_PLATFORM)
+  return fs->open(filename, "a");
+#else
+  return fs->open(filename, FILE_APPEND, true);
+#endif
+}
+
+File openArchiveAppendWithRecovery(ArchiveStorage* archive, const char* filename) {
+  if (archive == nullptr) {
+    return File();
+  }
+  FILESYSTEM* fs = archive->getFS();
+  if (fs == nullptr) {
+    return File();
+  }
+  File file = openArchiveAppend(fs, filename);
+  if (file) {
+    return file;
+  }
+  if (!archive->recover()) {
+    return File();
+  }
+  fs = archive->getFS();
+  return fs != nullptr ? openArchiveAppend(fs, filename) : File();
+}
+
 File openArchiveReadWithRecovery(ArchiveStorage* archive, const char* filename) {
   if (archive == nullptr) {
     return File();
@@ -1549,13 +1590,22 @@ bool MyMesh::restoreArchiveNeighbours() {
   }
 
   FILESYSTEM* fs = _archive->getFS();
-  if (fs == nullptr || !fs->exists(kArchiveNeighboursSnapshotPath)) {
+  if (fs == nullptr) {
     return false;
   }
 
-  File file = openArchiveReadWithRecovery(_archive, kArchiveNeighboursSnapshotPath);
+  const char* restore_path = nullptr;
+  if (fs->exists(kArchiveNeighboursLatestPath)) {
+    restore_path = kArchiveNeighboursLatestPath;
+  } else if (fs->exists(kArchiveNeighboursSnapshotPath)) {
+    restore_path = kArchiveNeighboursSnapshotPath;
+  } else {
+    return false;
+  }
+
+  File file = openArchiveReadWithRecovery(_archive, restore_path);
   if (!file) {
-    ARCHIVE_LOG("neighbours restore open failed path=%s", kArchiveNeighboursSnapshotPath);
+    ARCHIVE_LOG("neighbours restore open failed path=%s", restore_path);
     return false;
   }
 
@@ -1651,9 +1701,39 @@ void MyMesh::flushArchiveNeighbours() {
     return a->heard_timestamp > b->heard_timestamp;
   });
 
-  File file = openArchiveWriteWithRecovery(_archive, kArchiveNeighboursSnapshotPath);
+  char latest_block[1536];
+  size_t latest_len = 0;
+  uint32_t latest_epoch_secs = 0;
+  for (int i = 0; i < neighbours_count; ++i) {
+    char full_hex[65];
+    mesh::Utils::toHex(full_hex, sorted_neighbours[i]->id.pub_key, PUB_KEY_SIZE);
+    latest_len += snprintf(&latest_block[latest_len], sizeof(latest_block) - latest_len,
+                           "%s,%lu,%lu,%d\n",
+                           full_hex,
+                           static_cast<unsigned long>(sorted_neighbours[i]->advert_timestamp),
+                           static_cast<unsigned long>(sorted_neighbours[i]->heard_timestamp),
+                           static_cast<int>(sorted_neighbours[i]->snr));
+    latest_epoch_secs = max<uint32_t>(latest_epoch_secs, sorted_neighbours[i]->heard_timestamp);
+    if (latest_len >= sizeof(latest_block)) {
+      latest_len = sizeof(latest_block) - 1;
+      break;
+    }
+  }
+
+  File latest_file = openArchiveWriteWithRecovery(_archive, kArchiveNeighboursLatestPath);
+  if (!latest_file) {
+    ARCHIVE_LOG("neighbours open failed path=%s", kArchiveNeighboursLatestPath);
+    return;
+  }
+  const size_t latest_written = latest_file.print(latest_block);
+  latest_file.flush();
+  latest_file.close();
+
+  char daily_path[52];
+  buildUtcDailyArchivePath("neighbours", latest_epoch_secs, daily_path, sizeof(daily_path));
+  File file = openArchiveAppendWithRecovery(_archive, daily_path);
   if (!file) {
-    ARCHIVE_LOG("neighbours open failed path=%s", kArchiveNeighboursSnapshotPath);
+    ARCHIVE_LOG("neighbours open failed path=%s", daily_path);
     return;
   }
 
@@ -1669,8 +1749,10 @@ void MyMesh::flushArchiveNeighbours() {
   }
   file.flush();
   file.close();
-  ARCHIVE_LOG("neighbours flushed path=%s bytes=%u count=%d",
+  ARCHIVE_LOG("neighbours flushed latest=%s bytes=%u log=%s log_bytes=%u count=%d",
-              kArchiveNeighboursSnapshotPath,
+              kArchiveNeighboursLatestPath,
+              static_cast<unsigned>(latest_written),
+              daily_path,
               static_cast<unsigned>(total_written),
               static_cast<int>(neighbours_count));
   _archive_neighbours_dirty = false;

src/helpers/StatsHistory.cpp

@@ -2,6 +2,7 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <RTClib.h>
 
 #if defined(ESP32)
   #include <esp_heap_caps.h>
@@ -32,6 +33,11 @@ constexpr uint32_t kBootAutoCapturePsramMinBytes = 2000000UL;
 constexpr size_t kLiveOnlySampleCapacity = 24;
 constexpr size_t kLiveOnlyEventCapacity = 8;
 
+constexpr const char* kSummaryLatestPath = "/stats/summary.latest";
+constexpr const char* kSummaryLegacyLogPath = "/stats/summary.log";
+constexpr const char* kEventsLatestPath = "/stats/events.latest";
+constexpr const char* kEventsLegacyLogPath = "/stats/events.log";
+
 struct HistoryCapacityBucket {
   size_t sample_capacity;
   size_t event_capacity;
@@ -187,6 +193,19 @@ File openArchiveWriteWithRecovery(ArchiveStorage* archive, const char* filename)
   return fs != nullptr ? openArchiveWrite(fs, filename) : File();
 }
 
+bool buildUtcDailyLogPath(const char* prefix, uint32_t epoch_secs, char* path, size_t path_size) {
+  if (prefix == nullptr || path == nullptr || path_size == 0) {
+    return false;
+  }
+  if (epoch_secs == 0) {
+    snprintf(path, path_size, "/stats/%s-unknown.log", prefix);
+    return true;
+  }
+  const DateTime dt(epoch_secs);
+  snprintf(path, path_size, "/stats/%s-%04d-%02d-%02d.log", prefix, dt.year(), dt.month(), dt.day());
+  return true;
+}
+
 bool buildPointValue(const HistorySample& sample, const HistorySample* previous, const char* series, int& value) {
   if (strcmp(series, "battery") == 0) {
     value = static_cast<int>(sample.battery_mv);
@@ -752,17 +771,59 @@ bool StatsHistory::restoreSummaryLog() {
   }
 
   FILESYSTEM* fs = _archive->getFS();
-  if (fs == nullptr || !fs->exists("/stats/summary.log")) {
+  if (fs == nullptr) {
-    return false;
-  }
-
-  File file = openArchiveReadWithRecovery(_archive, "/stats/summary.log");
-  if (!file) {
-    ARCHIVE_LOG("summary restore open failed path=%s", "/stats/summary.log");
     return false;
   }
 
   _restored_sample_count = 0;
+
+  if (fs->exists(kSummaryLatestPath)) {
+    File latest_file = openArchiveReadWithRecovery(_archive, kSummaryLatestPath);
+    if (latest_file) {
+      char line[384];
+      size_t line_len = 0;
+      while (latest_file.available()) {
+        const int raw = latest_file.read();
+        if (raw < 0) {
+          break;
+        }
+        const char ch = static_cast<char>(raw);
+        if (ch == '\r') {
+          continue;
+        }
+        if (ch == '\n') {
+          break;
+        }
+        if (line_len + 1 < sizeof(line)) {
+          line[line_len++] = ch;
+        }
+      }
+      line[line_len] = 0;
+      latest_file.close();
+      HistorySample latest_sample{};
+      if (line_len > 0 && parseSummaryLine(line, latest_sample)) {
+        storeSample(latest_sample, false);
+        _restored_sample_count = 1;
+        return true;
+      }
+    } else {
+      ARCHIVE_LOG("summary restore open failed path=%s", kSummaryLatestPath);
+    }
+  }
+
+  const char* summary_restore_path = nullptr;
+  if (fs->exists(kSummaryLegacyLogPath)) {
+    summary_restore_path = kSummaryLegacyLogPath;
+  } else {
+    return false;
+  }
+
+  File file = openArchiveReadWithRecovery(_archive, summary_restore_path);
+  if (!file) {
+    ARCHIVE_LOG("summary restore open failed path=%s", summary_restore_path);
+    return false;
+  }
+
   const size_t size = static_cast<size_t>(file.size());
   const size_t start = (size > kSummaryRestoreWindowBytes) ? (size - kSummaryRestoreWindowBytes) : 0;
   if (start > 0 && !file.seek(start)) {
@@ -851,17 +912,28 @@ bool StatsHistory::restoreEventsLog() {
   }
 
   FILESYSTEM* fs = _archive->getFS();
-  if (fs == nullptr || !fs->exists("/stats/events.log")) {
+  if (fs == nullptr) {
     return false;
   }
 
-  File file = openArchiveRead(fs, "/stats/events.log");
+  const char* events_restore_path = nullptr;
+  if (fs->exists(kEventsLatestPath)) {
+    events_restore_path = kEventsLatestPath;
+  } else if (fs->exists(kEventsLegacyLogPath)) {
+    events_restore_path = kEventsLegacyLogPath;
+  } else {
+    return false;
+  }
+
+  File file = openArchiveReadWithRecovery(_archive, events_restore_path);
   if (!file) {
     return false;
   }
 
   const size_t size = static_cast<size_t>(file.size());
-  const size_t start = (size > kEventsRestoreWindowBytes) ? (size - kEventsRestoreWindowBytes) : 0;
+  const size_t start = (strcmp(events_restore_path, kEventsLatestPath) == 0)
+      ? 0
+      : ((size > kEventsRestoreWindowBytes) ? (size - kEventsRestoreWindowBytes) : 0);
   if (start > 0 && !file.seek(start)) {
     file.close();
     return false;
@@ -994,12 +1066,6 @@ void StatsHistory::flushSummaryLog() {
     return;
   }
 
-  File file = openArchiveAppendWithRecovery(_archive, "/stats/summary.log");
-  if (!file) {
-    ARCHIVE_LOG("summary open failed path=%s", "/stats/summary.log");
-    return;
-  }
-
   char line[384];
   snprintf(line, sizeof(line),
            "%lu,%lu,%u,%u,%d,%d,%d,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%d,%d,%u,%u,%d,%d,%ld,%ld,%u,%u\n",
@@ -1031,11 +1097,31 @@ void StatsHistory::flushSummaryLog() {
            static_cast<long>(latest.gps_lon_e6),
            static_cast<unsigned>(latest.sensor_flags),
            static_cast<unsigned>(latest.gps_satellites));
+
+  File latest_file = openArchiveWriteWithRecovery(_archive, kSummaryLatestPath);
+  if (!latest_file) {
+    ARCHIVE_LOG("summary latest open failed path=%s", kSummaryLatestPath);
+    return;
+  }
+  const size_t latest_written = latest_file.print(line);
+  latest_file.flush();
+  latest_file.close();
+
+  char daily_path[48];
+  buildUtcDailyLogPath("summary", latest.epoch_secs, daily_path, sizeof(daily_path));
+  File file = openArchiveAppendWithRecovery(_archive, daily_path);
+  if (!file) {
+    ARCHIVE_LOG("summary open failed path=%s", daily_path);
+    return;
+  }
+
   const size_t written = file.print(line);
   file.flush();
   file.close();
-  ARCHIVE_LOG("summary flushed path=%s bytes=%u sample_count=%u",
+  ARCHIVE_LOG("summary flushed latest=%s bytes=%u log=%s log_bytes=%u sample_count=%u",
-              "/stats/summary.log",
+              kSummaryLatestPath,
+              static_cast<unsigned>(latest_written),
+              daily_path,
               static_cast<unsigned>(written),
               static_cast<unsigned>(_sample_count));
   writeMetaFile();
@@ -1051,13 +1137,24 @@ void StatsHistory::flushEventsLog() {
     return;
   }
 
-  File file = openArchiveAppendWithRecovery(_archive, "/stats/events.log");
+  File latest_file = openArchiveWriteWithRecovery(_archive, kEventsLatestPath);
+  if (!latest_file) {
+    ARCHIVE_LOG("events latest open failed path=%s", kEventsLatestPath);
+    return;
+  }
+
+  char latest_epoch_path[48];
+  uint32_t latest_epoch_secs = _pending_events[_pending_event_count - 1].epoch_secs;
+  buildUtcDailyLogPath("events", latest_epoch_secs, latest_epoch_path, sizeof(latest_epoch_path));
+  File file = openArchiveAppendWithRecovery(_archive, latest_epoch_path);
   if (!file) {
-    ARCHIVE_LOG("events open failed path=%s", "/stats/events.log");
+    latest_file.close();
+    ARCHIVE_LOG("events open failed path=%s", latest_epoch_path);
     return;
   }
 
   size_t total_written = 0;
+  size_t latest_written = 0;
   for (size_t i = 0; i < _pending_event_count; ++i) {
     const HistoryEvent& event = _pending_events[i];
     char line[160];
@@ -1068,11 +1165,16 @@ void StatsHistory::flushEventsLog() {
              getEventTypeName(event.type),
              static_cast<int>(event.value));
     total_written += file.print(line);
+    latest_written += latest_file.print(line);
   }
+  latest_file.flush();
+  latest_file.close();
   file.flush();
   file.close();
-  ARCHIVE_LOG("events flushed path=%s bytes=%u count=%u",
+  ARCHIVE_LOG("events flushed latest=%s bytes=%u log=%s log_bytes=%u count=%u",
-              "/stats/events.log",
+              kEventsLatestPath,
+              static_cast<unsigned>(latest_written),
+              latest_epoch_path,
               static_cast<unsigned>(total_written),
               static_cast<unsigned>(_pending_event_count));
   _pending_event_count = 0;

src/helpers/web/WebPanelServer.cpp

@@ -1780,6 +1780,7 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
     }
     function sparkStrokeColor(key, points) {
       if (key === "packets") return "#d97706";
+      if (key === "gps_satellites") return "#2f8f4e";
       if (key === "signal") return "#3b82f6";
       if (key === "noise_floor") return "#94a3b8";
       if (key === "memory") {
@@ -1791,6 +1792,11 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
       }
       return "#2f8f4e";
     }
+    function sparkHoverColor(key, baseColor) {
+      if (key === "packets") return "#f59e0b";
+      if (key === "gps_satellites") return "#48b267";
+      return baseColor || "#2f8f4e";
+    }
     function sparkValueRange(key, values) {
       if (key === "signal") {
         return { min:(-125 * 4), max:(-30 * 4) };
@@ -1880,14 +1886,15 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
         y: scaleY(point[1])
       }));
       const strokeColor = sparkStrokeColor(key, points);
-      if (key === "packets") {
+      if (key === "packets" || key === "gps_satellites") {
         const slotWidth = (plotRight - plotLeft) / Math.max(points.length, 1);
         const barWidth = Math.max(3, Math.min(18, slotWidth * 0.68));
+        const hoverColor = sparkHoverColor(key, strokeColor);
         coords.forEach((point, index) => {
           const left = clamp(point.x - (barWidth / 2), plotLeft, plotRight - barWidth);
           const top = point.y;
           const barHeight = Math.max(1, plotBottom - top);
-          ctx.fillStyle = Number.isInteger(hoverIndex) && hoverIndex === index ? "#f59e0b" : strokeColor;
+          ctx.fillStyle = Number.isInteger(hoverIndex) && hoverIndex === index ? hoverColor : strokeColor;
           ctx.fillRect(left, top, barWidth, barHeight);
         });
         return;
@@ -1993,7 +2000,7 @@ const char kWebPanelAppHtml[] PROGMEM = R"HTML(
       if (gpsEnabled) {
         order.push("gps_satellites");
       }
-      return order.concat(["voltage", "sensor_temp", "humidity", "pressure", "pressure_altitude", "mcu_temp", "gps_altitude"]);
+      return order;
     }
     function initTrendCards(seriesOrder) {
       const trendsEl = document.getElementById("statsTrends");

variants/lilygo_tbeam_supreme_SX1262/target.cpp

@@ -27,6 +27,7 @@ AutoDiscoverRTCClock rtc_clock(fallback_clock);
 
 bool radio_init() {
   fallback_clock.begin();
+  Wire1.begin(PIN_BOARD_SDA1, PIN_BOARD_SCL1);
   rtc_clock.begin(Wire1);
   return radio.std_init(&spi);
 }