From 94d2ae57f1ae437178f3fb915673422a4c863988 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 20 Oct 2026 18:15:04 +0300 Subject: [PATCH] =?UTF-8?q?feat(build):=20gzip=20=D1=81=D1=82=D1=80=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D1=86=20=D0=B2=D0=B5=D0=B1-=D0=BF=D0=B0=D0=BD?= =?UTF-8?q?=D0=B5=D0=BB=D0=B8=20=D0=B8=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5?= =?UTF-8?q?=D1=80=D0=BA=D0=B0=20=D1=80=D0=B0=D0=B7=D0=BC=D0=B5=D1=80=D0=B0?= =?UTF-8?q?=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B7=D0=B0;=20=D0=BC=D0=BE=D0=B4?= =?UTF-8?q?=D1=83=D0=BB=D0=B8=20AdvertJail=20=D0=B8=20ChannelBudget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 ++ arch/esp32/extra_scripts/check_app_size.py | 52 ++++++++++++++ arch/esp32/extra_scripts/gzip_web_panel.py | 47 +++++++++++++ platformio.ini | 2 + src/helpers/AdvertJail.h | 79 ++++++++++++++++++++++ src/helpers/ChannelBudget.h | 66 ++++++++++++++++++ src/helpers/web/WebPanelServer.cpp | 44 ++++++++++-- 7 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 arch/esp32/extra_scripts/check_app_size.py create mode 100644 arch/esp32/extra_scripts/gzip_web_panel.py create mode 100644 src/helpers/AdvertJail.h create mode 100644 src/helpers/ChannelBudget.h diff --git a/README.md b/README.md index fa496512..72688981 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,12 @@ MQTT-репитер использует: - MQTT-целей стало 112 (42 uplink, 34 espnow, 36 bridge); после слияния проверены сборкой `heltec_v4_repeater_mqtt`, `LilyGo_TBeam_1W_repeater_mqtt`, `Meshadventurer_sx1262_repeater_mqtt`, `RAK_3112_repeater_mqtt`, `heltec_v4_repeater_mqtt_espnow`, `heltec_v4_repeater_mqtt_bridge`, `LilyGo_Tlora_C6_repeater_mqtt_espnow`, `LilyGo_TBeam_1W_repeater_mqtt_espnow`, `Waveshare_S3_LR1121_repeater_mqtt`, `Waveshare_S3_LR1121_repeater_mqtt_bridge`; - в `AGENTS.md` обновлён процесс: работаем в `main` (ветка `develop` — устаревшая, от эпохи EastMesh), upstream тянем релизным тегом, а не веткой `dev`, версия upstream — `v1.18.0`. +**Сжатие веб-панели и проверка размера образа** + +- страницы панели сжимаются gzip во время сборки (rch/esp32/extra_scripts/gzip_web_panel.py): 153 819 байт текста превращаются в 33 054 байта, образ репитера стал легче примерно на 118 КБ, браузер распаковывает страницы сам по Content-Encoding: gzip; +- добавлена проверка rch/esp32/extra_scripts/check_app_size.py: сборка падает, если образ больше своего app-раздела (PlatformIO считает только секции ELF и не учитывает выравнивание); +- в репозиторий добавлены модули из форка MeshCoreTel-moe: src/helpers/AdvertJail.h (лимит флуда анонсами по отправителю) и src/helpers/ChannelBudget.h (бюджет эфира на канал); подключение к обработке пакетов — следующий шаг. + **Исправлено** - в MQTT-телеметрию уходила только база версии (`v1.18.0`), потому что поле `firmware_version` бралось из `FIRMWARE_VERSION`, куда при сборке передаётся версия upstream. Теперь строка собирается в коде: `MQTT_FIRMWARE_VERSION = FIRMWARE_VERSION "-posadmesh-meshcoretel"`, то есть публикуется `v1.18.0-posadmesh-meshcoretel` — одинаково при локальной сборке и в CI; diff --git a/arch/esp32/extra_scripts/check_app_size.py b/arch/esp32/extra_scripts/check_app_size.py new file mode 100644 index 00000000..4baab798 --- /dev/null +++ b/arch/esp32/extra_scripts/check_app_size.py @@ -0,0 +1,52 @@ +Import("env") + +# Падаем на сборке, если образ прошивки больше своего app-раздела. +# PlatformIO показывает "Flash: N%" только по секциям ELF, а в образе ещё есть выравнивание +# (на ESP32-C6 это около 70 КБ): загрузчик отказывается стартовать образ длиннее раздела +# ("Image length ... doesn't fit in partition length ..."), и OTA его тоже не запишет. + +import os +import sys + + +def _parse_size(value): + value = value.strip() + if value[-1:].upper() in ("K", "M"): + return int(value[:-1], 0) * (1024 if value[-1].upper() == "K" else 1024 * 1024) + return int(value, 0) + + +def _smallest_app_partition(partitions_csv): + sizes = [] + with open(partitions_csv) as fp: + for line in fp: + line = line.strip() + if not line or line.startswith("#"): + continue + tokens = [t.strip() for t in line.split(",")] + if len(tokens) >= 5 and tokens[1] in ("app", "0"): + sizes.append(_parse_size(tokens[4])) + return min(sizes) if sizes else None + + +def check_app_size(source, target, env): + image = target[0].get_abspath() + partitions_csv = env.subst("$PARTITIONS_TABLE_CSV") + if not os.path.isfile(image) or not os.path.isfile(partitions_csv): + return 0 + slot = _smallest_app_partition(partitions_csv) + if slot is None: + return 0 + size = os.path.getsize(image) + if size > slot: + sys.stderr.write( + "Error: %s is %d bytes, %d more than its %d-byte app partition (%s); " + "the bootloader will not start it\n" + % (os.path.basename(image), size, size - slot, slot, os.path.basename(partitions_csv)) + ) + return 1 + print("App image %d of %d bytes, %d left" % (size, slot, slot - size)) + return 0 + + +env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", check_app_size) diff --git a/arch/esp32/extra_scripts/gzip_web_panel.py b/arch/esp32/extra_scripts/gzip_web_panel.py new file mode 100644 index 00000000..a556ae91 --- /dev/null +++ b/arch/esp32/extra_scripts/gzip_web_panel.py @@ -0,0 +1,47 @@ +Import("env") + +# Сжимаем страницы веб-панели прямо во время сборки: в прошивке они лежат gzip-архивами +# (HTML/JS ~146 КБ превращаются примерно в 30 КБ), а браузер распаковывает их сам по +# заголовку Content-Encoding: gzip. Исходники остаются читаемыми строками в +# WebPanelServer.cpp, скрипт только генерирует $BUILD_DIR/generated/WebPanelAssets.h +# и определяет WEB_PANEL_GZIP. + +import gzip +import re +from pathlib import Path + +PAGE = re.compile(r'const char (\w+)\[\] PROGMEM = R"HTML\((.*?)\)HTML";', re.S) +WANTED = ["kWebPanelLoginHtml", "kWebPanelStatsDisabledHtml", "kWebPanelAppHtml"] + + +def render(name, data): + rows = ",\n".join( + " " + ",".join("0x%02x" % b for b in data[i:i + 20]) for i in range(0, len(data), 20) + ) + return "static const uint8_t %sGz[%d] = {\n%s\n};\n" % (name, len(data), rows) + + +source = Path(env["PROJECT_DIR"]) / "src/helpers/web/WebPanelServer.cpp" +text = source.read_text(encoding="utf-8") +pages = {name: body for name, body in PAGE.findall(text)} + +missing = [name for name in WANTED if name not in pages] +if missing: + print("gzip_web_panel: страницы не найдены (%s), панель останется несжатой" % ", ".join(missing)) +else: + header = ( + "// Сгенерировано arch/esp32/extra_scripts/gzip_web_panel.py из WebPanelServer.cpp.\n" + "// Не редактировать: файл пересобирается при каждой сборке.\n" + "#pragma once\n\n#include \n\n" + + "\n".join(render(name, gzip.compress(pages[name].encode("utf-8"), 9, mtime=0)) for name in WANTED) + ) + out_dir = Path(env.subst("$BUILD_DIR")) / "generated" + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / "WebPanelAssets.h" + # перезаписываем только при изменении, иначе WebPanelServer.cpp пересобирался бы каждый раз + if not out.exists() or out.read_text(encoding="utf-8") != header: + out.write_text(header, encoding="utf-8") + env.Append(CPPPATH=[str(out_dir)], CPPDEFINES=[("WEB_PANEL_GZIP", 1)]) + total_raw = sum(len(pages[name]) for name in WANTED) + print("gzip_web_panel: %d страниц, %d байт текста сжато в %d" % ( + len(WANTED), total_raw, sum(len(gzip.compress(pages[name].encode("utf-8"), 9, mtime=0)) for name in WANTED))) diff --git a/platformio.ini b/platformio.ini index 3f5f9adc..f71b4f71 100644 --- a/platformio.ini +++ b/platformio.ini @@ -63,7 +63,9 @@ monitor_filters = esp32_exception_decoder extra_scripts = pre:arch/esp32/extra_scripts/generate_web_panel_cert.py pre:arch/esp32/extra_scripts/git_commit_hash.py + pre:arch/esp32/extra_scripts/gzip_web_panel.py merge-bin.py + post:arch/esp32/extra_scripts/check_app_size.py build_flags = ${arduino_base.build_flags} -D ESP32_PLATFORM ; -D ESP32_CPU_FREQ=80 ; change it to your need diff --git a/src/helpers/AdvertJail.h b/src/helpers/AdvertJail.h new file mode 100644 index 00000000..6693ed9a --- /dev/null +++ b/src/helpers/AdvertJail.h @@ -0,0 +1,79 @@ +#pragma once + +#include +#include + +#define ADVERT_JAIL_KEY_SIZE 8 // public key prefix: long enough that nobody grinds a lookalike key +#define ADVERT_JAIL_THRESHOLD 6 // strikes that put a sender in jail + +struct AdvertJailEntry { + uint8_t key[ADVERT_JAIL_KEY_SIZE]; + uint32_t first_seen; // millis() of the first counted advert + uint32_t last_seen; // millis() of the latest counted advert, 0 = free slot + uint32_t last_advert_time; // the sender's own timestamp on that advert + uint16_t adverts; // adverts counted + uint8_t strikes; // adverts that came sooner than the interval, capped at the threshold + uint8_t jailed; +}; + +namespace mesh { + +/** + * \brief Per-sender flood advert limit, ported from MeshSydney's advert.jail. + * An advert that comes sooner than 'interval_ms' after the sender's previous one is a strike; + * ADVERT_JAIL_THRESHOLD strikes put the sender in jail, and while jailed its early adverts are + * dropped. An advert that keeps to the interval takes a strike off; below 2 strikes the sender + * is let out. The same advert seen again (same timestamp) is not counted. + * \param table 'size' entries, zero-filled when empty + * \param pub_key the sender's public key (the first ADVERT_JAIL_KEY_SIZE bytes are kept) + * \param advert_time the timestamp inside the advert, by the sender's clock + * \param now_ms a millisecond clock (wrap-around is fine) + * \returns true if this advert must not be forwarded + */ +inline bool checkAdvertJail(AdvertJailEntry* table, int size, const uint8_t* pub_key, uint32_t advert_time, + uint32_t now_ms, uint32_t interval_ms) { + if (now_ms == 0) now_ms = 1; // 0 marks a free slot + + // find the sender, else the slot to reuse: a free one, then the fewest strikes, then the longest unheard + AdvertJailEntry* entry = NULL; + AdvertJailEntry* evict = &table[0]; + for (int i = 0; i < size; i++) { + AdvertJailEntry* e = &table[i]; + if (e->last_seen != 0 && memcmp(e->key, pub_key, ADVERT_JAIL_KEY_SIZE) == 0) { + entry = e; + break; + } + if (evict->last_seen == 0) continue; + if (e->last_seen == 0 || e->strikes < evict->strikes + || (e->strikes == evict->strikes && now_ms - e->last_seen > now_ms - evict->last_seen)) { + evict = e; + } + } + + if (entry == NULL) { // first advert from this sender + memset(evict, 0, sizeof(*evict)); + memcpy(evict->key, pub_key, ADVERT_JAIL_KEY_SIZE); + evict->first_seen = evict->last_seen = now_ms; + evict->last_advert_time = advert_time; + evict->adverts = 1; + return false; + } + if (advert_time == entry->last_advert_time) { // the same advert again + return entry->jailed != 0; + } + + bool interval_ok = now_ms - entry->last_seen >= interval_ms; + if (interval_ok && entry->strikes > 0) entry->strikes--; + if (entry->jailed && entry->strikes < 2) entry->jailed = 0; + + entry->last_seen = now_ms; + entry->last_advert_time = advert_time; + if (entry->adverts < 0xFFFF) entry->adverts++; + + if (!interval_ok && entry->strikes < ADVERT_JAIL_THRESHOLD) entry->strikes++; + if (!entry->jailed && entry->strikes >= ADVERT_JAIL_THRESHOLD) entry->jailed = 1; + + return entry->jailed && !interval_ok; +} + +} diff --git a/src/helpers/ChannelBudget.h b/src/helpers/ChannelBudget.h new file mode 100644 index 00000000..227021f2 --- /dev/null +++ b/src/helpers/ChannelBudget.h @@ -0,0 +1,66 @@ +#pragma once + +#include + +#define CHAN_BUDGET_CHANNELS 256 // one slot per 1-byte channel hash +#define CHAN_BUDGET_WINDOW_MS (10UL * 60UL * 1000UL) // the budget is per 10 minutes +#define CHAN_BUDGET_MAX 1000 + +namespace mesh { + +/** + * \brief Per-channel budget for flood group packets (GRP_TXT, GRP_DATA), keyed by the 1-byte channel hash. + * A token bucket kept in GCRA form (one "theoretical arrival time" per channel): a channel can send + * 'limit' packets back to back, then one more every window_ms / limit. A channel that never has more + * than 'limit' packets in any window_ms is never held back. Dropped packets cost nothing. + * The stored time is a fraction of the window rather than a token count, so a new limit applies + * to the same share of the window straight away. + * \param tat CHAN_BUDGET_CHANNELS entries, zero-filled at start + * \param limit packets per window, 0 = off + * \param now_ms a millisecond clock (wrap-around is fine) + * \returns true if the channel is over its budget and this packet must not be forwarded + */ +inline bool checkChannelBudget(uint32_t* tat, uint8_t channel_hash, uint32_t now_ms, uint16_t limit, + uint32_t window_ms) { + if (limit == 0) return false; + uint32_t interval = window_ms / limit; + if (interval == 0) interval = 1; + uint32_t tolerance = interval * (limit - 1); // lets 'limit' packets through back to back + + int32_t ahead = (int32_t)(tat[channel_hash] - now_ms); + if (ahead <= 0 || (uint32_t)ahead > tolerance + interval) { + ahead = 0; // bucket full; a time past the bucket is left over from weeks ago (see refreshChannelBudget) + } + if ((uint32_t)ahead > tolerance) return true; // bucket empty + + tat[channel_hash] = now_ms + (uint32_t)ahead + interval; + return false; +} + +/** + * \brief Call every few minutes: moves the stored time of idle channels up to now, so that it never gets old + * enough (~49 days of a 32-bit millisecond clock) to wrap around into the window and hold back + * a channel that has been quiet. + */ +inline void refreshChannelBudget(uint32_t* tat, uint32_t now_ms, uint32_t window_ms) { + for (int h = 0; h < CHAN_BUDGET_CHANNELS; h++) { + int32_t ahead = (int32_t)(tat[h] - now_ms); + if (ahead <= 0 || (uint32_t)ahead > window_ms) tat[h] = now_ms; + } +} + +/** + * \returns how much of the channel's budget is in use: 0 (idle) .. limit (over budget, packets are dropped) + */ +inline uint16_t getChannelBudgetUsed(const uint32_t* tat, uint8_t channel_hash, uint32_t now_ms, uint16_t limit, + uint32_t window_ms) { + if (limit == 0) return 0; + uint32_t interval = window_ms / limit; + if (interval == 0) interval = 1; + + int32_t ahead = (int32_t)(tat[channel_hash] - now_ms); + if (ahead <= 0 || (uint32_t)ahead > interval * limit) return 0; + return (uint16_t)(((uint32_t)ahead + interval - 1) / interval); +} + +} diff --git a/src/helpers/web/WebPanelServer.cpp b/src/helpers/web/WebPanelServer.cpp index 6d94056a..83198865 100644 --- a/src/helpers/web/WebPanelServer.cpp +++ b/src/helpers/web/WebPanelServer.cpp @@ -12,6 +12,11 @@ #include #include "../mqtt/generated/WebPanelCert.h" +#if defined(WEB_PANEL_GZIP) + // Страницы, сжатые gzip во время сборки (arch/esp32/extra_scripts/gzip_web_panel.py) + #include +#endif + namespace { #ifndef WEB_PANEL_STACK_SIZE @@ -120,6 +125,37 @@ esp_err_t sendWhole(httpd_req_t* req, const char* text, const size_t len) { return httpd_resp_send_chunk(req, nullptr, 0); } +// Отдача сжатой страницы: браузер распаковывает её сам по Content-Encoding: gzip, +// поэтому страницы панели занимают в прошивке вчетверо меньше места. +esp_err_t sendGzipped(httpd_req_t* req, const uint8_t* data, const size_t len) { + if (data == nullptr || len == 0) { + return httpd_resp_send_chunk(req, nullptr, 0); + } + + httpd_resp_set_hdr(req, "Content-Encoding", "gzip"); + + for (size_t offset = 0; offset < len; offset += kWebPageChunkSize) { + const size_t chunk = ((len - offset) < kWebPageChunkSize) ? (len - offset) : kWebPageChunkSize; + if (httpd_resp_send_chunk(req, reinterpret_cast(data + offset), chunk) != ESP_OK) { + return ESP_FAIL; + } + + vTaskDelay(1); + } + + return httpd_resp_send_chunk(req, nullptr, 0); +} + +#if defined(WEB_PANEL_GZIP) + #define sendLoginPage(req) sendGzipped(req, kWebPanelLoginHtmlGz, sizeof(kWebPanelLoginHtmlGz)) + #define sendStatsDisabledPage(req) sendGzipped(req, kWebPanelStatsDisabledHtmlGz, sizeof(kWebPanelStatsDisabledHtmlGz)) + #define sendAppPage(req) sendGzipped(req, kWebPanelAppHtmlGz, sizeof(kWebPanelAppHtmlGz)) +#else + #define sendLoginPage(req) sendProgmem(req, kWebPanelLoginHtml) + #define sendStatsDisabledPage(req) sendProgmem(req, kWebPanelStatsDisabledHtml) + #define sendAppPage(req) sendProgmem(req, kWebPanelAppHtml) +#endif + esp_err_t sendJsonEscapedChunk(httpd_req_t* req, const char* src) { char chunk[48]; size_t offset = 0; @@ -3534,7 +3570,7 @@ esp_err_t WebPanelServer::handleIndex(httpd_req_t* req) { ctx->self->noteActivity(); httpd_resp_set_type(req, "text/html; charset=utf-8"); markCloseAfterSend(req); - return finishAndClose(req, sendProgmem(req, kWebPanelLoginHtml)); + return finishAndClose(req, sendLoginPage(req)); } esp_err_t WebPanelServer::handleHttpRedirect(httpd_req_t* req) { @@ -3561,7 +3597,7 @@ esp_err_t WebPanelServer::handleApp(httpd_req_t* req) { ctx->self->noteActivity(); httpd_resp_set_type(req, "text/html; charset=utf-8"); markCloseAfterSend(req); - return finishAndClose(req, sendProgmem(req, kWebPanelAppHtml)); + return finishAndClose(req, sendAppPage(req)); } esp_err_t WebPanelServer::handleStatsPage(httpd_req_t* req) { @@ -3573,10 +3609,10 @@ esp_err_t WebPanelServer::handleStatsPage(httpd_req_t* req) { httpd_resp_set_type(req, "text/html; charset=utf-8"); if (ctx->self->_runner != nullptr && !ctx->self->_runner->isWebStatsEnabled()) { markCloseAfterSend(req); - return finishAndClose(req, sendProgmem(req, kWebPanelStatsDisabledHtml)); + return finishAndClose(req, sendStatsDisabledPage(req)); } markCloseAfterSend(req); - return finishAndClose(req, sendProgmem(req, kWebPanelAppHtml)); + return finishAndClose(req, sendAppPage(req)); } esp_err_t WebPanelServer::handleOtaRedirect(httpd_req_t* req) {