wadamesh/scripts/build/i18n-audit.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""i18n audit: every TR("...") key in the touch UI must be covered by every
deploy/apps/lang/*.lang file (the canonical translation source — translators
edit those files and PR them; the firmware ships no baked-in table).

Run from the repo root:  python3 scripts/build/i18n-audit.py
Exit code 1 if any language is missing keys (prints them).
"""
import glob, os, re, sys

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

def unescape_c(s):
    """Decode a C string literal body to the exact runtime text: escapes become
    BYTES (like the compiler emits), normal chars their UTF-8 bytes, then the
    whole buffer decodes as UTF-8 — so "\xE2\x80\xA6" and a literal … agree."""
    out, i = bytearray(), 0
    while i < len(s):
        if s[i] == '\\' and i + 1 < len(s):
            n = s[i + 1]
            if n == 'n': out.append(0x0A); i += 2
            elif n == 't': out.append(0x09); i += 2
            elif n == 'x': out.append(int(s[i + 2:i + 4], 16)); i += 4
            else: out += n.encode('utf-8'); i += 2
        else:
            out += s[i].encode('utf-8'); i += 1
    return out.decode('utf-8', 'replace')

def source_keys():
    keys = set()
    for f in glob.glob(os.path.join(ROOT, 'src/ui-touch/*.cpp')) + \
             glob.glob(os.path.join(ROOT, 'src/ui-touch/*.h')):
        src = strip_comments(open(f, encoding='utf-8').read())
        # An optional LV_SYMBOL_* macro may precede the literal: TR(LV_SYMBOL_OK "  Text").
        # Requiring the group to START with a quote silently skipped every such
        # call - the audit's worst blind spot (the sheet rows never got checked).
        for m in re.finditer(r'TR\(\s*((?:LV_SYMBOL_[A-Z0-9_]+\s+)?(?:"(?:[^"\\]|\\.)*"\s*)+)\)', src):
            parts = re.findall(r'"((?:[^"\\]|\\.)*)"', m.group(1))
            k = unescape_c(''.join(parts))
            # strip icon glyphs + their spacer, exactly as TR() does at runtime
            while k and 0xE000 <= ord(k[0]) <= 0xF8FF:
                k = k[1:]
            k = k.lstrip(' ')
            if k.strip():
                keys.add(k)
    # INDIRECT keys: TR(kSomeTable[i]) / TR(tbl[i].field). The regex above only sees
    # TR("literal"), so every string reached through a table looked unused — including
    # "About", "Backups" and every other settings-category name. Anyone pruning the
    # language files on that output would have deleted live translations, which is
    # exactly what a translator asked about before doing it (#262).
    #
    # So: find the table names that appear inside a TR(...) subscript, then pull every
    # string literal out of each table's initialiser. Deliberately greedy — over-
    # collecting keeps a translation alive, under-collecting deletes one.
    for f in glob.glob(os.path.join(ROOT, 'src/ui-touch/*.cpp')) + \
             glob.glob(os.path.join(ROOT, 'src/ui-touch/*.h')):
        src = strip_comments(open(f, encoding='utf-8').read())
        for name in set(re.findall(r'TR\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*\[', src)):
            for m in re.finditer(r'\b' + re.escape(name) + r'\s*\[[^\]]*\]\s*=\s*\{', src):
                i, depth = m.end(), 1
                while i < len(src) and depth:
                    if src[i] == '{': depth += 1
                    elif src[i] == '}': depth -= 1
                    i += 1
                for lit in re.findall(r'"((?:[^"\\]|\\.)*)"', src[m.end():i]):
                    k = unescape_c(lit)
                    if k.strip():
                        keys.add(k)

    # HELPER-WRAPPED keys: mk_row_btn("Reload tiles in view", cb) and its ~20
    # siblings take a raw literal and call TR() on it *inside* the helper. At
    # runtime they translate correctly -- but the extractor above only ever saw
    # TR("literal"), so the literal at the CALL SITE was never emitted as a key,
    # so it never landed in a .lang file, so no translator could ever supply it.
    # That is why a whole map/options/settings sheet stayed English in every
    # language while looking perfectly wrapped in the source (#257).
    #
    # Two passes: find every function or lambda that calls TR() on one of its own
    # parameters and note which parameter, then pull the literal out of that
    # argument position at every call site.
    srcs = [strip_comments(open(f, encoding='utf-8').read()) for f in _sources()]
    helpers = _tr_wrapping_helpers(srcs)
    for src in srcs:
        for name, idx in helpers:
            for arg in _call_args_at(src, name, idx):
                for lit in literal_groups(arg):
                    k = unescape_c(lit)
                    while k and 0xE000 <= ord(k[0]) <= 0xF8FF:
                        k = k[1:]
                    k = k.lstrip(' ')
                    if k.strip():
                        keys.add(k)

    # RANGE-FOR over a local table: `struct {...} rows[] = {{"Show contacts", ...}};`
    # then `for (auto& r : rows) TR(r.label)`. Same blind spot as the helper case --
    # correct at runtime, invisible to a TR("literal") scan, so untranslatable.
    for f in _sources():
        src = strip_comments(open(f, encoding='utf-8').read())
        for m in re.finditer(r'for\s*\(\s*(?:const\s+)?auto\s*&?\s*(\w+)\s*:\s*(\w+)\s*\)', src):
            var, tbl = m.group(1), m.group(2)
            brace = src.find('{', m.end())
            if brace < 0 or brace - m.end() > 8:   # single-statement body, no block
                brace = -1
            body = src[m.end():_balanced(src, brace)] if brace >= 0 else src[m.end():m.end() + 400]
            if not re.search(r'\bTR\(\s*' + re.escape(var) + r'\s*(?:\.|->)', body):
                continue
            init = None
            for d in re.finditer(r'\b' + re.escape(tbl) + r'\s*\[[^\]]*\]\s*=\s*\{', src[:m.start()]):
                init = d
            if init:
                seg = src[init.end():_balanced(src, init.end() - 1)]
                for lit in re.findall(r'"((?:[^"\\]|\\.)*)"', seg):
                    k = unescape_c(lit)
                    if k.strip():
                        keys.add(k)

    # TR(someFunc(...)): a helper that RETURNS one of several literals, translated
    # by the caller. Every literal it can return is a key.
    for f in _sources():
        src = strip_comments(open(f, encoding='utf-8').read())
        for name in set(re.findall(r'TR\(\s*([A-Za-z_]\w*)\s*\(', src)):
            for d in re.finditer(r'\b' + re.escape(name) + r'\s*\([^;{}]*\)\s*\{', src):
                seg = src[d.end() - 1:_balanced(src, d.end() - 1)]
                for lit in re.findall(r'"((?:[^"\\]|\\.)*)"', seg):
                    k = unescape_c(lit)
                    if k.strip():
                        keys.add(k)
    # LUA APPS: wada.sys.tr("...") (aliased to a local `tr` by convention). The
    # apps ship from the same catalog and their strings belong in the same files.
    for f in sorted(glob.glob(os.path.join(ROOT, 'deploy/apps/*/*/*.lua'))):
        src = strip_comments(open(f, encoding='utf-8').read())
        for lit in re.findall(r'\b(?:sys\.)?tr\(\s*"((?:[^"\\]|\\.)*)"', src):
            k = unescape_c(lit)
            if k.strip():
                keys.add(k)
    return keys


def strip_comments(src):
    """Blank out //... and /*...*/ while preserving string/char literals and line
    count. The extractor scans raw text, so a quoted phrase inside a comment was
    being collected as a translatable key -- that is how the Dutch string
    "Geblokkeerde gebruikers", which only appears in a comment explaining how a
    long translation degrades, ended up as a KEY in all thirteen files."""
    out, i, n = [], 0, len(src)
    while i < n:
        c = src[i]
        if c == '"' or c == "'":
            q = c; out.append(c); i += 1
            while i < n:
                out.append(src[i])
                if src[i] == '\\' and i + 1 < n:
                    out.append(src[i + 1]); i += 2; continue
                if src[i] == q: i += 1; break
                i += 1
            continue
        if c == '/' and i + 1 < n and src[i + 1] == '/':
            while i < n and src[i] != '\n': i += 1
            continue
        if c == '/' and i + 1 < n and src[i + 1] == '*':
            i += 2
            while i + 1 < n and not (src[i] == '*' and src[i + 1] == '/'):
                if src[i] == '\n': out.append('\n')
                i += 1
            i += 2
            continue
        out.append(c); i += 1
    return ''.join(out)


def literal_groups(arg):
    """Keys from one call argument.

    A C compiler concatenates ADJACENT string literals, so `"a" "b"` is one
    string. `cond ? "a" : "b"` is two, and joining them (which this used to do)
    invented keys that exist nowhere: "Paste (move)Paste (copy)",
    "Unblock  Block", "Unfav  Favorite". Those shipped to every translator in
    v18. Only whitespace and an LV_SYMBOL_* macro may sit between literals of
    one group; anything else starts a new one."""
    groups, cur, i, n = [], [], 0, len(arg)
    while i < n:
        c = arg[i]
        if c == '"':
            j, buf = i + 1, []
            while j < n:
                if arg[j] == '\\' and j + 1 < n: buf.append(arg[j:j + 2]); j += 2; continue
                if arg[j] == '"': break
                buf.append(arg[j]); j += 1
            cur.append(''.join(buf)); i = j + 1; continue
        if c.isspace(): i += 1; continue
        m = re.match(r'LV_SYMBOL_[A-Z0-9_]+', arg[i:])
        if m: i += m.end(); continue
        if cur: groups.append(cur); cur = []
        i += 1
    if cur: groups.append(cur)
    return [''.join(g) for g in groups]


def _sources():
    return sorted(glob.glob(os.path.join(ROOT, 'src/ui-touch/*.cpp')) +
                  glob.glob(os.path.join(ROOT, 'src/ui-touch/*.h')))


def _balanced(src, i):
    """Index just past the '}' closing the block whose '{' is at src[i]."""
    depth = 0
    while i < len(src):
        if src[i] == '{': depth += 1
        elif src[i] == '}':
            depth -= 1
            if depth == 0: return i + 1
        i += 1
    return len(src)


def _split_args(s):
    """Top-level comma split, ignoring commas inside (), [], {} or a string."""
    out, cur, depth, q, i = [], [], 0, None, 0
    while i < len(s):
        c = s[i]
        if q:
            cur.append(c)
            if c == '\\': 
                if i + 1 < len(s): cur.append(s[i+1]); i += 1
            elif c == q: q = None
        elif c in '"\'': q = c; cur.append(c)
        elif c in '([{': depth += 1; cur.append(c)
        elif c in ')]}': depth -= 1; cur.append(c)
        elif c == ',' and depth == 0: out.append(''.join(cur)); cur = []
        else: cur.append(c)
        i += 1
    out.append(''.join(cur))
    return out


# A function/lambda header: a name, a parenthesised parameter list, then '{'.
_HDR = re.compile(r'(?:auto\s+(?P<lam>[A-Za-z_]\w*)\s*=\s*\[[^\]]*\]\s*'
                  r'|\b(?P<fn>[A-Za-z_]\w*)\s*)\((?P<args>[^;{}]*)\)\s*'
                  r'(?:const\s*)?(?:->\s*[\w:*&<> ]+\s*)?\{')


def _tr_wrapping_helpers(srcs):
    """(helper name, argument index) for every helper that TR()s a parameter.

    Transitive: a helper that hands its parameter, untouched, to a translated
    position of another helper translates it too. The settings kit is built that
    way (setSwitchRow passes its title and help line to setRowText, which TR()s
    them), and without this every switch, choice and slider name on the rebuilt
    pages looked untranslated-by-design and never reached a .lang file."""
    funcs = []
    for src in srcs:
        for m in _HDR.finditer(src):
            name = m.group('lam') or m.group('fn')
            if not name or name in ('if', 'for', 'while', 'switch', 'catch', 'TR'):
                continue
            params = []
            for a in _split_args(m.group('args')):
                names = re.findall(r'[A-Za-z_]\w*', a)
                params.append(names[-1] if names else '')
            body = src[m.end() - 1:_balanced(src, m.end() - 1)]
            calls = set(re.findall(r'\b([A-Za-z_]\w*)\s*\(', body))
            funcs.append((name, params, body, calls))
    found = set()
    for name, params, body, _ in funcs:
        for i, pn in enumerate(params):
            if pn and re.search(r'\bTR\(\s*' + re.escape(pn) + r'\s*\)', body):
                found.add((name, i))
    changed = True
    while changed:
        changed = False
        by_name = {}
        for h, k in found:
            by_name.setdefault(h, set()).add(k)
        for name, params, body, calls in funcs:
            for i, pn in enumerate(params):
                if not pn or (name, i) in found:
                    continue
                for h in calls & by_name.keys():
                    if any(a.strip() == pn for k in by_name[h] for a in _call_args_at(body, h, k)):
                        found.add((name, i))
                        changed = True
                        break
    return found


def _call_args_at(src, name, idx):
    """Every argument at position idx across all calls to name(...)."""
    out = []
    for m in re.finditer(r'\b' + re.escape(name) + r'\s*\(', src):
        i = m.end()
        depth, q, start = 1, None, i
        while i < len(src) and depth:
            c = src[i]
            if q:
                if c == '\\': i += 1
                elif c == q: q = None
            elif c in '"\'': q = c
            elif c == '(': depth += 1
            elif c == ')': depth -= 1
            i += 1
        args = _split_args(src[start:i - 1])
        if idx < len(args):
            out.append(args[idx])
    return out

# Known untranslatable / identical-everywhere keys.
SKIP = {'...', '😊', 'Snake', '© OpenStreetMap', '© OpenTopoMap'}

def lang_keys(path):
    have = set()
    for line in open(path, encoding='utf-8'):
        # Header lines have no tab; '#' can legitimately START a key (LVGL
        # recolor markup), so never treat a leading '#' as a comment marker.
        if '\t' not in line:
            continue
        k = line.split('\t', 1)[0]
        have.add(k.replace('\\n', '\n').replace('\\t', '\t').replace('\\\\', '\\'))
    return have

def main():
    keys = source_keys() - SKIP

    # --obsolete: the REVERSE check. Rows in the .lang files that no TR() key matches
    # any more, i.e. translation work that is being carried for strings the firmware
    # no longer shows. Reported, never deleted automatically.
    #
    # Read the caveat before acting on it: this is static analysis. A string reached in
    # a way the extractor cannot see reads as unreferenced, and pruning it silently
    # deletes a working translation. That is not hypothetical -- before the indirect-
    # table support above, "About" and every other settings-category name appeared in
    # this list. Treat the output as candidates to CHECK, not a delete list.
    if "--obsolete" in sys.argv:
        for path in sorted(glob.glob(os.path.join(ROOT, 'deploy/apps/lang/*.lang'))):
            have = lang_keys(path)
            dead = sorted(k for k in have if k not in keys)
            tag = os.path.basename(path)
            print(f"{tag}: {len(have)} rows, {len(dead)} unreferenced")
            for k in dead:
                print(f"  {k!r}")
        print("\nCandidates only. Grep the source for one before removing it: a string "
              "reached indirectly can look unreferenced while being on screen.")
        return 0

    bad = 0
    for path in sorted(glob.glob(os.path.join(ROOT, 'deploy/apps/lang/*.lang'))):
        missing = sorted(k for k in keys if k not in lang_keys(path))
        tag = os.path.basename(path)
        if missing:
            bad += 1
            print(f"{tag}: MISSING {len(missing)}")
            for k in missing:
                print(f"  {k!r}")
        else:
            print(f"{tag}: ok ({len(keys)} keys covered)")
    sys.exit(1 if bad else 0)

if __name__ == '__main__':
    main()