#!/usr/bin/env python3 """Collect static evidence about web internationalization architecture. The scanner reports source-level risks and review prompts. It does not prove linguistic quality, locale completeness, runtime behavior, legal suitability, or accessibility conformance. """ from __future__ import annotations import argparse import json import os import re import sys from collections import Counter, defaultdict from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Iterable, Sequence SEVERITY_RANK = {"critical": 4, "high": 3, "medium": 2, "low": 1, "info": 0} TEXT_EXTENSIONS = { ".css", ".scss", ".sass", ".less", ".html", ".htm", ".xhtml", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".vue", ".svelte", ".astro", ".php", ".cfm", ".cfc", ".jsp", ".jspx", ".asp", ".aspx", ".cshtml", ".vbhtml", ".razor", ".json", ".yaml", ".yml", ".xml", ".svg", ".md", ".txt", ".properties", ".po", ".pot", ".resx", ".ftl", } STYLE_EXTENSIONS = {".css", ".scss", ".sass", ".less", ".vue", ".svelte", ".astro"} MARKUP_EXTENSIONS = { ".html", ".htm", ".xhtml", ".php", ".cfm", ".jsp", ".jspx", ".asp", ".aspx", ".cshtml", ".vbhtml", ".razor", ".vue", ".svelte", ".astro", } SCRIPT_EXTENSIONS = {".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".vue", ".svelte", ".astro"} SKIP_DIRS = { ".git", ".hg", ".svn", "node_modules", "vendor", "dist", "build", "coverage", ".next", ".nuxt", ".svelte-kit", ".angular", ".cache", ".parcel-cache", "target", "bin", "obj", } REPORT_BASENAMES = { "internationalization-report.md", "internationalization-report.json", "i18n-report.md", "i18n-report.json", } PACKAGE_SIGNALS = { "i18next": "i18next", "react-i18next": "react-i18next", "next-i18next": "next-i18next", "@formatjs/intl": "FormatJS", "react-intl": "React Intl", "@lingui/core": "Lingui", "@lingui/react": "Lingui", "vue-i18n": "Vue I18n", "@angular/localize": "Angular Localize", "@fluent/bundle": "Project Fluent", "@fluent/react": "Project Fluent", "globalize": "Globalize", "jquery-i18n": "jQuery i18n", "moment": "Moment.js", "moment-timezone": "Moment Timezone", "dayjs": "Day.js", "luxon": "Luxon", "date-fns": "date-fns", "@js-temporal/polyfill": "Temporal polyfill", "numeral": "Numeral.js", "accounting": "accounting.js", } FRAMEWORK_SIGNALS = { "react": "React", "vue": "Vue", "@angular/core": "Angular", "svelte": "Svelte", "next": "Next.js", "nuxt": "Nuxt", "astro": "Astro", } I18N_PRIMARY = { "i18next", "react-i18next", "next-i18next", "@formatjs/intl", "react-intl", "@lingui/core", "@lingui/react", "vue-i18n", "@angular/localize", "@fluent/bundle", "@fluent/react", "globalize", "jquery-i18n", } RTL_LANGS = {"ar", "arc", "dv", "fa", "ha", "he", "khw", "ks", "ku", "ps", "sd", "ug", "ur", "yi"} LOCALE_RE = re.compile(r"^[A-Za-z]{2,3}(?:[-_][A-Za-z]{4})?(?:[-_](?:[A-Za-z]{2}|\d{3}))?(?:[-_][A-Za-z0-9]{4,8})*$") PLACEHOLDER_RE = re.compile(r"\{\{\s*([A-Za-z_][\w.-]*)\s*\}\}|\{\s*([A-Za-z_][\w.-]*)\s*(?:,|\})") BIDI_OVERRIDE_CHARS = {"\u202a", "\u202b", "\u202c", "\u202d", "\u202e"} BIDI_ISOLATE_CHARS = {"\u2066", "\u2067", "\u2068", "\u2069"} PHYSICAL_PROPERTIES = { "margin-left", "margin-right", "padding-left", "padding-right", "border-left", "border-right", "border-left-width", "border-right-width", "border-left-color", "border-right-color", "border-left-style", "border-right-style", "left", "right", "float", "clear", } @dataclass(frozen=True) class Finding: rule_id: str severity: str confidence: str area: str file: str line: int title: str evidence: str impact: str remediation: str manual_check: str = "" @dataclass class Catalog: path: Path locale: str namespace: str values: dict[str, str] @dataclass class Inventory: root: str files_scanned: int = 0 bytes_scanned: int = 0 markup_files: int = 0 style_files: int = 0 script_files: int = 0 catalog_files: int = 0 catalog_keys: int = 0 locales: list[str] | None = None rtl_locales: list[str] | None = None libraries: list[str] | None = None frameworks: list[str] | None = None package_versions: dict[str, str] | None = None intl_api_calls: int = 0 physical_css_declarations: int = 0 logical_css_declarations: int = 0 direction_signals: int = 0 pseudo_locale_signals: int = 0 test_signals: int = 0 def __post_init__(self) -> None: self.locales = self.locales or [] self.rtl_locales = self.rtl_locales or [] self.libraries = self.libraries or [] self.frameworks = self.frameworks or [] self.package_versions = self.package_versions or {} class Scanner: def __init__(self, root: Path, output_path: Path | None = None, max_bytes: int = 2_000_000) -> None: self.root = root.resolve() self.output_path = output_path.resolve() if output_path else None self.max_bytes = max_bytes self.findings: list[Finding] = [] self.inventory = Inventory(root=str(self.root)) self.contents: dict[Path, str] = {} self.lines: dict[Path, list[str]] = {} self.package_versions: dict[str, str] = {} self.catalogs: list[Catalog] = [] self.html_roots: list[tuple[Path, str, str | None, str | None]] = [] self.rtl_support_signals = 0 self.lang_signals = 0 self.locale_resolution_signals = 0 self.timezone_signals = 0 self.logical_css_signals = 0 self.test_signals = 0 self.pseudo_signals = 0 def rel(self, path: Path) -> str: try: return str(path.resolve().relative_to(self.root)).replace(os.sep, "/") except ValueError: return str(path) def add( self, rule_id: str, severity: str, confidence: str, area: str, path: Path, line: int, title: str, evidence: str, impact: str, remediation: str, manual_check: str = "", ) -> None: self.findings.append( Finding( rule_id=rule_id, severity=severity, confidence=confidence, area=area, file=self.rel(path), line=max(1, line), title=title, evidence=evidence.strip()[:500], impact=impact, remediation=remediation, manual_check=manual_check, ) ) def iter_files(self) -> Iterable[Path]: if self.root.is_file(): yield self.root return for path in self.root.rglob("*"): if not path.is_file(): continue if any(part in SKIP_DIRS for part in path.parts): continue if self.output_path and path.resolve() == self.output_path: continue if path.name.lower() in REPORT_BASENAMES: continue if path.suffix.lower() in TEXT_EXTENSIONS or path.name in { "package.json", "composer.json", "bower.json", "Gemfile", "requirements.txt", }: yield path def load_files(self) -> None: for path in self.iter_files(): try: size = path.stat().st_size raw = path.read_bytes() except OSError: continue if size > self.max_bytes: continue try: text = raw.decode("utf-8") except UnicodeDecodeError as exc: self.add( "I18N001", "high", "high", "encoding", path, 1, "Text file is not valid UTF-8", f"UTF-8 decoding failed near byte {exc.start}.", "Inconsistent encodings can corrupt translated text, user input, exports, and source code.", "Convert the file deliberately to UTF-8, verify round-trip integrity, and align source, transport, database, and build encodings.", ) text = raw.decode("utf-8", errors="replace") if text.startswith("# Internationalization UI static review"): continue if text.lstrip().startswith('{') and '"inventory"' in text[:300] and '"findings"' in text: continue self.contents[path] = text self.lines[path] = text.splitlines() self.inventory.files_scanned += 1 self.inventory.bytes_scanned += size if path.suffix.lower() in MARKUP_EXTENSIONS: self.inventory.markup_files += 1 if path.suffix.lower() in STYLE_EXTENSIONS: self.inventory.style_files += 1 if path.suffix.lower() in SCRIPT_EXTENSIONS: self.inventory.script_files += 1 def line_number(self, path: Path, offset: int) -> int: return self.contents[path].count("\n", 0, offset) + 1 def line_text(self, path: Path, line: int) -> str: rows = self.lines.get(path, []) if 1 <= line <= len(rows): return rows[line - 1].strip() return "" def scan_packages(self) -> None: for path, text in self.contents.items(): if path.name != "package.json": continue try: data = json.loads(text) except json.JSONDecodeError: continue deps: dict[str, Any] = {} for section in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"): value = data.get(section, {}) if isinstance(value, dict): deps.update(value) for name, version in deps.items(): if isinstance(version, str): self.package_versions[name] = version libraries = sorted({label for name, label in PACKAGE_SIGNALS.items() if name in deps}) frameworks = sorted({label for name, label in FRAMEWORK_SIGNALS.items() if name in deps}) self.inventory.libraries = sorted(set(self.inventory.libraries or []) | set(libraries)) self.inventory.frameworks = sorted(set(self.inventory.frameworks or []) | set(frameworks)) self.inventory.package_versions = dict(sorted(self.package_versions.items())) primary = sorted(name for name in I18N_PRIMARY if name in deps) if len(primary) > 1: self.add( "I18N002", "medium", "medium", "architecture", path, 1, "Multiple internationalization frameworks are installed", ", ".join(primary), "Competing providers can produce inconsistent locale resolution, plural rules, catalogs, and hydration behavior.", "Document ownership boundaries, consolidate where practical, and add contract tests across integration seams.", ) if "moment" in deps: self.add( "I18N003", "low", "high", "dates", path, 1, "Moment.js is present", f"moment: {deps.get('moment')}", "Moment remains usable but carries a large mutable API and a legacy maintenance posture that can complicate modernization.", "Confirm that Moment is still required. For new code, compare platform Intl, Temporal or an appropriate maintained library, then migrate incrementally with date/time characterization tests.", ) if self.root.is_dir(): package_files = [p for p in self.contents if p.name == "package.json"] if package_files: lock_names = {"package-lock.json", "npm-shrinkwrap.json", "yarn.lock", "pnpm-lock.yaml", "bun.lock", "bun.lockb"} found_lock = any((self.root / name).exists() for name in lock_names) if not found_lock: self.add( "I18N004", "low", "medium", "dependencies", package_files[0], 1, "No JavaScript lockfile was detected", "A package manifest exists without a repository-level lockfile.", "Locale data, plural rules, and time-zone behavior can change through transitive dependency updates.", "Use a supported lockfile and define an update process for locale and time-zone dependencies.", ) def scan_unicode_controls(self) -> None: source_like = SCRIPT_EXTENSIONS | STYLE_EXTENSIONS | {".json", ".yaml", ".yml", ".xml", ".properties", ".resx"} for path, text in self.contents.items(): if "\ufffd" in text: idx = text.index("\ufffd") self.add( "I18N005", "high", "high", "encoding", path, self.line_number(path, idx), "Unicode replacement character is present", self.line_text(path, self.line_number(path, idx)), "The replacement character often indicates prior lossy decoding or corrupted text.", "Trace the encoding boundary, recover the original content where possible, and add round-trip tests.", ) overrides = [(i, ch) for i, ch in enumerate(text) if ch in BIDI_OVERRIDE_CHARS] isolates = [(i, ch) for i, ch in enumerate(text) if ch in BIDI_ISOLATE_CHARS] if overrides: idx, ch = overrides[0] severity = "critical" if path.suffix.lower() in source_like else "high" self.add( "I18N006", severity, "high", "unicode-security", path, self.line_number(path, idx), "Bidirectional override control is present", f"Found U+{ord(ch):04X} in source text.", "Invisible direction overrides can reorder displayed source, identifiers, logs, or content and may enable spoofing.", "Verify that the control is intentional. Prefer structural direction markup or isolation; reject unexpected controls in security-sensitive source and identifiers.", ) if isolates and path.suffix.lower() in source_like and not self.is_catalog_candidate(path): idx, ch = isolates[0] self.add( "I18N007", "medium", "medium", "unicode-security", path, self.line_number(path, idx), "Bidirectional isolate control is embedded in source", f"Found U+{ord(ch):04X} in source text.", "Invisible isolates may be legitimate but are difficult to review and can hide direction-sensitive behavior.", "Prefer visible structural markup or escaped forms in source. Document any required literal control characters.", ) def scan_markup(self) -> None: for path, text in self.contents.items(): if path.suffix.lower() not in MARKUP_EXTENSIONS: continue lower = text.lower() for match in re.finditer(r"]*)>", text, flags=re.I | re.S): attrs = match.group(1) lang_match = re.search(r"\blang\s*=\s*['\"]([^'\"]+)['\"]", attrs, flags=re.I) dir_match = re.search(r"\bdir\s*=\s*['\"]([^'\"]+)['\"]", attrs, flags=re.I) lang = lang_match.group(1).strip() if lang_match else None direction = dir_match.group(1).strip().lower() if dir_match else None self.html_roots.append((path, attrs, lang, direction)) line = self.line_number(path, match.start()) if not lang: self.add( "I18N010", "high", "high", "language", path, line, "Document root has no language declaration", self.line_text(path, line), "Browsers, screen readers, spell checking, translation tools, and typography cannot reliably determine the page language.", "Set a valid BCP 47 `lang` value on the `html` element and override it on content in other languages.", ) else: self.lang_signals += 1 if "_" in lang or not LOCALE_RE.match(lang): self.add( "I18N011", "medium", "high", "language", path, line, "Language tag appears non-canonical or malformed", f"lang=\"{lang}\"", "Invalid or non-web language tags can break language matching, pronunciation, and locale tooling.", "Validate and canonicalize the value as a BCP 47 tag; use hyphens rather than underscores.", ) base = re.split(r"[-_]", lang)[0].lower() if base in RTL_LANGS and direction != "rtl": self.add( "I18N012", "high", "high", "rtl", path, line, "RTL document language is not paired with `dir=\"rtl\"`", f"lang=\"{lang}\", dir={direction!r}", "Language metadata alone does not establish the correct base direction for the interface.", "Set `dir=\"rtl\"` on the document or the relevant subtree and test layout, focus, icons, and mixed-direction content.", ) if direction: self.rtl_support_signals += 1 self.inventory.direction_signals += 1 if direction not in {"ltr", "rtl", "auto"}: self.add( "I18N013", "high", "high", "rtl", path, line, "Invalid HTML direction value", f"dir=\"{direction}\"", "Invalid direction metadata can leave bidirectional text and layout behavior undefined or inconsistent.", "Use only `ltr`, `rtl`, or `auto` as appropriate.", ) if "` element was found.", "Language and direction metadata may be absent from a standalone document or fragment used as a page.", "Confirm whether the file is a fragment. If it is a document, provide an `html` element with `lang` and appropriate `dir`.", ) for match in re.finditer(r"\blang\s*=\s*['\"]([^'\"]+)['\"]", text, flags=re.I): value = match.group(1).strip() if "_" in value: line = self.line_number(path, match.start()) self.add( "I18N015", "medium", "high", "language", path, line, "Language attribute uses an underscore", self.line_text(path, line), "HTML language tags use BCP 47 syntax with hyphens.", "Replace underscores with a validated canonical BCP 47 tag.", ) for match in re.finditer(r" None: for path, text in self.contents.items(): if path.suffix.lower() not in STYLE_EXTENSIONS: continue physical_hits: list[tuple[str, int, str]] = [] for match in re.finditer(r"(?m)^\s*([a-z-]+)\s*:\s*([^;]+);?", text, flags=re.I): prop = match.group(1).lower() value = match.group(2).strip() line = self.line_number(path, match.start()) if prop in PHYSICAL_PROPERTIES or (prop == "text-align" and value.lower() in {"left", "right"}): physical_hits.append((prop, line, value)) if prop.startswith(("margin-inline", "padding-inline", "border-inline", "inset-inline")) or prop in { "inline-size", "min-inline-size", "max-inline-size", "block-size", "min-block-size", "max-block-size", "text-align", "float", "clear", } and value.lower() in {"start", "end", "inline-start", "inline-end"}: self.logical_css_signals += 1 self.inventory.logical_css_declarations += 1 if prop == "direction": self.rtl_support_signals += 1 self.inventory.direction_signals += 1 self.add( "I18N020", "medium", "high", "rtl", path, line, "Text direction is controlled in CSS", self.line_text(path, line), "The HTML `dir` attribute carries semantics and influences the bidirectional algorithm in ways CSS direction alone does not replace.", "Prefer structural `dir` metadata. Keep CSS direction only for narrowly documented presentation cases.", ) if prop == "unicode-bidi" and re.search(r"(?:bidi-override|isolate-override)", value, flags=re.I): self.add( "I18N021", "high", "high", "bidi", path, line, "CSS bidirectional override is enabled", self.line_text(path, line), "Bidirectional overrides can make rendered text order diverge from logical text order and create security or usability risks.", "Use HTML direction and isolation. Reserve overrides for a documented representation requirement with security review.", ) if prop == "content" and re.search(r"['\"][^'\"]*[A-Za-z]{2,}[^'\"]*['\"]", value): self.add( "I18N022", "medium", "high", "messages", path, line, "Human-readable text is generated by CSS", self.line_text(path, line), "CSS-generated text is difficult to translate, expose consistently to assistive technology, and manage in catalogs.", "Move user-facing copy into document content or a localized component message.", ) if prop in {"white-space"} and "nowrap" in value.lower(): self.add( "I18N023", "medium", "medium", "layout", path, line, "Text wrapping is disabled", self.line_text(path, line), "Longer translations and unspaced scripts can overflow, clip, or force horizontal scrolling.", "Allow wrapping for ordinary labels and content, or document a safe overflow and full-text access strategy.", ) if prop in {"overflow", "overflow-x", "overflow-y"} and "hidden" in value.lower(): context = "\n".join(self.lines[path][max(0, line - 8): line + 4]) if re.search(r"(?:height|max-height|line-clamp|-webkit-line-clamp|white-space\s*:\s*nowrap)", context, flags=re.I): self.add( "I18N024", "medium", "medium", "layout", path, line, "Text may be clipped by a fixed or truncated container", self.line_text(path, line), "Translation expansion, script metrics, zoom, and increased text spacing can hide essential content.", "Use content-driven sizing or provide an accessible expansion mechanism and test representative locales at zoom.", ) if prop in {"height", "max-height"} and re.search(r"\b\d+(?:\.\d+)?px\b", value): context = "\n".join(self.lines[path][max(0, line - 4): line + 6]) if re.search(r"(?:overflow\s*:\s*hidden|line-clamp|white-space\s*:\s*nowrap)", context, flags=re.I): self.add( "I18N025", "high", "medium", "layout", path, line, "Fixed text height is combined with clipping", self.line_text(path, line), "Localized content can become inaccessible when it exceeds an English-sized container.", "Remove the fixed height or separate decorative geometry from a content-driven text region.", ) self.inventory.physical_css_declarations += len(physical_hits) if physical_hits: first_prop, first_line, first_value = physical_hits[0] severity = "high" if len(physical_hits) >= 12 else "medium" self.add( "I18N026", severity, "high", "rtl", path, first_line, "Physical CSS properties create directional coupling", f"{len(physical_hits)} physical declarations; first: {first_prop}: {first_value}", "Left/right assumptions increase RTL maintenance cost and can produce incomplete mirroring.", "Migrate compatible declarations to logical properties, then test exceptions such as charts, maps, media, and physical controls.", ) if re.search(r"transform\s*:\s*[^;]*scaleX\(\s*-1\s*\)", text, flags=re.I): match = re.search(r"transform\s*:\s*[^;]*scaleX\(\s*-1\s*\)", text, flags=re.I) assert match is not None line = self.line_number(path, match.start()) self.add( "I18N027", "medium", "medium", "rtl", path, line, "Content is mirrored with `scaleX(-1)`", self.line_text(path, line), "Global or poorly scoped transforms can mirror text, logos, media controls, maps, charts, and other non-directional content.", "Scope mirroring to semantic directional assets and verify focus, hit testing, raster quality, and assistive behavior.", ) if re.search(r"\[dir\s*=\s*['\"]?rtl", text, flags=re.I) or ":dir(rtl)" in text: self.rtl_support_signals += 1 self.inventory.direction_signals += 1 def scan_scripts(self) -> None: intl_pattern = re.compile(r"\bIntl\.(?:DateTimeFormat|NumberFormat|PluralRules|Collator|ListFormat|RelativeTimeFormat|DisplayNames|Segmenter|Locale|DurationFormat)\b") for path, text in self.contents.items(): if path.suffix.lower() not in SCRIPT_EXTENSIONS: continue lower = text.lower() intl_calls = list(intl_pattern.finditer(text)) self.inventory.intl_api_calls += len(intl_calls) if intl_calls: self.locale_resolution_signals += 1 for match in re.finditer(r"Intl\.[A-Za-z]+\s*\(\s*['\"]([A-Za-z]{2,3}(?:[-_][A-Za-z0-9]+)*)['\"]", text): locale = match.group(1) line = self.line_number(path, match.start()) if "_" in locale: self.add( "I18N030", "medium", "high", "formatting", path, line, "Intl formatter uses a locale tag with underscores", self.line_text(path, line), "ECMA-402 locale identifiers use BCP 47 syntax and may reject malformed tags.", "Use a validated canonical BCP 47 tag with hyphens.", ) if locale.lower() in {"en", "en-us"}: self.add( "I18N031", "medium", "medium", "formatting", path, line, "Formatter is hard-coded to English", self.line_text(path, line), "A fixed locale can ignore the user's formatting preferences and produce inconsistent server/client output.", "Resolve the effective formatting locale through the application locale contract, except for explicitly machine-oriented or fixed-language output.", ) for match in re.finditer(r"new\s+Date\s*\(\s*['\"](\d{1,2}[/-]\d{1,2}[/-]\d{2,4})['\"]\s*\)", text): line = self.line_number(path, match.start()) self.add( "I18N032", "high", "high", "dates", path, line, "Ambiguous localized date string is parsed by the platform", self.line_text(path, line), "Strings such as month/day/year and day/month/year are ambiguous and parsing behavior can vary.", "Parse a defined machine format or use explicit locale-aware structured input with validation.", ) manual_date = re.search(r"get(?:UTC)?(?:Date|Month|FullYear)\s*\(\)[^\n]{0,180}(?:join\s*\(|['\"][/.-]['\"])", text) if manual_date: line = self.line_number(path, manual_date.start()) self.add( "I18N033", "high", "medium", "dates", path, line, "Date appears to be formatted manually", self.line_text(path, line), "Manual field ordering, separators, month numbering, calendars, and time-zone handling are error-prone across locales.", "Use a locale-aware date formatter with an explicit value type and time zone.", ) for match in re.finditer(r"['\"]\s*(?:\$|EUR|USD|GBP|CHF|JPY)\s*['\"]\s*\+|\+\s*['\"]\s*(?:\$|EUR|USD|GBP|CHF|JPY)\s*['\"]", text): line = self.line_number(path, match.start()) self.add( "I18N034", "high", "medium", "currency", path, line, "Currency output appears to be assembled manually", self.line_text(path, line), "Currency symbol placement, spacing, digits, fraction rules, and accounting notation vary by locale and currency.", "Keep amount and ISO currency code structured and format with a locale-aware currency formatter.", ) for match in re.finditer(r"\.toFixed\s*\(\s*2\s*\)", text): line = self.line_number(path, match.start()) context = text[max(0, match.start() - 120): match.end() + 120] if re.search(r"(?:price|amount|currency|total|subtotal|tax|\$|EUR|USD|GBP|CHF|JPY)", context, flags=re.I): self.add( "I18N035", "medium", "medium", "currency", path, line, "Money display assumes two fraction digits", self.line_text(path, line), "Currencies and domain rules do not universally use two decimal places, and binary floating-point rounding may be unsuitable for settlement.", "Use a domain money model and currency-aware formatting with explicit rounding rules.", ) manual_plural = re.search(r"(?:count|qty|quantity|total)\s*(?:===|==)\s*1\s*\?[^:\n]+:[^\n]+", text, flags=re.I) if manual_plural: line = self.line_number(path, manual_plural.start()) self.add( "I18N036", "high", "high", "messages", path, line, "Pluralization is implemented as singular versus plural", self.line_text(path, line), "Languages can require zero, one, two, few, many, and other categories, and fractional values may follow different rules.", "Use CLDR-backed plural or MessageFormat rules and give translators complete branches.", ) concat_translation = re.search(r"(?:\bt\s*\([^\n]+\)|\bformatMessage\s*\([^\n]+\))\s*\+|\+\s*(?:\bt\s*\(|\bformatMessage\s*\()", text) if concat_translation: line = self.line_number(path, concat_translation.start()) self.add( "I18N037", "high", "medium", "messages", path, line, "Translated fragments are concatenated", self.line_text(path, line), "Word order, agreement, case, punctuation, and spacing cannot be translated safely from fragments.", "Replace the fragments with one complete parameterized message.", ) raw_html = re.search(r"dangerouslySetInnerHTML\s*=\s*\{\s*\{[^}]*__html\s*:\s*(?:t\s*\(|intl\.|i18n\.)|v-html\s*=\s*['\"][^'\"]*(?:\$t|t\()", text, flags=re.I) if raw_html: line = self.line_number(path, raw_html.start()) self.add( "I18N038", "high", "medium", "messages-security", path, line, "Localized content is rendered as raw HTML", self.line_text(path, line), "Translator-controlled markup can create injection, broken semantics, inaccessible ordering, and unstable component contracts.", "Use a constrained rich-message component model with escaped variables and an allowlist of semantic placeholders.", ) accept_lang = re.search(r"accept-language[^\n]{0,160}(?:split\s*\(\s*['\"],|substring|slice|indexOf)", text, flags=re.I) if accept_lang: line = self.line_number(path, accept_lang.start()) self.add( "I18N039", "high", "medium", "negotiation", path, line, "Accept-Language appears to be parsed manually", self.line_text(path, line), "Language ranges include quality weights and require matching against supported locales, not simple first-token selection.", "Use a BCP 47-aware negotiation library or a documented lookup implementation with validation and fallback telemetry.", ) if re.search(r"navigator\.(?:language|languages)", text): self.locale_resolution_signals += 1 if re.search(r"useEffect\s*\([^)]{0,200}navigator\.(?:language|languages)|DOMContentLoaded[^\n]{0,200}navigator\.(?:language|languages)", text, flags=re.I | re.S): match = re.search(r"useEffect\s*\([^)]{0,200}navigator\.(?:language|languages)|DOMContentLoaded[^\n]{0,200}navigator\.(?:language|languages)", text, flags=re.I | re.S) assert match is not None line = self.line_number(path, match.start()) self.add( "I18N040", "high", "medium", "ssr", path, line, "Initial locale is resolved after rendering begins", self.line_text(path, line), "Late locale resolution can cause translation flashes, incorrect `lang` or `dir`, hydration mismatches, and layout shift.", "Resolve and serialize the initial locale before first paint, then hydrate with the same state.", ) if re.search(r"(?:timeZone|timezone|tz)\b", text, flags=re.I): self.timezone_signals += 1 for match in re.finditer(r"['\"](?:EST|EDT|CST|CDT|MST|MDT|PST|PDT|IST|BST|CET|CEST)['\"]", text): line = self.line_number(path, match.start()) self.add( "I18N041", "medium", "medium", "time-zones", path, line, "Ambiguous time-zone abbreviation is hard-coded", self.line_text(path, line), "Time-zone abbreviations can be ambiguous and do not encode historical or daylight-saving rules.", "Use an IANA time-zone identifier for authoritative behavior and localize the display label separately.", ) for match in re.finditer(r"(?:username|userName|identifier|login|slug|handle|email)[^\n]{0,120}\.toLowerCase\s*\(", text, flags=re.I): line = self.line_number(path, match.start()) self.add( "I18N042", "medium", "low", "unicode-security", path, line, "Security-sensitive identifier may use UI lowercase conversion", self.line_text(path, line), "Locale-sensitive or simplistic case conversion can disagree across services and create uniqueness or spoofing problems.", "Define normalization, case folding, script policy, and uniqueness for the identifier class, then implement the same policy at every boundary.", ) grapheme = re.search(r"(?:maxLength|character|chars?|remaining|truncate|ellipsis)[^\n]{0,160}(?:\.length\b|\.slice\s*\(|\.substring\s*\(|\.substr\s*\()", text, flags=re.I) if grapheme: line = self.line_number(path, grapheme.start()) self.add( "I18N043", "medium", "medium", "unicode", path, line, "User-visible character logic may operate on code units", self.line_text(path, line), "Naive length and slicing can split combining sequences, emoji, flags, and complex-script grapheme clusters.", "Use grapheme-aware segmentation for counters and truncation, and define transport limits separately.", ) ascii_validation = re.search(r"/(?:\^)?\[A-Za-z(?:0-9)?[^\]]*\][^/]*/[gimy]*", text) if ascii_validation: context = text[max(0, ascii_validation.start() - 100): ascii_validation.end() + 100] if re.search(r"(?:name|address|city|company|username|search|title|message)", context, flags=re.I): line = self.line_number(path, ascii_validation.start()) self.add( "I18N044", "high", "medium", "input", path, line, "Text validation appears limited to ASCII letters", self.line_text(path, line), "Legitimate names, addresses, search terms, and content in other scripts can be rejected.", "Validate according to the actual domain requirement. Prefer Unicode-aware properties or allow broad text with targeted security controls.", ) word_regex = re.search(r"/[^/]*\\w[^/]*/[gimy]*", text) if word_regex: context = text[max(0, word_regex.start() - 100): word_regex.end() + 100] if re.search(r"(?:word|name|search|token|slug|username)", context, flags=re.I): line = self.line_number(path, word_regex.start()) self.add( "I18N045", "medium", "low", "unicode", path, line, "Word processing may rely on ASCII-oriented regex semantics", self.line_text(path, line), "`\\w` and simple word boundaries do not represent all scripts or locale-specific segmentation.", "Use Unicode-aware properties or segmentation APIs and test supported scripts.", ) if re.search(r"(?:pseudo|en-XA|ar-XB|qps-ploc|pseudolocal)", text, flags=re.I): self.pseudo_signals += 1 self.inventory.pseudo_locale_signals += 1 if re.search(r"(?:describe\s*\(|it\s*\(|test\s*\(|playwright|cypress|vitest|jest)", text): self.test_signals += 1 self.inventory.test_signals += 1 def is_catalog_candidate(self, path: Path) -> bool: if path.suffix.lower() != ".json" or path.name == "package.json": return False parts = [part.lower() for part in path.parts] if any(part in {"locale", "locales", "i18n", "l10n", "translations", "translation", "messages"} for part in parts): return True stem = path.stem return bool(LOCALE_RE.match(stem)) def locale_from_path(self, path: Path) -> tuple[str | None, str]: parts = list(path.parts) candidates: list[tuple[int, str]] = [] for idx, part in enumerate(parts[:-1]): if LOCALE_RE.match(part): candidates.append((idx, part)) if LOCALE_RE.match(path.stem): candidates.append((len(parts) - 1, path.stem)) if not candidates: return None, path.stem idx, locale = candidates[-1] locale = locale.replace("_", "-") if idx == len(parts) - 1: namespace = "default" else: rel_tail = parts[idx + 1:] if rel_tail: rel_tail[-1] = Path(rel_tail[-1]).stem namespace = "/".join(rel_tail) or "default" return locale, namespace def flatten_json(self, value: Any, prefix: str = "") -> dict[str, str]: result: dict[str, str] = {} if isinstance(value, dict): for key, child in value.items(): next_prefix = f"{prefix}.{key}" if prefix else str(key) result.update(self.flatten_json(child, next_prefix)) elif isinstance(value, list): for idx, child in enumerate(value): next_prefix = f"{prefix}[{idx}]" result.update(self.flatten_json(child, next_prefix)) elif isinstance(value, (str, int, float, bool)) or value is None: result[prefix] = "" if value is None else str(value) return result def scan_catalogs(self) -> None: for path, text in self.contents.items(): if not self.is_catalog_candidate(path): continue locale, namespace = self.locale_from_path(path) if not locale: continue try: data = json.loads(text) except json.JSONDecodeError as exc: self.add( "I18N050", "high", "high", "catalogs", path, exc.lineno, "Translation catalog is invalid JSON", exc.msg, "The locale or namespace may fail to load at runtime and trigger fallback or blank UI.", "Fix the JSON syntax and validate every catalog in CI.", ) continue if not isinstance(data, (dict, list)): continue values = self.flatten_json(data) catalog = Catalog(path=path, locale=locale, namespace=namespace, values=values) self.catalogs.append(catalog) self.inventory.catalog_files += 1 self.inventory.catalog_keys += len(values) base = locale.split("-")[0].lower() if base in RTL_LANGS: self.rtl_support_signals += 1 for key, value in values.items(): if not value.strip(): self.add( "I18N051", "high", "high", "catalogs", path, 1, "Translation value is empty", f"{key} = {value!r}", "An empty localized value can remove labels, instructions, accessible names, or transactional content.", "Provide a reviewed value or use an explicit fallback state that remains observable.", ) if re.search(r"<\/?[A-Za-z][^>]*>", value): self.add( "I18N052", "medium", "medium", "messages-security", path, 1, "Translation contains HTML markup", f"{key} = {value[:160]!r}", "Raw markup in catalogs can create injection, semantic, accessibility, and translator-tooling risks.", "Use a constrained rich-message model or verify that markup and interpolated variables are parsed and escaped safely.", ) if any(ch in BIDI_OVERRIDE_CHARS for ch in value): self.add( "I18N053", "high", "high", "unicode-security", path, 1, "Translation contains a bidirectional override control", f"Key: {key}", "Invisible overrides can reorder text and adjacent values in unsafe or confusing ways.", "Confirm linguistic necessity and prefer structural isolation or direction metadata.", ) self.inventory.locales = sorted({catalog.locale for catalog in self.catalogs}) self.inventory.rtl_locales = sorted({catalog.locale for catalog in self.catalogs if catalog.locale.split("-")[0].lower() in RTL_LANGS}) groups: dict[str, list[Catalog]] = defaultdict(list) for catalog in self.catalogs: groups[catalog.namespace].append(catalog) for namespace, catalogs in groups.items(): if len(catalogs) < 2: continue reference = max(catalogs, key=lambda item: len(item.values)) ref_keys = set(reference.values) for catalog in catalogs: if catalog is reference: continue keys = set(catalog.values) missing = sorted(ref_keys - keys) extra = sorted(keys - ref_keys) if missing: self.add( "I18N054", "high", "high", "catalogs", catalog.path, 1, "Translation catalog is missing keys", f"Namespace {namespace}; missing {len(missing)} keys compared with {reference.locale}: {', '.join(missing[:8])}", "Missing messages can expose fallback language, raw keys, blank controls, or failed rendering.", "Add reviewed translations or define a deliberate observable fallback. Validate key parity in CI.", ) if extra: self.add( "I18N055", "low", "high", "catalogs", catalog.path, 1, "Translation catalog has keys absent from the reference catalog", f"Namespace {namespace}; extra {len(extra)} keys: {', '.join(extra[:8])}", "Extra keys may indicate obsolete messages, an incorrect reference locale, or deployed consumers on a different version.", "Verify ownership and deployed consumers before removing or reclassifying the keys.", ) common = ref_keys & keys mismatches: list[str] = [] for key in common: ref_placeholders = {a or b for a, b in PLACEHOLDER_RE.findall(reference.values[key])} cur_placeholders = {a or b for a, b in PLACEHOLDER_RE.findall(catalog.values[key])} if ref_placeholders != cur_placeholders: mismatches.append(key) if mismatches: self.add( "I18N056", "high", "high", "catalogs", catalog.path, 1, "Message placeholder sets differ across locales", f"Namespace {namespace}; {len(mismatches)} mismatches: {', '.join(mismatches[:8])}", "Missing or renamed placeholders can cause runtime failures, lost data, or untranslated fragments.", "Validate message syntax and placeholder names against the source contract during build and translation import.", ) def scan_global(self) -> None: if self.catalogs and not self.html_roots: anchor = self.catalogs[0].path self.add( "I18N060", "medium", "medium", "language", anchor, 1, "Localized catalogs exist but no document language declaration was detected", f"Detected {len(self.catalogs)} catalog files and no `` in scanned markup.", "Rendered documents may omit language metadata even though translations are available.", "Verify the application shell, server templates, and generated documents set `lang` and direction before first paint.", ) rtl_locales = self.inventory.rtl_locales or [] if rtl_locales and self.rtl_support_signals == 0: anchor = next(c.path for c in self.catalogs if c.locale in rtl_locales) self.add( "I18N061", "high", "high", "rtl", anchor, 1, "RTL locale catalogs exist without detectable direction support", f"RTL locales: {', '.join(rtl_locales)}", "Arabic, Hebrew, Persian, Urdu, or other RTL experiences can render with incorrect base direction, layout, punctuation, and focus behavior.", "Implement root and subtree direction, logical layout, bidi isolation, directional asset rules, and RTL regression tests.", ) if self.inventory.physical_css_declarations >= 20 and self.inventory.logical_css_declarations == 0: style_path = next((p for p in self.contents if p.suffix.lower() in STYLE_EXTENSIONS), self.root) self.add( "I18N062", "high", "medium", "rtl", style_path, 1, "CSS is strongly coupled to physical left and right without logical-property signals", f"Detected {self.inventory.physical_css_declarations} physical declarations and no logical declarations.", "RTL support will require many fragile overrides and may remain incomplete.", "Introduce logical properties incrementally at component boundaries and document physical exceptions.", ) if self.catalogs and self.pseudo_signals == 0: self.add( "I18N063", "low", "medium", "testing", self.catalogs[0].path, 1, "No pseudo-localization signal was detected", "Localized catalogs exist without en-XA, ar-XB, qps-ploc, or pseudolocalization references.", "Hard-coded strings, clipping, expansion, encoding, and RTL assumptions may reach translators or production late.", "Add expanded LTR and RTL pseudo-locales to component, browser, and visual regression workflows.", ) if self.catalogs and self.test_signals == 0: self.add( "I18N064", "medium", "low", "testing", self.catalogs[0].path, 1, "No internationalization test tooling was detected", "Catalogs are present but common test signals were not found in scanned source.", "Locale routing, fallback, formatting, RTL, hydration, and catalog failures may regress silently.", "Add catalog validation, locale unit tests, representative browser tests, and production fallback telemetry.", ) if self.catalogs and self.locale_resolution_signals == 0 and not (self.inventory.libraries or []): self.add( "I18N065", "medium", "low", "architecture", self.catalogs[0].path, 1, "Locale resolution architecture is not detectable", "Catalogs exist without common Intl APIs, i18n libraries, or locale-resolution signals.", "The application may rely on implicit defaults or scattered custom logic.", "Document and centralize locale negotiation, preference precedence, fallback, and formatting services.", ) def run(self) -> tuple[Inventory, list[Finding]]: self.load_files() self.scan_packages() self.scan_unicode_controls() self.scan_markup() self.scan_styles() self.scan_scripts() self.scan_catalogs() self.scan_global() unique: dict[tuple[str, str, int, str], Finding] = {} for finding in self.findings: key = (finding.rule_id, finding.file, finding.line, finding.evidence) unique[key] = finding findings = sorted( unique.values(), key=lambda item: (-SEVERITY_RANK[item.severity], item.file, item.line, item.rule_id), ) return self.inventory, findings def markdown_report(inventory: Inventory, findings: Sequence[Finding]) -> str: counts = Counter(item.severity for item in findings) lines = [ "# Internationalization UI static review", "", "Static evidence only. This report does not prove linguistic quality, locale completeness, runtime behavior, legal suitability, or accessibility conformance.", "", "## Summary", "", f"- Root: `{inventory.root}`", f"- Files scanned: {inventory.files_scanned}", f"- Translation catalogs: {inventory.catalog_files}", f"- Catalog keys: {inventory.catalog_keys}", f"- Locales: {', '.join(inventory.locales or []) or 'none detected'}", f"- RTL locales: {', '.join(inventory.rtl_locales or []) or 'none detected'}", f"- Libraries: {', '.join(inventory.libraries or []) or 'none detected'}", f"- Frameworks: {', '.join(inventory.frameworks or []) or 'none detected'}", f"- Intl API calls: {inventory.intl_api_calls}", f"- Physical CSS declarations: {inventory.physical_css_declarations}", f"- Logical CSS declarations: {inventory.logical_css_declarations}", f"- Findings: {len(findings)}", f"- Severity: critical {counts['critical']}, high {counts['high']}, medium {counts['medium']}, low {counts['low']}", "", "## Findings", "", ] if not findings: lines.append("No source-level findings were detected. Manual multilingual, RTL, formatting, native-speaker, and assistive-technology testing is still required.") return "\n".join(lines) + "\n" for finding in findings: lines.extend([ f"### [{finding.severity.upper()}] {finding.title}", "", f"- Rule: `{finding.rule_id}`", f"- Confidence: {finding.confidence}", f"- Area: {finding.area}", f"- Location: `{finding.file}:{finding.line}`", f"- Evidence: `{finding.evidence.replace('`', "'")}`", f"- Impact: {finding.impact}", f"- Remediation: {finding.remediation}", ]) if finding.manual_check: lines.append(f"- Manual check: {finding.manual_check}") lines.append("") return "\n".join(lines) def json_report(inventory: Inventory, findings: Sequence[Finding]) -> str: payload = { "inventory": asdict(inventory), "summary": { "finding_count": len(findings), "severity": dict(Counter(item.severity for item in findings)), }, "findings": [asdict(item) for item in findings], } return json.dumps(payload, ensure_ascii=False, indent=2) + "\n" def parse_args(argv: Sequence[str]) -> argparse.Namespace: parser = argparse.ArgumentParser(description="Inspect a repository for internationalization and localization risks.") parser.add_argument("path", type=Path, help="File or directory to inspect") parser.add_argument("--format", choices=("markdown", "json"), default="markdown") parser.add_argument("--output", type=Path, help="Write the report to this path") parser.add_argument("--fail-on", choices=("critical", "high", "medium", "low", "info"), help="Exit 2 when a finding at or above this severity exists") parser.add_argument("--max-bytes", type=int, default=2_000_000, help="Maximum bytes read per text file") return parser.parse_args(argv) def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv or sys.argv[1:]) if not args.path.exists(): print(f"Path does not exist: {args.path}", file=sys.stderr) return 1 scanner = Scanner(args.path, output_path=args.output, max_bytes=args.max_bytes) inventory, findings = scanner.run() report = markdown_report(inventory, findings) if args.format == "markdown" else json_report(inventory, findings) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(report, encoding="utf-8") else: sys.stdout.write(report) if args.fail_on: threshold = SEVERITY_RANK[args.fail_on] if any(SEVERITY_RANK[item.severity] >= threshold for item in findings): return 2 return 0 if __name__ == "__main__": raise SystemExit(main())