translations fixed to be safe

web/src/i18n/catalogue.rs

@@ -0,0 +1,277 @@
+//! What the `.ftl` catalogues have to hold true, checked against the files
+//! themselves rather than against what a reviewer remembers of them.
+//!
+//! Key parity and duplicates are the ordinary half. The other half guards the
+//! one place where catalogue text is trusted as markup:
+//!
+//! A handful of templates render a translation with Askama's `|safe`, because
+//! the message carries its own `<strong>` or `<code>`. That makes those
+//! messages the only strings in the catalogue that reach a page as HTML rather
+//! than as text — and the `sk` and `cs` catalogues were machine-translated, so
+//! "the translator would not write a `<script>`" is not something anyone
+//! checked. These tests check it, for every locale, every time they run.
+//!
+//! Two properties, together:
+//!
+//! 1. A message rendered `|safe` may contain only the tags below, balanced.
+//! 2. Every *other* message contains no markup at all, so moving a message to a
+//!    `|safe` site later cannot quietly bring markup along with it.
+
+use std::{collections::BTreeSet, fs, path::{Path, PathBuf}};
+
+/// The tags a translator may use: inline emphasis, plus links to elsewhere in
+/// this app. Nothing that can carry script, load a remote resource, or open an
+/// interactive control.
+const ALLOWED_TAGS: [&str; 4] = ["strong", "code", "em", "a"];
+
+const LOCALES: [&str; 3] = ["en", "sk", "cs"];
+
+fn crate_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+}
+
+/// Every message key some template renders with `|safe`.
+///
+/// Read out of the templates rather than listed here, so the test cannot drift
+/// from the markup it is guarding.
+fn keys_rendered_as_html() -> BTreeSet<String> {
+    let mut keys = BTreeSet::new();
+    let mut files = Vec::new();
+    collect_html(&crate_dir().join("templates"), &mut files);
+    assert!(!files.is_empty(), "found no templates to scan");
+
+    for file in files {
+        let markup = fs::read_to_string(&file).expect("template is readable");
+        for expression in markup.split("{{").skip(1) {
+            let Some(expression) = expression.split("}}").next() else {
+                continue;
+            };
+            if !expression.contains("|safe") {
+                continue;
+            }
+            // `nav.tr("k")`, `nav.tr_args("k", …)`, `locale.lookup("k")`: the
+            // key is the first quoted string of the call either way.
+            if let Some(key) = first_quoted(expression) {
+                keys.insert(key);
+            }
+        }
+    }
+    keys
+}
+
+fn collect_html(directory: &Path, found: &mut Vec<PathBuf>) {
+    for entry in fs::read_dir(directory).expect("template directory is readable") {
+        let path = entry.expect("directory entry is readable").path();
+        if path.is_dir() {
+            collect_html(&path, found);
+        } else if path.extension().is_some_and(|extension| extension == "html") {
+            found.push(path);
+        }
+    }
+}
+
+fn first_quoted(expression: &str) -> Option<String> {
+    let rest = expression.split_once('"')?.1;
+    Some(rest.split_once('"')?.0.to_string())
+}
+
+/// Every `key = value` in one catalogue, with continuation lines joined.
+///
+/// Deliberately a small hand parser rather than a Fluent one: the point is to
+/// look at the *source text* a translator wrote, before any placeable is
+/// resolved.
+fn messages(locale: &str) -> Vec<(String, String)> {
+    let path = crate_dir().join("locales").join(locale).join("main.ftl");
+    let catalogue = fs::read_to_string(&path)
+        .unwrap_or_else(|_| panic!("catalogue {} is readable", path.display()));
+
+    let mut messages: Vec<(String, String)> = Vec::new();
+    for line in catalogue.lines() {
+        let trimmed = line.trim_start();
+        if trimmed.is_empty() || trimmed.starts_with('#') {
+            continue;
+        }
+        let is_continuation = line.starts_with(' ') || line.starts_with('\t');
+        if is_continuation {
+            if let Some(last) = messages.last_mut() {
+                last.1.push(' ');
+                last.1.push_str(trimmed);
+            }
+            continue;
+        }
+        if let Some((key, value)) = line.split_once(" = ") {
+            messages.push((key.trim().to_string(), value.trim().to_string()));
+        }
+    }
+    assert!(!messages.is_empty(), "catalogue {locale} parsed as empty");
+    messages
+}
+
+/// Splits a message's `<…>` spans out, returning them in order.
+///
+/// A `<` that no name follows is a literal less-than sign — `ecb-less-than-1s`
+/// is the string `<1s` — and is not markup.
+fn tags(value: &str) -> Vec<String> {
+    let mut tags = Vec::new();
+    let mut rest = value;
+    while let Some((_, after)) = rest.split_once('<') {
+        if !after.starts_with(|character: char| character.is_ascii_alphabetic() || character == '/')
+        {
+            rest = after;
+            continue;
+        }
+        match after.split_once('>') {
+            Some((tag, remainder)) => {
+                tags.push(tag.trim().to_string());
+                rest = remainder;
+            }
+            None => break,
+        }
+    }
+    tags
+}
+
+/// Whether a tag's attributes are ones a translation may carry.
+///
+/// Only `<a href="…">`, and only to a path inside this app: a `javascript:` or
+/// off-site `href` in a machine-translated catalogue would be a redirect —
+/// or a script — that no reviewer of the `.ftl` diff is likely to notice.
+fn attributes_are_allowed(name: &str, attributes: &str) -> bool {
+    if attributes.is_empty() {
+        return true;
+    }
+    if name != "a" {
+        return false;
+    }
+    attributes
+        .strip_prefix("href=\"/")
+        .and_then(|target| target.strip_suffix('"'))
+        .is_some_and(|target| !target.starts_with('/'))
+}
+
+#[test]
+fn messages_rendered_as_html_use_only_balanced_inline_tags() {
+    let safe_keys = keys_rendered_as_html();
+    assert!(
+        !safe_keys.is_empty(),
+        "no `|safe` translations found — has the scan stopped matching?"
+    );
+
+    for locale in LOCALES {
+        for (key, value) in messages(locale) {
+            if !safe_keys.contains(&key) {
+                continue;
+            }
+            let mut open: Vec<String> = Vec::new();
+            for tag in tags(&value) {
+                let (tag, is_closing) = match tag.strip_prefix('/') {
+                    Some(name) => (name.to_string(), true),
+                    None => (tag.clone(), false),
+                };
+                let (name, attributes) = match tag.split_once(char::is_whitespace) {
+                    Some((name, attributes)) => (name.to_string(), attributes.trim().to_string()),
+                    None => (tag.clone(), String::new()),
+                };
+                assert!(
+                    ALLOWED_TAGS.contains(&name.as_str()),
+                    "{locale}/{key} renders as HTML and uses <{name}>, \
+                     which is not one of {ALLOWED_TAGS:?}"
+                );
+                assert!(
+                    attributes_are_allowed(&name, &attributes),
+                    "{locale}/{key} renders as HTML and its <{name}> carries \
+                     `{attributes}`; only a same-app `href=\"/…\"` is allowed"
+                );
+                if is_closing {
+                    assert_eq!(
+                        open.pop().as_deref(),
+                        Some(name.as_str()),
+                        "{locale}/{key} closes </{name}> that is not open"
+                    );
+                } else {
+                    open.push(name);
+                }
+            }
+            assert!(
+                open.is_empty(),
+                "{locale}/{key} leaves {open:?} unclosed"
+            );
+        }
+    }
+}
+
+#[test]
+fn messages_not_rendered_as_html_contain_no_markup() {
+    let safe_keys = keys_rendered_as_html();
+
+    for locale in LOCALES {
+        for (key, value) in messages(locale) {
+            if safe_keys.contains(&key) {
+                continue;
+            }
+            let found = tags(&value);
+            assert!(
+                found.is_empty(),
+                "{locale}/{key} contains {found:?} but is rendered as text, so \
+                 the tags would show as escaped source; drop them, or move the \
+                 message to a `|safe` site deliberately"
+            );
+        }
+    }
+}
+
+/// English is the source language and the fallback, so a key it lacks is a key
+/// nothing can render, and a key only *it* has is a page that silently falls
+/// back to English for a Slovak or Czech reader.
+#[test]
+fn every_catalogue_defines_the_same_keys() {
+    let english: BTreeSet<String> = messages("en").into_iter().map(|(key, _)| key).collect();
+
+    for locale in LOCALES {
+        if locale == "en" {
+            continue;
+        }
+        let translated: BTreeSet<String> =
+            messages(locale).into_iter().map(|(key, _)| key).collect();
+
+        let untranslated: Vec<&String> = english.difference(&translated).collect();
+        assert!(
+            untranslated.is_empty(),
+            "{locale} is missing {untranslated:?}"
+        );
+        let orphaned: Vec<&String> = translated.difference(&english).collect();
+        assert!(
+            orphaned.is_empty(),
+            "{locale} defines {orphaned:?}, which English does not"
+        );
+    }
+}
+
+/// A key defined twice silently keeps one of the two. Fluent does not complain.
+#[test]
+fn no_catalogue_defines_a_key_twice() {
+    for locale in LOCALES {
+        let mut seen = BTreeSet::new();
+        for (key, _) in messages(locale) {
+            assert!(seen.insert(key.clone()), "{locale} defines `{key}` twice");
+        }
+    }
+}
+
+/// The `|safe` keys have to exist in every catalogue, or a missing one falls
+/// back to the sentinel — which is fine as text and misleading as markup.
+#[test]
+fn every_key_rendered_as_html_exists_in_every_catalogue() {
+    let safe_keys = keys_rendered_as_html();
+
+    for locale in LOCALES {
+        let defined: BTreeSet<String> =
+            messages(locale).into_iter().map(|(key, _)| key).collect();
+        for key in &safe_keys {
+            assert!(
+                defined.contains(key),
+                "{locale} has no `{key}`, which a template renders as HTML"
+            );
+        }
+    }
+}

web/src/i18n/mod.rs

@@ -13,6 +13,12 @@ static_loader! {
     static LOCALES = {
         locales: "./locales",
         fallback_language: "en",
+        // Fluent wraps every interpolated value in U+2068/U+2069 by default, so
+        // that a right-to-left name cannot reorder the sentence around it. All
+        // three supported languages are left-to-right, and the marks are not
+        // free: they reach the page as invisible characters inside table and
+        // column names, which then fail to match anything a user copies out.
+        customise: |bundle| bundle.set_use_isolating(false),
     };
 }
 
@@ -42,12 +48,22 @@ impl Locale {
             .filter_map(|entry| {
                 let mut parts = entry.trim().split(';');
                 let language = Self::from_language_tag(parts.next()?.trim())?;
-                let quality = parts.find_map(|parameter| {
-                    parameter
-                        .trim()
-                        .strip_prefix("q=")
-                        .and_then(|value| value.parse::<f32>().ok())
-                }).unwrap_or(1.0);
+                // An entry with no `q=` at all is weight 1.0. An entry that
+                // *has* one the parser cannot read is unacceptable, not
+                // most-preferred (RFC 9110 §12.4.2) — otherwise a junk qvalue
+                // outranks every honest preference in the header.
+                let quality = match parts
+                    .find_map(|parameter| parameter.trim().strip_prefix("q="))
+                {
+                    Some(value) => value
+                        .parse::<f32>()
+                        .ok()
+                        // A qvalue is a number in 0..=1; anything outside that
+                        // is as malformed as text is.
+                        .filter(|quality| (0.0..=1.0).contains(quality))
+                        .unwrap_or(0.0),
+                    None => 1.0,
+                };
                 (quality > 0.0).then_some((language, quality))
             })
             // Keep the first language when qualities are equal: header order
@@ -83,16 +99,14 @@ impl Locale {
             .expect("supported locale codes are valid language identifiers")
     }
 
-    /// Looks up a Fluent message, falling back to English. Unknown keys remain
-    /// visible during development instead of silently rendering an empty label.
+    /// Looks up a Fluent message, falling back to English.
     pub(crate) fn lookup(self, key: &str) -> String {
         LOCALES
             .try_lookup(&self.language_identifier(), key)
-            .unwrap_or_else(|| format!("⟪{key}⟫"))
+            .unwrap_or_else(|| missing(key))
     }
 
     /// Looks up a message containing Fluent variables such as `{ $count }`.
-    #[allow(dead_code)]
     pub(crate) fn lookup_args(
         self,
         key: &str,
@@ -100,7 +114,21 @@ impl Locale {
     ) -> String {
         LOCALES
             .try_lookup_with_args(&self.language_identifier(), key, args)
-            .unwrap_or_else(|| format!("⟪{key}⟫"))
+            .unwrap_or_else(|| missing(key))
+    }
+}
+
+/// What an unknown key renders as.
+///
+/// In development it is loud on purpose, so a key that no catalogue defines is
+/// caught by looking at the page. A release build must not put that on a user's
+/// screen, so it degrades to the key's own words: `td-pick-table-delete` reads
+/// as "td pick table delete" — wrong, but a label rather than debug output.
+fn missing(key: &str) -> String {
+    if cfg!(debug_assertions) {
+        format!("⟪{key}⟫")
+    } else {
+        key.replace(['-', '_'], " ")
     }
 }
 
@@ -129,6 +157,9 @@ pub(crate) mod fluent_value {
     pub use fluent_templates::fluent_bundle::FluentValue;
 }
 
+#[cfg(test)]
+mod catalogue;
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -145,8 +176,52 @@ mod tests {
         assert_eq!(Locale::from_headers(&headers), Locale::Slovak);
     }
 
+    /// A `q=` the parser cannot read means "not acceptable", so it must not
+    /// beat a language the browser actually asked for.
+    #[test]
+    fn a_malformed_qvalue_does_not_outrank_a_real_preference() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::ACCEPT_LANGUAGE,
+            HeaderValue::from_static("cs;q=banana, sk;q=0.4"),
+        );
+
+        assert_eq!(Locale::from_headers(&headers), Locale::Slovak);
+    }
+
+    #[test]
+    fn an_out_of_range_qvalue_is_rejected_rather_than_trusted() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::ACCEPT_LANGUAGE,
+            HeaderValue::from_static("cs;q=999, sk;q=0.4"),
+        );
+
+        assert_eq!(Locale::from_headers(&headers), Locale::Slovak);
+    }
+
+    /// No `q=` at all still means "most preferred".
+    #[test]
+    fn a_missing_qvalue_is_full_weight() {
+        let mut headers = HeaderMap::new();
+        headers.insert(
+            header::ACCEPT_LANGUAGE,
+            HeaderValue::from_static("sk;q=0.9, cs"),
+        );
+
+        assert_eq!(Locale::from_headers(&headers), Locale::Czech);
+    }
+
     #[test]
     fn unknown_messages_are_visible_during_development() {
-        assert_eq!(Locale::Czech.lookup("missing-key"), "⟪missing-key⟫");
+        let rendered = Locale::Czech.lookup("missing-key");
+
+        if cfg!(debug_assertions) {
+            assert_eq!(rendered, "⟪missing-key⟫");
+        } else {
+            // Loud in development, but never on a user's screen in a release
+            // build.
+            assert_eq!(rendered, "missing key");
+        }
     }
 }