//! Edit classification — a typo fix is not a decision. //! //! Once a proposal is under vote, its author may still correct it. The engine //! tells two kinds of corrections apart: //! //! - **typographic**: accents, case, punctuation, spacing, emphasis, or at most //! two misspelt words fixed (edit distance ≤ 2, no negation touched) — and //! never a figure, a sign, a separator inside a number or a relation //! (`≥`, `+`, `%`…). The votes stay; the correction is shown with its diff. //! - **substantive**: anything else. It opens a new revision and every vote //! cast on the previous one waits for its author to reconfirm it. //! //! The rule is deliberately conservative: when in doubt, it is substantive. use serde::{Deserialize, Serialize}; use unicode_normalization::UnicodeNormalization; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum EditKind { Unchanged, Typographic, Substantive, } /// Words whose presence flips a meaning; touching one is never a typo. const NEGATIONS: &[&str] = &[ // fr "ne", "n", "pas", "jamais", "non", "sans", "aucun", "aucune", "ni", "plus", "rien", "nul", "nulle", "personne", // en "not", "no", "never", "without", "none", "nor", // es "nunca", "sin", "ningun", "ninguna", "ninguno", "tampoco", "nada", ]; const MAX_TYPO_WORDS: usize = 2; const MAX_TYPO_DISTANCE: usize = 2; /// Lowercase, strip diacritics. fn fold(s: &str) -> String { s.nfd().filter(|c| !is_combining(*c)).flat_map(char::to_lowercase).collect() } fn is_combining(c: char) -> bool { matches!(c as u32, 0x0300..=0x036F | 0x1AB0..=0x1AFF | 0x1DC0..=0x1DFF | 0x20D0..=0x20FF | 0xFE20..=0xFE2F) } /// Letters and digits only, folded: what survives every typographic change. fn skeleton(s: &str) -> String { fold(s).chars().filter(|c| c.is_alphanumeric()).collect() } /// Symbols that carry a value or a relation, wherever they stand. const RELATIONS: &[char] = &['+', '±', '×', '÷', '^', '<', '>', '=', '≤', '≥', '≠', '≈', '%', '‰', '€', '$', '£', '∞', '√']; /// Signs and fraction bars: meaningful next to a figure ("-5", "1/5", "5-10"). const SIGNS: &[char] = &['-', '−', '–', '/']; /// Separators: meaningful between two figures ("10 000", "10,000", "182.625"). const SEPARATORS: &[char] = &['.', ',', ' ', '\u{a0}', '\u{202f}', '\'', '’']; /// What the text says in figures and symbols, as written. Two texts with the /// same letters but a different figure, sign, separator inside a number or /// relation ("≥" → "≤", "+" → "-", "50" → "5,0", "182.625" → "182 625") do not /// say the same thing. fn figures(s: &str) -> String { let cs: Vec = s.chars().collect(); let is_figure = |i: Option| i.and_then(|i| cs.get(i)).is_some_and(|c| c.is_numeric()); cs.iter() .enumerate() .filter(|&(i, c)| { let (before, after) = (is_figure(i.checked_sub(1)), is_figure(Some(i + 1))); c.is_numeric() || RELATIONS.contains(c) || (SIGNS.contains(c) && (before || after)) || (SEPARATORS.contains(c) && before && after) }) .map(|(_, c)| *c) .collect() } /// Folded words (runs of letters/digits). fn words(s: &str) -> Vec { fold(s).split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).map(str::to_owned).collect() } pub fn classify(old: &str, new: &str) -> EditKind { if old == new { return EditKind::Unchanged; } // A figure, a sign or a relation is never a typo: it changes what the text says. if figures(old) != figures(new) { return EditKind::Substantive; } if skeleton(old) == skeleton(new) { return EditKind::Typographic; } let (a, b) = (words(old), words(new)); if a.len() != b.len() { return EditKind::Substantive; } let mut changed = 0; for (x, y) in a.iter().zip(&b) { if x == y { continue; } changed += 1; let touches_meaning = changed > MAX_TYPO_WORDS || x.chars().any(|c| c.is_ascii_digit()) || y.chars().any(|c| c.is_ascii_digit()) || NEGATIONS.contains(&x.as_str()) || NEGATIONS.contains(&y.as_str()) || x.chars().count().min(y.chars().count()) < 3 || levenshtein(x, y) > MAX_TYPO_DISTANCE; if touches_meaning { return EditKind::Substantive; } } EditKind::Typographic } pub fn levenshtein(a: &str, b: &str) -> usize { let b: Vec = b.chars().collect(); let mut prev: Vec = (0..=b.len()).collect(); let mut cur = vec![0; b.len() + 1]; for (i, ca) in a.chars().enumerate() { cur[0] = i + 1; for (j, cb) in b.iter().enumerate() { let cost = usize::from(ca != *cb); cur[j + 1] = (prev[j] + cost).min(prev[j + 1] + 1).min(cur[j] + 1); } std::mem::swap(&mut prev, &mut cur); } prev[b.len()] } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum DiffOp { Same, Added, Removed, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct DiffPart { pub op: DiffOp, pub text: String, } /// Tokens that keep every character: words, and the runs between them. fn tokens(s: &str) -> Vec<&str> { let mut out = Vec::new(); let mut start = 0; let mut in_word: Option = None; for (i, c) in s.char_indices() { let w = c.is_alphanumeric(); if in_word.is_some_and(|prev| prev != w) { out.push(&s[start..i]); start = i; } in_word = Some(w); } if start < s.len() { out.push(&s[start..]); } out } const MAX_DIFF_TOKENS: usize = 4000; /// Word-level diff for display. Concatenating `Same` + `Added` gives `new`, /// `Same` + `Removed` gives `old`. pub fn diff(old: &str, new: &str) -> Vec { let (a, b) = (tokens(old), tokens(new)); if a.len() * b.len() > MAX_DIFF_TOKENS * MAX_DIFF_TOKENS / 4 { return vec![ DiffPart { op: DiffOp::Removed, text: old.to_owned() }, DiffPart { op: DiffOp::Added, text: new.to_owned() }, ]; } // LCS table, suffix form. let (n, m) = (a.len(), b.len()); let mut lcs = vec![vec![0u32; m + 1]; n + 1]; for i in (0..n).rev() { for j in (0..m).rev() { lcs[i][j] = if a[i] == b[j] { lcs[i + 1][j + 1] + 1 } else { lcs[i + 1][j].max(lcs[i][j + 1]) }; } } let mut parts: Vec = Vec::new(); let mut push = |op: DiffOp, text: &str| match parts.last_mut() { Some(last) if last.op == op => last.text.push_str(text), _ => parts.push(DiffPart { op, text: text.to_owned() }), }; let (mut i, mut j) = (0, 0); while i < n && j < m { if a[i] == b[j] { push(DiffOp::Same, a[i]); i += 1; j += 1; } else if lcs[i + 1][j] >= lcs[i][j + 1] { push(DiffOp::Removed, a[i]); i += 1; } else { push(DiffOp::Added, b[j]); j += 1; } } for t in &a[i..] { push(DiffOp::Removed, t); } for t in &b[j..] { push(DiffOp::Added, t); } parts } #[cfg(test)] mod tests { use super::*; #[test] fn the_glued_word_of_the_seed_is_a_typo() { let old = "Toute Certification doit s'accompagner de latransmission de ce document"; let new = "Toute Certification doit s'accompagner de la transmission de ce document"; assert_eq!(classify(old, new), EditKind::Typographic); } #[test] fn accents_case_punctuation_emphasis() { assert_eq!(classify("Je m'engage a verifier.", "Je m’engage à vérifier !"), EditKind::Typographic); assert_eq!(classify("un **être humain**", "un être humain"), EditKind::Typographic); assert_eq!(classify("same", "same"), EditKind::Unchanged); } #[test] fn a_figure_a_sign_or_a_relation_is_never_a_typo() { // Same letters, different meaning (independent reviews, 2026-10-21). for (old, new) in [ ("au moins 50 voix", "au moins 5,0 voix"), ("1 5 jours", "15 jours"), ("10 000 membres", "10,000 membres"), ("un écart de -5", "un écart de 5"), ("1/5 des voix", "1.5 des voix"), ("50 % des voix", "50 ‰ des voix"), ("avoir >= 5 certifications", "avoir <= 5 certifications"), ("DU(n) + c² × M(n)", "DU(n) - c² × M(n)"), ("182.625 jours", "182 625 jours"), ("c²", "c2"), ] { assert_eq!(classify(old, new), EditKind::Substantive, "{old} → {new}"); } // Punctuation around figures, list markers, dashes between words stay typographic. assert_eq!(classify("50 voix pour", "50 voix pour !"), EditKind::Typographic); assert_eq!(classify("- une liste", "* une liste"), EditKind::Typographic); assert_eq!(classify("1. premier point", "1) premier point"), EditKind::Typographic); assert_eq!(classify("libre - le texte", "libre — le texte"), EditKind::Typographic); } #[test] fn misspelt_words() { assert_eq!(classify("un certifieurs existant", "un certifieur existant"), EditKind::Typographic); assert_eq!(classify("toute certifcation", "toute certification"), EditKind::Typographic); // « personne » is also a negation (ne … personne): conservative. assert_eq!(classify("la persone", "la personne"), EditKind::Substantive); } #[test] fn meaning_changes() { // A number. assert_eq!(classify("tous les 5 jours", "tous les 6 jours"), EditKind::Substantive); // A negation, even one letter apart. assert_eq!(classify("je peux certifier", "je ne peux certifier"), EditKind::Substantive); assert_eq!(classify("plus de deux", "pas de deux"), EditKind::Substantive); // A different word. assert_eq!(classify("au moins deux canaux", "au moins trois canaux"), EditKind::Substantive); // Three fixes are no longer a typo. assert_eq!(classify("aaaa bbbb cccc", "aaab bbbc cccd"), EditKind::Substantive); // A word added. assert_eq!(classify("en visio", "en visio directe"), EditKind::Substantive); } #[test] fn diff_reconstructs_both_sides() { let old = "J'ai rencontré la personne physiquement (préférable)."; let new = "J'ai rencontré la personne en visio (préférable)."; let parts = diff(old, new); let rebuild = |keep: DiffOp| -> String { parts.iter().filter(|p| p.op == DiffOp::Same || p.op == keep).map(|p| p.text.as_str()).collect() }; assert_eq!(rebuild(DiffOp::Removed), old); assert_eq!(rebuild(DiffOp::Added), new); assert!(parts.iter().any(|p| p.op == DiffOp::Removed && p.text.contains("physiquement"))); } #[test] fn levenshtein_basics() { assert_eq!(levenshtein("kitten", "sitting"), 3); assert_eq!(levenshtein("", "abc"), 3); assert_eq!(levenshtein("été", "ete"), 2); } }