//! Edit classification — a typo fix is not a decision. //! //! Once a proposal is under vote, its author may still correct it. The engine //! tells two kinds of corrections apart: //! //! - **typographic**: accents, case, punctuation, spacing, emphasis, or at most //! two misspelt words fixed by a single slip each (one letter added, removed //! or swapped, never at the start of the word, never on a negation or a //! quantity) — and never a figure, a sign, a separator inside a number, a //! relation, a bracket or anything between backticks. The votes stay; the //! correction is shown with its diff. //! - **substantive**: anything else. It opens a new revision and every vote //! cast on the previous one waits for its author to reconfirm it. //! //! The rule is deliberately conservative: when in doubt, it is substantive. use serde::{Deserialize, Serialize}; use unicode_normalization::UnicodeNormalization; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum EditKind { Unchanged, Typographic, Substantive, } /// Words whose presence flips a meaning; touching one is never a typo. const NEGATIONS: &[&str] = &[ // fr "ne", "n", "pas", "jamais", "non", "sans", "aucun", "aucune", "ni", "plus", "rien", "nul", "nulle", "personne", "guere", "point", // en "not", "no", "never", "without", "none", "nor", "nothing", "nobody", "nowhere", "neither", // es "nunca", "sin", "ningun", "ninguna", "ninguno", "tampoco", "nada", "nadie", "jamas", "ni", ]; /// Words that carry a quantity, a scope or a relation (folded); changing one is /// never a typo. #[rustfmt::skip] const QUANTITIES: &[&str] = &[ // fr "un", "une", "deux", "trois", "quatre", "cinq", "six", "sept", "huit", "neuf", "dix", "onze", "douze", "treize", "quatorze", "quinze", "seize", "vingt", "trente", "quarante", "cinquante", "soixante", "septante", "huitante", "octante", "nonante", "cent", "cents", "mille", "million", "millions", "milliard", "milliards", "premier", "premiere", "premiers", "premieres", "second", "seconde", "deuxieme", "troisieme", "dernier", "derniere", "double", "triple", "moitie", "tiers", "quart", "moins", "minimum", "maximum", "majorite", "minorite", "unanimite", "tous", "toutes", "tout", "toute", "chaque", "seul", "seule", "seuls", "seules", "dessus", "dessous", "superieur", "superieure", "inferieur", "inferieure", "avant", "apres", "sous", "entre", "hors", "sauf", "inclus", "exclus", // en "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "twenty", "thirty", "forty", "fifty", "sixty", "hundred", "thousand", "million", "first", "second", "third", "last", "half", "twice", "less", "more", "least", "most", "majority", "minority", "all", "every", "each", "only", "above", "below", "over", "under", "before", "after", "within", "except", // es "uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve", "diez", "once", "doce", "veinte", "treinta", "cuarenta", "cincuenta", "cien", "ciento", "mil", "millon", "primero", "primera", "segundo", "segunda", "ultimo", "ultima", "mitad", "doble", "menos", "mas", "minimo", "maximo", "mayoria", "minoria", "todos", "todas", "todo", "toda", "cada", "solo", "sola", "encima", "debajo", "antes", "despues", "sobre", "bajo", "excepto", ]; const MAX_TYPO_WORDS: usize = 2; /// Shorter words have too many neighbours one letter away ("can" / "cant"). const MIN_TYPO_WORD: usize = 5; /// Letters at each end of a word that a slip never touches: the start carries /// prefixes ("valide" / "invalide"), the end carries number, gender, tense and /// mood ("certifierai" / "certifierais", "será" / "sería"). const FIXED_ENDS: usize = 2; /// Lowercase, strip diacritics. fn fold(s: &str) -> String { s.nfd().filter(|c| !is_combining(*c)).flat_map(char::to_lowercase).collect() } fn is_combining(c: char) -> bool { matches!(c as u32, 0x0300..=0x036F | 0x1AB0..=0x1AFF | 0x1DC0..=0x1DFF | 0x20D0..=0x20FF | 0xFE20..=0xFE2F) } /// Letters and digits only, folded: what survives every typographic change. fn skeleton(s: &str) -> String { fold(s).chars().filter(|c| c.is_alphanumeric()).collect() } /// A letter of the Latin alphabet, precomposed accents included (after NFC): /// the only letters a typo fix may touch. Any other script may hold look-alikes /// (Cyrillic « а ») or reorder the display (Hebrew, Arabic). fn is_latin_letter(c: char) -> bool { c.is_alphabetic() && matches!(c as u32, 0x41..=0x5A | 0x61..=0x7A | 0xC0..=0x24F | 0x1E00..=0x1EFF) } /// Ordinary punctuation, free to change in a typo fix — except where noted below. const PUNCTUATION: &[char] = &['.', ',', ';', ':', '!', '?', '\'', '’', '‘', '"', '“', '”', '«', '»', '…', '‐', '-', '–', '—', '*', '_']; /// The only spaces free to change; any other (hair, thin, ogham…) is kept as it is. const FREE_SPACES: &[char] = &[' ', '\u{a0}', '\u{202f}', '\n', '\r', '\t']; /// Dashes become a sign when a figure follows or precedes, even across spaces ("- 5"). const DASHES: &[char] = &['‐', '-', '–', '—']; /// Invisible format characters (zero-width, bidirectional overrides and /// isolates, byte order mark): they can make "12" display as "21" ("Trojan /// Source"). A text of reference never needs them; they are refused. pub fn has_hidden_controls(s: &str) -> bool { s.chars().any( |c| matches!(c as u32, 0x200B..=0x200F | 0x202A..=0x202E | 0x2060..=0x2064 | 0x2066..=0x2069 | 0x061C | 0xFEFF), ) } /// Everything a typo fix may not change, in order. Built the other way round /// from a list of dangerous symbols: what is kept is everything that is NOT a /// Latin letter, a space or ordinary punctuation — figures in any script, signs, /// relations, brackets, look-alikes (U+201A for a comma, U+2044 for a slash, /// full-width forms), stray combining marks (U+0336, U+0338), other alphabets. /// Anything that touches a figure counts too, spaces and punctuation included: /// "5,0", "5,\u{202f}0", "1:5", "1…5" never read like "50" or "15". fn signature(s: &str) -> String { let cs: Vec = s.nfc().collect(); let is_figure = |i: Option| i.and_then(|i| cs.get(i)).is_some_and(|c| c.is_numeric()); // The next character that is not a free space, on either side. let across = |i: usize, step: isize| { let mut j = i as isize + step; while j >= 0 && (j as usize) < cs.len() && FREE_SPACES.contains(&cs[j as usize]) { j += step; } j >= 0 && is_figure(Some(j as usize)) }; cs.iter() .enumerate() .filter(|&(i, &c)| { let touches_figure = is_figure(i.checked_sub(1)) || is_figure(Some(i + 1)); if is_latin_letter(c) { false } else if touches_figure { true } else if FREE_SPACES.contains(&c) { false } else if DASHES.contains(&c) { across(i, -1) || across(i, 1) } else { !PUNCTUATION.contains(&c) } }) .map(|(_, &c)| c) .collect() } /// What stands between backticks: formulas and identifiers, compared exactly. fn code_spans(s: &str) -> Vec<&str> { s.split('`').skip(1).step_by(2).collect() } /// Web and mail addresses, domains, paths: compared exactly ("duniter" / /// "dunniter" is a typo of a word, not of an address). fn addresses(s: &str) -> Vec { s.split_whitespace() .map(|t| t.trim_matches(|c: char| !c.is_alphanumeric())) .filter(|t| { let cs: Vec = t.chars().collect(); t.contains("://") || t.contains('@') || (1..cs.len().saturating_sub(1)) .any(|i| matches!(cs[i], '.' | '/') && cs[i - 1].is_alphanumeric() && cs[i + 1].is_alphanumeric()) }) .map(str::to_owned) .collect() } /// A single slip of the keyboard inside a word: one letter added, removed, or /// two neighbours swapped — never at either end (see `FIXED_ENDS`), and never a /// letter replaced by another ("deux" / "dix", "majorité" / "minorité"). fn one_slip(x: &str, y: &str) -> bool { let (a, b): (Vec, Vec) = (x.chars().collect(), y.chars().collect()); if a.len().min(b.len()) < MIN_TYPO_WORD { return false; } let inside = |i: usize, len: usize| i >= FIXED_ENDS && i + FIXED_ENDS < len; match a.len().abs_diff(b.len()) { 1 => { let (short, long) = if a.len() < b.len() { (&a, &b) } else { (&b, &a) }; (0..long.len()).any(|i| { let mut l = long.clone(); l.remove(i); inside(i, long.len()) && &l == short }) } 0 => { let at: Vec = (0..a.len()).filter(|&i| a[i] != b[i]).collect(); at.len() == 2 && at[1] == at[0] + 1 && a[at[0]] == b[at[1]] && a[at[1]] == b[at[0]] && inside(at[0], a.len()) && inside(at[1], a.len()) } _ => false, } } /// Folded words (runs of letters/digits). fn words(s: &str) -> Vec { fold(s).split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).map(str::to_owned).collect() } /// Classify `new` against `old`. For a text under vote, `old` must be the text /// that was **voted** (its last initial or substantive revision), not the /// current one: typo fixes are measured together, so slips never add up. pub fn classify(old: &str, new: &str) -> EditKind { if old == new { return EditKind::Unchanged; } if has_hidden_controls(new) || signature(old) != signature(new) || code_spans(old) != code_spans(new) || addresses(old) != addresses(new) { return EditKind::Substantive; } if skeleton(old) == skeleton(new) { return EditKind::Typographic; } let (a, b) = (words(old), words(new)); if a.len() != b.len() { return EditKind::Substantive; } let mut changed = 0; for (x, y) in a.iter().zip(&b) { if x == y { continue; } changed += 1; let touches_meaning = changed > MAX_TYPO_WORDS || x.chars().any(char::is_numeric) || y.chars().any(char::is_numeric) || [x, y].iter().any(|w| NEGATIONS.contains(&w.as_str()) || QUANTITIES.contains(&w.as_str())) || !one_slip(x, y); if touches_meaning { return EditKind::Substantive; } } EditKind::Typographic } pub fn levenshtein(a: &str, b: &str) -> usize { let b: Vec = b.chars().collect(); let mut prev: Vec = (0..=b.len()).collect(); let mut cur = vec![0; b.len() + 1]; for (i, ca) in a.chars().enumerate() { cur[0] = i + 1; for (j, cb) in b.iter().enumerate() { let cost = usize::from(ca != *cb); cur[j + 1] = (prev[j] + cost).min(prev[j + 1] + 1).min(cur[j] + 1); } std::mem::swap(&mut prev, &mut cur); } prev[b.len()] } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum DiffOp { Same, Added, Removed, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct DiffPart { pub op: DiffOp, pub text: String, } /// Tokens that keep every character: words, and the runs between them. fn tokens(s: &str) -> Vec<&str> { let mut out = Vec::new(); let mut start = 0; let mut in_word: Option = None; for (i, c) in s.char_indices() { let w = c.is_alphanumeric(); if in_word.is_some_and(|prev| prev != w) { out.push(&s[start..i]); start = i; } in_word = Some(w); } if start < s.len() { out.push(&s[start..]); } out } const MAX_DIFF_TOKENS: usize = 4000; /// Word-level diff for display. Concatenating `Same` + `Added` gives `new`, /// `Same` + `Removed` gives `old`. pub fn diff(old: &str, new: &str) -> Vec { let (a, b) = (tokens(old), tokens(new)); if a.len() * b.len() > MAX_DIFF_TOKENS * MAX_DIFF_TOKENS / 4 { return vec![ DiffPart { op: DiffOp::Removed, text: old.to_owned() }, DiffPart { op: DiffOp::Added, text: new.to_owned() }, ]; } // LCS table, suffix form. let (n, m) = (a.len(), b.len()); let mut lcs = vec![vec![0u32; m + 1]; n + 1]; for i in (0..n).rev() { for j in (0..m).rev() { lcs[i][j] = if a[i] == b[j] { lcs[i + 1][j + 1] + 1 } else { lcs[i + 1][j].max(lcs[i][j + 1]) }; } } let mut parts: Vec = Vec::new(); let mut push = |op: DiffOp, text: &str| match parts.last_mut() { Some(last) if last.op == op => last.text.push_str(text), _ => parts.push(DiffPart { op, text: text.to_owned() }), }; let (mut i, mut j) = (0, 0); while i < n && j < m { if a[i] == b[j] { push(DiffOp::Same, a[i]); i += 1; j += 1; } else if lcs[i + 1][j] >= lcs[i][j + 1] { push(DiffOp::Removed, a[i]); i += 1; } else { push(DiffOp::Added, b[j]); j += 1; } } for t in &a[i..] { push(DiffOp::Removed, t); } for t in &b[j..] { push(DiffOp::Added, t); } parts } #[cfg(test)] mod tests { use super::*; #[test] fn the_glued_word_of_the_seed_is_a_typo() { let old = "Toute Certification doit s'accompagner de latransmission de ce document"; let new = "Toute Certification doit s'accompagner de la transmission de ce document"; assert_eq!(classify(old, new), EditKind::Typographic); } #[test] fn accents_case_punctuation_emphasis() { assert_eq!(classify("Je m'engage a verifier.", "Je m’engage à vérifier !"), EditKind::Typographic); assert_eq!(classify("un **être humain**", "un être humain"), EditKind::Typographic); assert_eq!(classify("same", "same"), EditKind::Unchanged); } #[test] fn a_figure_a_sign_or_a_relation_is_never_a_typo() { // Same letters, different meaning (independent reviews, 2026-10-21). for (old, new) in [ ("au moins 50 voix", "au moins 5,0 voix"), ("1 5 jours", "15 jours"), ("10 000 membres", "10,000 membres"), ("un écart de -5", "un écart de 5"), ("1/5 des voix", "1.5 des voix"), ("50 % des voix", "50 ‰ des voix"), ("avoir >= 5 certifications", "avoir <= 5 certifications"), ("DU(n) + c² × M(n)", "DU(n) - c² × M(n)"), ("182.625 jours", "182 625 jours"), ("c²", "c2"), ] { assert_eq!(classify(old, new), EditKind::Substantive, "{old} → {new}"); } // Punctuation around figures, list markers, dashes between words stay typographic. assert_eq!(classify("50 voix pour", "50 voix pour !"), EditKind::Typographic); assert_eq!(classify("- une liste", "* une liste"), EditKind::Typographic); // A bracket always counts, even as a list marker: conservative. assert_eq!(classify("1. premier point", "1) premier point"), EditKind::Substantive); assert_eq!(classify("1. premier point", "1. premier point."), EditKind::Typographic); assert_eq!(classify("libre - le texte", "libre — le texte"), EditKind::Typographic); } #[test] fn a_word_turned_around_is_never_a_typo() { // Third independent review, 2026-10-21: one or two letters that invert a meaning. for (old, new) in [ ("ces deux présents engagements", "ces dix présents engagements"), ("Nouveau compte valide si", "Nouveau compte invalide si"), ("100 certifications possibles", "100 certifications impossibles"), ("à la majorité des voix", "à la minorité des voix"), ("un minimum de garanties", "un maximum de garanties"), ("les membres inclus", "les membres exclus"), ("une personne majeure", "une personne mineure"), ("une durée limitée", "une durée illimitée"), ("un mandat révocable", "un mandat irrévocable"), ("des comptes indépendants", "des comptes dépendants"), ("pendant six mois", "pendant dix mois"), ("sept certifications", "cent certifications"), ("seize jours", "treize jours"), ("five members", "nine members"), ("you can certify", "you cant certify"), ("Y = ceil(N^(1/5))", "Y = ceil(N^1/5)"), ("M(n)/N(n)", "M(n)N(n)"), ("DU(n) - c", "DU(n) / c"), ("la formule `a - b`", "la formule `a * b`"), // Fourth independent review: look-alikes, overlays, addresses, endings. ("au moins 50 voix", "au moins 5\u{201a}0 voix"), ("**12 mois**", "**1\u{201a}2 mois**"), ("15 jours", "1\u{2044}5 jours"), ("2 ans", "\u{ff0d}2 ans"), (">= 5", "\u{226f}= 5"), ("a = b", "a \u{2260} b"), ("a = b", "a =\u{0338} b"), ("ne jamais certifier", "ne j\u{0336}a\u{0336}m\u{0336}a\u{0336}i\u{0336}s\u{0336} certifier"), ("la clé est vérifiée", "la clé est ~~vérifiée~~"), ("une personne connue", "une personne conn\u{05d5}ue"), ("une personne connue", "une personne c\u{043e}nnue"), ( "https://git.duniter.org/documents/g1_monetary_license", "https://git.dunniter.org/documents/g1_monetary_license", ), ("au-dessus de 50 %", "au-dessous de 50 %"), ("je certifierai", "je certifierais"), ("será válido", "sería válido"), ("they must certify", "they mustn certify"), // Fifth independent review: anything touching a figure, special spaces, signs across spaces. ("au moins 50 voix", "au moins 5,\u{202f}0 voix"), ("**12 mois**", "**1,\u{202f}2 mois**"), ("100 certifications", "10,\u{200a}0 certifications"), ("80%", "8,\u{200a}0%"), ("15 jours", "1:5 jours"), ("15 jours", "1…5 jours"), ("15 jours", "1*5 jours"), ("15 jours", "1;5 jours"), ("15 jours", "1‘5 jours"), ("un écart de 5", "un écart de -\u{200a}5"), ("un écart de 5", "un écart de - 5"), ("un écart de 5", "un écart de \u{1680}5"), ] { assert_eq!(classify(old, new), EditKind::Substantive, "{old} → {new}"); } } #[test] fn hidden_controls_are_spotted() { assert!(has_hidden_controls("**\u{202e}12\u{202c} mois**")); assert!(has_hidden_controls("5\u{200b}0")); assert!(!has_hidden_controls("Je m’engage — « à 50 % ».")); } #[test] fn misspelt_words() { // The end of a word carries number and tense: a plural fixed is a revision. assert_eq!(classify("un certifieurs existant", "un certifieur existant"), EditKind::Substantive); assert_eq!(classify("toute certifcation", "toute certification"), EditKind::Typographic); assert_eq!(classify("un acceuil chaleureux", "un accueil chaleureux"), EditKind::Typographic); // A letter replaced by another is not a slip: the author revises. assert_eq!(classify("une certifixation", "une certification"), EditKind::Substantive); // « personne » is also a negation (ne … personne): conservative. assert_eq!(classify("la persone", "la personne"), EditKind::Substantive); } #[test] fn meaning_changes() { // A number. assert_eq!(classify("tous les 5 jours", "tous les 6 jours"), EditKind::Substantive); // A negation, even one letter apart. assert_eq!(classify("je peux certifier", "je ne peux certifier"), EditKind::Substantive); assert_eq!(classify("plus de deux", "pas de deux"), EditKind::Substantive); // A different word. assert_eq!(classify("au moins deux canaux", "au moins trois canaux"), EditKind::Substantive); // Three fixes are no longer a typo. assert_eq!(classify("aaaa bbbb cccc", "aaab bbbc cccd"), EditKind::Substantive); // A word added. assert_eq!(classify("en visio", "en visio directe"), EditKind::Substantive); } #[test] fn diff_reconstructs_both_sides() { let old = "J'ai rencontré la personne physiquement (préférable)."; let new = "J'ai rencontré la personne en visio (préférable)."; let parts = diff(old, new); let rebuild = |keep: DiffOp| -> String { parts.iter().filter(|p| p.op == DiffOp::Same || p.op == keep).map(|p| p.text.as_str()).collect() }; assert_eq!(rebuild(DiffOp::Removed), old); assert_eq!(rebuild(DiffOp::Added), new); assert!(parts.iter().any(|p| p.op == DiffOp::Removed && p.text.contains("physiquement"))); } #[test] fn levenshtein_basics() { assert_eq!(levenshtein("kitten", "sitting"), 3); assert_eq!(levenshtein("", "abc"), 3); assert_eq!(levenshtein("été", "ete"), 2); } }