//! Edit classification — a typo fix is not a decision. //! //! Once a proposal is under vote, its author may still correct it. The engine //! tells two kinds of corrections apart: //! //! - **typographic**: letters and spaces only — accents, case, the style of //! apostrophes and quotes, words glued or split, or at most two misspelt //! words fixed by a single slip each (one letter added, removed or swapped //! in the middle of the word, never on a negation or a quantity). Never a //! punctuation mark, a figure, a sign, an emphasis, a formula or an address. //! The votes stay; the correction is shown with its diff. //! - **substantive**: anything else. It opens a new revision and every vote //! cast on the previous one waits for its author to reconfirm it. //! //! The rule is deliberately conservative: when in doubt, it is substantive. use serde::{Deserialize, Serialize}; use unicode_normalization::UnicodeNormalization; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum EditKind { Unchanged, Typographic, Substantive, } /// Words whose presence flips a meaning; touching one is never a typo. const NEGATIONS: &[&str] = &[ // fr "ne", "n", "pas", "jamais", "non", "sans", "aucun", "aucune", "ni", "plus", "rien", "nul", "nulle", "personne", "guere", "point", // en "not", "no", "never", "without", "none", "nor", "nothing", "nobody", "nowhere", "neither", // es "nunca", "sin", "ningun", "ninguna", "ninguno", "tampoco", "nada", "nadie", "jamas", "ni", ]; /// Words that carry a quantity, a scope or a relation (folded); changing one is /// never a typo. #[rustfmt::skip] const QUANTITIES: &[&str] = &[ // fr "un", "une", "deux", "trois", "quatre", "cinq", "six", "sept", "huit", "neuf", "dix", "onze", "douze", "treize", "quatorze", "quinze", "seize", "vingt", "trente", "quarante", "cinquante", "soixante", "septante", "huitante", "octante", "nonante", "cent", "cents", "mille", "million", "millions", "milliard", "milliards", "premier", "premiere", "premiers", "premieres", "second", "seconde", "deuxieme", "troisieme", "dernier", "derniere", "double", "triple", "moitie", "tiers", "quart", "moins", "minimum", "maximum", "majorite", "minorite", "unanimite", "tous", "toutes", "tout", "toute", "chaque", "seul", "seule", "seuls", "seules", "dessus", "dessous", "superieur", "superieure", "inferieur", "inferieure", "avant", "apres", "sous", "entre", "hors", "sauf", "inclus", "exclus", // en "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "twenty", "thirty", "forty", "fifty", "sixty", "hundred", "thousand", "million", "first", "second", "third", "last", "half", "twice", "less", "more", "least", "most", "majority", "minority", "all", "every", "each", "only", "above", "below", "over", "under", "before", "after", "within", "except", // es "uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve", "diez", "once", "doce", "veinte", "treinta", "cuarenta", "cincuenta", "cien", "ciento", "mil", "millon", "primero", "primera", "segundo", "segunda", "ultimo", "ultima", "mitad", "doble", "menos", "mas", "minimo", "maximo", "mayoria", "minoria", "todos", "todas", "todo", "toda", "cada", "solo", "sola", "encima", "debajo", "antes", "despues", "sobre", "bajo", "excepto", ]; const MAX_TYPO_WORDS: usize = 2; /// Shorter words have too many neighbours one letter away ("can" / "cant"). const MIN_TYPO_WORD: usize = 5; /// Letters at each end of a word that a slip never touches: the start carries /// prefixes ("valide" / "invalide"), the end carries number, gender, tense and /// mood ("certifierai" / "certifierais", "será" / "sería"). const FIXED_ENDS: usize = 2; /// Lowercase, strip diacritics. fn fold(s: &str) -> String { s.nfd().filter(|c| !is_combining(*c)).flat_map(char::to_lowercase).collect() } fn is_combining(c: char) -> bool { matches!(c as u32, 0x0300..=0x036F | 0x1AB0..=0x1AFF | 0x1DC0..=0x1DFF | 0x20D0..=0x20FF | 0xFE20..=0xFE2F) } /// Letters and digits only, folded: what survives every typographic change. fn skeleton(s: &str) -> String { fold(s).chars().filter(|c| c.is_alphanumeric()).collect() } /// A letter of the Latin alphabet as French, English and Spanish write it, /// precomposed accents included (after NFC, Latin-1 and Latin Extended-A): the /// only letters a typo fix may touch. Any other letter may be a look-alike /// (Cyrillic « а », « ǃ » for « ! », « Ʒ » for « 3 ») or reorder the display /// (Hebrew, Arabic): it is kept as it is. fn is_latin_letter(c: char) -> bool { c.is_alphabetic() && matches!(c as u32, 0x41..=0x5A | 0x61..=0x7A | 0xC0..=0x17F) } /// The only spaces free to change; any other (carriage return, tab, hair, /// thin, ogham…) is kept as it is. const FREE_SPACES: &[char] = &[' ', '\u{a0}', '\u{202f}', '\n']; /// Invisible format characters (zero-width, bidirectional overrides and /// isolates, byte order mark): they can make "12" display as "21" ("Trojan /// Source"). A text of reference never needs them; they are refused. pub fn has_hidden_controls(s: &str) -> bool { s.chars().any( |c| matches!(c as u32, 0x200B..=0x200F | 0x202A..=0x202E | 0x2060..=0x2064 | 0x2066..=0x2069 | 0x061C | 0xFEFF), ) } /// Apostrophes and quotation marks of every style are one and the same sign: /// « Je m'engage » → « Je m’engage » is typography. fn same_mark(c: char) -> char { match c { '’' | '‘' | 'ʼ' => '\'', '“' | '”' | '«' | '»' => '"', other => other, } } /// The shape of a text: every run of Latin letters becomes one mark, free spaces /// vanish, and **everything else stays as it is, in order** — punctuation, /// figures, signs, emphasis, brackets, look-alikes, other alphabets. A typo fix /// keeps the shape: it touches letters and spaces only (2026-10-21, after six /// independent reviews showed that any free punctuation can carry a meaning: /// a minus sign before a bold number, a list merged into one condition…). fn shape(s: &str) -> String { let cs: Vec = s.nfc().collect(); let is_figure = |i: Option| i.and_then(|i| cs.get(i)).is_some_and(|c| c.is_numeric()); let mut out = String::new(); let mut in_word = false; for (i, &c) in cs.iter().enumerate() { if is_latin_letter(c) { if !in_word { out.push('a'); } in_word = true; } else if FREE_SPACES.contains(&c) && !is_figure(i.checked_sub(1)) && !is_figure(Some(i + 1)) { // A space between letters may come and go: « latransmission » and // « la transmission » share a shape (segmentation is checked apart). // Next to a figure it stays: « 1 5 » is not « 15 ». } else { out.push(same_mark(c)); in_word = false; } } out } /// What stands between backticks: formulas and identifiers, compared exactly. fn code_spans(s: &str) -> Vec<&str> { s.split('`').skip(1).step_by(2).collect() } /// Addresses (links, domains, mails, paths) and identifiers that hold a figure /// (keys, fingerprints, references): compared exactly, case included. fn exact_tokens(s: &str) -> Vec { s.split_whitespace() .map(|t| t.trim_matches(|c: char| !c.is_alphanumeric())) .filter(|t| { let cs: Vec = t.chars().collect(); t.contains("://") || t.contains('@') || cs.iter().any(|c| c.is_numeric()) && cs.iter().any(|c| c.is_alphabetic()) || (1..cs.len().saturating_sub(1)) .any(|i| matches!(cs[i], '.' | '/') && cs[i - 1].is_alphanumeric() && cs[i + 1].is_alphanumeric()) }) .map(str::to_owned) .collect() } /// A single slip of the keyboard inside a word: one letter added, removed, or /// two neighbours swapped — never at either end (see `FIXED_ENDS`), and never a /// letter replaced by another ("deux" / "dix", "majorité" / "minorité"). fn one_slip(x: &str, y: &str) -> bool { let (a, b): (Vec, Vec) = (x.chars().collect(), y.chars().collect()); if a.len().min(b.len()) < MIN_TYPO_WORD { return false; } let inside = |i: usize, len: usize| i >= FIXED_ENDS && i + FIXED_ENDS < len; match a.len().abs_diff(b.len()) { 1 => { let (short, long) = if a.len() < b.len() { (&a, &b) } else { (&b, &a) }; (0..long.len()).any(|i| { let mut l = long.clone(); l.remove(i); inside(i, long.len()) && &l == short }) } 0 => { let at: Vec = (0..a.len()).filter(|&i| a[i] != b[i]).collect(); at.len() == 2 && at[1] == at[0] + 1 && a[at[0]] == b[at[1]] && a[at[1]] == b[at[0]] && inside(at[0], a.len()) && inside(at[1], a.len()) } _ => false, } } /// Folded words (runs of letters/digits). fn words(s: &str) -> Vec { fold(s).split(|c: char| !c.is_alphanumeric()).filter(|w| !w.is_empty()).map(str::to_owned).collect() } /// Same letters, at most two words glued or split, never a negation: /// « latransmission » → « la transmission ». A boundary moved is not a typo /// (« les avoir » → « le savoir »), nor « Si non » → « Sinon ». fn only_glued_or_split(a: &[String], b: &[String]) -> bool { let joined = |x: &String, y: &String| !NEGATIONS.contains(&x.as_str()) && !NEGATIONS.contains(&y.as_str()); let (mut i, mut j, mut moves) = (0, 0, 0); while i < a.len() && j < b.len() { if a[i] == b[j] { (i, j) = (i + 1, j + 1); continue; } if j + 1 < b.len() && a[i] == format!("{}{}", b[j], b[j + 1]) && joined(&b[j], &b[j + 1]) { (i, j) = (i + 1, j + 2); } else if i + 1 < a.len() && b[j] == format!("{}{}", a[i], a[i + 1]) && joined(&a[i], &a[i + 1]) { (i, j) = (i + 2, j + 1); } else { return false; } moves += 1; if moves > MAX_TYPO_WORDS { return false; } } i == a.len() && j == b.len() } /// Classify `new` against `old`. For a text under vote, `old` must be the text /// that was **voted** (its last initial or substantive revision), not the /// current one: typo fixes are measured together, so slips never add up. pub fn classify(old: &str, new: &str) -> EditKind { if old == new { return EditKind::Unchanged; } if has_hidden_controls(new) || shape(old) != shape(new) || code_spans(old) != code_spans(new) || exact_tokens(old) != exact_tokens(new) { return EditKind::Substantive; } let (a, b) = (words(old), words(new)); if skeleton(old) == skeleton(new) { // Accents, case, apostrophes; words glued or split, never re-cut. return if only_glued_or_split(&a, &b) { EditKind::Typographic } else { EditKind::Substantive }; } if a.len() != b.len() { return EditKind::Substantive; } let mut changed = 0; for (x, y) in a.iter().zip(&b) { if x == y { continue; } changed += 1; let touches_meaning = changed > MAX_TYPO_WORDS || x.chars().any(char::is_numeric) || y.chars().any(char::is_numeric) || [x, y].iter().any(|w| NEGATIONS.contains(&w.as_str()) || QUANTITIES.contains(&w.as_str())) || !one_slip(x, y); if touches_meaning { return EditKind::Substantive; } } EditKind::Typographic } pub fn levenshtein(a: &str, b: &str) -> usize { let b: Vec = b.chars().collect(); let mut prev: Vec = (0..=b.len()).collect(); let mut cur = vec![0; b.len() + 1]; for (i, ca) in a.chars().enumerate() { cur[0] = i + 1; for (j, cb) in b.iter().enumerate() { let cost = usize::from(ca != *cb); cur[j + 1] = (prev[j] + cost).min(prev[j + 1] + 1).min(cur[j] + 1); } std::mem::swap(&mut prev, &mut cur); } prev[b.len()] } #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum DiffOp { Same, Added, Removed, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct DiffPart { pub op: DiffOp, pub text: String, } /// Tokens that keep every character: words, and the runs between them. fn tokens(s: &str) -> Vec<&str> { let mut out = Vec::new(); let mut start = 0; let mut in_word: Option = None; for (i, c) in s.char_indices() { let w = c.is_alphanumeric(); if in_word.is_some_and(|prev| prev != w) { out.push(&s[start..i]); start = i; } in_word = Some(w); } if start < s.len() { out.push(&s[start..]); } out } const MAX_DIFF_TOKENS: usize = 4000; /// Word-level diff for display. Concatenating `Same` + `Added` gives `new`, /// `Same` + `Removed` gives `old`. pub fn diff(old: &str, new: &str) -> Vec { let (a, b) = (tokens(old), tokens(new)); if a.len() * b.len() > MAX_DIFF_TOKENS * MAX_DIFF_TOKENS / 4 { return vec![ DiffPart { op: DiffOp::Removed, text: old.to_owned() }, DiffPart { op: DiffOp::Added, text: new.to_owned() }, ]; } // LCS table, suffix form. let (n, m) = (a.len(), b.len()); let mut lcs = vec![vec![0u32; m + 1]; n + 1]; for i in (0..n).rev() { for j in (0..m).rev() { lcs[i][j] = if a[i] == b[j] { lcs[i + 1][j + 1] + 1 } else { lcs[i + 1][j].max(lcs[i][j + 1]) }; } } let mut parts: Vec = Vec::new(); let mut push = |op: DiffOp, text: &str| match parts.last_mut() { Some(last) if last.op == op => last.text.push_str(text), _ => parts.push(DiffPart { op, text: text.to_owned() }), }; let (mut i, mut j) = (0, 0); while i < n && j < m { if a[i] == b[j] { push(DiffOp::Same, a[i]); i += 1; j += 1; } else if lcs[i + 1][j] >= lcs[i][j + 1] { push(DiffOp::Removed, a[i]); i += 1; } else { push(DiffOp::Added, b[j]); j += 1; } } for t in &a[i..] { push(DiffOp::Removed, t); } for t in &b[j..] { push(DiffOp::Added, t); } parts } #[cfg(test)] mod tests { use super::*; #[test] fn the_glued_word_of_the_seed_is_a_typo() { let old = "Toute Certification doit s'accompagner de latransmission de ce document"; let new = "Toute Certification doit s'accompagner de la transmission de ce document"; assert_eq!(classify(old, new), EditKind::Typographic); } #[test] fn glued_or_split_words_stay_few_and_never_touch_a_negation() { // Seventh independent review, 2026-10-21: a text in force disfigured at will. for (old, new) in [ ("à ne certifier que", "à necertifier que"), ("à ne certifier que", "àne certifier que"), ("Si non, la demande attend", "Sinon, la demande attend"), ("la transmission de ce document à chacun", "latransmission dece documentà chacun"), ] { assert_eq!(classify(old, new), EditKind::Substantive, "{old} → {new}"); } assert_eq!( classify("de latransmission dece document", "de la transmission de ce document"), EditKind::Typographic ); } #[test] fn odd_spaces_and_look_alike_letters_are_kept() { assert_eq!(classify("un texte\tde référence", "un texte de référence"), EditKind::Substantive); assert_eq!(classify("un texte\rde référence", "un texte de référence"), EditKind::Substantive); assert_eq!(classify("toute certification", "toute certiǃfication"), EditKind::Substantive); assert_eq!(classify("toute certification", "toute certifiƷcation"), EditKind::Substantive); assert_eq!(classify("le coeur de l'ŒUVRE, niño", "le Coeur de l'œuvre, Niño"), EditKind::Typographic); } #[test] fn accents_case_apostrophes_quotes() { assert_eq!(classify("Je m'engage a verifier.", "Je m’engage à vérifier."), EditKind::Typographic); assert_eq!(classify("le \"Pacte\" de la monnaie", "le « Pacte » de la Monnaie"), EditKind::Typographic); assert_eq!(classify("same", "same"), EditKind::Unchanged); } #[test] fn punctuation_and_emphasis_go_through_a_proposal() { // Sixth independent review: free punctuation can always carry a meaning. for (old, new) in [ ("Je m'engage à vérifier.", "Je m'engage à vérifier !"), ("Je m'engage à vérifier.", "Je m'engage à vérifier ?"), ("un **être humain**", "un être humain"), ("tous les **5 jours**", "tous les -**5 jours**"), ("Seuil = C + B^W", "Seuil = C + B^-W"), ("max(0, T-C)", "max(0, TC)"), ("(Y = ceil(N))", "(Y = -ceil(N))"), ("- avoir un compte\n- être joignable", "- avoir un compte, être joignable"), ("(sauf le cas a) et le cas b", "(sauf le cas a et le cas b)"), ("libre - le texte", "libre — le texte"), ("- une liste", "* une liste"), ("clé DsEx1pS33abc", "clé dSeX1Ps33abc"), ("les avoir vérifiés", "le savoir vérifiés"), ] { assert_eq!(classify(old, new), EditKind::Substantive, "{old} → {new}"); } assert_eq!(classify("toute latransmission", "toute la transmission"), EditKind::Typographic); assert_eq!(classify("la trans mission", "la transmission"), EditKind::Typographic); } #[test] fn a_figure_a_sign_or_a_relation_is_never_a_typo() { // Same letters, different meaning (independent reviews, 2026-10-21). for (old, new) in [ ("au moins 50 voix", "au moins 5,0 voix"), ("1 5 jours", "15 jours"), ("10 000 membres", "10,000 membres"), ("un écart de -5", "un écart de 5"), ("1/5 des voix", "1.5 des voix"), ("50 % des voix", "50 ‰ des voix"), ("avoir >= 5 certifications", "avoir <= 5 certifications"), ("DU(n) + c² × M(n)", "DU(n) - c² × M(n)"), ("182.625 jours", "182 625 jours"), ("c²", "c2"), ] { assert_eq!(classify(old, new), EditKind::Substantive, "{old} → {new}"); } assert_eq!(classify("50 voix pour", "50 voix Pour"), EditKind::Typographic); // A bracket always counts, even as a list marker: conservative. assert_eq!(classify("1. premier point", "1) premier point"), EditKind::Substantive); assert_eq!(classify("1. premier point", "1. Premier point"), EditKind::Typographic); } #[test] fn a_word_turned_around_is_never_a_typo() { // Third independent review, 2026-10-21: one or two letters that invert a meaning. for (old, new) in [ ("ces deux présents engagements", "ces dix présents engagements"), ("Nouveau compte valide si", "Nouveau compte invalide si"), ("100 certifications possibles", "100 certifications impossibles"), ("à la majorité des voix", "à la minorité des voix"), ("un minimum de garanties", "un maximum de garanties"), ("les membres inclus", "les membres exclus"), ("une personne majeure", "une personne mineure"), ("une durée limitée", "une durée illimitée"), ("un mandat révocable", "un mandat irrévocable"), ("des comptes indépendants", "des comptes dépendants"), ("pendant six mois", "pendant dix mois"), ("sept certifications", "cent certifications"), ("seize jours", "treize jours"), ("five members", "nine members"), ("you can certify", "you cant certify"), ("Y = ceil(N^(1/5))", "Y = ceil(N^1/5)"), ("M(n)/N(n)", "M(n)N(n)"), ("DU(n) - c", "DU(n) / c"), ("la formule `a - b`", "la formule `a * b`"), // Fourth independent review: look-alikes, overlays, addresses, endings. ("au moins 50 voix", "au moins 5\u{201a}0 voix"), ("**12 mois**", "**1\u{201a}2 mois**"), ("15 jours", "1\u{2044}5 jours"), ("2 ans", "\u{ff0d}2 ans"), (">= 5", "\u{226f}= 5"), ("a = b", "a \u{2260} b"), ("a = b", "a =\u{0338} b"), ("ne jamais certifier", "ne j\u{0336}a\u{0336}m\u{0336}a\u{0336}i\u{0336}s\u{0336} certifier"), ("la clé est vérifiée", "la clé est ~~vérifiée~~"), ("une personne connue", "une personne conn\u{05d5}ue"), ("une personne connue", "une personne c\u{043e}nnue"), ( "https://git.duniter.org/documents/g1_monetary_license", "https://git.dunniter.org/documents/g1_monetary_license", ), ("au-dessus de 50 %", "au-dessous de 50 %"), ("je certifierai", "je certifierais"), ("será válido", "sería válido"), ("they must certify", "they mustn certify"), // Fifth independent review: anything touching a figure, special spaces, signs across spaces. ("au moins 50 voix", "au moins 5,\u{202f}0 voix"), ("**12 mois**", "**1,\u{202f}2 mois**"), ("100 certifications", "10,\u{200a}0 certifications"), ("80%", "8,\u{200a}0%"), ("15 jours", "1:5 jours"), ("15 jours", "1…5 jours"), ("15 jours", "1*5 jours"), ("15 jours", "1;5 jours"), ("15 jours", "1‘5 jours"), ("un écart de 5", "un écart de -\u{200a}5"), ("un écart de 5", "un écart de - 5"), ("un écart de 5", "un écart de \u{1680}5"), ] { assert_eq!(classify(old, new), EditKind::Substantive, "{old} → {new}"); } } #[test] fn hidden_controls_are_spotted() { assert!(has_hidden_controls("**\u{202e}12\u{202c} mois**")); assert!(has_hidden_controls("5\u{200b}0")); assert!(!has_hidden_controls("Je m’engage — « à 50 % ».")); } #[test] fn misspelt_words() { // The end of a word carries number and tense: a plural fixed is a revision. assert_eq!(classify("un certifieurs existant", "un certifieur existant"), EditKind::Substantive); assert_eq!(classify("toute certifcation", "toute certification"), EditKind::Typographic); assert_eq!(classify("un acceuil chaleureux", "un accueil chaleureux"), EditKind::Typographic); // A letter replaced by another is not a slip: the author revises. assert_eq!(classify("une certifixation", "une certification"), EditKind::Substantive); // « personne » is also a negation (ne … personne): conservative. assert_eq!(classify("la persone", "la personne"), EditKind::Substantive); } #[test] fn meaning_changes() { // A number. assert_eq!(classify("tous les 5 jours", "tous les 6 jours"), EditKind::Substantive); // A negation, even one letter apart. assert_eq!(classify("je peux certifier", "je ne peux certifier"), EditKind::Substantive); assert_eq!(classify("plus de deux", "pas de deux"), EditKind::Substantive); // A different word. assert_eq!(classify("au moins deux canaux", "au moins trois canaux"), EditKind::Substantive); // Three fixes are no longer a typo. assert_eq!(classify("aaaa bbbb cccc", "aaab bbbc cccd"), EditKind::Substantive); // A word added. assert_eq!(classify("en visio", "en visio directe"), EditKind::Substantive); } #[test] fn diff_reconstructs_both_sides() { let old = "J'ai rencontré la personne physiquement (préférable)."; let new = "J'ai rencontré la personne en visio (préférable)."; let parts = diff(old, new); let rebuild = |keep: DiffOp| -> String { parts.iter().filter(|p| p.op == DiffOp::Same || p.op == keep).map(|p| p.text.as_str()).collect() }; assert_eq!(rebuild(DiffOp::Removed), old); assert_eq!(rebuild(DiffOp::Added), new); assert!(parts.iter().any(|p| p.op == DiffOp::Removed && p.text.contains("physiquement"))); } #[test] fn levenshtein_basics() { assert_eq!(levenshtein("kitten", "sitting"), 3); assert_eq!(levenshtein("", "abc"), 3); assert_eq!(levenshtein("été", "ete"), 2); } }