diff --git a/server b/server index 891aa003..b7504c89 160000 --- a/server +++ b/server @@ -1 +1 @@ -Subproject commit 891aa0038bb4c5083a36cc02470f088776078f99 +Subproject commit b7504c894c2f3a962c289a03bff284e840747efc diff --git a/web/locales/cs/main.ftl b/web/locales/cs/main.ftl index 851abeb0..84409dbe 100644 --- a/web/locales/cs/main.ftl +++ b/web/locales/cs/main.ftl @@ -573,6 +573,7 @@ import-target-tables = Cílová tabulka import-choose-table = Vyberte tabulku import-csv-file = Soubor CSV import-csv-data = Data CSV +import-csv-format-hint = Použijte CSV v UTF-8 s oddělovačem čárka a každé pole uzavřete do dvojitých uvozovek, například: "name","number","active" import-error-title = CSV se nepodařilo importovat import-import-rows = Importovat řádky import-success-title = Import dokončen @@ -617,6 +618,7 @@ import-success-ignored-system = { $columns } přiděluje server, importované ř import-err-missing-type = Chybí typ sloupce '{ $column }' import-err-permission = Vyžaduje se oprávnění k importu. Tabulka navíc potřebuje oprávnění k vkládání, aby do ní šlo načítat. import-err-unterminated-quote = CSV obsahuje neuzavřenou uvozovkovou hodnotu. +import-err-strict-csv = Neplatný formát CSV. Každé pole musí být uzavřeno v dvojitých uvozovkách, pole musí být oddělena čárkami a každý záznam musí zůstat na jednom řádku. import-err-empty = CSV je prázdné. import-err-bad-boolean = Neplatná booleovská hodnota '{ $value }' import-err-bad-integer = Neplatná celočíselná hodnota '{ $value }' diff --git a/web/locales/en/main.ftl b/web/locales/en/main.ftl index 7b942c01..2de7015f 100644 --- a/web/locales/en/main.ftl +++ b/web/locales/en/main.ftl @@ -563,6 +563,7 @@ import-target-tables = Target table import-choose-table = Choose a table import-csv-file = CSV file import-csv-data = CSV data +import-csv-format-hint = Use UTF-8 CSV with a comma separator and every field in double quotes, for example: "name","number","active" import-error-title = Could not import CSV import-import-rows = Import rows import-success-title = Import complete @@ -604,6 +605,7 @@ import-success-ignored-system = { $columns } are assigned by the server, so the import-err-missing-type = Missing type for column '{ $column }' import-err-permission = Import permission is required. A table also needs insert permission before it can be loaded. import-err-unterminated-quote = CSV contains an unterminated quoted value. +import-err-strict-csv = Invalid CSV format. Every field must be enclosed in double quotes, fields must be separated by commas, and each record must stay on one line. import-err-empty = CSV is empty. import-err-bad-boolean = Invalid boolean value '{ $value }' import-err-bad-integer = Invalid integer value '{ $value }' diff --git a/web/locales/sk/main.ftl b/web/locales/sk/main.ftl index c838f6a5..3ae8875b 100644 --- a/web/locales/sk/main.ftl +++ b/web/locales/sk/main.ftl @@ -573,6 +573,7 @@ import-target-tables = Cieľová tabuľka import-choose-table = Vyberte tabuľku import-csv-file = Súbor CSV import-csv-data = Údaje CSV +import-csv-format-hint = Použite CSV v UTF-8 s oddeľovačom čiarka a každé pole uzavrite do dvojitých úvodzoviek, napríklad: "name","number","active" import-error-title = CSV sa nepodarilo importovať import-import-rows = Importovať riadky import-success-title = Import dokončený @@ -617,6 +618,7 @@ import-success-ignored-system = { $columns } prideľuje server, importované ria import-err-missing-type = Chýba typ stĺpca '{ $column }' import-err-permission = Vyžaduje sa oprávnenie na import. Tabuľka navyše potrebuje oprávnenie na vkladanie, aby sa do nej dalo načítať. import-err-unterminated-quote = CSV obsahuje neuzavretú úvodzovkovú hodnotu. +import-err-strict-csv = Neplatný formát CSV. Každé pole musí byť uzavreté v dvojitých úvodzovkách, polia musia byť oddelené čiarkami a každý záznam musí zostať na jednom riadku. import-err-empty = CSV je prázdny. import-err-bad-boolean = Neplatná boolovská hodnota '{ $value }' import-err-bad-integer = Neplatná celočíselná hodnota '{ $value }' diff --git a/web/src/pages/import_export/common/csv.rs b/web/src/pages/import_export/common/csv.rs index a1885d91..548c8311 100644 --- a/web/src/pages/import_export/common/csv.rs +++ b/web/src/pages/import_export/common/csv.rs @@ -1,51 +1,57 @@ use crate::{i18n::Locale, tr}; +/// Parses the one CSV shape accepted by data transfer: UTF-8, comma separated, +/// and every field enclosed in double quotes. Keeping one spelling makes the +/// file self-describing and leaves commas inside values unambiguous. pub(crate) fn parse_csv(locale: Locale, input: &str) -> Result>, String> { - // The byte-order mark Excel writes at the head of a "CSV UTF-8" file is - // encoding metadata, not a character of the first field, so decoding - // consumes it. Nothing else about the file is touched here: what the - // fields contain is the user's, and the import validates it rather than - // tidying it up. let input = input.strip_prefix('\u{feff}').unwrap_or(input); + if input.is_empty() { + return Err(tr!(locale, "import-err-empty")); + } + + let mut chars = input.chars().peekable(); let mut rows = Vec::new(); let mut row = Vec::new(); - let mut field = String::new(); - let mut chars = input.chars().peekable(); - let mut quoted = false; - while let Some(ch) = chars.next() { - match ch { - '"' if quoted && chars.peek() == Some(&'"') => { - field.push('"'); - let _ = chars.next(); - } - '"' => quoted = !quoted, - ',' if !quoted => row.push(std::mem::take(&mut field)), - '\n' if !quoted => { - if field.ends_with('\r') { - field.pop(); - } - row.push(std::mem::take(&mut field)); - if row.iter().any(|value| !value.is_empty()) { - rows.push(std::mem::take(&mut row)); - } else { - row.clear(); - } - } - _ => field.push(ch), + + loop { + match chars.next() { + Some('"') => {} + Some(_) => return Err(tr!(locale, "import-err-strict-csv")), + None if row.is_empty() => break, + None => return Err(tr!(locale, "import-err-strict-csv")), + } + + let mut field = String::new(); + loop { + match chars.next() { + Some('"') if chars.peek() == Some(&'"') => { + field.push('"'); + let _ = chars.next(); + } + Some('"') => break, + // A strict import row is one physical line. This also avoids + // hiding a missing closing quote many lines above the error. + Some('\n' | '\r') => return Err(tr!(locale, "import-err-strict-csv")), + Some(ch) => field.push(ch), + None => return Err(tr!(locale, "import-err-unterminated-quote")), + } } - } - if quoted { - return Err(tr!(locale, "import-err-unterminated-quote")); - } - if field.ends_with('\r') { - field.pop(); - } - if !field.is_empty() || !row.is_empty() { row.push(field); - if row.iter().any(|value| !value.is_empty()) { - rows.push(row); + + match chars.next() { + Some(',') => continue, + Some('\n') => rows.push(std::mem::take(&mut row)), + Some('\r') if chars.next() == Some('\n') => { + rows.push(std::mem::take(&mut row)); + } + Some(_) => return Err(tr!(locale, "import-err-strict-csv")), + None => { + rows.push(row); + break; + } } } + if rows.is_empty() { Err(tr!(locale, "import-err-empty")) } else { @@ -53,18 +59,15 @@ pub(crate) fn parse_csv(locale: Locale, input: &str) -> Result>, } } +/// Writes the same fully quoted format the importer accepts. pub(crate) fn write_record(output: &mut String, fields: &[String]) { for (index, field) in fields.iter().enumerate() { if index > 0 { output.push(','); } - if field.contains([',', '"', '\n', '\r']) { - output.push('"'); - output.push_str(&field.replace('"', "\"\"")); - output.push('"'); - } else { - output.push_str(field); - } + output.push('"'); + output.push_str(&field.replace('"', "\"\"")); + output.push('"'); } output.push('\n'); } @@ -74,10 +77,48 @@ mod tests { use super::*; #[test] - fn quoted_multiline_csv_round_trips() { - let fields = vec!["name".to_string(), "hello, \"world\"\nnext".to_string()]; + fn fully_quoted_csv_round_trips_commas_and_quotes() { + let fields = vec![ + "Acme, s.r.o.".to_string(), + "Customer said \"send it today\"".to_string(), + ]; let mut csv = String::new(); write_record(&mut csv, &fields); - assert_eq!(parse_csv(crate::i18n::Locale::default(), &csv).unwrap(), vec![fields]); + + assert_eq!( + csv, + "\"Acme, s.r.o.\",\"Customer said \"\"send it today\"\"\"\n" + ); + assert_eq!( + parse_csv(crate::i18n::Locale::default(), &csv).unwrap(), + vec![fields] + ); + } + + #[test] + fn every_field_must_be_quoted() { + for csv in [ + "name,num\nAcme,123\n", + "\"name\",num\n\"Acme\",\"123\"\n", + "\"name\",\"num\"\n\"Acme\",123\n", + "\"name\" ,\"num\"\n", + ] { + assert!(parse_csv(crate::i18n::Locale::default(), csv).is_err(), "{csv}"); + } + } + + #[test] + fn accepts_utf8_bom_and_crlf() { + let csv = "\u{feff}\"name\",\"num\"\r\n\"Acme\",\"123\"\r\n"; + assert_eq!( + parse_csv(crate::i18n::Locale::default(), csv).unwrap(), + vec![vec!["name", "num"], vec!["Acme", "123"]] + ); + } + + #[test] + fn multiline_fields_are_rejected() { + let csv = "\"name\",\"note\"\n\"Acme\",\"first\nsecond\"\n"; + assert!(parse_csv(crate::i18n::Locale::default(), csv).is_err()); } } diff --git a/web/src/pages/import_export/import/logic.rs b/web/src/pages/import_export/import/logic.rs index 0e93ec7d..62f8a8c9 100644 --- a/web/src/pages/import_export/import/logic.rs +++ b/web/src/pages/import_export/import/logic.rs @@ -635,10 +635,18 @@ mod tests { /// header is left exactly as it arrived. #[test] fn the_byte_order_mark_is_decoded_away_and_nothing_else_is() { - let rows = parse_csv(Locale::default(), "\u{feff}label,amount\nAcme,10\n").unwrap(); + let rows = parse_csv( + Locale::default(), + "\u{feff}\"label\",\"amount\"\n\"Acme\",\"10\"\n", + ) + .unwrap(); assert_eq!(rows[0], vec!["label", "amount"]); - let rows = parse_csv(Locale::default(), "LABEL, amount \nAcme,10\n").unwrap(); + let rows = parse_csv( + Locale::default(), + "\"LABEL\",\" amount \"\n\"Acme\",\"10\"\n", + ) + .unwrap(); assert_eq!(rows[0], vec!["LABEL", " amount "]); } diff --git a/web/templates/pages/import_export/import/fields.html b/web/templates/pages/import_export/import/fields.html index 9d407598..b08bb094 100644 --- a/web/templates/pages/import_export/import/fields.html +++ b/web/templates/pages/import_export/import/fields.html @@ -27,6 +27,7 @@ onchange="this.files[0]?.text().then(value => document.getElementById('csv-data').value = value)"> + {{ nav.tr("import-csv-format-hint") }}