filipriec/komp_ac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

import improvements

Browse Source
This commit is contained in:
Priec
2026-08-17 14:33:45 +02:00
parent 69dd0137ab
commit 1b9136f1de
17 changed files with 2162 additions and 905 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
web/src/lib.rs
View File

@@ -469,6 +469,11 @@ mod tests {
("/admin/validation/rules", ""),
("/admin/validation/sets", ""),
("/admin/import", ""),
("/admin/import/source", ""),
("/admin/import/prepare", ""),
("/admin/import/preview", ""),
("/admin/import/prepared.csv", ""),
("/admin/import/template.csv", ""),
("/admin/export.csv", ""),
] {
let response = test_router()

131
web/src/pages/import_export/common/schema.rs
View File

@@ -6,12 +6,12 @@ use crate::{i18n::Locale, tr};
use crate::definitions::table_structure::TableStructureResponse;
/// The columns an import can write, and the export's default header.
/// The export's default header.
///
/// Both ends use this list, so a file the default export writes is a file the
/// import accepts. That only holds if every system column is left out: a row is
/// inserted with `post_table_data`, which takes user columns and nothing else,
/// so exporting `row_revision` or `created_at` produced a file whose own
/// A file the default export writes is a file the import can be pointed at
/// column for column. That only holds if every system column is left out: a row
/// is inserted with `post_table_data`, which takes user columns and nothing
/// else, so exporting `row_revision` or `created_at` produced a file whose own
/// re-import the server answered with `Invalid column`. The names come from
/// the server's declarations rather than a list spelled out here, so a system
/// column added there is excluded here too.
@@ -80,24 +80,49 @@ fn is_read_omitted_column(name: &str) -> bool {
.any(|column| column.name == name)
}
/// Column names keyed by their lowercased form, for the one caller allowed to
/// match loosely: the normalizer, which rewrites a header into the column's own
/// spelling and shows the user the result before anything is imported.
/// The columns an import may write into: the destinations the mapping step
/// offers, in the order the table declares them.
///
/// Two columns that fold to the same key leave that key out entirely. There is
/// no answer to "which one did the header mean", and inventing one would be the
/// normalizer quietly choosing for the user.
pub(crate) fn folded_column_lookup(columns: &[String]) -> HashMap<String, String> {
let mut lookup: HashMap<String, Option<String>> = HashMap::new();
for column in columns {
lookup
.entry(column.to_lowercase())
.and_modify(|entry| *entry = None)
.or_insert_with(|| Some(column.clone()));
}
lookup
.into_iter()
.filter_map(|(folded, column)| Some((folded, column?)))
/// This is the list the whole import rests on. A source position lands in a
/// column because the user picked it from here, so what is not here cannot be
/// written to at all — which is why the exclusions are the server's own flags
/// rather than a guess:
///
/// * `is_primary_key` — `id` comes from a sequence.
/// * `read_only` — the server sets this for a quantity-ledger column and for a
/// link projection, and refuses an insert that names either. Accounting
/// columns are marked `generated` but *not* read-only, so they stay: they are
/// a user's to fill in.
/// * system columns, except the ones an insert actually takes. `deleted` is
/// offered, because writing it is how a file that recorded deleted rows loads
/// back as deleted rows; `row_revision` and `created_at` are not, because the
/// server assigns them and answers `Invalid column` to anything else.
pub(crate) fn importable_columns(schema: &TableStructureResponse) -> Vec<String> {
schema
.columns
.iter()
.filter(|column| !column.is_primary_key && !column.read_only)
.filter(|column| {
!is_system_column(&column.name) || is_importable_system_column(&column.name)
})
.map(|column| column.name.clone())
.collect()
}
/// The writable columns the table declares `NOT NULL`.
///
/// Shown in the preview as a warning rather than enforced as a rule: a
/// `NOT NULL` column may still have a default, and `information_schema` does
/// not say which do. Refusing here would block imports the server would have
/// accepted, so the server stays the one that decides and this only tells the
/// user which columns are the likely reason if it refuses.
pub(crate) fn required_columns(schema: &TableStructureResponse) -> Vec<String> {
schema
.columns
.iter()
.filter(|column| !column.is_primary_key && !column.read_only)
.filter(|column| !is_system_column(&column.name) && !column.is_nullable)
.map(|column| column.name.clone())
.collect()
}
@@ -160,6 +185,7 @@ mod tests {
name: name.to_string(),
data_type: "TEXT".to_string(),
is_primary_key,
is_nullable: true,
..Default::default()
}
}
@@ -207,4 +233,65 @@ mod tests {
]
);
}
/// What the mapping step may offer as a destination. `deleted` is in,
/// because an insert takes it; the columns the server assigns and the ones
/// it marks read-only are out, because an insert naming them is refused.
#[test]
fn only_the_columns_an_insert_takes_are_offered_as_destinations() {
let ledger = TableColumn {
read_only: true,
..column("stock", false)
};
// Accounting columns are generated companions, but the server leaves
// them writable — so they are a destination like any other.
let accounting = TableColumn {
generated: true,
generated_from: "accounting".to_string(),
..column("debit", false)
};
let schema = TableStructureResponse {
columns: vec![
column("id", true),
column("deleted", false),
column("row_revision", false),
column("number", false),
ledger,
accounting,
column("created_at", false),
],
};
assert_eq!(
importable_columns(&schema),
vec![
"deleted".to_string(),
"number".to_string(),
"debit".to_string(),
]
);
}
/// Only the user's own `NOT NULL` columns are reported as required.
/// `deleted` is `NOT NULL` on every managed table and has a default, so
/// reporting it would warn about every import ever prepared.
#[test]
fn required_columns_are_the_users_own_not_null_ones() {
let schema = TableStructureResponse {
columns: vec![
column("id", true),
TableColumn {
is_nullable: false,
..column("deleted", false)
},
TableColumn {
is_nullable: false,
..column("number", false)
},
column("note", false),
],
};
assert_eq!(required_columns(&schema), vec!["number".to_string()]);
}
}

6
web/src/pages/import_export/import/loader.rs
View File

@@ -4,20 +4,20 @@ use crate::AppState;
use super::{
super::common::loader::{LoadError, load_catalog},
state::{ImportForm, ImportPageState},
state::{ImportForm, ImportPageState, Step},
};
pub(crate) async fn load_page(
state: AppState,
headers: &HeaderMap,
form: ImportForm,
error: Option<String>,
step: Step,
) -> Result<ImportPageState, LoadError> {
let catalog = load_catalog(state, headers, crate::authz::IMPORT, "insert").await?;
Ok(ImportPageState {
nav: crate::ui::Nav::from_authorization(headers, "", &catalog.authorization),
catalog,
form,
error,
step,
})
}

1211
web/src/pages/import_export/import/logic.rs
View File

File diff suppressed because it is too large Load Diff

17
web/src/pages/import_export/import/mod.rs
View File

@@ -1,16 +1,29 @@
mod loader;
mod logic;
mod prepare;
mod state;
mod ui;
use axum::{Router, extract::DefaultBodyLimit, routing::{get, post}};
use axum::{
Router,
extract::DefaultBodyLimit,
routing::{get, post},
};
use crate::AppState;
pub(crate) fn router() -> Router<AppState> {
Router::new()
.route("/admin/import", get(logic::import_page))
// The preparation, step by step. Each one swaps the step block and
// leaves the rest of the form alone.
.route("/admin/import/source", post(logic::source_step))
.route("/admin/import/prepare", post(logic::prepare_step))
.route("/admin/import/preview", post(logic::preview_step))
// The two things a prepared import can be: rows in the table, or a file
// to look at first. Both read the same preparation.
.route("/admin/import", post(logic::import_csv))
.route("/admin/import/normalize", post(logic::normalize_headers))
.route("/admin/import/prepared.csv", post(logic::download_prepared))
.route("/admin/import/template.csv", post(logic::download_template))
.layer(DefaultBodyLimit::max(128 * 1024 * 1024))
}

565
web/src/pages/import_export/import/prepare.rs Normal file
View File

@@ -0,0 +1,565 @@
//! Turning what the user uploaded into the one CSV the import understands.
//!
//! Everything here is about *position*. A value lands in a column because the
//! user pointed position 3 at `active`, and for no other reason — not because
//! a source header spells something similar, not because the words look alike.
//! The source's own header, when it has one, is read out for orientation and
//! then thrown away.
//!
//! The output is the canonical form: fully quoted CSV whose header names real
//! destination columns, in the order the source presents them. From there the
//! import is the strict one it always was — types, validations, scripts, links
//! and permissions are all the server's, unchanged.
use std::collections::HashSet;
use crate::{i18n::Locale, tr};
use super::super::common::csv::{parse_csv, write_record};
/// How the uploaded text is laid out.
///
/// Always the user's answer, never inferred. A data row can hold words that
/// read exactly like column names — `"name","num"` is a perfectly good pair of
/// customer records — so deciding for them is how a real row gets silently
/// eaten as a header.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(crate) enum SourceMode {
/// The first row is the source's own header. It labels the mapping rows
/// and is not imported.
#[default]
Header,
/// Every row is data.
Data,
/// No source at all: pick and order destination columns, and take away a
/// header to fill in elsewhere.
Template,
}
impl SourceMode {
pub(crate) fn parse(locale: Locale, raw: &str) -> Result<Self, String> {
match raw {
"header" => Ok(Self::Header),
"data" => Ok(Self::Data),
"template" => Ok(Self::Template),
_ => Err(tr!(locale, "import-err-source-mode")),
}
}
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Header => "header",
Self::Data => "data",
Self::Template => "template",
}
}
/// Whether a source file is needed at all. A template is generated from the
/// table, so there is nothing to upload.
pub(crate) fn reads_a_file(self) -> bool {
!matches!(self, Self::Template)
}
}
/// The source file, split into the header it states and the rows that carry
/// values.
#[derive(Debug)]
pub(crate) struct Source {
/// The source's own header row, kept only to label the mapping rows. It is
/// `None` for a data-only file, and it never decides anything.
pub header: Option<Vec<String>>,
pub rows: Vec<Vec<String>>,
/// How many positions every row has. The mapping has exactly this many
/// entries, which is what makes "position 3" mean one thing.
pub width: usize,
}
impl Source {
/// What the first data row holds at each position, for the mapping step to
/// show beside the destination picker. An example is the one thing that
/// reliably tells a user which column they are looking at.
pub(crate) fn examples(&self) -> Vec<String> {
(0..self.width)
.map(|position| {
self.rows
.first()
.and_then(|row| row.get(position))
.cloned()
.unwrap_or_default()
})
.collect()
}
/// The source's name for a position, when it has one.
pub(crate) fn source_name(&self, position: usize) -> Option<String> {
self.header
.as_ref()
.and_then(|header| header.get(position))
.cloned()
}
}
/// Reads the uploaded text as `mode` says it is laid out.
pub(crate) fn read_source(locale: Locale, csv: &str, mode: SourceMode) -> Result<Source, String> {
let mut rows = parse_csv(locale, csv)?;
let header = match mode {
SourceMode::Header => {
if rows.len() < 2 {
return Err(tr!(locale, "import-err-no-data-rows"));
}
Some(rows.remove(0))
}
SourceMode::Data => None,
// Nothing to read: the caller decides the columns from the table.
SourceMode::Template => return Err(tr!(locale, "import-err-template-has-no-source")),
};
if rows.is_empty() {
return Err(tr!(locale, "import-err-no-data-rows"));
}
let width = header
.as_ref()
.map_or_else(|| rows[0].len(), |header| header.len());
if width == 0 {
return Err(tr!(locale, "import-err-empty"));
}
// Positions only mean anything if every row has the same ones. A short row
// would otherwise shift every value after the gap into the wrong column.
if let Some(row) = rows.iter().find(|row| row.len() != width) {
return Err(tr!(
locale,
"import-err-row-width",
"expected" => width as i64,
"found" => row.len() as i64,
));
}
Ok(Source { header, rows, width })
}
/// Where each source position is to be written, `None` being "ignore".
#[derive(Debug)]
pub(crate) struct Mapping(Vec<Option<String>>);
impl Mapping {
/// The destination chosen for a position, for the form to re-render with
/// the user's own answers still selected.
pub(crate) fn target(&self, position: usize) -> Option<&str> {
self.0.get(position).and_then(Option::as_deref)
}
pub(crate) fn len(&self) -> usize {
self.0.len()
}
}
/// Reads the posted mapping and checks the rules that make "proper data in the
/// proper place" true rather than hoped for.
///
/// `writable` is the destination list the table itself declares, loaded again
/// on every step: a column dropped or renamed while the form was open is caught
/// here instead of writing values into a column that no longer means what the
/// user chose.
pub(crate) fn read_mapping(
locale: Locale,
posted: &[String],
width: usize,
writable: &[String],
) -> Result<Mapping, String> {
// One select per position, always posted, so the form's shape and the
// file's have to agree. They will not if the CSV was edited after the
// mapping was built.
if posted.len() != width {
return Err(tr!(
locale,
"import-err-mapping-stale",
"positions" => width as i64,
"mapped" => posted.len() as i64,
));
}
let known = writable.iter().map(String::as_str).collect::<HashSet<_>>();
let mut used = HashSet::new();
let mut targets = Vec::with_capacity(width);
for (position, target) in posted.iter().enumerate() {
let target = target.trim();
if target.is_empty() {
targets.push(None);
continue;
}
if !known.contains(target) {
return Err(tr!(
locale,
"import-err-mapping-unknown-column",
"column" => target.to_string(),
"position" => (position + 1) as i64,
));
}
// Two positions pointed at one column let whichever is read second
// decide the value, silently. Refused instead.
if !used.insert(target.to_string()) {
return Err(tr!(
locale,
"import-err-mapping-duplicate",
"column" => target.to_string(),
));
}
targets.push(Some(target.to_string()));
}
if used.is_empty() {
return Err(tr!(locale, "import-err-mapping-empty"));
}
Ok(Mapping(targets))
}
/// The canonical import, and what it leaves behind.
pub(crate) struct Prepared {
/// The destination header, in source-position order — which is the only
/// order the data can be written in, since the values arrive in it.
pub columns: Vec<String>,
/// Data rows holding only the mapped positions, aligned to `columns`.
pub rows: Vec<Vec<String>>,
/// The source positions sent nowhere, numbered from 1 as the mapping step
/// numbers them.
pub ignored: Vec<usize>,
/// Destination columns this import does not write. The server fills them
/// with their defaults, or refuses the row if it cannot.
pub omitted: Vec<String>,
}
impl Prepared {
pub(crate) fn row_count(&self) -> usize {
self.rows.len()
}
/// The first `limit` rows, for the preview. A preview is for recognising
/// your own data, and nobody recognises it on row 40 000.
pub(crate) fn preview_rows(&self, limit: usize) -> &[Vec<String>] {
&self.rows[..self.rows.len().min(limit)]
}
pub(crate) fn hidden_rows(&self, limit: usize) -> usize {
self.rows.len().saturating_sub(limit)
}
}
/// Applies the mapping: the values that were pointed somewhere, under the names
/// they were pointed at.
pub(crate) fn prepare(mapping: &Mapping, source: &Source, writable: &[String]) -> Prepared {
let taken = (0..mapping.len())
.filter_map(|position| Some((position, mapping.target(position)?.to_string())))
.collect::<Vec<_>>();
let columns = taken
.iter()
.map(|(_, column)| column.clone())
.collect::<Vec<_>>();
let rows = source
.rows
.iter()
.map(|row| {
taken
.iter()
.map(|(position, _)| row.get(*position).cloned().unwrap_or_default())
.collect()
})
.collect();
let ignored = (0..source.width)
.filter(|position| mapping.target(*position).is_none())
.map(|position| position + 1)
.collect();
let omitted = writable
.iter()
.filter(|column| !columns.contains(column))
.cloned()
.collect();
Prepared {
columns,
rows,
ignored,
omitted,
}
}
/// The prepared import as text: exactly what the importer reads, and exactly
/// what "Download prepared CSV" hands over, so inspecting the file and
/// importing it cannot disagree.
pub(crate) fn canonical_csv(prepared: &Prepared) -> String {
let mut csv = String::new();
write_record(&mut csv, &prepared.columns);
for row in &prepared.rows {
write_record(&mut csv, row);
}
csv
}
/// A header and nothing else: the columns the user picked, in the order they
/// arranged them, ready to be filled in elsewhere and brought back as a file
/// with a header.
pub(crate) fn template_csv(columns: &[String]) -> String {
let mut csv = String::new();
write_record(&mut csv, columns);
csv
}
/// The destination a position starts out pointed at when the mapping step first
/// opens.
///
/// Only an exact match counts, and only against a real destination column. That
/// is not name interpretation: a header cell that *is* the column's name is the
/// column's name, which is the case whenever the file came from this system's
/// own template or export. Anything else starts at "Ignore", and every row is
/// in front of the user to confirm or change before a single value moves.
pub(crate) fn suggest_mapping(source: &Source, writable: &[String]) -> Vec<String> {
let mut used = HashSet::new();
(0..source.width)
.map(|position| {
let Some(name) = source.source_name(position) else {
return String::new();
};
if writable.contains(&name) && used.insert(name.clone()) {
name
} else {
String::new()
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn strings(values: &[&str]) -> Vec<String> {
values.iter().map(|value| value.to_string()).collect()
}
fn writable() -> Vec<String> {
strings(&["name", "company_number", "active", "note"])
}
/// A data-only file is all rows, and nothing in it is read as a header —
/// even when the first row happens to hold the column names.
#[test]
fn a_data_only_file_keeps_its_first_row() {
let csv = "\"name\",\"num\"\n\"Acme\",\"12345678\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Data).unwrap();
assert!(source.header.is_none());
assert_eq!(source.rows.len(), 2);
assert_eq!(source.examples(), strings(&["name", "num"]));
}
/// A file declared to have a header loses its first row, and that row is
/// only ever shown back to the user.
#[test]
fn a_header_file_loses_its_first_row_to_labels() {
let csv = "\"Company\",\"Registration number\"\n\"Acme\",\"12345678\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(source.rows.len(), 1);
assert_eq!(source.source_name(0).as_deref(), Some("Company"));
assert_eq!(source.examples(), strings(&["Acme", "12345678"]));
}
/// A header with no rows under it is not an import, and says so rather than
/// preparing an empty one.
#[test]
fn a_header_with_nothing_under_it_is_refused() {
let csv = "\"Company\",\"Registration number\"\n";
assert!(read_source(Locale::default(), csv, SourceMode::Header).is_err());
}
/// Positions only mean something if every row has the same ones.
#[test]
fn rows_of_different_widths_are_refused() {
let csv = "\"Acme\",\"12345678\"\n\"Example\"\n";
let error = read_source(Locale::default(), csv, SourceMode::Data)
.expect_err("a short row shifts every value after it");
assert!(error.contains('2') && error.contains('1'), "{error}");
}
/// The heart of it: the user's choices decide where values go, source names
/// are irrelevant, and an ignored position takes its value nowhere.
#[test]
fn values_land_where_the_user_pointed_them() {
let csv = "\"whatever\",\"old field 7\",\"enabled value\",\"legacy\"\n\
\"Acme\",\"12345678\",\"true\",\"drop me\"\n\
\"Example\",\"87654321\",\"false\",\"drop me too\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
let mapping = read_mapping(
Locale::default(),
&strings(&["name", "company_number", "active", ""]),
source.width,
&writable(),
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
assert_eq!(
prepared.columns,
strings(&["name", "company_number", "active"])
);
assert_eq!(prepared.rows[0], strings(&["Acme", "12345678", "true"]));
assert_eq!(prepared.ignored, vec![4]);
assert_eq!(prepared.omitted, strings(&["note"]));
assert_eq!(
canonical_csv(&prepared),
"\"name\",\"company_number\",\"active\"\n\
\"Acme\",\"12345678\",\"true\"\n\
\"Example\",\"87654321\",\"false\"\n"
);
}
/// Out-of-order and sparse mappings are the normal case, not an edge one:
/// the header follows the source's positions, because the values do.
#[test]
fn the_prepared_header_follows_the_source_order() {
let csv = "\"Acme\",\"unused legacy value\",\"true\",\"12345678\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Data).unwrap();
let mapping = read_mapping(
Locale::default(),
&strings(&["name", "", "active", "company_number"]),
source.width,
&writable(),
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
assert_eq!(
canonical_csv(&prepared),
"\"name\",\"active\",\"company_number\"\n\"Acme\",\"true\",\"12345678\"\n"
);
assert_eq!(prepared.ignored, vec![2]);
}
/// What is downloaded and what is imported are the same file. The import
/// reads `Prepared` directly rather than its own text, so this is the check
/// that the two cannot drift apart.
#[test]
fn the_downloaded_csv_parses_back_to_the_rows_that_get_imported() {
let csv = "\"Acme, s.r.o.\",\"said \"\"yes\"\"\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Data).unwrap();
let mapping = read_mapping(
Locale::default(),
&strings(&["name", "note"]),
source.width,
&writable(),
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
let reparsed = parse_csv(Locale::default(), &canonical_csv(&prepared)).unwrap();
assert_eq!(reparsed[0], prepared.columns);
assert_eq!(reparsed[1..], prepared.rows[..]);
}
/// One destination cannot be filled from two positions: the second would
/// silently win.
#[test]
fn a_destination_cannot_be_chosen_twice() {
let error = read_mapping(
Locale::default(),
&strings(&["name", "name"]),
2,
&writable(),
)
.expect_err("two positions cannot both be `name`");
assert!(error.contains("name"), "{error}");
}
/// A destination that is not a writable column of this table is refused,
/// whatever the form posted — the table is asked again on every step.
#[test]
fn a_destination_the_table_does_not_offer_is_refused() {
let error = read_mapping(
Locale::default(),
&strings(&["name", "id"]),
2,
&writable(),
)
.expect_err("`id` is the server's");
assert!(error.contains("id"), "{error}");
}
/// A mapping built against a different file is refused rather than applied
/// to whatever positions happen to line up.
#[test]
fn a_mapping_that_does_not_cover_the_file_is_refused() {
assert!(read_mapping(Locale::default(), &strings(&["name"]), 3, &writable()).is_err());
}
/// Ignoring everything is not an import.
#[test]
fn a_mapping_that_writes_nothing_is_refused() {
assert!(read_mapping(Locale::default(), &strings(&["", ""]), 2, &writable()).is_err());
}
/// A file this system generated the template for comes back with its own
/// column names, so every row starts out pointed at the right place — still
/// shown, still confirmed, never applied on its own.
#[test]
fn an_exact_header_starts_the_mapping_off() {
let csv = "\"name\",\"active\"\n\"Acme\",\"true\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(suggest_mapping(&source, &writable()), strings(&["name", "active"]));
}
/// Anything that is not exactly a column name starts at "Ignore". There is
/// no folding, no trimming and no similarity: ` name` and `Name` are not
/// the column `name`, and "Company" is not a guess anyone should make.
#[test]
fn nothing_but_an_exact_name_is_suggested() {
let csv = "\"Company\",\" name\",\"NAME\",\"note\"\n\"Acme\",\"x\",\"y\",\"z\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(
suggest_mapping(&source, &writable()),
strings(&["", "", "", "note"])
);
}
/// A header naming one column twice suggests it once: the second would be
/// a duplicate the mapping step refuses anyway.
#[test]
fn a_repeated_header_is_only_suggested_once() {
let csv = "\"name\",\"name\"\n\"Acme\",\"Other\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(suggest_mapping(&source, &writable()), strings(&["name", ""]));
}
/// The generated template is a header and nothing else, in the order it was
/// arranged in.
#[test]
fn a_template_is_the_chosen_columns_in_the_chosen_order() {
assert_eq!(
template_csv(&strings(&["active", "name", "company_number"])),
"\"active\",\"name\",\"company_number\"\n"
);
}
/// And it is a file this page can read straight back: the header mode plus
/// the exact-name suggestion close the loop.
#[test]
fn a_generated_template_comes_back_ready_to_import() {
let mut csv = template_csv(&strings(&["name", "active"]));
csv.push_str("\"Acme\",\"true\"\n");
let source = read_source(Locale::default(), &csv, SourceMode::Header).unwrap();
let mapping = read_mapping(
Locale::default(),
&suggest_mapping(&source, &writable()),
source.width,
&writable(),
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
assert_eq!(prepared.columns, strings(&["name", "active"]));
assert_eq!(prepared.rows, vec![strings(&["Acme", "true"])]);
}
#[test]
fn a_source_mode_round_trips_and_an_unknown_one_is_refused() {
for mode in [SourceMode::Header, SourceMode::Data, SourceMode::Template] {
assert_eq!(SourceMode::parse(Locale::default(), mode.as_str()), Ok(mode));
}
assert!(SourceMode::parse(Locale::default(), "guess").is_err());
}
}

223
web/src/pages/import_export/import/state.rs
View File

@@ -1,53 +1,214 @@
use crate::{i18n::Locale, tr};
use super::prepare::SourceMode;
/// Every field the import form carries, at every step.
///
/// The whole preparation is one form posted back and forth: the page keeps no
/// server-side session, so each step re-renders the fields the next one needs
/// and the user can go back without losing what they chose.
#[derive(Clone, Debug, Default, serde::Deserialize)]
pub(crate) struct ImportForm {
#[serde(default)]
pub profile_name: String,
/// One entry per checked table. The form posts the key once per checked
/// box, which only `axum_extra`'s `Form` decodes into a `Vec`.
/// One table. An import writes into one table, and a file for two tables is
/// two prepared imports rather than one file with a second header row.
#[serde(default)]
pub table_names: Vec<String>,
pub table_name: String,
/// `header`, `data` or `template` — see [`SourceMode`]. Kept as text so an
/// unrecognised value is a message rather than a form rejection.
#[serde(default)]
pub source_mode: String,
#[serde(default)]
pub csv_data: String,
/// Whether a system column in the file is written rather than left to the
/// server. Only `deleted` can be: see
/// [`IMPORTABLE_SYSTEM_COLUMNS`](super::super::common::schema::IMPORTABLE_SYSTEM_COLUMNS).
/// An unchecked box is not posted at all, so its absence is the `false`.
/// The destination chosen for each source position, in position order, one
/// entry per position; empty means "ignore". Posted once per `<select>`,
/// which only `axum_extra`'s `Form` decodes into a `Vec`.
#[serde(default)]
pub import_system_columns: Option<String>,
pub mapping: Vec<String>,
/// Every destination column in the order the template step has them
/// arranged, chosen or not, so the arrangement survives a re-render.
#[serde(default)]
pub template_order: Vec<String>,
/// The columns ticked in the template step, which the checkboxes post in
/// the arranged order — so this *is* the generated header.
#[serde(default)]
pub template_columns: Vec<String>,
/// `up` or `down` when a move button posted, with `index` naming the row.
#[serde(default)]
pub action: Option<String>,
#[serde(default)]
pub index: Option<usize>,
}
impl ImportForm {
pub(crate) fn mode(&self, locale: Locale) -> Result<SourceMode, String> {
if self.source_mode.is_empty() {
return Ok(SourceMode::default());
}
SourceMode::parse(locale, &self.source_mode)
}
/// The scope and table this import writes into, checked for being answered
/// at all. Whether they are the user's to write into is the catalog's
/// answer, and then the backend's.
pub(crate) fn target(&self, locale: Locale) -> Result<(String, String), String> {
let profile = self.profile_name.trim();
if profile.is_empty() {
return Err(tr!(locale, "import-err-select-profile"));
}
let table = self.table_name.trim();
if table.is_empty() {
return Err(tr!(locale, "import-err-tables-required"));
}
if self.mode(locale)?.reads_a_file() && self.csv_data.trim().is_empty() {
return Err(tr!(locale, "import-err-csv-required"));
}
Ok((profile.to_string(), table.to_string()))
}
/// Whether `name` is the table the form has selected.
pub(crate) fn is_table(&self, name: &str) -> bool {
self.table_name == name
}
/// Whether the radio for `mode` is the one selected, for re-rendering the
/// choice the user made.
pub(crate) fn is_mode(&self, mode: &str) -> bool {
if self.source_mode.is_empty() {
return mode == SourceMode::default().as_str();
}
self.source_mode == mode
}
}
/// Which of the preparation's steps the page is showing.
///
/// One page, three stops: say what the source is, say where each position goes,
/// then look at the result before anything is written.
pub(crate) enum Step {
/// Choose the destination table and describe the source.
Source,
/// Point each source position at a destination column.
Mapping(MappingStep),
/// Pick and arrange destination columns, and take away the header.
Template(TemplateStep),
/// The prepared import, as it will be sent.
Preview(PreviewStep),
}
pub(crate) struct MappingStep {
pub table_name: String,
/// The destinations to offer, in the order the table declares them.
pub columns: Vec<String>,
pub rows: Vec<MappingRow>,
pub source_rows: usize,
/// Whether the source stated a header, so the table can show that column at
/// all — and say that it decides nothing.
pub has_source_names: bool,
}
/// One source position, and what the user has pointed it at.
pub(crate) struct MappingRow {
/// Numbered from 1, the way the source is read by a person.
pub position: usize,
/// What the source called this position, when it said. Orientation only.
pub source_name: Option<String>,
/// The first data row's value here — usually the fastest way to recognise
/// which column this is.
pub example: String,
/// The chosen destination, empty for "ignore".
pub target: String,
}
impl MappingRow {
/// Whether this position is pointed at `column`, for re-rendering the
/// picker with the user's own answer selected.
pub(crate) fn targets(&self, column: &str) -> bool {
self.target == column
}
}
pub(crate) struct TemplateStep {
pub table_name: String,
pub rows: Vec<TemplateRow>,
/// The generated header as it stands, empty when nothing is ticked.
pub header: String,
pub chosen: usize,
}
pub(crate) struct TemplateRow {
/// Position in the arrangement, from 0, for the move buttons.
pub index: usize,
pub name: String,
pub chosen: bool,
/// Declared `NOT NULL`, so leaving it out of the template is likely to
/// produce rows the server refuses.
pub required: bool,
pub first: bool,
pub last: bool,
}
pub(crate) struct PreviewStep {
pub table_name: String,
pub columns: Vec<String>,
pub rows: Vec<Vec<String>>,
/// Rows the preview does not show, so a large file says how large.
pub hidden_rows: usize,
pub source_rows: usize,
/// Source positions written nowhere, labelled with the source's own name
/// when it gave one.
pub ignored: Vec<String>,
/// Destination columns this import does not write.
pub omitted: Vec<String>,
/// Of those, the ones the table declares `NOT NULL`. A warning, not a
/// refusal: the column may have a default, and only the server knows.
pub missing_required: Vec<String>,
/// The canonical CSV, which is both what gets imported and what the
/// download hands over.
pub csv: String,
}
pub(crate) struct ImportPageState {
pub nav: crate::ui::Nav,
pub catalog: super::super::common::loader::Catalog,
pub form: ImportForm,
pub error: Option<String>,
pub step: Step,
}
impl ImportForm {
pub(crate) fn import_system_columns(&self) -> bool {
self.import_system_columns.is_some()
#[cfg(test)]
mod tests {
use super::*;
/// A template needs no file, so the form does not insist on one — and the
/// other two modes do.
#[test]
fn only_a_file_import_needs_a_file() {
let form = ImportForm {
profile_name: "acme".to_string(),
table_name: "customers".to_string(),
source_mode: "template".to_string(),
..ImportForm::default()
};
assert_eq!(
form.target(Locale::default()).unwrap(),
("acme".to_string(), "customers".to_string())
);
let form = ImportForm {
source_mode: "data".to_string(),
..form
};
assert!(form.target(Locale::default()).is_err());
}
pub(crate) fn targets(&self, locale: Locale) -> Result<(String, Vec<String>), String> {
let profile = self.profile_name.trim();
if profile.is_empty() {
return Err(tr!(locale, "import-err-select-profile"));
}
let tables = self
.table_names
.iter()
.map(|table| table.trim())
.filter(|table| !table.is_empty())
.map(str::to_string)
.collect::<Vec<_>>();
if tables.is_empty() {
return Err(tr!(locale, "import-err-tables-required"));
}
if self.csv_data.trim().is_empty() {
return Err(tr!(locale, "import-err-csv-required"));
}
Ok((profile.to_string(), tables))
/// The default is a file with a header, and the radio renders as such
/// before anything has been posted.
#[test]
fn the_source_mode_defaults_to_a_header_file() {
let form = ImportForm::default();
assert_eq!(form.mode(Locale::default()).unwrap(), SourceMode::Header);
assert!(form.is_mode("header"));
assert!(!form.is_mode("data"));
}
}

218
web/src/pages/import_export/import/ui.rs
View File

@@ -1,50 +1,44 @@
use askama::Template;
use crate::{i18n::Locale, tr};
use crate::ui::{Alert, Nav, render};
use crate::{i18n::Locale, tr};
use super::state::ImportPageState;
// `Step` is named by the step template's `{% match %}`, so it has to be in
// scope here — the derive expands into this module.
use super::state::{ImportPageState, Step};
/// GET /admin/import
/// GET /admin/import — the whole page, whichever step it is showing.
#[derive(Template)]
#[template(path = "pages/import_export/import/import.html")]
struct ImportPage<'a> {
nav: Nav,
page: &'a ImportPageState,
/// `None` on a fresh page: the included fields template shows its
/// "normalized" line only after a rewrite has actually run.
changed: Option<bool>,
}
/// POST /admin/import/normalize — the form comes back with the rewritten CSV
/// in it, so the user reads what changed before importing anything.
/// The step block on its own. Every button that advances or goes back swaps
/// this, so the form around it — and the file the user chose — stays put.
#[derive(Template)]
#[template(path = "pages/import_export/import/fields.html")]
struct ImportFields<'a> {
#[template(path = "pages/import_export/import/step.html")]
struct ImportStep<'a> {
nav: Nav,
page: &'a ImportPageState,
/// Whether the rewrite actually changed the text, so a file that was
/// already exact says so instead of looking like it was edited.
changed: Option<bool>,
}
pub(crate) fn render_page(page: &ImportPageState) -> String {
render(&ImportPage {
nav: page.nav.clone(),
page,
changed: None,
})
}
pub(crate) fn render_normalized(page: &ImportPageState, changed: bool) -> String {
render(&ImportFields {
pub(crate) fn render_step(page: &ImportPageState) -> String {
render(&ImportStep {
nav: page.nav.clone(),
page,
changed: Some(changed),
})
}
/// POST /admin/import — the #submission-status swaps.
/// The #submission-status swaps.
pub(crate) fn render_error(locale: Locale, message: &str) -> String {
render(&Alert::error(
locale,
@@ -56,30 +50,176 @@ pub(crate) fn render_error(locale: Locale, message: &str) -> String {
pub(crate) fn render_success(
locale: Locale,
inserted: usize,
source_rows: usize,
table_count: usize,
ignored_system_columns: &[String],
prepared_rows: usize,
table_name: &str,
) -> String {
let mut message = tr!(
locale,
"import-success-message",
"inserted" => inserted as i64,
"source_rows" => source_rows as i64,
"table_count" => table_count as i64,
);
// What the file said and what was written differ here, so the result says
// so rather than leaving the user to assume their ids came across.
if !ignored_system_columns.is_empty() {
message.push('\n');
message.push_str(&tr!(
locale,
"import-success-ignored-system",
"columns" => ignored_system_columns.join(", "),
));
}
render(&Alert::success(
locale,
&tr!(locale, "import-success-title"),
&message,
&tr!(
locale,
"import-success-message",
"inserted" => inserted as i64,
"source_rows" => prepared_rows as i64,
"table" => table_name.to_string(),
),
))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::auth::AuthorizationSnapshot;
use crate::pages::import_export::common::loader::{Catalog, Profile};
use super::super::state::{
ImportForm, MappingRow, MappingStep, PreviewStep, TemplateRow, TemplateStep,
};
fn strings(values: &[&str]) -> Vec<String> {
values.iter().map(|value| value.to_string()).collect()
}
fn page(step: Step) -> ImportPageState {
let authorization = AuthorizationSnapshot {
role: "bookkeeper".to_string(),
permissions: Vec::new(),
};
ImportPageState {
nav: Nav::from_authorization(&axum::http::HeaderMap::new(), "", &authorization),
catalog: Catalog {
profiles: vec![Profile {
name: "acme".to_string(),
label: "acme".to_string(),
tables: strings(&["customers"]),
}],
authorization,
},
form: ImportForm {
profile_name: "acme".to_string(),
table_name: "customers".to_string(),
source_mode: "header".to_string(),
csv_data: "\"Company\"\n\"Acme\"\n".to_string(),
mapping: strings(&["name"]),
..ImportForm::default()
},
step,
}
}
/// The first step asks where the data goes and how the source is laid out,
/// and asks the second question outright — the page never decides for
/// itself whether the first row is a header.
#[test]
fn the_first_step_asks_how_the_source_is_laid_out() {
let html = render_page(&page(Step::Source));
for mode in ["header", "data", "template"] {
assert!(
html.contains(&format!(r#"name="source_mode" value="{mode}""#)),
"{html}"
);
}
assert!(html.contains(r#"name="table_name""#), "{html}");
assert!(html.contains(r#"name="csv_data""#), "{html}");
}
/// The mapping step is one picker per source position, and every picker
/// offers Ignore alongside the table's own columns. The source's own name
/// is shown next to it and is not what any option is built from.
#[test]
fn the_mapping_step_offers_a_destination_for_every_position() {
let html = render_step(&page(Step::Mapping(MappingStep {
table_name: "customers".to_string(),
columns: strings(&["name", "company_number"]),
rows: vec![
MappingRow {
position: 1,
source_name: Some("Company".to_string()),
example: "Acme".to_string(),
target: "name".to_string(),
},
MappingRow {
position: 2,
source_name: Some("Internal note".to_string()),
example: "old customer".to_string(),
target: String::new(),
},
],
source_rows: 2,
has_source_names: true,
})));
assert_eq!(html.matches(r#"name="mapping""#).count(), 2);
assert!(html.contains(r#"<option value="name" selected>"#), "{html}");
assert!(html.contains("Internal note"), "{html}");
assert!(html.contains("old customer"), "{html}");
// The destination and the source it came from are carried forward, so
// going back does not lose the file.
assert!(html.contains(r#"name="csv_data""#), "{html}");
}
/// The template step is a chooser with an arrangement, and it shows the
/// header it would generate rather than describing it.
#[test]
fn the_template_step_shows_the_header_it_generates() {
let html = render_step(&page(Step::Template(TemplateStep {
table_name: "customers".to_string(),
rows: vec![
TemplateRow {
index: 0,
name: "name".to_string(),
chosen: true,
required: true,
first: true,
last: false,
},
TemplateRow {
index: 1,
name: "note".to_string(),
chosen: false,
required: false,
first: false,
last: true,
},
],
header: "\"name\"\n".to_string(),
chosen: 1,
})));
assert!(html.contains("&#34;name&#34;"), "{html}");
assert_eq!(html.matches(r#"name="template_order""#).count(), 2);
assert!(html.contains(r#"formaction="/admin/import/template.csv""#), "{html}");
// The ends of the arrangement have nowhere to move to.
assert_eq!(html.matches("disabled").count(), 2);
}
/// The preview shows the values under the columns they will actually be
/// written to, says what is being left out, and offers both exits.
#[test]
fn the_preview_shows_the_prepared_import_and_both_ways_out() {
let html = render_step(&page(Step::Preview(PreviewStep {
table_name: "customers".to_string(),
columns: strings(&["name", "active"]),
rows: vec![strings(&["Acme", "true"])],
hidden_rows: 3,
source_rows: 4,
ignored: strings(&["2 (Internal note)"]),
omitted: strings(&["note"]),
missing_required: strings(&["company_number"]),
csv: "\"name\",\"active\"\n\"Acme\",\"true\"\n".to_string(),
})));
assert!(html.contains("<th>name</th>"), "{html}");
assert!(html.contains("<td>Acme</td>"), "{html}");
assert!(html.contains("2 (Internal note)"), "{html}");
assert!(html.contains("company_number"), "{html}");
// Import, and download the very same file.
assert!(html.contains(r#"hx-post="/admin/import""#), "{html}");
assert!(html.contains("&#34;name&#34;,&#34;active&#34;"), "{html}");
// The mapping travels with it, so the download and the import prepare
// the identical file.
assert!(html.contains(r#"name="mapping" value="name""#), "{html}");
}
}