filipriec/komp_ac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

import improvements2

Browse Source
This commit is contained in:
Priec
2026-08-17 15:20:31 +02:00
parent 1b9136f1de
commit 71807f9a6d
15 changed files with 944 additions and 1292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
web/locales/cs/main.ftl
View File

@@ -566,23 +566,18 @@ analytics-result-meta-truncated = { $rows ->
# --- Import / export -------------------------------------------------------------
transfer-eyebrow = Přenos dat
import-title = Import CSV
import-lead = Import připravíte ve třech krocích: určíte, kam data směřují, řeknete, co je který sloupec vašeho souboru, a výsledek si prohlédnete dřív, než se cokoli zapíše.
import-lead = Import připravíte ve třech krocích: vyberete tabulku a předáte soubor, řeknete, odkud bere hodnotu každý sloupec tabulky, a výsledek si prohlédnete dřív, než se cokoli zapíše.
import-scope = Rozsah
import-choose-scope = Vyberte globální nebo profil
import-target-tables = Cílová tabulka
import-choose-table = Vyberte tabulku
import-csv-file = Soubor CSV
import-csv-data = Data CSV
import-csv-format-hint = Použijte CSV v UTF-8 s oddělovačem čárka a každé pole uzavřete do dvojitých uvozovek, například: "Acme","12345678","true"
import-source-mode = Jak je zdroj uspořádán
import-source-header = Soubor má řádek záhlaví
import-source-data = Soubor obsahuje pouze data
import-source-template = Vygenerovat prázdnou šablonu
import-source-mode-hint = Toto je vždy vaše odpověď, nikdy odhad. Skutečný datový řádek může obsahovat slova, která vypadají přesně jako názvy sloupců, takže se zde nic samo nerozhoduje, zda je první řádek záhlaví.
import-csv-format-hint = První řádek je vlastní záhlaví souboru a každý další řádek jsou data. Použijte CSV v UTF-8 s oddělovačem čárka a každé pole uzavřete do dvojitých uvozovek, například: "hl","he","s"
import-continue = Pokračovat
import-carried = Import do tabulky { $table } v rozsahu { $scope }.
import-error-title = CSV se nepodařilo importovat
import-import-rows = Importovat připravená data
import-import-rows = Importovat
import-success-title = Import dokončen
import-success-message = Vloženo { $inserted ->
[one] { $inserted } řádek
@@ -594,43 +589,32 @@ import-success-message = Vloženo { $inserted ->
*[other] { $source_rows } připravených řádků
}.
# --- Krok 2a: mapování -----------------------------------------------------
import-mapping-heading = Pošlete každou pozici do sloupce tabulky { $table }
import-mapping-hint = Hodnoty se umisťují podle pozice, nikoli podle názvu. Název ze zdroje se zobrazuje jen proto, abyste pozice rozeznali; o ničem nerozhoduje. Pozice ponechaná na Ignorovat se neimportuje a sloupec, který zde nevyberete, se ponechá tabulce.
# --- Krok 2: mapování ------------------------------------------------------
import-mapping-heading = Odkud bere hodnotu každý sloupec tabulky { $table }?
import-mapping-hint = Každý sloupec tabulky je uveden jednou, takže žádný z nich nelze naplnit dvakrát. Hodnoty se berou podle pozice: název zdrojového sloupce se zobrazuje proto, abyste jej rozeznali, a o ničem nerozhoduje. Sloupec ponechaný na Neimportovat se nezapíše a ostatní sloupce souboru se ignorují.
import-mapping-rows = Soubor obsahuje { $rows } datových řádků.
import-th-position = Pozice
import-th-source-name = Název ve zdroji
import-th-example = Ukázková hodnota
import-th-destination = Poslat do
import-destination-ignore = Ignorovat
import-back-to-source = Zpět na zdroj
import-th-destination = Sloupec tabulky
import-th-required = Povinný
import-th-source = Bere hodnotu z
import-th-example = Ukázka
import-source-none = Neimportovat
import-source-option = { $name } — pozice { $position }
import-source-unnamed = pozice { $position }
import-back-to-source = Zpět na soubor
import-to-preview = Připravit a zobrazit
# --- Krok 2b: generátor šablony --------------------------------------------
import-template-heading = Sestavte šablonu pro tabulku { $table }
import-template-hint = Zaškrtněte sloupce, které má soubor obsahovat, a uspořádejte je v pořadí, v jakém je chcete vyplňovat.
import-th-order = Pořadí
import-th-include = Zahrnout
import-required-column = povinný
import-template-generated = Záhlaví, které se vygeneruje:
import-template-nothing-chosen = Zaškrtněte alespoň jeden sloupec.
import-template-round-trip = Vyplňte tento soubor jinde a pak jej přineste zpět s vybranou možností "Soubor má řádek záhlaví". Protože jde o vlastní názvy sloupců tabulky, každá pozice už bude namířena na správný sloupec a vy ji jen potvrdíte.
import-download-template = Stáhnout šablonu
# --- Krok 3: náhled --------------------------------------------------------
import-preview-heading = Připravený import pro tabulku { $table }
import-summary-rows = Řádky zdroje
import-summary-columns-used = Použité sloupce
import-summary-ignored = Ignorované pozice zdroje
import-summary-omitted = Vynechané cílové sloupce
import-summary-columns-used = Namapované sloupce tabulky
import-summary-ignored = Ignorované zdrojové sloupce
import-summary-omitted = Vynechané sloupce tabulky
import-summary-missing-required = Tento import je nevyplňuje a tabulka je deklaruje jako NOT NULL: { $columns }. Pokud tyto sloupce nemají výchozí hodnotu, server řádky odmítne.
import-preview-more-rows = Dalších { $rows } řádků je připraveno a zde se nezobrazuje.
import-prepared-csv = Připravené CSV
import-prepared-csv-hint = Toto je soubor, který se importuje, a tentýž soubor, který předá stažení.
import-download-prepared = Stáhnout připravené CSV
import-back-to-mapping = Zpět na mapování
import-move-up = Posunout { $name } nahoru
import-move-down = Posunout { $name } dolů
# --- Odmítnutí -------------------------------------------------------------
import-err-select-profile = Vyberte rozsah.
@@ -641,15 +625,12 @@ import-err-tables-not-in-profile = Cílová tabulka nepatří do vybraného rozs
import-err-missing-structure = Backend vynechal požadovanou strukturu tabulky.
import-err-no-importable-columns = Tabulka '{ $table }' nemá žádné sloupce, do kterých by import mohl zapisovat.
import-err-csv-row = Řádek { $row }: { $error }
import-err-row-width = Každý řádek musí obsahovat stejný počet pozic. Tento soubor začíná s { $expected } a později má řádek s { $found }.
import-err-no-data-rows = Soubor neobsahuje žádné datové řádky.
import-err-source-mode = Uveďte, jak je zdroj uspořádán.
import-err-template-has-no-source = Šablona se generuje z tabulky, takže není co připravovat ze souboru.
import-err-template-empty = Zaškrtněte pro šablonu alespoň jeden sloupec.
import-err-mapping-stale = Mapování bylo sestaveno pro soubor s { $mapped } pozicemi a tento má { $positions }. Připravte jej znovu.
import-err-mapping-unknown-column = Pozice { $position } míří na '{ $column }', což není sloupec, do kterého lze do této tabulky importovat. Tabulka se mohla změnit, zatímco byl formulář otevřený.
import-err-mapping-duplicate = Dvě pozice míří na '{ $column }'. Sloupec lze naplnit jen z jedné pozice.
import-err-mapping-empty = Všechny pozice jsou nastaveny na Ignorovat, takže není co importovat.
import-err-row-width = Každý řádek musí obsahovat tolik hodnot jako záhlaví. Záhlaví má { $expected } a pozdější řádek má { $found }.
import-err-no-data-rows = Soubor potřebuje řádek záhlaví a pod ním alespoň jeden datový řádek.
import-err-mapping-empty = Žádný sloupec tabulky není namapován, takže není co importovat.
import-err-source-used-twice = Zdrojový sloupec '{ $name }' na pozici { $position } se používá pro dva sloupce tabulky. Zdrojový sloupec může naplnit jen jeden sloupec tabulky.
import-err-source-position-missing = Sloupec '{ $column }' má brát pozici { $position }, a soubor má jen { $positions }. Připravte mapování znovu.
import-err-destination-missing = Namapovaný sloupec už v tabulce '{ $table }' neexistuje nebo do něj už import nemůže zapisovat. Připravte mapování znovu.
import-err-missing-type = Chybí typ sloupce '{ $column }'
import-err-permission = Vyžaduje se oprávnění k importu. Tabulka navíc potřebuje oprávnění ke vkládání, aby do ní šlo načítat.
import-err-unterminated-quote = CSV obsahuje neuzavřenou uvozovkovou hodnotu.

65
web/locales/en/main.ftl
View File

@@ -556,23 +556,18 @@ analytics-result-meta-truncated = { $rows ->
# --- Import / export -------------------------------------------------------
transfer-eyebrow = Data transfer
import-title = Import CSV
import-lead = Prepare an import in three steps: say where the data is going, say what each column of your file is, then look at the result before anything is written.
import-lead = Prepare an import in three steps: choose the table and hand over the file, say where each column of the table takes its value from, then look at the result before anything is written.
import-scope = Scope
import-choose-scope = Choose global or a profile
import-target-tables = Target table
import-choose-table = Choose a table
import-csv-file = CSV file
import-csv-data = CSV data
import-csv-format-hint = Use UTF-8 CSV with a comma separator and every field in double quotes, for example: "Acme","12345678","true"
import-source-mode = How the source is laid out
import-source-header = The file has a header row
import-source-data = The file is data only
import-source-template = Generate an empty template
import-source-mode-hint = This is always your answer, never a guess. A real data row can hold words that read exactly like column names, so nothing here decides on its own whether the first row is a header.
import-csv-format-hint = The first row is the file's own header, and every row after it is data. Use UTF-8 CSV with a comma separator and every field in double quotes, for example: "hl","he","s"
import-continue = Continue
import-carried = Importing into { $table } in { $scope }.
import-error-title = Could not import CSV
import-import-rows = Import prepared data
import-import-rows = Import
import-success-title = Import complete
import-success-message = Inserted { $inserted ->
[one] { $inserted } row
@@ -582,43 +577,32 @@ import-success-message = Inserted { $inserted ->
*[other] { $source_rows } prepared rows
}.
# --- Step 2a: mapping ------------------------------------------------------
import-mapping-heading = Send each position to a column of { $table }
import-mapping-hint = Values are placed by position, not by name. A source name is shown only so you can tell the positions apart; it decides nothing. A position left on Ignore is not imported, and a column not chosen here is left to the table.
# --- Step 2: mapping -------------------------------------------------------
import-mapping-heading = Where does each column of { $table } take its value from?
import-mapping-hint = Every column of the table is listed once, so none of them can be filled twice. Values are taken by position: a source column's name is shown so you can recognise it, and decides nothing. A column left on Do not import is not written, and the file's other columns are ignored.
import-mapping-rows = The file holds { $rows } data rows.
import-th-position = Position
import-th-source-name = Source name
import-th-example = Example value
import-th-destination = Send to
import-destination-ignore = Ignore
import-back-to-source = Back to the source
import-th-destination = Table column
import-th-required = Required
import-th-source = Takes its value from
import-th-example = Example
import-source-none = Do not import
import-source-option = { $name } — position { $position }
import-source-unnamed = position { $position }
import-back-to-source = Back to the file
import-to-preview = Prepare and preview
# --- Step 2b: template generator -------------------------------------------
import-template-heading = Build a template for { $table }
import-template-hint = Tick the columns the file should contain and arrange them in the order you want to fill them in.
import-th-order = Order
import-th-include = Include
import-required-column = required
import-template-generated = The header this generates:
import-template-nothing-chosen = Tick at least one column.
import-template-round-trip = Fill this in elsewhere, then bring it back with "The file has a header row" selected. Because these are the table's own column names, every position will already be pointed at the right column for you to confirm.
import-download-template = Download template
# --- Step 3: preview -------------------------------------------------------
import-preview-heading = Prepared import for { $table }
import-summary-rows = Source rows
import-summary-columns-used = Columns used
import-summary-ignored = Source positions ignored
import-summary-omitted = Destination columns omitted
import-summary-columns-used = Mapped table columns
import-summary-ignored = Ignored source columns
import-summary-omitted = Omitted table columns
import-summary-missing-required = Not filled in by this import, and declared NOT NULL by the table: { $columns }. If those columns have no default, the server will refuse the rows.
import-preview-more-rows = { $rows } further rows are prepared and not shown here.
import-prepared-csv = The prepared CSV
import-prepared-csv-hint = This is the file that gets imported, and the same file the download hands over.
import-download-prepared = Download prepared CSV
import-back-to-mapping = Back to the mapping
import-move-up = Move { $name } up
import-move-down = Move { $name } down
# --- Refusals --------------------------------------------------------------
import-err-select-profile = Select a scope.
@@ -629,15 +613,12 @@ import-err-tables-not-in-profile = The target table does not belong to the selec
import-err-missing-structure = The backend omitted a requested table structure.
import-err-no-importable-columns = Table '{ $table }' has no columns an import can write into.
import-err-csv-row = Row { $row }: { $error }
import-err-row-width = Every row must hold the same number of positions. This file starts with { $expected } and later has a row of { $found }.
import-err-no-data-rows = The file holds no data rows.
import-err-source-mode = Say how the source is laid out.
import-err-template-has-no-source = A template is generated from the table, so there is no file to prepare.
import-err-template-empty = Tick at least one column for the template.
import-err-mapping-stale = The mapping was built for a file of { $mapped } positions and this one has { $positions }. Prepare it again.
import-err-mapping-unknown-column = Position { $position } is pointed at '{ $column }', which is not a column this table can be imported into. The table may have changed while the form was open.
import-err-mapping-duplicate = Two positions are pointed at '{ $column }'. A column can be filled from one position only.
import-err-mapping-empty = Every position is set to Ignore, so there is nothing to import.
import-err-row-width = Every row must hold as many values as the header. The header has { $expected } and a later row has { $found }.
import-err-no-data-rows = The file needs a header row and at least one data row under it.
import-err-mapping-empty = No column of the table is mapped, so there is nothing to import.
import-err-source-used-twice = Source column '{ $name }' at position { $position } is used for two table columns. A source column can fill one table column only.
import-err-source-position-missing = Column '{ $column }' is set to take position { $position }, and the file only has { $positions }. Prepare the mapping again.
import-err-destination-missing = A mapped column no longer exists in table '{ $table }', or is no longer one an import can write into. Prepare the mapping again.
import-err-missing-type = Missing type for column '{ $column }'
import-err-permission = Import permission is required. A table also needs insert permission before it can be loaded.
import-err-unterminated-quote = CSV contains an unterminated quoted value.

65
web/locales/sk/main.ftl
View File

@@ -566,23 +566,18 @@ analytics-result-meta-truncated = { $rows ->
# --- Import / export ---------------------------------------------------------
transfer-eyebrow = Prenos údajov
import-title = Import CSV
import-lead = Import pripravíte v troch krokoch: určíte, kam údaje smerujú, poviete, čo je ktorý stĺpec vášho súboru, a výsledok si pozriete skôr, než sa čokoľvek zapíše.
import-lead = Import pripravíte v troch krokoch: vyberiete tabuľku a odovzdáte súbor, poviete, odkiaľ berie hodnotu každý stĺpec tabuľky, a výsledok si pozriete skôr, než sa čokoľvek zapíše.
import-scope = Rozsah
import-choose-scope = Vyberte globálne alebo profil
import-target-tables = Cieľová tabuľka
import-choose-table = Vyberte tabuľku
import-csv-file = Súbor CSV
import-csv-data = Údaje CSV
import-csv-format-hint = Použite CSV v UTF-8 s oddeľovačom čiarka a každé pole uzavrite do dvojitých úvodzoviek, napríklad: "Acme","12345678","true"
import-source-mode = Ako je zdroj usporiadaný
import-source-header = Súbor má riadok hlavičky
import-source-data = Súbor obsahuje len údaje
import-source-template = Vygenerovať prázdnu šablónu
import-source-mode-hint = Toto je vždy vaša odpoveď, nikdy odhad. Skutočný riadok údajov môže obsahovať slová, ktoré vyzerajú presne ako názvy stĺpcov, takže sa tu nič samo nerozhoduje, či je prvý riadok hlavička.
import-csv-format-hint = Prvý riadok je vlastná hlavička súboru a každý ďalší riadok sú údaje. Použite CSV v UTF-8 s oddeľovačom čiarka a každé pole uzavrite do dvojitých úvodzoviek, napríklad: "hl","he","s"
import-continue = Pokračovať
import-carried = Import do tabuľky { $table } v rozsahu { $scope }.
import-error-title = CSV sa nepodarilo importovať
import-import-rows = Importovať pripravené údaje
import-import-rows = Importovať
import-success-title = Import dokončený
import-success-message = Vložený { $inserted ->
[one] { $inserted } riadok
@@ -592,43 +587,32 @@ import-success-message = Vložený { $inserted ->
*[other] { $source_rows } pripravených riadkov
}.
# --- Krok 2a: mapovanie ----------------------------------------------------
import-mapping-heading = Pošlite každú pozíciu do stĺpca tabuľky { $table }
import-mapping-hint = Hodnoty sa umiestňujú podľa pozície, nie podľa názvu. Názov zo zdroja sa zobrazuje len preto, aby ste pozície rozoznali; nerozhoduje o ničom. Pozícia ponechaná na Ignorovať sa neimportuje a stĺpec, ktorý tu nevyberiete, sa ponechá tabuľke.
# --- Krok 2: mapovanie -----------------------------------------------------
import-mapping-heading = Odkiaľ berie hodnotu každý stĺpec tabuľky { $table }?
import-mapping-hint = Každý stĺpec tabuľky je uvedený raz, takže žiadny z nich sa nedá naplniť dvakrát. Hodnoty sa berú podľa pozície: názov zdrojového stĺpca sa zobrazuje preto, aby ste ho rozoznali, a o ničom nerozhoduje. Stĺpec ponechaný na Neimportovať sa nezapíše a ostatné stĺpce súboru sa ignorujú.
import-mapping-rows = Súbor obsahuje { $rows } riadkov údajov.
import-th-position = Pozícia
import-th-source-name = Názov v zdroji
import-th-example = Ukážková hodnota
import-th-destination = Poslať do
import-destination-ignore = Ignorovať
import-back-to-source = Späť na zdroj
import-th-destination = Stĺpec tabuľky
import-th-required = Povinný
import-th-source = Berie hodnotu z
import-th-example = Ukážka
import-source-none = Neimportovať
import-source-option = { $name } — pozícia { $position }
import-source-unnamed = pozícia { $position }
import-back-to-source = Späť na súbor
import-to-preview = Pripraviť a zobraziť
# --- Krok 2b: generátor šablóny --------------------------------------------
import-template-heading = Zostavte šablónu pre tabuľku { $table }
import-template-hint = Zaškrtnite stĺpce, ktoré má súbor obsahovať, a usporiadajte ich v poradí, v akom ich chcete vypĺňať.
import-th-order = Poradie
import-th-include = Zahrnúť
import-required-column = povinný
import-template-generated = Hlavička, ktorá sa vygeneruje:
import-template-nothing-chosen = Zaškrtnite aspoň jeden stĺpec.
import-template-round-trip = Vyplňte tento súbor inde a potom ho prineste späť s vybranou možnosťou "Súbor má riadok hlavičky". Keďže ide o vlastné názvy stĺpcov tabuľky, každá pozícia už bude nasmerovaná na správny stĺpec a vy ju len potvrdíte.
import-download-template = Stiahnuť šablónu
# --- Krok 3: náhľad --------------------------------------------------------
import-preview-heading = Pripravený import pre tabuľku { $table }
import-summary-rows = Riadky zdroja
import-summary-columns-used = Použité stĺpce
import-summary-ignored = Ignorované pozície zdroja
import-summary-omitted = Vynechané cieľové stĺpce
import-summary-columns-used = Namapované stĺpce tabuľky
import-summary-ignored = Ignorované zdrojové stĺpce
import-summary-omitted = Vynechané stĺpce tabuľky
import-summary-missing-required = Tento import ich nevypĺňa a tabuľka ich deklaruje ako NOT NULL: { $columns }. Ak tieto stĺpce nemajú predvolenú hodnotu, server riadky odmietne.
import-preview-more-rows = Ďalších { $rows } riadkov je pripravených a tu sa nezobrazuje.
import-prepared-csv = Pripravené CSV
import-prepared-csv-hint = Toto je súbor, ktorý sa importuje, a ten istý súbor, ktorý odovzdá stiahnutie.
import-download-prepared = Stiahnuť pripravené CSV
import-back-to-mapping = Späť na mapovanie
import-move-up = Posunúť { $name } nahor
import-move-down = Posunúť { $name } nadol
# --- Odmietnutia -----------------------------------------------------------
import-err-select-profile = Vyberte rozsah.
@@ -639,15 +623,12 @@ import-err-tables-not-in-profile = Cieľová tabuľka nepatrí do vybraného roz
import-err-missing-structure = Backend vynechal požadovanú štruktúru tabuľky.
import-err-no-importable-columns = Tabuľka '{ $table }' nemá žiadne stĺpce, do ktorých by import mohol zapisovať.
import-err-csv-row = Riadok { $row }: { $error }
import-err-row-width = Každý riadok musí obsahovať rovnaký počet pozícií. Tento súbor začína s { $expected } a neskôr má riadok s { $found }.
import-err-no-data-rows = Súbor neobsahuje žiadne riadky údajov.
import-err-source-mode = Uveďte, ako je zdroj usporiadaný.
import-err-template-has-no-source = Šablóna sa generuje z tabuľky, takže nie je čo pripravovať zo súboru.
import-err-template-empty = Zaškrtnite pre šablónu aspoň jeden stĺpec.
import-err-mapping-stale = Mapovanie bolo zostavené pre súbor s { $mapped } pozíciami a tento má { $positions }. Pripravte ho znova.
import-err-mapping-unknown-column = Pozícia { $position } smeruje na '{ $column }', čo nie je stĺpec, do ktorého sa dá do tejto tabuľky importovať. Tabuľka sa mohla zmeniť, kým bol formulár otvorený.
import-err-mapping-duplicate = Dve pozície smerujú na '{ $column }'. Stĺpec sa dá naplniť len z jednej pozície.
import-err-mapping-empty = Všetky pozície sú nastavené na Ignorovať, takže nie je čo importovať.
import-err-row-width = Každý riadok musí obsahovať toľko hodnôt ako hlavička. Hlavička má { $expected } a neskorší riadok má { $found }.
import-err-no-data-rows = Súbor potrebuje riadok hlavičky a pod ním aspoň jeden riadok údajov.
import-err-mapping-empty = Žiadny stĺpec tabuľky nie je namapovaný, takže nie je čo importovať.
import-err-source-used-twice = Zdrojový stĺpec '{ $name }' na pozícii { $position } sa používa pre dva stĺpce tabuľky. Zdrojový stĺpec môže naplniť len jeden stĺpec tabuľky.
import-err-source-position-missing = Stĺpec '{ $column }' má brať pozíciu { $position }, a súbor má len { $positions }. Pripravte mapovanie znova.
import-err-destination-missing = Namapovaný stĺpec už v tabuľke '{ $table }' neexistuje alebo do neho už import nemôže zapisovať. Pripravte mapovanie znova.
import-err-missing-type = Chýba typ stĺpca '{ $column }'
import-err-permission = Vyžaduje sa oprávnenie na import. Tabuľka navyše potrebuje oprávnenie na vkladanie, aby sa do nej dalo načítať.
import-err-unterminated-quote = CSV obsahuje neuzavretú úvodzovkovú hodnotu.

1
web/src/lib.rs
View File

@@ -473,7 +473,6 @@ mod tests {
("/admin/import/prepare", ""),
("/admin/import/preview", ""),
("/admin/import/prepared.csv", ""),
("/admin/import/template.csv", ""),
("/admin/export.csv", ""),
] {
let response = test_router()

105
web/src/pages/import_export/common/schema.rs
View File

@@ -80,52 +80,6 @@ fn is_read_omitted_column(name: &str) -> bool {
.any(|column| column.name == name)
}
/// The columns an import may write into: the destinations the mapping step
/// offers, in the order the table declares them.
///
/// This is the list the whole import rests on. A source position lands in a
/// column because the user picked it from here, so what is not here cannot be
/// written to at all — which is why the exclusions are the server's own flags
/// rather than a guess:
///
/// * `is_primary_key` — `id` comes from a sequence.
/// * `read_only` — the server sets this for a quantity-ledger column and for a
/// link projection, and refuses an insert that names either. Accounting
/// columns are marked `generated` but *not* read-only, so they stay: they are
/// a user's to fill in.
/// * system columns, except the ones an insert actually takes. `deleted` is
/// offered, because writing it is how a file that recorded deleted rows loads
/// back as deleted rows; `row_revision` and `created_at` are not, because the
/// server assigns them and answers `Invalid column` to anything else.
pub(crate) fn importable_columns(schema: &TableStructureResponse) -> Vec<String> {
schema
.columns
.iter()
.filter(|column| !column.is_primary_key && !column.read_only)
.filter(|column| {
!is_system_column(&column.name) || is_importable_system_column(&column.name)
})
.map(|column| column.name.clone())
.collect()
}
/// The writable columns the table declares `NOT NULL`.
///
/// Shown in the preview as a warning rather than enforced as a rule: a
/// `NOT NULL` column may still have a default, and `information_schema` does
/// not say which do. Refusing here would block imports the server would have
/// accepted, so the server stays the one that decides and this only tells the
/// user which columns are the likely reason if it refuses.
pub(crate) fn required_columns(schema: &TableStructureResponse) -> Vec<String> {
schema
.columns
.iter()
.filter(|column| !column.is_primary_key && !column.read_only)
.filter(|column| !is_system_column(&column.name) && !column.is_nullable)
.map(|column| column.name.clone())
.collect()
}
pub(crate) fn column_types(schema: &TableStructureResponse) -> HashMap<String, String> {
schema
.columns
@@ -234,64 +188,5 @@ mod tests {
);
}
/// What the mapping step may offer as a destination. `deleted` is in,
/// because an insert takes it; the columns the server assigns and the ones
/// it marks read-only are out, because an insert naming them is refused.
#[test]
fn only_the_columns_an_insert_takes_are_offered_as_destinations() {
let ledger = TableColumn {
read_only: true,
..column("stock", false)
};
// Accounting columns are generated companions, but the server leaves
// them writable — so they are a destination like any other.
let accounting = TableColumn {
generated: true,
generated_from: "accounting".to_string(),
..column("debit", false)
};
let schema = TableStructureResponse {
columns: vec![
column("id", true),
column("deleted", false),
column("row_revision", false),
column("number", false),
ledger,
accounting,
column("created_at", false),
],
};
assert_eq!(
importable_columns(&schema),
vec![
"deleted".to_string(),
"number".to_string(),
"debit".to_string(),
]
);
}
/// Only the user's own `NOT NULL` columns are reported as required.
/// `deleted` is `NOT NULL` on every managed table and has a default, so
/// reporting it would warn about every import ever prepared.
#[test]
fn required_columns_are_the_users_own_not_null_ones() {
let schema = TableStructureResponse {
columns: vec![
column("id", true),
TableColumn {
is_nullable: false,
..column("deleted", false)
},
TableColumn {
is_nullable: false,
..column("number", false)
},
column("note", false),
],
};
assert_eq!(required_columns(&schema), vec!["number".to_string()]);
}
}

227
web/src/pages/import_export/import/destination.rs Normal file
View File

@@ -0,0 +1,227 @@
//! The columns an import may write into, and how the form names one.
//!
//! The mapping is read destination-first: every writable column of the table
//! gets one row, and the row asks where its value comes from. That is what
//! makes "one destination filled twice" impossible to express rather than
//! merely refused — each destination appears exactly once, by construction.
use crate::definitions::table_structure::{TableColumn, TableStructureResponse};
use super::super::common::schema::{is_importable_system_column, is_system_column};
/// How a destination column is named in the form.
///
/// Not by its display name. A user column has a stable identity the server
/// assigns once — `column_id` — and it keeps that identity through a rename, so
/// a mapping built before a rename still means the same column afterwards
/// rather than silently pointing at whatever now answers to the old name.
///
/// The system columns have no such id (`column_id` is zero for them), so they
/// are named by the one thing they do have. That is safe for exactly the reason
/// the id exists: a system column's name is the server's and cannot be renamed.
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum DestinationKey {
Column(i64),
System(String),
}
impl DestinationKey {
fn of(column: &TableColumn) -> Self {
if column.column_id == 0 {
Self::System(column.name.clone())
} else {
Self::Column(column.column_id)
}
}
/// What the hidden field carries.
pub(crate) fn encode(&self) -> String {
match self {
Self::Column(id) => format!("id:{id}"),
Self::System(name) => format!("system:{name}"),
}
}
pub(crate) fn parse(raw: &str) -> Option<Self> {
if let Some(id) = raw.strip_prefix("id:") {
return id.parse().ok().map(Self::Column);
}
raw.strip_prefix("system:")
.map(|name| Self::System(name.to_string()))
}
}
/// One column an import may write into.
#[derive(Clone, Debug)]
pub(crate) struct DestinationColumn {
pub key: DestinationKey,
/// The name the table currently shows, which is what the page displays and
/// what the prepared CSV's header says. The identity is `key`.
pub name: String,
/// Declared `NOT NULL`. A warning in the preview rather than a rule: the
/// column may have a default, and `information_schema` does not say which
/// do, so refusing here would block imports the server would accept.
pub required: bool,
}
/// The columns of `schema` an import may write into, in the order the table
/// declares them.
///
/// The exclusions are the server's own flags rather than a guess:
///
/// * `is_primary_key` — `id` comes from a sequence.
/// * `read_only` — set for a quantity-ledger column and for a link projection,
/// and an insert naming either is refused. Accounting columns are marked
/// `generated` but *not* read-only, so they stay: they are a user's to fill
/// in.
/// * system columns, except the ones an insert actually takes. `deleted` is
/// offered, because writing it is how a file that recorded deleted rows loads
/// back as deleted rows; `row_revision` and `created_at` are not, because the
/// server assigns them and answers `Invalid column` to anything else.
pub(crate) fn destination_columns(schema: &TableStructureResponse) -> Vec<DestinationColumn> {
schema
.columns
.iter()
.filter(|column| !column.is_primary_key && !column.read_only)
.filter(|column| {
!is_system_column(&column.name) || is_importable_system_column(&column.name)
})
.map(|column| DestinationColumn {
key: DestinationKey::of(column),
name: column.name.clone(),
// A system column's nullability is the server's business, and
// `deleted` is NOT NULL with a default on every managed table —
// reporting it would warn about every import ever prepared.
required: !column.is_nullable && !is_system_column(&column.name),
})
.collect()
}
/// The column a posted key names, in the table as it stands right now.
///
/// `None` means the table no longer has it — dropped, or made read-only, while
/// the form sat open. The caller turns that into a refusal; it must never
/// become "write it somewhere else".
pub(crate) fn resolve<'a>(
columns: &'a [DestinationColumn],
raw: &str,
) -> Option<&'a DestinationColumn> {
let key = DestinationKey::parse(raw)?;
columns.iter().find(|column| column.key == key)
}
#[cfg(test)]
mod tests {
use super::*;
fn column(name: &str, column_id: i64) -> TableColumn {
TableColumn {
name: name.to_string(),
data_type: "TEXT".to_string(),
is_nullable: true,
column_id,
..Default::default()
}
}
fn schema() -> TableStructureResponse {
TableStructureResponse {
columns: vec![
TableColumn {
is_primary_key: true,
..column("id", 0)
},
column("deleted", 0),
column("row_revision", 0),
TableColumn {
is_nullable: false,
..column("number", 42)
},
// A quantity-ledger column: the server refuses an insert that
// names it, so it is not a destination.
TableColumn {
read_only: true,
..column("stock", 57)
},
// Accounting columns are generated companions but stay
// writable, so they are destinations like any other.
TableColumn {
generated: true,
generated_from: "accounting".to_string(),
..column("debit", 81)
},
column("created_at", 0),
],
}
}
#[test]
fn only_the_columns_an_insert_takes_are_offered() {
let columns = destination_columns(&schema());
assert_eq!(
columns
.iter()
.map(|column| column.name.as_str())
.collect::<Vec<_>>(),
vec!["deleted", "number", "debit"]
);
}
/// A user column is identified by the id the server gave it, so a rename
/// cannot move a mapping onto a different column. A system column has no
/// id and no rename, so its name is identity enough.
#[test]
fn a_user_column_is_named_by_its_stable_id_and_a_system_one_by_its_name() {
let columns = destination_columns(&schema());
assert_eq!(columns[0].key.encode(), "system:deleted");
assert_eq!(columns[1].key.encode(), "id:42");
for column in &columns {
assert_eq!(
DestinationKey::parse(&column.key.encode()).as_ref(),
Some(&column.key)
);
}
}
/// The point of the id: the mapping survives the rename, and follows the
/// column rather than the name.
#[test]
fn a_renamed_column_is_still_the_same_destination() {
let posted = destination_columns(&schema())[1].key.encode();
let mut renamed = schema();
renamed.columns[3].name = "invoice_number".to_string();
let columns = destination_columns(&renamed);
let resolved =
resolve(&columns, &posted).expect("column 42 is column 42 whatever it is called");
assert_eq!(resolved.name, "invoice_number");
}
/// A destination that is gone, or that the table has since made read-only,
/// resolves to nothing — which the caller has to refuse rather than guess
/// past.
#[test]
fn a_destination_the_table_no_longer_offers_resolves_to_nothing() {
let columns = destination_columns(&schema());
assert!(resolve(&columns, "id:57").is_none());
assert!(resolve(&columns, "id:999").is_none());
assert!(resolve(&columns, "system:row_revision").is_none());
assert!(resolve(&columns, "number").is_none());
assert!(resolve(&columns, "").is_none());
}
/// Only the user's own `NOT NULL` columns are reported as required.
#[test]
fn required_is_the_users_own_not_null_columns() {
let columns = destination_columns(&schema());
assert_eq!(
columns
.iter()
.filter(|column| column.required)
.map(|column| column.name.as_str())
.collect::<Vec<_>>(),
vec!["number"]
);
}
}

496
web/src/pages/import_export/import/logic.rs
View File

@@ -5,8 +5,9 @@ use axum::{
http::{HeaderMap, HeaderValue, header},
response::{Html, IntoResponse, Redirect, Response},
};
// A mapping posts `mapping` once per source position, and `axum::Form`
// (serde_urlencoded) cannot decode repeated keys into a `Vec`.
// The mapping posts `destination` and `source_position` once per destination
// row, and `axum::Form` (serde_urlencoded) cannot decode repeated keys into a
// `Vec`.
use axum_extra::extract::Form;
use crate::{
@@ -22,16 +23,12 @@ use crate::{
use super::{
super::common::{
loader::LoadError,
schema::{column_types, csv_value, importable_columns, required_columns},
schema::{column_types, csv_value},
},
destination::{DestinationColumn, destination_columns, resolve},
loader::load_page,
prepare::{
Prepared, SourceMode, canonical_csv, prepare, read_mapping, read_source, suggest_mapping,
template_csv,
},
state::{
ImportForm, MappingRow, MappingStep, PreviewStep, Step, TemplateRow, TemplateStep,
},
prepare::{Prepared, Source, canonical_csv, prepare, read_mapping, read_source},
state::{ImportForm, MappingRow, MappingStep, PreviewStep, SourceOption, Step},
ui,
};
@@ -42,17 +39,25 @@ const PREVIEW_ROWS: usize = 20;
///
/// Loaded again at every step rather than carried in the form. The destination
/// list is what makes a mapping mean anything, so it comes from the table each
/// time: a column dropped or renamed while the form sat open turns into a
/// refusal on the next click instead of a value written somewhere else.
/// time: a column dropped or made read-only while the form sat open turns into
/// a refusal on the next click instead of a value written somewhere else.
struct Destination {
table_name: String,
/// The columns a mapping may point at.
columns: Vec<String>,
/// Of those, the ones declared `NOT NULL`.
required: Vec<String>,
columns: Vec<DestinationColumn>,
types: HashMap<String, String>,
}
impl Destination {
/// The destination names, for the parts of the preparation that only need
/// to know which columns exist.
fn names(&self) -> Vec<String> {
self.columns
.iter()
.map(|column| column.name.clone())
.collect()
}
}
/// GET /admin/import
pub(crate) async fn import_page(State(state): State<AppState>, headers: HeaderMap) -> Response {
match load_page(state, &headers, ImportForm::default(), Step::Source).await {
@@ -74,11 +79,8 @@ pub(crate) async fn source_step(
render_step(state, &headers, form, Step::Source).await
}
/// POST /admin/import/prepare — the second step: the mapping table for a file,
/// or the column chooser for a template.
///
/// Also where the template's move buttons land, because reordering is a change
/// to the same step rather than a step of its own.
/// POST /admin/import/prepare — the mapping: one row per writable destination
/// column, asking where its value comes from.
pub(crate) async fn prepare_step(
State(state): State<AppState>,
headers: HeaderMap,
@@ -92,43 +94,48 @@ pub(crate) async fn prepare_step(
Ok(destination) => destination,
Err(response) => return response,
};
let mode = match form.mode(locale) {
Ok(mode) => mode,
let source = match read_source(locale, &form.csv_data) {
Ok(source) => source,
Err(message) => return reject(&headers, message),
};
let step = match mode {
SourceMode::Template => template_step(&form, &destination),
SourceMode::Header | SourceMode::Data => {
let source = match read_source(locale, &form.csv_data, mode) {
Ok(source) => source,
Err(message) => return reject(&headers, message),
};
// A mapping that does not cover this file was built against a
// different one — the user edited the CSV and came back. Start it
// over rather than lining up positions that no longer correspond.
let chosen = if form.mapping.len() == source.width {
form.mapping.clone()
} else {
suggest_mapping(&source, &destination.columns)
};
let examples = source.examples();
Step::Mapping(MappingStep {
table_name: destination.table_name.clone(),
columns: destination.columns.clone(),
rows: (0..source.width)
.map(|position| MappingRow {
position: position + 1,
source_name: source.source_name(position),
example: examples.get(position).cloned().unwrap_or_default(),
target: chosen.get(position).cloned().unwrap_or_default(),
})
.collect(),
source_rows: source.rows.len(),
has_source_names: source.header.is_some(),
// What the user has already answered, if they are coming back from the
// preview. Keyed by the destination's identity rather than by row order, so
// a table that gained or lost a column keeps the answers for the rest.
let answered = form
.rows()
.into_iter()
.filter_map(|(key, position)| Some((key.to_string(), position?)))
.collect::<HashMap<_, _>>();
let step = Step::Mapping(MappingStep {
table_name: destination.table_name.clone(),
rows: destination
.columns
.iter()
.map(|column| {
let key = column.key.encode();
// Nothing starts out mapped. An exact name match would be a
// reasonable guess, but a guess is what this page exists to
// avoid — every destination is the user's to answer.
let chosen = answered
.get(&key)
.copied()
.filter(|index| *index < source.width());
MappingRow {
key,
name: column.name.clone(),
required: column.required,
example: chosen
.map(|index| source.example(index).to_string())
.unwrap_or_default(),
chosen: chosen.map(|index| (index + 1).to_string()).unwrap_or_default(),
}
})
}
};
.collect(),
sources: source_options(locale, &source),
source_rows: source.rows.len(),
});
render_step(state, &headers, form, step).await
}
@@ -142,12 +149,12 @@ pub(crate) async fn preview_step(
return rejection;
}
let locale = Locale::from_headers(&headers);
let (destination, prepared) = match prepared(state.clone(), &headers, &form).await {
let (destination, source, prepared) = match prepared(state.clone(), &headers, &form).await {
Ok(parts) => parts,
Err(response) => return response,
};
let source_names = source_names(locale, &form);
let sources = source_options(locale, &source);
let step = Step::Preview(PreviewStep {
table_name: destination.table_name.clone(),
columns: prepared.columns.clone(),
@@ -157,16 +164,13 @@ pub(crate) async fn preview_step(
ignored: prepared
.ignored
.iter()
.map(|position| match source_names.get(position - 1) {
Some(name) if !name.is_empty() => format!("{position} ({name})"),
_ => position.to_string(),
})
.filter_map(|position| Some(sources.get(position - 1)?.label.clone()))
.collect(),
missing_required: destination
.required
.columns
.iter()
.filter(|column| !prepared.columns.contains(column))
.cloned()
.filter(|column| column.required && !prepared.columns.contains(&column.name))
.map(|column| column.name.clone())
.collect(),
omitted: prepared.omitted.clone(),
csv: canonical_csv(&prepared),
@@ -192,7 +196,7 @@ pub(crate) async fn import_csv(
Ok(target) => target,
Err(message) => return reject(&headers, message),
};
let (destination, prepared) = match prepared(state.clone(), &headers, &form).await {
let (destination, _, prepared) = match prepared(state.clone(), &headers, &form).await {
Ok(parts) => parts,
Err(response) => return response,
};
@@ -206,7 +210,9 @@ pub(crate) async fn import_csv(
tr!(
locale,
"import-err-csv-row",
"row" => (index + 1) as i64,
// The header is row 1 of the file the user is looking at,
// so the first data row is row 2.
"row" => (index + 2) as i64,
"error" => error,
)
})
@@ -261,39 +267,24 @@ pub(crate) async fn download_prepared(
if let Some(rejection) = reject_cross_site(&headers) {
return rejection;
}
let (destination, prepared) = match prepared(state, &headers, &form).await {
let (destination, _, prepared) = match prepared(state, &headers, &form).await {
Ok(parts) => parts,
Err(response) => return response,
};
attachment(
canonical_csv(&prepared),
&format!("{}_prepared.csv", destination.table_name),
)
}
let csv = canonical_csv(&prepared);
let filename = format!("{}_prepared.csv", destination.table_name);
/// POST /admin/import/template.csv — the chosen columns as a header row, and
/// nothing else.
pub(crate) async fn download_template(
State(state): State<AppState>,
headers: HeaderMap,
Form(form): Form<ImportForm>,
) -> Response {
if let Some(rejection) = reject_cross_site(&headers) {
return rejection;
let mut response = csv.into_response();
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("text/csv; charset=utf-8"),
);
if let Ok(value) = HeaderValue::try_from(format!("attachment; filename=\"{filename}\"")) {
response
.headers_mut()
.insert(header::CONTENT_DISPOSITION, value);
}
let locale = Locale::from_headers(&headers);
let destination = match destination(state, &headers, &form).await {
Ok(destination) => destination,
Err(response) => return response,
};
let columns = match template_selection(locale, &form, &destination) {
Ok(columns) => columns,
Err(message) => return reject(&headers, message),
};
attachment(
template_csv(&columns),
&format!("{}_template.csv", destination.table_name),
)
response
}
/// The table this import writes into, once the form's scope and table have been
@@ -340,7 +331,7 @@ async fn destination(
.remove(&table_name)
.ok_or_else(|| unavailable(headers, tr!(locale, "import-err-missing-structure")))?;
let columns = importable_columns(&structure);
let columns = destination_columns(&structure);
if columns.is_empty() {
return Err(reject(
headers,
@@ -353,131 +344,72 @@ async fn destination(
}
Ok(Destination {
table_name,
required: required_columns(&structure),
types: column_types(&structure),
columns,
})
}
/// The prepared import: the source read as the user said it is laid out, the
/// mapping checked against the table as it stands, and the two applied.
/// The prepared import: the file read, the mapping resolved against the table
/// as it stands, and the two applied.
///
/// Shared by preview, download and import so that all three are the same file
/// — looking at it and importing it cannot disagree about what it contains.
/// Shared by preview, download and import so that all three are the same file —
/// looking at it and importing it cannot disagree about what it contains.
async fn prepared(
state: AppState,
headers: &HeaderMap,
form: &ImportForm,
) -> Result<(Destination, Prepared), Response> {
) -> Result<(Destination, Source, Prepared), Response> {
let locale = Locale::from_headers(headers);
let destination = destination(state, headers, form).await?;
let mode = form.mode(locale).map_err(|message| reject(headers, message))?;
if !mode.reads_a_file() {
return Err(reject(
headers,
tr!(locale, "import-err-template-has-no-source"),
));
}
let source = read_source(locale, &form.csv_data, mode)
.map_err(|message| reject(headers, message))?;
let mapping = read_mapping(locale, &form.mapping, source.width, &destination.columns)
.map_err(|message| reject(headers, message))?;
let prepared = prepare(&mapping, &source, &destination.columns);
Ok((destination, prepared))
}
let source = read_source(locale, &form.csv_data).map_err(|message| reject(headers, message))?;
/// The column chooser, with the arrangement the form carries and whatever move
/// button was pressed applied to it.
fn template_step(form: &ImportForm, destination: &Destination) -> Step {
let mut order = arranged_columns(form, destination);
if let (Some(action), Some(index)) = (form.action.as_deref(), form.index) {
let swap_with = match action {
"up" if index > 0 => Some(index - 1),
"down" if index + 1 < order.len() => Some(index + 1),
_ => None,
};
if let Some(other) = swap_with {
order.swap(index, other);
}
// Every posted destination has to still exist and still be writable. A key
// that resolves to nothing is refused rather than skipped: skipping it
// would quietly import a subset of what the preview promised.
let mut chosen = Vec::new();
for (key, position) in form.rows() {
let column = resolve(&destination.columns, key).ok_or_else(|| {
reject(
headers,
tr!(
locale,
"import-err-destination-missing",
"table" => destination.table_name.clone(),
),
)
})?;
chosen.push((column.name.clone(), position));
}
let chosen = order
.iter()
.filter(|column| form.template_columns.contains(column))
.cloned()
.collect::<Vec<_>>();
let last = order.len().saturating_sub(1);
Step::Template(TemplateStep {
table_name: destination.table_name.clone(),
rows: order
.iter()
.enumerate()
.map(|(index, name)| TemplateRow {
index,
chosen: form.template_columns.contains(name),
required: destination.required.contains(name),
name: name.clone(),
first: index == 0,
last: index == last,
})
.collect(),
header: if chosen.is_empty() {
String::new()
} else {
template_csv(&chosen)
},
chosen: chosen.len(),
})
let assignments =
read_mapping(locale, &chosen, &source).map_err(|message| reject(headers, message))?;
let prepared = prepare(&assignments, &source, &destination.names());
Ok((destination, source, prepared))
}
/// The destination columns in the order the template step has them arranged.
///
/// The table's own order until the user moves something, and thereafter what
/// the form carries — minus anything the table no longer has, plus anything it
/// has gained, so a table changed mid-arrangement neither drops a column from
/// the chooser nor offers one that is gone.
fn arranged_columns(form: &ImportForm, destination: &Destination) -> Vec<String> {
let mut order = form
.template_order
.iter()
.filter(|column| destination.columns.contains(column))
.cloned()
.collect::<Vec<_>>();
order.dedup();
for column in &destination.columns {
if !order.contains(column) {
order.push(column.clone());
}
}
order
}
/// The template's columns, in the arranged order, checked for being answered.
fn template_selection(
locale: Locale,
form: &ImportForm,
destination: &Destination,
) -> Result<Vec<String>, String> {
let chosen = arranged_columns(form, destination)
.into_iter()
.filter(|column| form.template_columns.contains(column))
.collect::<Vec<_>>();
if chosen.is_empty() {
return Err(tr!(locale, "import-err-template-empty"));
}
Ok(chosen)
}
/// The source's own names for its positions, for labelling the ignored ones in
/// the summary. Best effort: a source that cannot be read at all has already
/// been refused elsewhere.
fn source_names(locale: Locale, form: &ImportForm) -> Vec<String> {
form.mode(locale)
.ok()
.filter(|mode| *mode == SourceMode::Header)
.and_then(|mode| read_source(locale, &form.csv_data, mode).ok())
.and_then(|source| source.header)
.unwrap_or_default()
/// The file's columns as every picker offers them: `hl — position 3`, and just
/// the position when the file left the name blank.
fn source_options(locale: Locale, source: &Source) -> Vec<SourceOption> {
(0..source.width())
.map(|index| {
let position = (index + 1) as i64;
let name = source.name(index).trim();
SourceOption {
position: index + 1,
label: if name.is_empty() {
tr!(locale, "import-source-unnamed", "position" => position)
} else {
tr!(
locale,
"import-source-option",
"name" => name.to_string(),
"position" => position,
)
},
example: source.example(index).to_string(),
}
})
.collect()
}
/// One prepared row as the bulk insert takes it: values under their destination
@@ -519,22 +451,8 @@ async fn render_step(
}
}
fn attachment(csv: String, filename: &str) -> Response {
let mut response = csv.into_response();
response.headers_mut().insert(
header::CONTENT_TYPE,
HeaderValue::from_static("text/csv; charset=utf-8"),
);
if let Ok(value) = HeaderValue::try_from(format!("attachment; filename=\"{filename}\"")) {
response
.headers_mut()
.insert(header::CONTENT_DISPOSITION, value);
}
response
}
/// Something this crate refused before any of it reached the backend: an
/// unknown scope, a mapping that does not fit the file, a cell no type accepts.
/// unknown scope, a position the file does not have, a cell no type accepts.
/// The buttons target `#submission-status`, so the answer is the alert fragment
/// rather than a whole page.
fn reject(headers: &HeaderMap, message: String) -> Response {
@@ -569,134 +487,34 @@ fn unavailable(headers: &HeaderMap, message: String) -> Response {
mod tests {
use super::*;
fn strings(values: &[&str]) -> Vec<String> {
values.iter().map(|value| value.to_string()).collect()
fn source() -> Source {
read_source(
Locale::default(),
"\"1\",\"1b\",\"hl\",\"\",\"s\"\n\"x1\",\"x2\",\"value-a\",\"value-b\",\"value-c\"\n",
)
.unwrap()
}
fn destination() -> Destination {
Destination {
table_name: "customers".to_string(),
columns: strings(&["name", "company_number", "active"]),
required: strings(&["name"]),
types: HashMap::new(),
}
}
fn template_rows(step: &Step) -> Vec<(String, bool)> {
match step {
Step::Template(step) => step
.rows
.iter()
.map(|row| (row.name.clone(), row.chosen))
.collect(),
_ => panic!("not the template step"),
}
}
/// The chooser starts in the table's own order, with nothing ticked.
/// Every picker offers the file's columns by name *and* position, because
/// the name is what a person recognises and the position is what is stored.
#[test]
fn the_template_starts_in_table_order() {
let step = template_step(&ImportForm::default(), &destination());
assert_eq!(
template_rows(&step),
vec![
("name".to_string(), false),
("company_number".to_string(), false),
("active".to_string(), false),
]
);
fn a_source_column_is_offered_by_name_and_position() {
let options = source_options(Locale::default(), &source());
assert_eq!(options.len(), 5);
assert_eq!(options[2].position, 3);
assert!(options[2].label.contains("hl"), "{}", options[2].label);
assert!(options[2].label.contains('3'), "{}", options[2].label);
assert_eq!(options[2].example, "value-a");
}
/// Moving a row rearranges the chooser, and the generated header follows
/// the arrangement rather than the table.
/// A file that left a column unnamed still has that column, and it is still
/// selectable — by the one thing it does have.
#[test]
fn moving_a_row_rearranges_the_generated_header() {
let form = ImportForm {
template_order: strings(&["name", "company_number", "active"]),
template_columns: strings(&["name", "active"]),
action: Some("down".to_string()),
index: Some(0),
..ImportForm::default()
};
let step = template_step(&form, &destination());
assert_eq!(
template_rows(&step)
.into_iter()
.map(|(name, _)| name)
.collect::<Vec<_>>(),
strings(&["company_number", "name", "active"])
);
match step {
Step::Template(step) => {
assert_eq!(step.header, "\"name\",\"active\"\n");
assert_eq!(step.chosen, 2);
}
_ => panic!("not the template step"),
}
}
fn an_unnamed_source_column_is_offered_by_position_alone() {
let options = source_options(Locale::default(), &source());
/// A move off either end does nothing rather than wrapping around.
#[test]
fn a_move_past_the_end_is_ignored() {
for (action, index) in [("up", 0), ("down", 2)] {
let form = ImportForm {
template_order: strings(&["name", "company_number", "active"]),
action: Some(action.to_string()),
index: Some(index),
..ImportForm::default()
};
assert_eq!(
template_rows(&template_step(&form, &destination()))
.into_iter()
.map(|(name, _)| name)
.collect::<Vec<_>>(),
strings(&["name", "company_number", "active"])
);
}
}
/// A table changed while the chooser was open: the column it lost leaves
/// the arrangement, and the one it gained joins the end of it.
#[test]
fn the_arrangement_follows_the_table_it_is_for() {
let form = ImportForm {
template_order: strings(&["active", "gone", "name"]),
..ImportForm::default()
};
assert_eq!(
arranged_columns(&form, &destination()),
strings(&["active", "name", "company_number"])
);
}
/// Generating a template with nothing ticked is refused rather than
/// producing an empty header.
#[test]
fn a_template_needs_at_least_one_column() {
assert!(
template_selection(Locale::default(), &ImportForm::default(), &destination()).is_err()
);
}
/// The ignored positions are labelled with the source's own names when the
/// file gave any, which is what makes the summary readable.
#[test]
fn a_header_files_names_are_read_back_for_the_summary() {
let form = ImportForm {
source_mode: "header".to_string(),
csv_data: "\"Company\",\"Internal note\"\n\"Acme\",\"old\"\n".to_string(),
..ImportForm::default()
};
assert_eq!(
source_names(Locale::default(), &form),
strings(&["Company", "Internal note"])
);
// A data-only file has none, and the summary falls back to positions.
let form = ImportForm {
source_mode: "data".to_string(),
..form
};
assert!(source_names(Locale::default(), &form).is_empty());
assert!(options[3].label.contains('4'), "{}", options[3].label);
assert_eq!(options[3].example, "value-b");
}
}

2
web/src/pages/import_export/import/mod.rs
View File

@@ -1,3 +1,4 @@
mod destination;
mod loader;
mod logic;
mod prepare;
@@ -24,6 +25,5 @@ pub(crate) fn router() -> Router<AppState> {
// to look at first. Both read the same preparation.
.route("/admin/import", post(logic::import_csv))
.route("/admin/import/prepared.csv", post(logic::download_prepared))
.route("/admin/import/template.csv", post(logic::download_template))
.layer(DefaultBodyLimit::max(128 * 1024 * 1024))
}

611
web/src/pages/import_export/import/prepare.rs
View File

@@ -1,15 +1,19 @@
//! Turning what the user uploaded into the one CSV the import understands.
//! Turning the uploaded file plus the user's mapping into the one CSV the
//! import understands.
//!
//! Everything here is about *position*. A value lands in a column because the
//! user pointed position 3 at `active`, and for no other reason — not because
//! a source header spells something similar, not because the words look alike.
//! The source's own header, when it has one, is read out for orientation and
//! then thrown away.
//! There is one shape of input: a header row, then data rows. The header is
//! read so the user can tell the file's columns apart, and is then done with —
//! it is never imported, and it never decides anything. What decides is the
//! mapping: destination column `a` takes source position 3, because the user
//! said so.
//!
//! Positions, not names, all the way through. A file may perfectly well have
//! two columns called `name`; the page shows them as `name — position 1` and
//! `name — position 2`, and what it stores is 1 and 2.
//!
//! The output is the canonical form: fully quoted CSV whose header names real
//! destination columns, in the order the source presents them. From there the
//! import is the strict one it always was — types, validations, scripts, links
//! and permissions are all the server's, unchanged.
//! destination columns. From there the import is the strict one it always was —
//! types, validations, scripts, links and permissions are all the server's.
use std::collections::HashSet;
@@ -17,210 +21,128 @@ use crate::{i18n::Locale, tr};
use super::super::common::csv::{parse_csv, write_record};
/// How the uploaded text is laid out.
///
/// Always the user's answer, never inferred. A data row can hold words that
/// read exactly like column names — `"name","num"` is a perfectly good pair of
/// customer records — so deciding for them is how a real row gets silently
/// eaten as a header.
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub(crate) enum SourceMode {
/// The first row is the source's own header. It labels the mapping rows
/// and is not imported.
#[default]
Header,
/// Every row is data.
Data,
/// No source at all: pick and order destination columns, and take away a
/// header to fill in elsewhere.
Template,
}
impl SourceMode {
pub(crate) fn parse(locale: Locale, raw: &str) -> Result<Self, String> {
match raw {
"header" => Ok(Self::Header),
"data" => Ok(Self::Data),
"template" => Ok(Self::Template),
_ => Err(tr!(locale, "import-err-source-mode")),
}
}
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Header => "header",
Self::Data => "data",
Self::Template => "template",
}
}
/// Whether a source file is needed at all. A template is generated from the
/// table, so there is nothing to upload.
pub(crate) fn reads_a_file(self) -> bool {
!matches!(self, Self::Template)
}
}
/// The source file, split into the header it states and the rows that carry
/// values.
/// The uploaded file: its header, and its data.
#[derive(Debug)]
pub(crate) struct Source {
/// The source's own header row, kept only to label the mapping rows. It is
/// `None` for a data-only file, and it never decides anything.
pub header: Option<Vec<String>>,
/// The names the file gives its own columns. Shown to the user so they can
/// recognise the file; never matched against anything.
pub header: Vec<String>,
pub rows: Vec<Vec<String>>,
/// How many positions every row has. The mapping has exactly this many
/// entries, which is what makes "position 3" mean one thing.
pub width: usize,
}
impl Source {
/// What the first data row holds at each position, for the mapping step to
/// show beside the destination picker. An example is the one thing that
/// reliably tells a user which column they are looking at.
pub(crate) fn examples(&self) -> Vec<String> {
(0..self.width)
.map(|position| {
self.rows
.first()
.and_then(|row| row.get(position))
.cloned()
.unwrap_or_default()
})
.collect()
/// How many positions the file has. Every row has exactly these, which is
/// what makes "position 3" mean one thing.
pub(crate) fn width(&self) -> usize {
self.header.len()
}
/// The source's name for a position, when it has one.
pub(crate) fn source_name(&self, position: usize) -> Option<String> {
self.header
.as_ref()
.and_then(|header| header.get(position))
.cloned()
/// The name the file gives position `index`, empty when it gave none.
pub(crate) fn name(&self, index: usize) -> &str {
self.header.get(index).map(String::as_str).unwrap_or_default()
}
/// What the first data row holds at `index`. An example is the fastest way
/// to tell which column of a file you are looking at.
pub(crate) fn example(&self, index: usize) -> &str {
self.rows
.first()
.and_then(|row| row.get(index))
.map(String::as_str)
.unwrap_or_default()
}
}
/// Reads the uploaded text as `mode` says it is laid out.
pub(crate) fn read_source(locale: Locale, csv: &str, mode: SourceMode) -> Result<Source, String> {
/// Reads the uploaded text: header row, then at least one data row, every row
/// the same width.
pub(crate) fn read_source(locale: Locale, csv: &str) -> Result<Source, String> {
let mut rows = parse_csv(locale, csv)?;
let header = match mode {
SourceMode::Header => {
if rows.len() < 2 {
return Err(tr!(locale, "import-err-no-data-rows"));
}
Some(rows.remove(0))
}
SourceMode::Data => None,
// Nothing to read: the caller decides the columns from the table.
SourceMode::Template => return Err(tr!(locale, "import-err-template-has-no-source")),
};
if rows.is_empty() {
// One row is a header with nothing under it, which is not an import.
if rows.len() < 2 {
return Err(tr!(locale, "import-err-no-data-rows"));
}
let width = header
.as_ref()
.map_or_else(|| rows[0].len(), |header| header.len());
if width == 0 {
let header = rows.remove(0);
if header.is_empty() {
return Err(tr!(locale, "import-err-empty"));
}
// Positions only mean anything if every row has the same ones. A short row
// would otherwise shift every value after the gap into the wrong column.
if let Some(row) = rows.iter().find(|row| row.len() != width) {
if let Some(row) = rows.iter().find(|row| row.len() != header.len()) {
return Err(tr!(
locale,
"import-err-row-width",
"expected" => width as i64,
"expected" => header.len() as i64,
"found" => row.len() as i64,
));
}
Ok(Source { header, rows, width })
Ok(Source { header, rows })
}
/// Where each source position is to be written, `None` being "ignore".
#[derive(Debug)]
pub(crate) struct Mapping(Vec<Option<String>>);
impl Mapping {
/// The destination chosen for a position, for the form to re-render with
/// the user's own answers still selected.
pub(crate) fn target(&self, position: usize) -> Option<&str> {
self.0.get(position).and_then(Option::as_deref)
}
pub(crate) fn len(&self) -> usize {
self.0.len()
}
/// One destination column, and the source position its values come from.
#[derive(Debug, Eq, PartialEq)]
pub(crate) struct Assignment {
/// The destination's current name, which is what the prepared header says.
pub column: String,
/// Index into a source row, from 0. The page numbers positions from 1.
pub source: usize,
}
/// Reads the posted mapping and checks the rules that make "proper data in the
/// proper place" true rather than hoped for.
/// Checks the mapping the user built and reduces it to the assignments that
/// actually write something.
///
/// `writable` is the destination list the table itself declares, loaded again
/// on every step: a column dropped or renamed while the form was open is caught
/// here instead of writing values into a column that no longer means what the
/// user chose.
/// `chosen` is one entry per writable destination column, in the order the
/// mapping step rendered them, `None` meaning "do not import". A destination
/// cannot appear twice because it only ever has one row, so what is left to
/// check is the source side.
pub(crate) fn read_mapping(
locale: Locale,
posted: &[String],
width: usize,
writable: &[String],
) -> Result<Mapping, String> {
// One select per position, always posted, so the form's shape and the
// file's have to agree. They will not if the CSV was edited after the
// mapping was built.
if posted.len() != width {
return Err(tr!(
locale,
"import-err-mapping-stale",
"positions" => width as i64,
"mapped" => posted.len() as i64,
));
}
let known = writable.iter().map(String::as_str).collect::<HashSet<_>>();
chosen: &[(String, Option<usize>)],
source: &Source,
) -> Result<Vec<Assignment>, String> {
let mut used = HashSet::new();
let mut targets = Vec::with_capacity(width);
for (position, target) in posted.iter().enumerate() {
let target = target.trim();
if target.is_empty() {
targets.push(None);
continue;
}
if !known.contains(target) {
let mut assignments = Vec::new();
for (column, position) in chosen {
let Some(index) = position else { continue };
// A position the file does not have — the CSV was edited after the
// mapping was built, or the form was tampered with.
if *index >= source.width() {
return Err(tr!(
locale,
"import-err-mapping-unknown-column",
"column" => target.to_string(),
"position" => (position + 1) as i64,
"import-err-source-position-missing",
"position" => (*index + 1) as i64,
"column" => column.clone(),
"positions" => source.width() as i64,
));
}
// Two positions pointed at one column let whichever is read second
// decide the value, silently. Refused instead.
if !used.insert(target.to_string()) {
// One source column feeding two destinations is more often a misclick
// than an intention, and the page stops offering a position once it is
// taken. Saying so beats copying a value into two columns quietly.
if !used.insert(*index) {
return Err(tr!(
locale,
"import-err-mapping-duplicate",
"column" => target.to_string(),
"import-err-source-used-twice",
"position" => (*index + 1) as i64,
"name" => source.name(*index).to_string(),
));
}
targets.push(Some(target.to_string()));
assignments.push(Assignment {
column: column.clone(),
source: *index,
});
}
if used.is_empty() {
if assignments.is_empty() {
return Err(tr!(locale, "import-err-mapping-empty"));
}
Ok(Mapping(targets))
Ok(assignments)
}
/// The canonical import, and what it leaves behind.
pub(crate) struct Prepared {
/// The destination header, in source-position order — which is the only
/// order the data can be written in, since the values arrive in it.
/// The destination header, in the order the mapping step lists the columns.
pub columns: Vec<String>,
/// Data rows holding only the mapped positions, aligned to `columns`.
/// Data rows holding only the mapped values, aligned to `columns`.
pub rows: Vec<Vec<String>>,
/// The source positions sent nowhere, numbered from 1 as the mapping step
/// numbers them.
/// The source positions no destination takes, from 1.
pub ignored: Vec<usize>,
/// Destination columns this import does not write. The server fills them
/// with their defaults, or refuses the row if it cannot.
@@ -232,8 +154,8 @@ impl Prepared {
self.rows.len()
}
/// The first `limit` rows, for the preview. A preview is for recognising
/// your own data, and nobody recognises it on row 40 000.
/// The first `limit` rows. A preview is for recognising your own data, and
/// nobody recognises it on row 40 000.
pub(crate) fn preview_rows(&self, limit: usize) -> &[Vec<String>] {
&self.rows[..self.rows.len().min(limit)]
}
@@ -243,42 +165,47 @@ impl Prepared {
}
}
/// Applies the mapping: the values that were pointed somewhere, under the names
/// they were pointed at.
pub(crate) fn prepare(mapping: &Mapping, source: &Source, writable: &[String]) -> Prepared {
let taken = (0..mapping.len())
.filter_map(|position| Some((position, mapping.target(position)?.to_string())))
.collect::<Vec<_>>();
let columns = taken
/// Applies the mapping: the values that were asked for, under the names they
/// were asked for.
///
/// `writable` is every destination column the table offers, so the result can
/// say which ones this import leaves alone.
pub(crate) fn prepare(
assignments: &[Assignment],
source: &Source,
writable: &[String],
) -> Prepared {
let columns = assignments
.iter()
.map(|(_, column)| column.clone())
.map(|assignment| assignment.column.clone())
.collect::<Vec<_>>();
let rows = source
.rows
.iter()
.map(|row| {
taken
assignments
.iter()
.map(|(position, _)| row.get(*position).cloned().unwrap_or_default())
.map(|assignment| row.get(assignment.source).cloned().unwrap_or_default())
.collect()
})
.collect();
let ignored = (0..source.width)
.filter(|position| mapping.target(*position).is_none())
.map(|position| position + 1)
.collect();
let omitted = writable
let taken = assignments
.iter()
.filter(|column| !columns.contains(column))
.cloned()
.collect();
.map(|assignment| assignment.source)
.collect::<HashSet<_>>();
Prepared {
ignored: (0..source.width())
.filter(|index| !taken.contains(index))
.map(|index| index + 1)
.collect(),
omitted: writable
.iter()
.filter(|column| !columns.contains(column))
.cloned()
.collect(),
columns,
rows,
ignored,
omitted,
}
}
@@ -294,39 +221,6 @@ pub(crate) fn canonical_csv(prepared: &Prepared) -> String {
csv
}
/// A header and nothing else: the columns the user picked, in the order they
/// arranged them, ready to be filled in elsewhere and brought back as a file
/// with a header.
pub(crate) fn template_csv(columns: &[String]) -> String {
let mut csv = String::new();
write_record(&mut csv, columns);
csv
}
/// The destination a position starts out pointed at when the mapping step first
/// opens.
///
/// Only an exact match counts, and only against a real destination column. That
/// is not name interpretation: a header cell that *is* the column's name is the
/// column's name, which is the case whenever the file came from this system's
/// own template or export. Anything else starts at "Ignore", and every row is
/// in front of the user to confirm or change before a single value moves.
pub(crate) fn suggest_mapping(source: &Source, writable: &[String]) -> Vec<String> {
let mut used = HashSet::new();
(0..source.width)
.map(|position| {
let Some(name) = source.source_name(position) else {
return String::new();
};
if writable.contains(&name) && used.insert(name.clone()) {
name
} else {
String::new()
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
@@ -335,101 +229,94 @@ mod tests {
values.iter().map(|value| value.to_string()).collect()
}
fn writable() -> Vec<String> {
strings(&["name", "company_number", "active", "note"])
/// The worked example: six source columns, three of them wanted.
fn example() -> Source {
read_source(
Locale::default(),
"\"1\",\"1b\",\"hl\",\"he\",\"s\",\"aa\"\n\
\"x1\",\"x2\",\"value-a\",\"value-b\",\"value-c\",\"unused\"\n\
\"y1\",\"y2\",\"other-a\",\"other-b\",\"other-c\",\"unused\"\n",
)
.unwrap()
}
/// A data-only file is all rows, and nothing in it is read as a header —
/// even when the first row happens to hold the column names.
#[test]
fn a_data_only_file_keeps_its_first_row() {
let csv = "\"name\",\"num\"\n\"Acme\",\"12345678\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Data).unwrap();
assert!(source.header.is_none());
fn the_first_row_is_the_header_and_the_rest_is_data() {
let source = example();
assert_eq!(source.width(), 6);
assert_eq!(source.rows.len(), 2);
assert_eq!(source.examples(), strings(&["name", "num"]));
assert_eq!(source.name(2), "hl");
assert_eq!(source.example(2), "value-a");
}
/// A file declared to have a header loses its first row, and that row is
/// only ever shown back to the user.
#[test]
fn a_header_file_loses_its_first_row_to_labels() {
let csv = "\"Company\",\"Registration number\"\n\"Acme\",\"12345678\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(source.rows.len(), 1);
assert_eq!(source.source_name(0).as_deref(), Some("Company"));
assert_eq!(source.examples(), strings(&["Acme", "12345678"]));
}
/// A header with no rows under it is not an import, and says so rather than
/// A header with nothing under it is not an import, and says so rather than
/// preparing an empty one.
#[test]
fn a_header_with_nothing_under_it_is_refused() {
let csv = "\"Company\",\"Registration number\"\n";
assert!(read_source(Locale::default(), csv, SourceMode::Header).is_err());
assert!(read_source(Locale::default(), "\"a\",\"b\"\n").is_err());
}
/// Positions only mean something if every row has the same ones.
#[test]
fn rows_of_different_widths_are_refused() {
let csv = "\"Acme\",\"12345678\"\n\"Example\"\n";
let error = read_source(Locale::default(), csv, SourceMode::Data)
fn rows_of_a_different_width_than_the_header_are_refused() {
let error = read_source(Locale::default(), "\"a\",\"b\"\n\"only-one\"\n")
.expect_err("a short row shifts every value after it");
assert!(error.contains('2') && error.contains('1'), "{error}");
}
/// The heart of it: the user's choices decide where values go, source names
/// are irrelevant, and an ignored position takes its value nowhere.
/// The whole point, end to end: three destinations take three positions,
/// the other three source columns go nowhere.
#[test]
fn values_land_where_the_user_pointed_them() {
let csv = "\"whatever\",\"old field 7\",\"enabled value\",\"legacy\"\n\
\"Acme\",\"12345678\",\"true\",\"drop me\"\n\
\"Example\",\"87654321\",\"false\",\"drop me too\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
let mapping = read_mapping(
fn only_the_mapped_destinations_are_prepared() {
let source = example();
let writable = strings(&["a", "b", "c", "note"]);
let assignments = read_mapping(
Locale::default(),
&strings(&["name", "company_number", "active", ""]),
source.width,
&writable(),
&[
("a".to_string(), Some(2)),
("b".to_string(), Some(3)),
("c".to_string(), Some(4)),
("note".to_string(), None),
],
&source,
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
let prepared = prepare(&assignments, &source, &writable);
assert_eq!(
prepared.columns,
strings(&["name", "company_number", "active"])
);
assert_eq!(prepared.rows[0], strings(&["Acme", "12345678", "true"]));
assert_eq!(prepared.ignored, vec![4]);
assert_eq!(prepared.columns, strings(&["a", "b", "c"]));
assert_eq!(prepared.ignored, vec![1, 2, 6]);
assert_eq!(prepared.omitted, strings(&["note"]));
assert_eq!(
canonical_csv(&prepared),
"\"name\",\"company_number\",\"active\"\n\
\"Acme\",\"12345678\",\"true\"\n\
\"Example\",\"87654321\",\"false\"\n"
"\"a\",\"b\",\"c\"\n\
\"value-a\",\"value-b\",\"value-c\"\n\
\"other-a\",\"other-b\",\"other-c\"\n"
);
}
/// Out-of-order and sparse mappings are the normal case, not an edge one:
/// the header follows the source's positions, because the values do.
/// Destinations may take their values in any order, and out of order is the
/// normal case — the prepared header follows the destination rows, and each
/// row's values follow it.
#[test]
fn the_prepared_header_follows_the_source_order() {
let csv = "\"Acme\",\"unused legacy value\",\"true\",\"12345678\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Data).unwrap();
let mapping = read_mapping(
fn a_destination_may_take_any_position() {
let source = example();
let assignments = read_mapping(
Locale::default(),
&strings(&["name", "", "active", "company_number"]),
source.width,
&writable(),
&[
("a".to_string(), Some(5)),
("b".to_string(), None),
("c".to_string(), Some(0)),
],
&source,
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
let prepared = prepare(&assignments, &source, &strings(&["a", "b", "c"]));
assert_eq!(
canonical_csv(&prepared),
"\"name\",\"active\",\"company_number\"\n\"Acme\",\"true\",\"12345678\"\n"
"\"a\",\"c\"\n\"unused\",\"x1\"\n\"unused\",\"y1\"\n"
);
assert_eq!(prepared.ignored, vec![2]);
assert_eq!(prepared.omitted, strings(&["b"]));
}
/// What is downloaded and what is imported are the same file. The import
@@ -437,129 +324,79 @@ mod tests {
/// that the two cannot drift apart.
#[test]
fn the_downloaded_csv_parses_back_to_the_rows_that_get_imported() {
let csv = "\"Acme, s.r.o.\",\"said \"\"yes\"\"\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Data).unwrap();
let mapping = read_mapping(
let source = read_source(
Locale::default(),
&strings(&["name", "note"]),
source.width,
&writable(),
"\"name\",\"note\"\n\"Acme, s.r.o.\",\"said \"\"yes\"\"\"\n",
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
let assignments = read_mapping(
Locale::default(),
&[("name".to_string(), Some(0)), ("note".to_string(), Some(1))],
&source,
)
.unwrap();
let prepared = prepare(&assignments, &source, &strings(&["name", "note"]));
let reparsed = parse_csv(Locale::default(), &canonical_csv(&prepared)).unwrap();
assert_eq!(reparsed[0], prepared.columns);
assert_eq!(reparsed[1..], prepared.rows[..]);
}
/// One destination cannot be filled from two positions: the second would
/// silently win.
/// A file may name two columns the same. Nothing here cares: the mapping is
/// positions, and the two are 1 and 2.
#[test]
fn a_destination_cannot_be_chosen_twice() {
let error = read_mapping(
fn duplicate_source_names_are_not_ambiguous() {
let source = read_source(
Locale::default(),
&strings(&["name", "name"]),
2,
&writable(),
)
.expect_err("two positions cannot both be `name`");
assert!(error.contains("name"), "{error}");
}
/// A destination that is not a writable column of this table is refused,
/// whatever the form posted — the table is asked again on every step.
#[test]
fn a_destination_the_table_does_not_offer_is_refused() {
let error = read_mapping(
Locale::default(),
&strings(&["name", "id"]),
2,
&writable(),
)
.expect_err("`id` is the server's");
assert!(error.contains("id"), "{error}");
}
/// A mapping built against a different file is refused rather than applied
/// to whatever positions happen to line up.
#[test]
fn a_mapping_that_does_not_cover_the_file_is_refused() {
assert!(read_mapping(Locale::default(), &strings(&["name"]), 3, &writable()).is_err());
}
/// Ignoring everything is not an import.
#[test]
fn a_mapping_that_writes_nothing_is_refused() {
assert!(read_mapping(Locale::default(), &strings(&["", ""]), 2, &writable()).is_err());
}
/// A file this system generated the template for comes back with its own
/// column names, so every row starts out pointed at the right place — still
/// shown, still confirmed, never applied on its own.
#[test]
fn an_exact_header_starts_the_mapping_off() {
let csv = "\"name\",\"active\"\n\"Acme\",\"true\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(suggest_mapping(&source, &writable()), strings(&["name", "active"]));
}
/// Anything that is not exactly a column name starts at "Ignore". There is
/// no folding, no trimming and no similarity: ` name` and `Name` are not
/// the column `name`, and "Company" is not a guess anyone should make.
#[test]
fn nothing_but_an_exact_name_is_suggested() {
let csv = "\"Company\",\" name\",\"NAME\",\"note\"\n\"Acme\",\"x\",\"y\",\"z\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(
suggest_mapping(&source, &writable()),
strings(&["", "", "", "note"])
);
}
/// A header naming one column twice suggests it once: the second would be
/// a duplicate the mapping step refuses anyway.
#[test]
fn a_repeated_header_is_only_suggested_once() {
let csv = "\"name\",\"name\"\n\"Acme\",\"Other\"\n";
let source = read_source(Locale::default(), csv, SourceMode::Header).unwrap();
assert_eq!(suggest_mapping(&source, &writable()), strings(&["name", ""]));
}
/// The generated template is a header and nothing else, in the order it was
/// arranged in.
#[test]
fn a_template_is_the_chosen_columns_in_the_chosen_order() {
assert_eq!(
template_csv(&strings(&["active", "name", "company_number"])),
"\"active\",\"name\",\"company_number\"\n"
);
}
/// And it is a file this page can read straight back: the header mode plus
/// the exact-name suggestion close the loop.
#[test]
fn a_generated_template_comes_back_ready_to_import() {
let mut csv = template_csv(&strings(&["name", "active"]));
csv.push_str("\"Acme\",\"true\"\n");
let source = read_source(Locale::default(), &csv, SourceMode::Header).unwrap();
let mapping = read_mapping(
Locale::default(),
&suggest_mapping(&source, &writable()),
source.width,
&writable(),
"\"name\",\"name\",\"status\"\n\"first\",\"second\",\"live\"\n",
)
.unwrap();
let prepared = prepare(&mapping, &source, &writable());
assert_eq!(prepared.columns, strings(&["name", "active"]));
assert_eq!(prepared.rows, vec![strings(&["Acme", "true"])]);
let assignments = read_mapping(
Locale::default(),
&[("a".to_string(), Some(1)), ("b".to_string(), Some(0))],
&source,
)
.unwrap();
let prepared = prepare(&assignments, &source, &strings(&["a", "b"]));
assert_eq!(prepared.rows, vec![strings(&["second", "first"])]);
}
/// One source column feeding two destinations is refused: the page stops
/// offering a position once it is taken, so arriving here means something
/// went wrong rather than that the user meant it.
#[test]
fn a_source_mode_round_trips_and_an_unknown_one_is_refused() {
for mode in [SourceMode::Header, SourceMode::Data, SourceMode::Template] {
assert_eq!(SourceMode::parse(Locale::default(), mode.as_str()), Ok(mode));
}
assert!(SourceMode::parse(Locale::default(), "guess").is_err());
fn a_source_position_cannot_be_used_twice() {
let source = example();
let error = read_mapping(
Locale::default(),
&[("a".to_string(), Some(2)), ("b".to_string(), Some(2))],
&source,
)
.expect_err("position 3 cannot fill both");
assert!(error.contains("hl") && error.contains('3'), "{error}");
}
/// A position the file does not have is refused rather than read as blank.
#[test]
fn a_position_past_the_end_of_the_file_is_refused() {
let source = example();
let error = read_mapping(Locale::default(), &[("a".to_string(), Some(9))], &source)
.expect_err("the file has six positions");
assert!(error.contains("10") && error.contains('6'), "{error}");
}
/// Mapping nothing is not an import.
#[test]
fn a_mapping_that_writes_nothing_is_refused() {
let source = example();
assert!(
read_mapping(
Locale::default(),
&[("a".to_string(), None), ("b".to_string(), None)],
&source
)
.is_err()
);
}
}

237
web/src/pages/import_export/import/state.rs
View File

@@ -1,54 +1,37 @@
use crate::{i18n::Locale, tr};
use super::prepare::SourceMode;
/// Every field the import form carries, at every step.
///
/// The whole preparation is one form posted back and forth: the page keeps no
/// server-side session, so each step re-renders the fields the next one needs
/// and the user can go back without losing what they chose.
/// server-side draft, so each step re-states the answers the next one needs and
/// the user can go back without losing what they chose.
#[derive(Clone, Debug, Default, serde::Deserialize)]
pub(crate) struct ImportForm {
#[serde(default)]
pub profile_name: String,
/// One table. An import writes into one table, and a file for two tables is
/// two prepared imports rather than one file with a second header row.
/// two prepared imports.
#[serde(default)]
pub table_name: String,
/// `header`, `data` or `template` — see [`SourceMode`]. Kept as text so an
/// unrecognised value is a message rather than a form rejection.
#[serde(default)]
pub source_mode: String,
/// The uploaded file, header row and all.
#[serde(default)]
pub csv_data: String,
/// The destination chosen for each source position, in position order, one
/// entry per position; empty means "ignore". Posted once per `<select>`,
/// which only `axum_extra`'s `Form` decodes into a `Vec`.
/// One entry per writable destination column, in the order the mapping step
/// renders them: the column's stable identity, not its name. See
/// [`DestinationKey`](super::destination::DestinationKey).
///
/// Posted once per row, which only `axum_extra`'s `Form` decodes into a
/// `Vec`.
#[serde(default)]
pub mapping: Vec<String>,
/// Every destination column in the order the template step has them
/// arranged, chosen or not, so the arrangement survives a re-render.
pub destination: Vec<String>,
/// The source position chosen for the destination at the same index, from
/// 1; empty means "do not import". The two vectors are filled in document
/// order, so they line up row for row.
#[serde(default)]
pub template_order: Vec<String>,
/// The columns ticked in the template step, which the checkboxes post in
/// the arranged order — so this *is* the generated header.
#[serde(default)]
pub template_columns: Vec<String>,
/// `up` or `down` when a move button posted, with `index` naming the row.
#[serde(default)]
pub action: Option<String>,
#[serde(default)]
pub index: Option<usize>,
pub source_position: Vec<String>,
}
impl ImportForm {
pub(crate) fn mode(&self, locale: Locale) -> Result<SourceMode, String> {
if self.source_mode.is_empty() {
return Ok(SourceMode::default());
}
SourceMode::parse(locale, &self.source_mode)
}
/// The scope and table this import writes into, checked for being answered
/// at all. Whether they are the user's to write into is the catalog's
/// answer, and then the backend's.
@@ -61,7 +44,7 @@ impl ImportForm {
if table.is_empty() {
return Err(tr!(locale, "import-err-tables-required"));
}
if self.mode(locale)?.reads_a_file() && self.csv_data.trim().is_empty() {
if self.csv_data.trim().is_empty() {
return Err(tr!(locale, "import-err-csv-required"));
}
Ok((profile.to_string(), table.to_string()))
@@ -72,81 +55,80 @@ impl ImportForm {
self.table_name == name
}
/// Whether the radio for `mode` is the one selected, for re-rendering the
/// choice the user made.
pub(crate) fn is_mode(&self, mode: &str) -> bool {
if self.source_mode.is_empty() {
return mode == SourceMode::default().as_str();
}
self.source_mode == mode
/// The mapping as posted: the destination key of each row, and the source
/// position it was given.
///
/// A row whose position is blank, unparseable or zero is "do not import".
/// Positions are one-based on the page and zero-based here, since what they
/// index is a row of the file.
pub(crate) fn rows(&self) -> Vec<(&str, Option<usize>)> {
self.destination
.iter()
.enumerate()
.map(|(index, key)| {
let position = self
.source_position
.get(index)
.and_then(|raw| raw.trim().parse::<usize>().ok())
.and_then(|position| position.checked_sub(1));
(key.as_str(), position)
})
.collect()
}
}
/// Which of the preparation's steps the page is showing.
///
/// One page, three stops: say what the source is, say where each position goes,
/// then look at the result before anything is written.
/// One page, three stops: say where the data is going and hand over the file,
/// say where each destination column's value comes from, then look at the
/// result before anything is written.
pub(crate) enum Step {
/// Choose the destination table and describe the source.
Source,
/// Point each source position at a destination column.
Mapping(MappingStep),
/// Pick and arrange destination columns, and take away the header.
Template(TemplateStep),
/// The prepared import, as it will be sent.
Preview(PreviewStep),
}
pub(crate) struct MappingStep {
pub table_name: String,
/// The destinations to offer, in the order the table declares them.
pub columns: Vec<String>,
/// One row per writable destination column. A destination cannot be filled
/// twice because it appears exactly once.
pub rows: Vec<MappingRow>,
/// The file's columns, as they are offered in every picker.
pub sources: Vec<SourceOption>,
pub source_rows: usize,
/// Whether the source stated a header, so the table can show that column at
/// all — and say that it decides nothing.
pub has_source_names: bool,
}
/// One source position, and what the user has pointed it at.
/// One destination column, asking where its value comes from.
pub(crate) struct MappingRow {
/// Numbered from 1, the way the source is read by a person.
pub position: usize,
/// What the source called this position, when it said. Orientation only.
pub source_name: Option<String>,
/// The first data row's value here — usually the fastest way to recognise
/// which column this is.
/// The stable identity, for the hidden field.
pub key: String,
/// The name the table shows, and the name the prepared header will use.
pub name: String,
pub required: bool,
/// The chosen source position as the form carries it — one-based, empty for
/// "do not import".
pub chosen: String,
/// The first data row's value at the chosen position, so the row shows what
/// it is actually going to import.
pub example: String,
/// The chosen destination, empty for "ignore".
pub target: String,
}
impl MappingRow {
/// Whether this position is pointed at `column`, for re-rendering the
/// picker with the user's own answer selected.
pub(crate) fn targets(&self, column: &str) -> bool {
self.target == column
/// Whether this row takes `option`'s position, for re-rendering the picker
/// with the user's own answer selected.
pub(crate) fn takes(&self, option: &SourceOption) -> bool {
self.chosen == option.position.to_string()
}
}
pub(crate) struct TemplateStep {
pub table_name: String,
pub rows: Vec<TemplateRow>,
/// The generated header as it stands, empty when nothing is ticked.
pub header: String,
pub chosen: usize,
}
pub(crate) struct TemplateRow {
/// Position in the arrangement, from 0, for the move buttons.
pub index: usize,
pub name: String,
pub chosen: bool,
/// Declared `NOT NULL`, so leaving it out of the template is likely to
/// produce rows the server refuses.
pub required: bool,
pub first: bool,
pub last: bool,
/// One column of the uploaded file, as the pickers offer it.
pub(crate) struct SourceOption {
/// One-based, which is how the page counts and how the form posts.
pub position: usize,
/// `hl — position 3`, or just the position when the file left the name
/// blank. Built in Rust so it is translated once.
pub label: String,
pub example: String,
}
pub(crate) struct PreviewStep {
@@ -156,8 +138,8 @@ pub(crate) struct PreviewStep {
/// Rows the preview does not show, so a large file says how large.
pub hidden_rows: usize,
pub source_rows: usize,
/// Source positions written nowhere, labelled with the source's own name
/// when it gave one.
/// The file's columns no destination takes, labelled the way the pickers
/// labelled them.
pub ignored: Vec<String>,
/// Destination columns this import does not write.
pub omitted: Vec<String>,
@@ -180,35 +162,84 @@ pub(crate) struct ImportPageState {
mod tests {
use super::*;
/// A template needs no file, so the form does not insist on one — and the
/// other two modes do.
#[test]
fn only_a_file_import_needs_a_file() {
let form = ImportForm {
fn a_target_needs_a_scope_a_table_and_a_file() {
let complete = ImportForm {
profile_name: "acme".to_string(),
table_name: "customers".to_string(),
source_mode: "template".to_string(),
csv_data: "\"a\"\n\"1\"\n".to_string(),
..ImportForm::default()
};
assert_eq!(
form.target(Locale::default()).unwrap(),
complete.target(Locale::default()).unwrap(),
("acme".to_string(), "customers".to_string())
);
let form = ImportForm {
source_mode: "data".to_string(),
..form
};
assert!(form.target(Locale::default()).is_err());
for missing in [
ImportForm {
profile_name: String::new(),
..complete.clone()
},
ImportForm {
table_name: String::new(),
..complete.clone()
},
ImportForm {
csv_data: " ".to_string(),
..complete.clone()
},
] {
assert!(missing.target(Locale::default()).is_err());
}
}
/// The default is a file with a header, and the radio renders as such
/// before anything has been posted.
/// The two posted vectors line up row for row, and the page's one-based
/// positions become the zero-based indexes a row is read with.
#[test]
fn the_source_mode_defaults_to_a_header_file() {
let form = ImportForm::default();
assert_eq!(form.mode(Locale::default()).unwrap(), SourceMode::Header);
assert!(form.is_mode("header"));
assert!(!form.is_mode("data"));
fn the_posted_rows_pair_up_and_lose_their_offset() {
let form = ImportForm {
destination: vec![
"id:42".to_string(),
"id:57".to_string(),
"system:deleted".to_string(),
],
source_position: vec!["3".to_string(), String::new(), "1".to_string()],
..ImportForm::default()
};
assert_eq!(
form.rows(),
vec![
("id:42", Some(2)),
("id:57", None),
("system:deleted", Some(0)),
]
);
}
/// Anything that is not a position is "do not import" rather than a guess.
/// Zero is not a position either: the page counts from 1.
#[test]
fn an_unusable_position_means_do_not_import() {
let form = ImportForm {
destination: vec!["id:1".to_string(), "id:2".to_string(), "id:3".to_string()],
source_position: vec!["0".to_string(), "x".to_string(), "-1".to_string()],
..ImportForm::default()
};
assert!(form.rows().iter().all(|(_, position)| position.is_none()));
}
/// A row with no position posted at all is still a row, so the pairing
/// cannot slide by one.
#[test]
fn a_missing_position_does_not_shift_the_rows() {
let form = ImportForm {
destination: vec!["id:1".to_string(), "id:2".to_string()],
source_position: vec!["2".to_string()],
..ImportForm::default()
};
assert_eq!(form.rows(), vec![("id:1", Some(1)), ("id:2", None)]);
}
}

168
web/src/pages/import_export/import/ui.rs
View File

@@ -74,7 +74,7 @@ mod tests {
use crate::pages::import_export::common::loader::{Catalog, Profile};
use super::super::state::{
ImportForm, MappingRow, MappingStep, PreviewStep, TemplateRow, TemplateStep,
ImportForm, MappingRow, MappingStep, PreviewStep, SourceOption,
};
fn strings(values: &[&str]) -> Vec<String> {
@@ -99,127 +99,113 @@ mod tests {
form: ImportForm {
profile_name: "acme".to_string(),
table_name: "customers".to_string(),
source_mode: "header".to_string(),
csv_data: "\"Company\"\n\"Acme\"\n".to_string(),
mapping: strings(&["name"]),
..ImportForm::default()
csv_data: "\"hl\",\"he\"\n\"value-a\",\"value-b\"\n".to_string(),
destination: strings(&["id:42", "id:57"]),
source_position: strings(&["1", ""]),
},
step,
}
}
/// The first step asks where the data goes and how the source is laid out,
/// and asks the second question outright — the page never decides for
/// itself whether the first row is a header.
fn mapping() -> Step {
Step::Mapping(MappingStep {
table_name: "customers".to_string(),
rows: vec![
MappingRow {
key: "id:42".to_string(),
name: "a".to_string(),
required: true,
chosen: "1".to_string(),
example: "value-a".to_string(),
},
MappingRow {
key: "id:57".to_string(),
name: "b".to_string(),
required: false,
chosen: String::new(),
example: String::new(),
},
],
sources: vec![
SourceOption {
position: 1,
label: "hl \u{2014} position 1".to_string(),
example: "value-a".to_string(),
},
SourceOption {
position: 2,
label: "he \u{2014} position 2".to_string(),
example: "value-b".to_string(),
},
],
source_rows: 1,
})
}
/// The first step asks for a destination and a file, and nothing else —
/// there is one shape of input, so there is nothing to choose about it.
#[test]
fn the_first_step_asks_how_the_source_is_laid_out() {
fn the_first_step_asks_for_a_table_and_a_file() {
let html = render_page(&page(Step::Source));
for mode in ["header", "data", "template"] {
assert!(
html.contains(&format!(r#"name="source_mode" value="{mode}""#)),
"{html}"
);
}
assert!(html.contains(r#"name="table_name""#), "{html}");
assert!(html.contains(r#"name="csv_data""#), "{html}");
assert!(!html.contains("source_mode"), "{html}");
}
/// The mapping step is one picker per source position, and every picker
/// offers Ignore alongside the table's own columns. The source's own name
/// is shown next to it and is not what any option is built from.
/// The mapping is read destination-first: one row per column of the table,
/// each asking where its value comes from. A destination cannot be filled
/// twice because it appears exactly once.
#[test]
fn the_mapping_step_offers_a_destination_for_every_position() {
let html = render_step(&page(Step::Mapping(MappingStep {
table_name: "customers".to_string(),
columns: strings(&["name", "company_number"]),
rows: vec![
MappingRow {
position: 1,
source_name: Some("Company".to_string()),
example: "Acme".to_string(),
target: "name".to_string(),
},
MappingRow {
position: 2,
source_name: Some("Internal note".to_string()),
example: "old customer".to_string(),
target: String::new(),
},
],
source_rows: 2,
has_source_names: true,
})));
fn the_mapping_step_asks_every_destination_column_once() {
let html = render_step(&page(mapping()));
assert_eq!(html.matches(r#"name="mapping""#).count(), 2);
assert!(html.contains(r#"<option value="name" selected>"#), "{html}");
assert!(html.contains("Internal note"), "{html}");
assert!(html.contains("old customer"), "{html}");
// The destination and the source it came from are carried forward, so
// going back does not lose the file.
assert!(html.contains(r#"name="csv_data""#), "{html}");
assert_eq!(html.matches(r#"name="source_position""#).count(), 2);
assert_eq!(html.matches(r#"name="destination""#).count(), 2);
assert!(html.contains("<code>a</code>"), "{html}");
assert!(html.contains("<code>b</code>"), "{html}");
// The file's columns are offered by name and position together.
assert!(html.contains("hl \u{2014} position 1"), "{html}");
assert!(html.contains(r#"<option value="1" data-example="value-a" selected>"#), "{html}");
// Nothing is mapped unless the user said so.
assert!(html.contains(">Do not import</option>"), "{html}");
}
/// The template step is a chooser with an arrangement, and it shows the
/// header it would generate rather than describing it.
/// The destination travels as its stable identity, never as its name, so a
/// rename between the mapping and the import moves with the column.
#[test]
fn the_template_step_shows_the_header_it_generates() {
let html = render_step(&page(Step::Template(TemplateStep {
table_name: "customers".to_string(),
rows: vec![
TemplateRow {
index: 0,
name: "name".to_string(),
chosen: true,
required: true,
first: true,
last: false,
},
TemplateRow {
index: 1,
name: "note".to_string(),
chosen: false,
required: false,
first: false,
last: true,
},
],
header: "\"name\"\n".to_string(),
chosen: 1,
})));
fn a_destination_is_carried_as_its_identity() {
let html = render_step(&page(mapping()));
assert!(html.contains("&#34;name&#34;"), "{html}");
assert_eq!(html.matches(r#"name="template_order""#).count(), 2);
assert!(html.contains(r#"formaction="/admin/import/template.csv""#), "{html}");
// The ends of the arrangement have nowhere to move to.
assert_eq!(html.matches("disabled").count(), 2);
assert!(html.contains(r#"name="destination" value="id:42""#), "{html}");
assert!(!html.contains(r#"name="destination" value="a""#), "{html}");
}
/// The preview shows the values under the columns they will actually be
/// written to, says what is being left out, and offers both exits.
/// The preview shows the values under the columns they will be written to,
/// says what is being left out, and offers both exits.
#[test]
fn the_preview_shows_the_prepared_import_and_both_ways_out() {
let html = render_step(&page(Step::Preview(PreviewStep {
table_name: "customers".to_string(),
columns: strings(&["name", "active"]),
rows: vec![strings(&["Acme", "true"])],
columns: strings(&["a", "b", "c"]),
rows: vec![strings(&["value-a", "value-b", "value-c"])],
hidden_rows: 3,
source_rows: 4,
ignored: strings(&["2 (Internal note)"]),
ignored: strings(&["1 \u{2014} position 1", "aa \u{2014} position 6"]),
omitted: strings(&["note"]),
missing_required: strings(&["company_number"]),
csv: "\"name\",\"active\"\n\"Acme\",\"true\"\n".to_string(),
missing_required: strings(&["number"]),
csv: "\"a\",\"b\",\"c\"\n".to_string(),
})));
assert!(html.contains("<th>name</th>"), "{html}");
assert!(html.contains("<td>Acme</td>"), "{html}");
assert!(html.contains("2 (Internal note)"), "{html}");
assert!(html.contains("company_number"), "{html}");
// Import, and download the very same file.
assert!(html.contains("<th>a</th>"), "{html}");
assert!(html.contains("<td>value-a</td>"), "{html}");
assert!(html.contains("aa \u{2014} position 6"), "{html}");
assert!(html.contains("number"), "{html}");
assert!(html.contains(r#"hx-post="/admin/import""#), "{html}");
assert!(html.contains("&#34;name&#34;,&#34;active&#34;"), "{html}");
assert!(html.contains("&#34;a&#34;,&#34;b&#34;,&#34;c&#34;"), "{html}");
// The mapping travels with it, so the download and the import prepare
// the identical file.
assert!(html.contains(r#"name="mapping" value="name""#), "{html}");
assert!(html.contains(r#"name="destination" value="id:42""#), "{html}");
assert!(html.contains(r#"name="source_position" value="1""#), "{html}");
}
}

20
web/static/app.css
View File

@@ -317,21 +317,13 @@
/* ---------- Import preparation (pages/import_export/import) ---------- */
/* The one question the page must not answer for the user: whether the first
row is a header. It is a fieldset rather than a row of loose checkboxes so
the three answers read as one choice. */
.source-mode { margin: 0; border: 1px solid #e1e6ee; border-radius: 8px; padding: 12px 14px; background: #fbfcfe; }
.source-mode legend { padding: 0 6px; color: #33415c; font-size: 13px; font-weight: 600; }
.source-mode .check { margin: 4px 0; font-size: 13px; color: #33415c; }
.source-mode .hint { margin: 8px 0 0; font-size: 12px; }
/* A mapping row is read left to right as a sentence: position 3, which holds
"12345678", goes to company_number. The example is what people actually
recognise their data by, so it gets the room. */
.mapping-table .source-name { color: #7a8496; font-size: 12px; }
/* A mapping row is read left to right as a sentence: the column `a` takes its
value from "hl — position 3", which on the first row holds "value-a". The
example is what people actually recognise their data by, so it gets the
room, and it is the last thing on the line because it is the confirmation
rather than the choice. */
.mapping-table select { width: 100%; min-width: 220px; }
.mapping-table .example { max-width: 320px; overflow: hidden; text-overflow: ellipsis; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; color: #33415c; }
.mapping-table select { width: 100%; min-width: 180px; }
.template-table .check { justify-content: center; }
/* A secondary action in a form's action row — "back a step". The row's own
rule paints every button as the primary one, so this has to say otherwise

38
web/templates/pages/import_export/import/behaviour.html
View File

@@ -1,5 +1,5 @@
{#
Two things the mapping form does in the browser, both of them conveniences
Two things the mapping table does in the browser, both of them conveniences
over rules the server enforces anyway.
Everything is delegated from `document`, because each step swaps the whole
@@ -7,38 +7,28 @@
#}
<script>
(function () {
// A template is generated from the table, so there is no file to give.
// Hiding the fields is not the check — the server refuses a file import
// with no CSV either way.
function syncSourceFields() {
const mode = document.querySelector("[data-source-mode]:checked");
const fields = document.querySelector("[data-source-fields]");
if (!mode || !fields) return;
fields.hidden = mode.value === "template";
}
// A destination column can be filled from one source position only. The
// mapping step refuses a repeat outright; this stops it being offered, so
// the refusal is not how the user finds out.
function syncDestinations() {
const selects = Array.from(document.querySelectorAll("[data-destination]"));
function sync() {
const selects = Array.from(document.querySelectorAll("[data-source-select]"));
// A source column can fill one destination only. The mapping refuses a
// repeat outright; this stops it being offered, so the refusal is not
// how the user finds out.
const taken = new Set(selects.map((select) => select.value).filter(Boolean));
for (const select of selects) {
for (const option of select.options) {
option.disabled = option.value !== "" && option.value !== select.value && taken.has(option.value);
}
// The example beside the row is the first data row's value at the
// chosen position, so the row shows what it is actually going to
// import rather than only which column it named.
const cell = select.closest("tr")?.querySelector("[data-example-cell]");
if (cell) cell.textContent = select.selectedOptions[0]?.dataset.example ?? "";
}
}
document.addEventListener("change", function (event) {
if (event.target.matches("[data-source-mode]")) syncSourceFields();
if (event.target.matches("[data-destination]")) syncDestinations();
if (event.target.matches("[data-source-select]")) sync();
});
document.body.addEventListener("htmx:afterSwap", function () {
syncSourceFields();
syncDestinations();
});
syncSourceFields();
syncDestinations();
document.body.addEventListener("htmx:afterSwap", sync);
sync();
})();
</script>

7
web/templates/pages/import_export/import/carried.html
View File

@@ -2,12 +2,11 @@
What the first step answered, carried forward.
Steps 2 and 3 replace the fields that hold these, so they re-state them: the
form is the whole state of the preparation, and the destination and the source
text have to survive every swap. The summary line above them is so the user
can still see what they are preparing after the selects are gone.
form is the whole state of the preparation, and the destination and the file
have to survive every swap. The summary line above them is so the user can
still see what they are preparing after the selects are gone.
#}
<p class="hint">{{ nav.tr_args("import-carried", [("scope", page.form.profile_name.clone()), ("table", page.form.table_name.clone())]) }}</p>
<input type="hidden" name="profile_name" value="{{ page.form.profile_name }}">
<input type="hidden" name="table_name" value="{{ page.form.table_name }}">
<input type="hidden" name="source_mode" value="{{ page.form.source_mode }}">
<input type="hidden" name="csv_data" value="{{ page.form.csv_data }}">

129
web/templates/pages/import_export/import/step.html
View File

@@ -11,7 +11,7 @@
{% match page.step %}
{#- ------------------------------------------------------------------
Step 1 — where the data is going, and how the source is laid out.
Step 1 — where the data is going, and the file it comes from.
------------------------------------------------------------------ -#}
{% when Step::Source %}
<div class="form-grid">
@@ -31,33 +31,12 @@
{%- endfor %}{% endfor %}
</select>
</label>
{#
Always asked, never guessed. A data row can hold words that read exactly
like column names, so deciding for the user is how a real row gets eaten
as a header.
#}
<fieldset class="wide source-mode">
<legend>{{ nav.tr("import-source-mode") }}</legend>
<label class="check"><input type="radio" name="source_mode" value="header" data-source-mode
{% if page.form.is_mode("header") %} checked{% endif %}>{{ nav.tr("import-source-header") }}</label>
<label class="check"><input type="radio" name="source_mode" value="data" data-source-mode
{% if page.form.is_mode("data") %} checked{% endif %}>{{ nav.tr("import-source-data") }}</label>
<label class="check"><input type="radio" name="source_mode" value="template" data-source-mode
{% if page.form.is_mode("template") %} checked{% endif %}>{{ nav.tr("import-source-template") }}</label>
<p class="hint">{{ nav.tr("import-source-mode-hint") }}</p>
</fieldset>
<div class="wide" data-source-fields>
<div class="form-grid">
<label class="wide">{{ nav.tr("import-csv-file") }}
<input type="file" accept=".csv,text/csv"
onchange="this.files[0]?.text().then(value =&gt; document.getElementById('csv-data').value = value)">
</label>
<label class="wide">{{ nav.tr("import-csv-data") }}<textarea id="csv-data" name="csv_data" rows="12">{{ page.form.csv_data }}</textarea></label>
<small class="wide">{{ nav.tr("import-csv-format-hint") }}</small>
</div>
</div>
<label class="wide">{{ nav.tr("import-csv-file") }}
<input type="file" accept=".csv,text/csv"
onchange="this.files[0]?.text().then(value =&gt; document.getElementById('csv-data').value = value)">
</label>
<label class="wide">{{ nav.tr("import-csv-data") }}<textarea id="csv-data" name="csv_data" rows="12">{{ page.form.csv_data }}</textarea></label>
<small class="wide">{{ nav.tr("import-csv-format-hint") }}</small>
</div>
<div class="form-actions">
<a href="/">{{ nav.tr("common-cancel") }}</a>
@@ -65,7 +44,9 @@
</div>
{#- ------------------------------------------------------------------
Step 2a — one row per source position, pointed wherever the user says.
Step 2 — one row per destination column, asking where its value comes
from. Read this way round, a destination cannot be filled twice: it
appears exactly once.
------------------------------------------------------------------ -#}
{% when Step::Mapping with (step) %}
{% include "pages/import_export/import/carried.html" %}
@@ -76,26 +57,32 @@
<table class="builder-table mapping-table">
<thead>
<tr>
<th>{{ nav.tr("import-th-position") }}</th>
{% if step.has_source_names %}<th>{{ nav.tr("import-th-source-name") }}</th>{% endif %}
<th>{{ nav.tr("import-th-example") }}</th>
<th>{{ nav.tr("import-th-destination") }}</th>
<th>{{ nav.tr("import-th-required") }}</th>
<th>{{ nav.tr("import-th-source") }}</th>
<th>{{ nav.tr("import-th-example") }}</th>
</tr>
</thead>
<tbody>
{% for row in step.rows %}
<tr>
<td class="mark">{{ row.position }}</td>
{% if step.has_source_names %}<td class="source-name">{% if let Some(name) = row.source_name %}{{ name }}{% endif %}</td>{% endif %}
<td class="example">{{ row.example }}</td>
<td><code>{{ row.name }}</code></td>
<td>{% if row.required %}{{ nav.tr("common-yes") }}{% else %}<span class="hint">{{ nav.tr("common-no") }}</span>{% endif %}</td>
<td>
<select name="mapping" data-destination>
<option value="">{{ nav.tr("import-destination-ignore") }}</option>
{% for column in step.columns %}
<option value="{{ column }}"{% if row.targets(column) %} selected{% endif %}>{{ column }}</option>
{#
The destination travels as its stable identity, so a rename
between here and Import moves the mapping with the column
instead of leaving it pointing at a name.
#}
<input type="hidden" name="destination" value="{{ row.key }}">
<select name="source_position" data-source-select>
<option value="">{{ nav.tr("import-source-none") }}</option>
{% for option in step.sources %}
<option value="{{ option.position }}" data-example="{{ option.example }}"{% if row.takes(option) %} selected{% endif %}>{{ option.label }}</option>
{% endfor %}
</select>
</td>
<td class="example" data-example-cell>{{ row.example }}</td>
</tr>
{% endfor %}
</tbody>
@@ -106,69 +93,17 @@
<button type="button" hx-post="/admin/import/preview" hx-include="closest form">{{ nav.tr("import-to-preview") }}</button>
</div>
{#- ------------------------------------------------------------------
Step 2b — the template generator: pick the columns, arrange them, take
away the header.
------------------------------------------------------------------ -#}
{% when Step::Template with (step) %}
{% include "pages/import_export/import/carried.html" %}
<div class="builder-section">
<h2>{{ nav.tr_args("import-template-heading", [("table", step.table_name.clone())]) }}</h2>
<p class="hint">{{ nav.tr("import-template-hint") }}</p>
<table class="builder-table template-table">
<thead>
<tr>
<th>{{ nav.tr("import-th-order") }}</th>
<th>{{ nav.tr("import-th-include") }}</th>
<th>{{ nav.tr("import-th-destination") }}</th>
</tr>
</thead>
<tbody>
{% for row in step.rows %}
<tr>
<td class="order">
<button type="button" class="toggle move" aria-label="{{ nav.tr_args("import-move-up", [("name", row.name.clone())]) }}"
{% if row.first %}disabled{% else %}hx-post="/admin/import/prepare" hx-include="closest form"
hx-vals='{"action": "up", "index": "{{ row.index }}"}'{% endif %}>&uarr;</button>
<button type="button" class="toggle move" aria-label="{{ nav.tr_args("import-move-down", [("name", row.name.clone())]) }}"
{% if row.last %}disabled{% else %}hx-post="/admin/import/prepare" hx-include="closest form"
hx-vals='{"action": "down", "index": "{{ row.index }}"}'{% endif %}>&darr;</button>
</td>
<td>
{#
Both fields post in document order, so moving a row moves the
generated header with it: `template_order` remembers where the
unticked ones sit, `template_columns` is the header itself.
#}
<input type="hidden" name="template_order" value="{{ row.name }}">
<label class="check"><input type="checkbox" name="template_columns" value="{{ row.name }}"{% if row.chosen %} checked{% endif %}
hx-post="/admin/import/prepare" hx-include="closest form"></label>
</td>
<td><code>{{ row.name }}</code>{% if row.required %} <span class="hint">{{ nav.tr("import-required-column") }}</span>{% endif %}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% if step.chosen > 0 %}
<p class="hint">{{ nav.tr("import-template-generated") }}</p>
<pre class="sql-preview">{{ step.header }}</pre>
{% else %}
<p class="hint">{{ nav.tr("import-template-nothing-chosen") }}</p>
{% endif %}
<p class="hint">{{ nav.tr("import-template-round-trip") }}</p>
</div>
<div class="form-actions">
<button type="button" class="secondary" hx-post="/admin/import/source" hx-include="closest form">{{ nav.tr("import-back-to-source") }}</button>
{# A real submit: the answer is a file. #}
<button type="submit" formaction="/admin/import/template.csv" formnovalidate>{{ nav.tr("import-download-template") }}</button>
</div>
{#- ------------------------------------------------------------------
Step 3 — the prepared import, exactly as it will be sent.
------------------------------------------------------------------ -#}
{% when Step::Preview with (step) %}
{% include "pages/import_export/import/carried.html" %}
{% for value in page.form.mapping %}<input type="hidden" name="mapping" value="{{ value }}">{% endfor %}
{#
The mapping travels with the preview, so Import and Download prepare the
identical file from the identical answers.
#}
{% for key in page.form.destination %}<input type="hidden" name="destination" value="{{ key }}">{% endfor %}
{% for position in page.form.source_position %}<input type="hidden" name="source_position" value="{{ position }}">{% endfor %}
<div class="builder-section">
<h2>{{ nav.tr_args("import-preview-heading", [("table", step.table_name.clone())]) }}</h2>
<dl class="postable-grid">