filipriec/komp_ac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bulk import2

Browse Source
This commit is contained in:
Filipriec
2026-08-21 22:37:30 +02:00
parent 0756e99959
commit adaa2c4ac6
8 changed files with 421 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client

Submodule client updated: 9b57d475cc...de93a2a5e1

48
common/proto/tables_data.proto
View File

@@ -29,17 +29,6 @@ service TablesData {
rpc PostAccountingTableData(PostAccountingTableDataRequest) rpc PostAccountingTableData(PostAccountingTableDataRequest)
returns (PostTableDataResponse); returns (PostTableDataResponse);
// Insert multiple rows atomically by applying PostTableData behavior to each row.
//
// Behavior:
// - Accepts 1..10,000 rows in one gRPC request
// - Processes rows in request order
// - Each row is inserted through the same validation, script execution,
// typed binding, database insert, and indexing path as PostTableData
// - Stops at the first failing row and returns that row's gRPC error code
// - Commits only when every row succeeds; a failure rolls the entire request back
rpc PostTableDataBulk(PostTableDataBulkRequest) returns (PostTableDataBulkResponse);
// Starts a durable, profile-scoped import staging session. Staging never changes // Starts a durable, profile-scoped import staging session. Staging never changes
// profile data; CommitTableDataImport applies every staged chunk atomically. // profile data; CommitTableDataImport applies every staged chunk atomically.
rpc BeginTableDataImport(BeginTableDataImportRequest) rpc BeginTableDataImport(BeginTableDataImportRequest)
@@ -189,36 +178,6 @@ message PostTableDataResponse {
optional int64 journal_id = 5; optional int64 journal_id = 5;
} }
// One row in a bulk insert request.
message PostTableDataBulkRow {
// Required. Same data payload as PostTableDataRequest.data.
map<string, google.protobuf.Value> data = 1;
}
// Bulk insert request.
message PostTableDataBulkRequest {
// Required. Profile (PostgreSQL schema) name that owns the table.
string profile_name = 1;
// Required. Logical table (definition) name within the profile.
string table_name = 2;
// Required. Rows to insert. Must contain at least 1 and at most 10,000 rows.
repeated PostTableDataBulkRow rows = 3;
}
// Bulk insert response.
message PostTableDataBulkResponse {
// True if all rows were inserted successfully.
bool success = 1;
// Human-readable message.
string message = 2;
// Per-row responses from the underlying PostTableData logic, in request order.
repeated PostTableDataResponse responses = 3;
}
message BeginTableDataImportRequest { message BeginTableDataImportRequest {
string profile_name = 1; string profile_name = 1;
} }
@@ -230,7 +189,12 @@ message BeginTableDataImportResponse {
message StageTableDataImportRequest { message StageTableDataImportRequest {
string import_id = 1; string import_id = 1;
string table_name = 2; string table_name = 2;
repeated PostTableDataBulkRow rows = 3; repeated TableDataImportRow rows = 3;
}
message TableDataImportRow {
// Required. Data payload for one staged row.
map<string, google.protobuf.Value> data = 1;
} }
message StageTableDataImportResponse { message StageTableDataImportResponse {

3
common/src/grpc_error.rs
View File

@@ -4,8 +4,7 @@ pub const ACCOUNT_PATH_METADATA_KEY: &str = "komp-ac-account-path-bin";
pub const ACCOUNT_PREFIX_METADATA_KEY: &str = "komp-ac-account-prefix-bin"; pub const ACCOUNT_PREFIX_METADATA_KEY: &str = "komp-ac-account-prefix-bin";
pub const ACCOUNT_CURRENCY_METADATA_KEY: &str = "komp-ac-account-currency"; pub const ACCOUNT_CURRENCY_METADATA_KEY: &str = "komp-ac-account-currency";
pub const ACCOUNT_FIELD_METADATA_KEY: &str = "komp-ac-account-field-bin"; pub const ACCOUNT_FIELD_METADATA_KEY: &str = "komp-ac-account-field-bin";
pub const BULK_FAILED_ROW_INDEX_METADATA_KEY: &str = "komp-ac-bulk-failed-row-index"; pub const IMPORT_FAILED_ROW_INDEX_METADATA_KEY: &str = "komp-ac-import-failed-row-index";
pub const BULK_INSERTED_ROWS_METADATA_KEY: &str = "komp-ac-bulk-inserted-rows";
pub const COMPUTED_VALUE_MISMATCH_REASON: &str = "computed-value-mismatch"; pub const COMPUTED_VALUE_MISMATCH_REASON: &str = "computed-value-mismatch";
pub const ACCOUNT_NOT_FOUND_REASON: &str = "account-not-found"; pub const ACCOUNT_NOT_FOUND_REASON: &str = "account-not-found";
pub const ROW_STALE_REASON: &str = "row-stale"; pub const ROW_STALE_REASON: &str = "row-stale";

BIN
common/src/proto/descriptor.bin
View File

Binary file not shown.

149
common/src/proto/komp_ac.tables_data.rs
View File

@@ -77,42 +77,6 @@ pub struct PostTableDataResponse {
#[prost(int64, optional, tag = "5")] #[prost(int64, optional, tag = "5")]
pub journal_id: ::core::option::Option<i64>, pub journal_id: ::core::option::Option<i64>,
} }
/// One row in a bulk insert request.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct PostTableDataBulkRow {
/// Required. Same data payload as PostTableDataRequest.data.
#[prost(map = "string, message", tag = "1")]
pub data: ::std::collections::HashMap<
::prost::alloc::string::String,
::prost_types::Value,
>,
}
/// Bulk insert request.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct PostTableDataBulkRequest {
/// Required. Profile (PostgreSQL schema) name that owns the table.
#[prost(string, tag = "1")]
pub profile_name: ::prost::alloc::string::String,
/// Required. Logical table (definition) name within the profile.
#[prost(string, tag = "2")]
pub table_name: ::prost::alloc::string::String,
/// Required. Rows to insert. Must contain at least 1 and at most 10,000 rows.
#[prost(message, repeated, tag = "3")]
pub rows: ::prost::alloc::vec::Vec<PostTableDataBulkRow>,
}
/// Bulk insert response.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct PostTableDataBulkResponse {
/// True if all rows were inserted successfully.
#[prost(bool, tag = "1")]
pub success: bool,
/// Human-readable message.
#[prost(string, tag = "2")]
pub message: ::prost::alloc::string::String,
/// Per-row responses from the underlying PostTableData logic, in request order.
#[prost(message, repeated, tag = "3")]
pub responses: ::prost::alloc::vec::Vec<PostTableDataResponse>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BeginTableDataImportRequest { pub struct BeginTableDataImportRequest {
#[prost(string, tag = "1")] #[prost(string, tag = "1")]
@@ -130,7 +94,16 @@ pub struct StageTableDataImportRequest {
#[prost(string, tag = "2")] #[prost(string, tag = "2")]
pub table_name: ::prost::alloc::string::String, pub table_name: ::prost::alloc::string::String,
#[prost(message, repeated, tag = "3")] #[prost(message, repeated, tag = "3")]
pub rows: ::prost::alloc::vec::Vec<PostTableDataBulkRow>, pub rows: ::prost::alloc::vec::Vec<TableDataImportRow>,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct TableDataImportRow {
/// Required. Data payload for one staged row.
#[prost(map = "string, message", tag = "1")]
pub data: ::std::collections::HashMap<
::prost::alloc::string::String,
::prost_types::Value,
>,
} }
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StageTableDataImportResponse { pub struct StageTableDataImportResponse {
@@ -631,45 +604,6 @@ pub mod tables_data_client {
); );
self.inner.unary(req, path, codec).await self.inner.unary(req, path, codec).await
} }
/// Insert multiple rows atomically by applying PostTableData behavior to each row.
///
/// Behavior:
///
/// * Accepts 1..10,000 rows in one gRPC request
/// * Processes rows in request order
/// * Each row is inserted through the same validation, script execution,
/// typed binding, database insert, and indexing path as PostTableData
/// * Stops at the first failing row and returns that row's gRPC error code
/// * Commits only when every row succeeds; a failure rolls the entire request back
pub async fn post_table_data_bulk(
&mut self,
request: impl tonic::IntoRequest<super::PostTableDataBulkRequest>,
) -> std::result::Result<
tonic::Response<super::PostTableDataBulkResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::unknown(
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/komp_ac.tables_data.TablesData/PostTableDataBulk",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(
GrpcMethod::new(
"komp_ac.tables_data.TablesData",
"PostTableDataBulk",
),
);
self.inner.unary(req, path, codec).await
}
/// Starts a durable, profile-scoped import staging session. Staging never changes /// Starts a durable, profile-scoped import staging session. Staging never changes
/// profile data; CommitTableDataImport applies every staged chunk atomically. /// profile data; CommitTableDataImport applies every staged chunk atomically.
pub async fn begin_table_data_import( pub async fn begin_table_data_import(
@@ -1162,23 +1096,6 @@ pub mod tables_data_server {
tonic::Response<super::PostTableDataResponse>, tonic::Response<super::PostTableDataResponse>,
tonic::Status, tonic::Status,
>; >;
/// Insert multiple rows atomically by applying PostTableData behavior to each row.
///
/// Behavior:
///
/// * Accepts 1..10,000 rows in one gRPC request
/// * Processes rows in request order
/// * Each row is inserted through the same validation, script execution,
/// typed binding, database insert, and indexing path as PostTableData
/// * Stops at the first failing row and returns that row's gRPC error code
/// * Commits only when every row succeeds; a failure rolls the entire request back
async fn post_table_data_bulk(
&self,
request: tonic::Request<super::PostTableDataBulkRequest>,
) -> std::result::Result<
tonic::Response<super::PostTableDataBulkResponse>,
tonic::Status,
>;
/// Starts a durable, profile-scoped import staging session. Staging never changes /// Starts a durable, profile-scoped import staging session. Staging never changes
/// profile data; CommitTableDataImport applies every staged chunk atomically. /// profile data; CommitTableDataImport applies every staged chunk atomically.
async fn begin_table_data_import( async fn begin_table_data_import(
@@ -1511,52 +1428,6 @@ pub mod tables_data_server {
}; };
Box::pin(fut) Box::pin(fut)
} }
"/komp_ac.tables_data.TablesData/PostTableDataBulk" => {
#[allow(non_camel_case_types)]
struct PostTableDataBulkSvc<T: TablesData>(pub Arc<T>);
impl<
T: TablesData,
> tonic::server::UnaryService<super::PostTableDataBulkRequest>
for PostTableDataBulkSvc<T> {
type Response = super::PostTableDataBulkResponse;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::PostTableDataBulkRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as TablesData>::post_table_data_bulk(&inner, request)
.await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = PostTableDataBulkSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/komp_ac.tables_data.TablesData/BeginTableDataImport" => { "/komp_ac.tables_data.TablesData/BeginTableDataImport" => {
#[allow(non_camel_case_types)] #[allow(non_camel_case_types)]
struct BeginTableDataImportSvc<T: TablesData>(pub Arc<T>); struct BeginTableDataImportSvc<T: TablesData>(pub Arc<T>);

2
server

Submodule server updated: d4731d5194...65f54df500

388
test_luna/README.md Normal file
View File

@@ -0,0 +1,388 @@
# Chromium ERP import test
Test the web application as a real ERP workflow in Chromium. Do not begin with
a toy CSV. First build and inspect the complete table ecosystem through the web
admin. Create each CSV only after its destination schema and all referenced
parent rows exist.
Use a disposable database. Log in as `superadmin` with an empty password.
## Start the application and browser
From the repository root:
```sh
cargo run -p server -- server
```
This starts gRPC on `[::1]:50051` and the embedded `web` crate on
`http://127.0.0.1:3000`.
Open an isolated Chromium profile:
```sh
chromium \
--user-data-dir=/tmp/komp-ac-test-luna-chromium \
--remote-debugging-port=9222 \
--no-first-run \
http://127.0.0.1:3000/login
```
The DevTools targets are at `http://127.0.0.1:9222/json`. Capture console and
Network failures throughout the run.
## Build the ERP schema first
Create the tables through `/admin/tables/new`, in the order below. Use the
builder controls for required, indexed, currency, rounding, quantity ledger,
column ordering, and row-display columns. After every table creation, verify
the redirect selects the new table and that the admin workspace reports the
expected types and flags.
Use a unique suffix if these names already exist.
### 1. Shared units catalogue
Create global table `units`:
| Column | Type | Options |
| --- | --- | --- |
| `code` | text | required, indexed, row display |
| `description` | text | required |
| `active` | boolean | required |
This exercises global scope. Confirm account, accounting, accounting-transfer,
and quantity-ledger choices cannot be applied to a global table.
### 2. Profile and chart of accounts
Create profile `luna_erp` with currency `EUR` by creating `chart_accounts`:
| Column | Type | Options |
| --- | --- | --- |
| `account_path` | account | required, row display; automatically unique/indexed |
| `label` | text | required, row display |
| `active` | boolean | required |
The account catalogue must be created before any table using `accounting` or
`accounting_transfer`.
### 3. Business partners
Create profile table `partners`:
| Column | Type | Options |
| --- | --- | --- |
| `partner_code` | text | required, indexed, row display |
| `legal_name` | text | required, indexed, row display |
| `active` | boolean | required |
| `email` | email_address | optional |
| `phone` | phone | optional; inspect generated companion columns |
| `iban` | iban | optional; inspect generated companion columns |
| `payment_card` | credit_card | optional |
| `credit_limit` | money | EUR, half-up rounding |
| `registered_on` | temporal → date | optional |
| `preferred_call_time` | temporal → time | optional |
| `synced_at` | temporal → instant | optional |
| `source_timestamp` | temporal → raw_datetime | optional |
| `payment_window` | duration | optional |
| `contract_period` | period | optional |
| `external_number` | bigint | indexed |
| `risk_score` | numeric | optional |
Rename at least one generated PHONE or IBAN companion using the generated-name
panel, apply it, and confirm the preview and final table use the alias.
### 4. Warehouses
Create profile table `warehouses`:
| Column | Type | Options |
| --- | --- | --- |
| `warehouse_code` | text | required, indexed, row display |
| `name` | text | required, row display |
| `manager` | link(partners) | required, automatically indexed |
| `unit` | link(units) | required, automatically indexed; cross-scope link |
| `capacity` | int | optional |
Confirm links cannot target the table currently being created and that the UI
does not offer a redundant manual-index control for a link.
### 5. Products and quantity ledger
Create profile table `products`:
| Column | Type | Options |
| --- | --- | --- |
| `sku` | text | required, indexed, row display |
| `name` | text | required, row display |
| `unit_gtin` | gtin → 8 | optional |
| `case_gtin` | gtin → 12 | optional |
| `retail_gtin` | gtin → 13 | optional |
| `pallet_gtin` | gtin → 14 | optional |
| `supplier` | link(partners) | required, auto-indexed |
| `home_warehouse` | link(warehouses) | required, auto-indexed |
| `unit` | link(units) | required, auto-indexed |
| `sale_price` | money | EUR, exact rounding |
| `purchase_price` | money | USD, half-up rounding |
| `vat_rate` | decimal(5,2) | required |
| `package_count` | int | optional |
| `legacy_id` | bigint | indexed |
| `measured_weight` | numeric | optional |
| `stock` | decimal(18,3) | quantity ledger enabled |
| `active` | boolean | required |
Confirm `stock` is displayed as quantity-ledger/read-only and starts at zero.
The import mapper must not offer it as a writable destination.
### 6. Stock transactions with multiple links
Create `stock_receipts` with:
- `receipt_no`: required, indexed text and row display;
- `product`: required link(products);
- `warehouse`: required link(warehouses);
- `supplier`: required link(partners);
- `quantity`: required decimal(18,3);
- `received_on`: required date;
- `processed`: int.
Create `stock_issues` with the same shape, replacing `supplier` with required
`customer` link(partners), and `received_on` with required `issued_on`.
Through `/admin/logic/new`, attach these scripts:
- `stock_receipts.processed`:
`(quantity-add "product" "stock" (steel_get_column "stock_receipts" "quantity"))`
- `stock_issues.processed`:
`(quantity-sub "product" "stock" (steel_get_column "stock_issues" "quantity"))`
These tables test three foreign keys per imported row and indirect updates to a
read-only quantity-ledger column.
### 7. Sales documents and link chains
Create `sales_orders`:
- `order_no`: required/indexed text and row display;
- `customer`: required link(partners);
- `warehouse`: required link(warehouses);
- `ordered_on`: required date;
- `status`: required text;
- `total`: EUR money with half-up rounding;
- `paid`: required boolean.
Create `sales_order_lines`:
- `sales_order`: required link(sales_orders);
- `product`: required link(products);
- `quantity`: required decimal(18,3);
- `unit_price`: required EUR money, exact rounding;
- `discount_rate`: decimal(5,2);
- `line_position`: required int.
This supplies a parent → child → product link chain and composite business
rows for import ordering and FK rollback tests.
### 8. Accounting-only table-definition coverage
Create `sales_journal` with an `accounting` definition row, EUR currency, and
half-up rounding. Inspect its generated name, tax-point date, debit, credit,
and account fields; rename at least one alias and apply it. Add an invoice
number and a link to `partners`.
Create `period_transfers` with an `accounting_transfer` definition row and a
required text reference.
Confirm each compound definition expands into its managed columns, cannot be
indexed, and can only occur once. These tables cover creation-only accounting
options. Do not attempt a positive accounting import unless the required
accounting periods and accounts have also been configured; otherwise use the
expected refusal as an atomic-failure case.
## Confirm option/type coverage
Before making CSVs, verify the ecosystem covers:
- text, boolean, int, bigint, numeric, decimal and money;
- date, time, instant, raw datetime, duration and period;
- phone, IBAN, email address and credit card;
- GTIN-8, GTIN-12, GTIN-13 and GTIN-14;
- account, accounting and accounting transfer;
- profile-local, global, and cross-scope link targets;
- required and optional fields;
- manual indexes and automatic FK/account indexes;
- exact and half-up money rounding, with EUR and USD metadata;
- row-display selection and ordering;
- generated-column aliases;
- quantity-ledger and ordinary numeric columns.
If the live backend offers another declarable type, add it to a suitable table
and record it. The backend's live type catalogue is authoritative.
## Create CSVs last, in dependency order
Only now create CSV files under a temporary directory such as
`/tmp/komp-ac-test-luna/`. Derive every header from the verified table shown in
the admin UI. Do not commit fixed parent IDs into this directory.
Import in this order:
1. `units`;
2. `chart_accounts`;
3. `partners`;
4. `warehouses` using real partner and unit IDs;
5. `products` using real partner, warehouse, and unit IDs;
6. `sales_orders` using real partner and warehouse IDs;
7. `sales_order_lines` using real order and product IDs;
8. `stock_receipts` and `stock_issues` using real linked IDs.
After each parent import, obtain its inserted IDs from the Analytics page or a
read-only database query, then generate the child CSV. Include quoted commas,
quotes, Unicode, optional blanks, booleans, money values, dates, and decimal
quantities. Use valid GTIN checksums and valid PHONE/IBAN values so a failure is
attributable to the feature being tested rather than bad fixture data.
For every import, verify upload → mapping → preview → asynchronous progress →
final result. Map by destination name, not blindly with **Map in order**,
because `deleted` may be the first offered destination and read-only/generated
fields change positional alignment.
## Required import situations
### Successful relational import
Import at least two parents and multiple children. Verify persisted joins:
- warehouse → manager and unit;
- product → supplier, warehouse, and unit;
- order line → order → customer;
- order line → product.
The displayed link values must resolve to the configured row-display columns,
not merely expose numeric IDs.
### Quantity-ledger effects
Import receipts for the same product in multiple rows, then import issues.
Verify:
`products.stock = sum(receipts.quantity) - sum(issues.quantity)`
Also confirm `stock` was never writable in the products import mapping. A
failed receipt/issue import must add no quantity contribution.
### Mapping and type conversion
Reorder source columns, intentionally ignore one source field, leave optional
destinations blank, and use each offered date format. Confirm preview and the
canonical prepared CSV agree exactly.
### Foreign-key atomic failure
Create a child CSV containing valid rows plus one nonexistent product, partner,
warehouse, or order ID. The entire import must fail and commit zero rows and
zero quantity-ledger effects.
### Validation/type atomic failure
Repeat with one invalid boolean, date, decimal, money, GTIN, IBAN, or required
value. Assert the reported CSV row is correct and no partial rows remain.
### Multi-chunk rollback
Generate more than 1,000 stock or order-line rows, with an invalid row after
the first 1,000. This crosses the import chunk boundary. The final commit must
still be atomic: zero imported rows and no ledger contribution.
### Schema drift
Prepare a mapping in one tab. In another tab rename, append, or otherwise
change a destination column before Preview or Import. The stable column IDs
must preserve a rename safely, while removed/read-only destinations must be
refused rather than silently retargeted.
### Session and concurrency
Try to start a second import in the same session while a large one is running;
it must be refused. Confirm a different session cannot poll the first session's
job and that logout/session loss is handled without swapping a login page into
the progress card.
## Pass evidence
Record:
- Chromium version and console/network failures;
- every created table and its final column flags;
- generated aliases and link targets;
- each CSV header, row count, mapping, preview, and final alert;
- before/after row counts for successful and failed imports;
- joined FK results;
- quantity-ledger balances and the arithmetic that produced them;
- screenshots of complex mappings, previews, progress, success, and expected
failures.
An HTTP 200 or an “Import complete” alert alone is not a pass. The database
state, link resolution, atomic rollback, and quantity-ledger effects must agree
with the imported ERP transactions.
## Executed run and evidence
The staged run was completed against a freshly built server and a disposable
database. `cargo build -p server` passed before the server was started. Chromium
logged in as `superadmin` with an empty password.
The live schema created through the browser was:
- global `units`;
- profile `luna_erp`: `chart_accounts`, `partners`, `warehouses`, `products`,
`stock_receipts`, `stock_issues`, `sales_orders`, `order_lines`,
`journal_entries`, and `account_transfers`;
- the profile's managed accounting tables (`general_ledger`, `journal_lines`,
`ledger_accounts`, and `quantity_ledger`).
The live tables deliberately cover text, boolean, int, bigint, numeric,
decimal, EUR/USD money with both rounding modes, date/time/instant, duration,
period, phone, IBAN, email, credit card, GTIN-8/12/13/14, account, FK links,
accounting, accounting transfer, generated companions, and a
`products.stock` `decimal(12,3)` quantity ledger. The stock transaction tables
have `stock_effect decimal(12,3)` computed targets, because a script cannot
target its own source `quantity` column.
The saved Steel scripts are:
```scheme
(quantity-add "product" "stock"
(steel_get_column "stock_receipts" "quantity"))
(quantity-subtract "product" "stock"
(steel_get_column "stock_issues" "quantity"))
```
The following imports completed and were checked with read-only database
queries: partners, warehouses, products, stock receipts, stock issues, sales
orders, order lines, chart accounts, and journal entries. Receipt quantity
`25.500` followed by issue quantity `20.500` left `products.stock = 5.000` and
created two quantity-ledger entries. The journal import created one journal
row and one general-ledger row. The order-line links resolved through the
sales-order and customer/product chain.
The mixed order-line failure (one valid row followed by product ID `999999`)
reported `Link 'product' points to a missing row`, inserted zero rows, and left
the existing order-line count and quantity effects unchanged. This is the
atomic FK rollback checkpoint for the ladder.
Observed failures are part of the test evidence: missing required values are
rejected atomically; duration requires an elapsed form such as `PT720H`, period
requires a calendar form such as `P3M`, and `instant` requires a timezone-free
civil datetime. A missing computed `stock_effect` is rejected, and the
subtraction script expects the positive computed amount while applying a
negative ledger contribution. Accounting account values must be paths such as
`4000/SALES`, not numeric ledger IDs.
Two product issues are also reproducible. Required user columns are displayed
as nullable and generated DDL permits NULL even though backend validation still
enforces `required`. The progress card remained at `0%` while the backend job
had already completed; repeated submissions therefore created duplicate test
rows. The final database counts and ledger arithmetic, rather than the stale
card, were used as the pass criteria.

36
web/src/pages/import_export/import/logic.rs
View File

@@ -15,7 +15,7 @@ use crate::{
table_structure::GetTableStructureRequest, table_structure::GetTableStructureRequest,
tables_data::{ tables_data::{
AbortTableDataImportRequest, BeginTableDataImportRequest, CommitTableDataImportRequest, AbortTableDataImportRequest, BeginTableDataImportRequest, CommitTableDataImportRequest,
PostTableDataBulkRow, StageTableDataImportRequest, StageTableDataImportRequest, TableDataImportRow,
}, },
}, },
services::{authenticated_request, reject_cross_site}, services::{authenticated_request, reject_cross_site},
@@ -40,7 +40,7 @@ use super::{
/// How many prepared rows the preview shows. /// How many prepared rows the preview shows.
const PREVIEW_ROWS: usize = 20; const PREVIEW_ROWS: usize = 20;
/// How many rows one bulk insert carries. Also the granularity of the progress /// How many rows one staged import chunk carries. Also the granularity of the progress
/// the page shows, since a chunk is what the running total counts. /// the page shows, since a chunk is what the running total counts.
const CHUNK_ROWS: usize = 1_000; const CHUNK_ROWS: usize = 1_000;
@@ -221,7 +221,7 @@ pub(crate) async fn import_csv(
.iter() .iter()
.enumerate() .enumerate()
.map(|(index, row)| { .map(|(index, row)| {
bulk_row(locale, &destination, &prepared.columns, row).map_err(|error| { import_row(locale, &destination, &prepared.columns, row).map_err(|error| {
tr!( tr!(
locale, locale,
"import-err-csv-row", "import-err-csv-row",
@@ -350,7 +350,7 @@ struct Running {
id: String, id: String,
profile_name: String, profile_name: String,
table_name: String, table_name: String,
rows: Vec<PostTableDataBulkRow>, rows: Vec<TableDataImportRow>,
} }
/// The insert, chunk by chunk, reporting after each one. /// The insert, chunk by chunk, reporting after each one.
@@ -610,14 +610,14 @@ fn source_options(locale: Locale, source: &Source) -> Vec<SourceOption> {
.collect() .collect()
} }
/// One prepared row as the bulk insert takes it: values under their destination /// One prepared row as the staged import takes it: values under their destination
/// names, converted to the column's type. /// names, converted to the column's type.
fn bulk_row( fn import_row(
locale: Locale, locale: Locale,
destination: &Destination, destination: &Destination,
columns: &[String], columns: &[String],
row: &[String], row: &[String],
) -> Result<PostTableDataBulkRow, String> { ) -> Result<TableDataImportRow, String> {
let mut data = HashMap::new(); let mut data = HashMap::new();
for (index, column) in columns.iter().enumerate() { for (index, column) in columns.iter().enumerate() {
let data_type = destination.types.get(column).ok_or_else(|| { let data_type = destination.types.get(column).ok_or_else(|| {
@@ -634,7 +634,7 @@ fn bulk_row(
)?; )?;
data.insert(column.clone(), value); data.insert(column.clone(), value);
} }
Ok(PostTableDataBulkRow { data }) Ok(TableDataImportRow { data })
} }
async fn render_step( async fn render_step(
@@ -679,8 +679,8 @@ fn import_failure(
return (0, Outcome::SessionLost); return (0, Outcome::SessionLost);
} }
let status = failure.status_code(); let status = failure.status_code();
match bulk_failure(error) { match import_row_failure(error) {
Some((failed_row_index, _inserted_in_chunk)) => ( Some(failed_row_index) => (
0, 0,
Outcome::RowFailed { Outcome::RowFailed {
status, status,
@@ -700,22 +700,14 @@ fn import_failure(
} }
} }
fn bulk_failure(error: &tonic::Status) -> Option<(usize, usize)> { fn import_row_failure(error: &tonic::Status) -> Option<usize> {
let failed_row_index = error error
.metadata() .metadata()
.get(crate::grpc_error::BULK_FAILED_ROW_INDEX_METADATA_KEY)? .get(crate::grpc_error::IMPORT_FAILED_ROW_INDEX_METADATA_KEY)?
.to_str() .to_str()
.ok()? .ok()?
.parse() .parse()
.ok()?; .ok()
let inserted_rows = error
.metadata()
.get(crate::grpc_error::BULK_INSERTED_ROWS_METADATA_KEY)?
.to_str()
.ok()?
.parse()
.ok()?;
Some((failed_row_index, inserted_rows))
} }
fn load_error(headers: &HeaderMap, error: LoadError) -> Response { fn load_error(headers: &HeaderMap, error: LoadError) -> Response {