From adaa2c4ac633da0263ed35e2addee9352a131827 Mon Sep 17 00:00:00 2001 From: Filipriec Date: Fri, 21 Aug 2026 22:37:30 +0200 Subject: [PATCH] bulk import2 --- client | 2 +- common/proto/tables_data.proto | 48 +-- common/src/grpc_error.rs | 3 +- common/src/proto/descriptor.bin | Bin 189349 -> 187473 bytes common/src/proto/komp_ac.tables_data.rs | 149 +------- server | 2 +- test_luna/README.md | 388 ++++++++++++++++++++ web/src/pages/import_export/import/logic.rs | 36 +- 8 files changed, 421 insertions(+), 207 deletions(-) create mode 100644 test_luna/README.md diff --git a/client b/client index 9b57d475..de93a2a5 160000 --- a/client +++ b/client @@ -1 +1 @@ -Subproject commit 9b57d475cc83dd5c9f7540ff279ac98deb9e0d48 +Subproject commit de93a2a5e1af71bced9a211e6e34f8af1cb0e4bd diff --git a/common/proto/tables_data.proto b/common/proto/tables_data.proto index ed69872b..2a0bce6d 100644 --- a/common/proto/tables_data.proto +++ b/common/proto/tables_data.proto @@ -29,17 +29,6 @@ service TablesData { rpc PostAccountingTableData(PostAccountingTableDataRequest) returns (PostTableDataResponse); - // Insert multiple rows atomically by applying PostTableData behavior to each row. - // - // Behavior: - // - Accepts 1..10,000 rows in one gRPC request - // - Processes rows in request order - // - Each row is inserted through the same validation, script execution, - // typed binding, database insert, and indexing path as PostTableData - // - Stops at the first failing row and returns that row's gRPC error code - // - Commits only when every row succeeds; a failure rolls the entire request back - rpc PostTableDataBulk(PostTableDataBulkRequest) returns (PostTableDataBulkResponse); - // Starts a durable, profile-scoped import staging session. Staging never changes // profile data; CommitTableDataImport applies every staged chunk atomically. rpc BeginTableDataImport(BeginTableDataImportRequest) @@ -189,36 +178,6 @@ message PostTableDataResponse { optional int64 journal_id = 5; } -// One row in a bulk insert request. -message PostTableDataBulkRow { - // Required. Same data payload as PostTableDataRequest.data. - map data = 1; -} - -// Bulk insert request. -message PostTableDataBulkRequest { - // Required. Profile (PostgreSQL schema) name that owns the table. - string profile_name = 1; - - // Required. Logical table (definition) name within the profile. - string table_name = 2; - - // Required. Rows to insert. Must contain at least 1 and at most 10,000 rows. - repeated PostTableDataBulkRow rows = 3; -} - -// Bulk insert response. -message PostTableDataBulkResponse { - // True if all rows were inserted successfully. - bool success = 1; - - // Human-readable message. - string message = 2; - - // Per-row responses from the underlying PostTableData logic, in request order. - repeated PostTableDataResponse responses = 3; -} - message BeginTableDataImportRequest { string profile_name = 1; } @@ -230,7 +189,12 @@ message BeginTableDataImportResponse { message StageTableDataImportRequest { string import_id = 1; string table_name = 2; - repeated PostTableDataBulkRow rows = 3; + repeated TableDataImportRow rows = 3; +} + +message TableDataImportRow { + // Required. Data payload for one staged row. + map data = 1; } message StageTableDataImportResponse { diff --git a/common/src/grpc_error.rs b/common/src/grpc_error.rs index 904a391e..4c200935 100644 --- a/common/src/grpc_error.rs +++ b/common/src/grpc_error.rs @@ -4,8 +4,7 @@ pub const ACCOUNT_PATH_METADATA_KEY: &str = "komp-ac-account-path-bin"; pub const ACCOUNT_PREFIX_METADATA_KEY: &str = "komp-ac-account-prefix-bin"; pub const ACCOUNT_CURRENCY_METADATA_KEY: &str = "komp-ac-account-currency"; pub const ACCOUNT_FIELD_METADATA_KEY: &str = "komp-ac-account-field-bin"; -pub const BULK_FAILED_ROW_INDEX_METADATA_KEY: &str = "komp-ac-bulk-failed-row-index"; -pub const BULK_INSERTED_ROWS_METADATA_KEY: &str = "komp-ac-bulk-inserted-rows"; +pub const IMPORT_FAILED_ROW_INDEX_METADATA_KEY: &str = "komp-ac-import-failed-row-index"; pub const COMPUTED_VALUE_MISMATCH_REASON: &str = "computed-value-mismatch"; pub const ACCOUNT_NOT_FOUND_REASON: &str = "account-not-found"; pub const ROW_STALE_REASON: &str = "row-stale"; diff --git a/common/src/proto/descriptor.bin b/common/src/proto/descriptor.bin index 8035716c24a0b94e0c47b774abcad2a75e1af12f..3a6f09feaca92937a3b7b3113c8ac480e080a54b 100644 GIT binary patch delta 7438 zcmZXZdvH}%mWS_N=j7z(ax30OUjy}Ml53EjFL3c z#Y{O>7sPIgDJ;5W;7FJTrU3y74;3Z|3XTJcv~&S2;G;%isHWOh3zZps(BgdS?7fq! zng8;!*Kh5$)?WK@51U>uKe4xb;S*-J{Bofj=;?WCp`4NYt#Q-T_Mbd7H=Vv(>Q=lx zV|beWTy6w$;cr_Gu_ z>yAe{JLb*5t7E}knbojv6DR+FFu5%bXJg%9GA9!Z}Vd?ZuvU$9iBp4bs z#uagijtJEB7oLJ77*;(to|&nFBpe|l)6vh!eG>vyBYITA7ICgII;%Lq4Citgs9*%P{( z2i@Qa-7VF_gX8zp(LDCBdAYv%4fT zZB?k|MD@2pkZMA z1j_+3_E>u3Zuv(iHAo-pl^Pu`t8tDD(#PuR<6p&@l%3brb8)xSloY#6ZdhEI&4$OY z%M2gok)76M#*D=U)k&gXo7{S1;_^5MaUTBKb1`pR~1XG7@D<{t-u;Olg%z9BnfFlXz4nVQ@>@?-d!I&fPCj%N9cwwIm{%=oVAf z&m+1mrlvL?o9QUyw)9{3OH0c(Bc;wptGLbNYdj*^W+?r4GYPaKy|Q0!XxY&x;TQ{d zn0&31aB_k54e>phgp=#J^nsV;-r{pcVrLU_GCgO?ePFB)*uTk-;^8?ncu2N%K@#mY zxqov@r*rK#`Qcgp1FhB`BX)5hyn;4+5ZgV5{?lRvQltT4I#4?zOrpIex6irN0QQ>v zHJRH`-M(m~J`+N?1+7~U+ZRn);z#lNq8Xs4iI7(GlF7Z|ymI7*FZp0git6@d@3tg! z3tG1z#xHxfAZ}l_Zo{xdSLJ|_hHOrVh(IsZWjx9yH~HoK`0~Xa?Io#VD>0ni0BFB#e^iRg*jHlH+CwUp4v0_y%T)G`B-W zS{ug|?rQ((&mNlBKD)J`H+7+N);~PlF>7|=!D7d}LdV?p!lUzN{iJ<%f!n>+27#C$ zP+yY;0pj|Q8F8&gf;eQxwPZn*CeiCAchm*JH50z>*R528IO3}WQ6y4PD4#@cn%wVPL|kIwn?9nvMs&V>Jrr|rjPUY(MU7rwyThJy1V*9R72*mBX_V|;qB8lEN zxi3w+{g4c;<+csqH~FS`JY}j-uV+nq<+zNU1mN_nj}^r9tUnzIJv?#`ra>dPiAoRl9HEA$wGT=Rk`FZ<>N zQC{}VTbV>(o7}ftQ0C~a2Ln&5aFMv{p-SuWxVXAN z{2(f>!M#0JuSI^aqHX<%cmu8*Q%e^akaSD1%y}7o^>F$tA*D^cCW&? zT`k$`A`JU!(p};%YzQsrwOY$AaThkDNV~*c*!}bbStC-Ntr&z9vSC9bL$;eUJ zBuNteMsl0oB#9{|;cq0bk2RJ-XW8Iu2H_U8d4ss!pf%H{2#DJazGR7Jw$YaiLJE4R zmTaRh8KX!y+L9G@T57h**9<}n+M0oAH~E@@XgArCsT6uf+{F(e1-(>z@ELIzKcktk zyZCF8XshIQ6ej9?geO9-j*04sy-6mq6WDru& zMg(S~dHWOu1M0s`n)Ii(YAHmwh<&I)XhB;<5N)@V*_S%VwB5R+tQ6Yr8>B|`>;!Ft zfOy*O8wAADcH1Du5~kI{Y_}bvQJ~!+V($hBE$B1Q{kTK&_Ib@H(jC&M-!oK-?asag zN-TP*zNmNlBLQ2aJF7K=Yw7;lBzi$|2h;yJF4xr5V)=sP2WKyVT3wAj>A6EPZX$o< zbB`|yh~*yNlpsF$XjAG#pjIokH@)?+OhTwDvA0jCE3wzNB%=uTYD?}0_c&tyTOK{9D0iy_y z=@LvJxjJ2fotp0 z{(OV*3flbzVtdNx0pj(P=8?UL>UGZ3{(ytff?lhsoc5_OiuSa>6Y4dUGyZ&okb-ts zfJphUYPTwg^o&2>>XjzHEn+u4h!*r(ZK<~+g&S^}qUp&_qBV1Yc7@g7{}XH!&|K(jO$9d`4Cl{~+S0=i@2= zoJ}ubLlXT_a-X<$Z@}Ll{c+x)8Gq=D2vJ1P77@huLtjLY$Uf8}>WS8%S^P=d_a&&v zdl9ymAe;@?*>4yi+CPc=z|^1_ez;f<4ikTzamN4 zm_(N)cg2;d5zEVZa#zH>R~ofWU-rc>eE7Bi~B^=nEf~f?K=ZR`n5FL zZ>V5qJZyM;GlghMqN|cy6PU)f0{>%ENbh`4+G?9nT-Bc}pYcgLY|?@(>G|b*@`#xP z5ZIEytv!hClEAG!i1#G{TjYR#Np8}xm!%W_Q}z}4oAp{2xb*@l?6Sc9paRu2a61IV z(XznpkS49plY#CKdLF_PXjdM@)02IkbkZjSw?P=i)006JJ7lR%2VT+h_neq!VpCSE z6@l2of@oI+L)+X$TC5epbra+1SvE|&D&6&g+({ku;j}7neE}j|6}WG+Ai`Avx7`X` z2aLPY31?;b_nL&~b=cJxy*{J50{4rEQ5PI_c#I{3WI7(Xa$iJg9RK!iJky!|o(BHS6& P)IJb(&g+=pG4X!^XsKpS delta 8835 zcmZvid301&p2zF{?yE=TWq%1-DhWIYic8i8ZNUY_wvlNS8XP^&(5Fw46ebc<>8eDV zL-#2~)T85q3Oq&J=s{3i;$emo4nzkSKrgZ?yDwdSqwrk=yAoRrL$6d}b=sdwpVVSMn!`-o(_t zuKBZ4w}l%YrnADP#Z>63tPfwflP+@}6ZpMn1(mN@LO=Q8TnfLJORo7Y+FMMUi^K1h(aDOm zSNeB}`HkVy=DYO|2~rE!mb9{e;f$ zAl01=Zk{#s${?NmRbMjG%iU+DQ*FsiCYc%PWg3H2x;>fZE?1dH1)Z6oQzf5l4|+S& zslJ;#@ZV&RNi0YPwSWgK^3B%*8=M z1?)@n)O2-a)Ps`Uy`B7nHZGW(Xq)eO1zwbAR$yw~N<;N(rmI?PqeSDy*L&-QS_RT~ z8mKXR;8|MRBB5(=g#pip!P%6iFTBtQ_Urjf)I#9I?sT`(S#uLAL=i22Ud zH2o~o7K83;x7HGLSJzFp1l{!FdheG?m*>lC!iKf5qit{ECv9yHesjVlkDYp%S} zyH;sDU(N_~Yw2gr9yBwCM&&^>qh^A6jt9+*%Wm{=!f}H6zMLJluA|4B^C6n;R+w)2 z5Y4V^vINoWNk8|FlM)s9a!&5_I=V$P7eF<~9clU%KsBeL!4gz+etezxojxE6ec6$F z2`EIp|T?Ebi_(V(4%%3vb zyIbd1?8{89_9b#eb1_tzp|x8KRc2`E7DJV}_(tzVtt#>5qTGy^={(U?0@b3Si7J6= z(a=PdK(*+qo4mjyC*nu5k>|p-uTW5&jn#U;@*F2Z5#L!X!gpSwj!A`Hlz+fNT`VZ( z85E3pBQ0XjEEZMO-d&gy^hWF5B24$wZ!XgP1>|=R%Z2KYS7zh_^1DSvrT4I>#zzU1#`@E=(TSV07McpVPy&SDC2)f*~3TT1#7FFxGi!DO?g1Bso@|Aq|Wf9pL zezB287r!iGbzUQqAx@@0tj|$%W4|C*De)r$^@|Zj77?gll=|LyB|+ncFq@-u8#h=B zaiH<9BCo_q*uo8>tlT?aN#Nkd@SPkrHf|I&!t`MSH;R~coroaW$gW?cB%n=U^J_G@ zanrB_=by7l#L7(s$(=-Hm3O_8;LO_+u6T`Z$ZiqjnLZAtdW*=niD7Dhr8Tw{5w?g? zqrFzln)KZ_L}a@V@%mr}?ma%@aRnxVxu{~ym1B={+@UvTFYknH9*K~1CL z7YDO&?Bn=vZwhAfRmV3ME~tfFsYHAD-vd;aMFd`q5MdUf8fOt9w~4dPwTKM2iH1gR z95Qr#cc+NFjSMRSu1S74DIVia5evK-NXp@)b_a@9ReVZ#Yd=j1TXu?Q7QZ3{ezqZaJq!x7rO=8HgFLza zq(Wdl>=krw{F=W?X%^Qss8V#f&=g$Thikem)!m!mIp__#k_r6Ks;+$C0u-JBS1Vks zWT7>FVohTmAiA~}2z&R6e7)a*NNleNwA(-=wpWZl$2$x5Mt%2yi2NS*R{o8>y1F!z zj*HPbAYx;@OE8!y``dL$P+7bwxIW#7+GJjUqKsEhCb%t$3Sg)JQuRV6Gp`Rdn@JaP z7zerDdq4y~WcMBr`5m%*FUrXu=Dk;+#orJfwvGe2&F(-Tgok+t&QVFi@%IHa$FB=! zCew|)vTezn3Ff6!3sl9@he|5_Lseu~>gLY21`Nb5{~)VMy`e#{=ZLR8k1EgK7x{r& zUO=Swz8G_!MWpt=XymJq9M^Y06pb!H=+K;ewl@5df1Jx4CI$(7E z$SzYL!hXcdl*5kj-9L!P$0jVw3+E3aHco{d!C{XIs#IY?2n*CJ7l^(`MV@b45%s7T z$+tK;G2i{8i2Mmr!#*~U2RCkPv!A5c@4&-xe-{8Q`|6$^wX0riuBs@3cf zAfEIo?-4Dq2-GJu5Pgr^6B>vFkJ|+E_?e#z8l@5hK?=0eED!E;5v#V%(0(pz>r_Ub z@BUdt{>Nm5+RFK}&B)`7PTI)kGj^bbzWa@cd}pGf@^iki z_7!r}Z!ylvYM?-b1?zzV(fM0DP$0s7t4|klihOr5MOIOG(^eW&j^fZ+OtBiT1@;wj z*d-J`xRuVI2td#!WO4k=~67Z5?05cVNo;-ae$`Ngd&o-6GAdqE)d#~%#{mNopI&PR#NP|mEqu9G$UKYSA?^2_@R8$t|YU_ z(F);8GK;*}cb}rj??~5xsG6Op$gIXFYqb>8^@Nh|r+8{9XZ%w5_;#8)5r5+$FIhu@==l;@s2WW*4TQ3vr{X;INX+7fuzd%e4I$6r2D>1E z5N^ObvN9eB;ReFx^VRc=eES6#Nt!G!H2kSyc8EV>=zEoS(AGdImyRpdL36VRzY68HljE ztYzgqvfb7)5TrmWWy^M3%g_qxZf#kX9}Cakw4Q;W1!~WL(7tIs148?zwv45qx5(5C zAV`5$%4_f~dzzwEbvCEzNZ;KNUKz*(O(RZ)ihCqbf z%iEAW9LedvO;o353Ir)oX9QHOFkStw*?4VpdA74 z#{(48wJusAJwS2(sK-+DJvcmos75R0*PMg)NI+jm4;J$X&c*7l@ZG}{`8Z6yN#n{Z z(DN|GMyXn%f@kABQ!4-v^gZhe5MAH1s}hKy@7Y?Rf-U<%*9sE=P<}A1DvpF*Z&3q;Jgg(u0w9D(D5mcwAcRM#e5AVFE0~U|+kGPbhVbaHkbOUD z*CSfFE8p(?s#xi}pHSpa;dO%)lvSeRC%i4EB9}@Y|FQ6nLAq=r{zk}Sb}a(Y^O#+W zK!iNTYw-evtn%IC6!|jz-$80AuR_P;w*0E%kSC0#0K`0DjRB(T32P}3F;7@at9bAy zjimr6PYx^D(v#Lwv_g53Ep5dxs@T#~;n)8~jamE+;VJ735W-V-lLH|R7m8u8u5GIE~`=k28(*~K!d&W))0oO!1t^?8j6F@Sk|1TiAnJ7qy3I|fAK z5=muERALwIk|m|4Gg7^KI0+sI&?bQ>KqRpY zUr_N}oH6&x@UypRdG=mO_VM_q1pKTXPfpx-@0XE>rLK45hw3R#>0Vj8H4APc=-T zBt3EvLIG_kwkaGA#Wsb~*I$XyS!wI}BMmvVzPnOJo;EhsqUTD9_pAaB6-X`n^n^53 z8wg>6+9x17KOxPVAP`}nkg8~7DH>PV1VOL~sFx2=wbtm2fRL_|DkGM{rYB|A3=RnG zlhR!OKxm(oajRt~o*dSKo|5K;vsS%-0(Ip8g!CyH*PkeXs`J=-?@}01=ew(Ah1<2I1+0)LdQ%#cGL{kdyeA7N?F4S)aS)5G@cB0Z44UoqHhqu9s%+fr!4I=YD|y zaiosZ?hikFhc;*NH|DF~&KD5I_S^Xaa#uU^Km_WyGhfHw8x80-K2Xr0?KYW)Sy0VV&j^PXpXU~PvfHdzo zXhom_Y2I;w2s9w^jx!$9H41M9GO{~-A$cS}lo#c6NXe9+WZt^#KUsL0MKl*X>ECdQ%hrAJ?YY*8l(j diff --git a/common/src/proto/komp_ac.tables_data.rs b/common/src/proto/komp_ac.tables_data.rs index 9be0e14b..dccca5db 100644 --- a/common/src/proto/komp_ac.tables_data.rs +++ b/common/src/proto/komp_ac.tables_data.rs @@ -77,42 +77,6 @@ pub struct PostTableDataResponse { #[prost(int64, optional, tag = "5")] pub journal_id: ::core::option::Option, } -/// One row in a bulk insert request. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PostTableDataBulkRow { - /// Required. Same data payload as PostTableDataRequest.data. - #[prost(map = "string, message", tag = "1")] - pub data: ::std::collections::HashMap< - ::prost::alloc::string::String, - ::prost_types::Value, - >, -} -/// Bulk insert request. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PostTableDataBulkRequest { - /// Required. Profile (PostgreSQL schema) name that owns the table. - #[prost(string, tag = "1")] - pub profile_name: ::prost::alloc::string::String, - /// Required. Logical table (definition) name within the profile. - #[prost(string, tag = "2")] - pub table_name: ::prost::alloc::string::String, - /// Required. Rows to insert. Must contain at least 1 and at most 10,000 rows. - #[prost(message, repeated, tag = "3")] - pub rows: ::prost::alloc::vec::Vec, -} -/// Bulk insert response. -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PostTableDataBulkResponse { - /// True if all rows were inserted successfully. - #[prost(bool, tag = "1")] - pub success: bool, - /// Human-readable message. - #[prost(string, tag = "2")] - pub message: ::prost::alloc::string::String, - /// Per-row responses from the underlying PostTableData logic, in request order. - #[prost(message, repeated, tag = "3")] - pub responses: ::prost::alloc::vec::Vec, -} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BeginTableDataImportRequest { #[prost(string, tag = "1")] @@ -130,7 +94,16 @@ pub struct StageTableDataImportRequest { #[prost(string, tag = "2")] pub table_name: ::prost::alloc::string::String, #[prost(message, repeated, tag = "3")] - pub rows: ::prost::alloc::vec::Vec, + pub rows: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableDataImportRow { + /// Required. Data payload for one staged row. + #[prost(map = "string, message", tag = "1")] + pub data: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost_types::Value, + >, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct StageTableDataImportResponse { @@ -631,45 +604,6 @@ pub mod tables_data_client { ); self.inner.unary(req, path, codec).await } - /// Insert multiple rows atomically by applying PostTableData behavior to each row. - /// - /// Behavior: - /// - /// * Accepts 1..10,000 rows in one gRPC request - /// * Processes rows in request order - /// * Each row is inserted through the same validation, script execution, - /// typed binding, database insert, and indexing path as PostTableData - /// * Stops at the first failing row and returns that row's gRPC error code - /// * Commits only when every row succeeds; a failure rolls the entire request back - pub async fn post_table_data_bulk( - &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - self.inner - .ready() - .await - .map_err(|e| { - tonic::Status::unknown( - format!("Service was not ready: {}", e.into()), - ) - })?; - let codec = tonic_prost::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/komp_ac.tables_data.TablesData/PostTableDataBulk", - ); - let mut req = request.into_request(); - req.extensions_mut() - .insert( - GrpcMethod::new( - "komp_ac.tables_data.TablesData", - "PostTableDataBulk", - ), - ); - self.inner.unary(req, path, codec).await - } /// Starts a durable, profile-scoped import staging session. Staging never changes /// profile data; CommitTableDataImport applies every staged chunk atomically. pub async fn begin_table_data_import( @@ -1162,23 +1096,6 @@ pub mod tables_data_server { tonic::Response, tonic::Status, >; - /// Insert multiple rows atomically by applying PostTableData behavior to each row. - /// - /// Behavior: - /// - /// * Accepts 1..10,000 rows in one gRPC request - /// * Processes rows in request order - /// * Each row is inserted through the same validation, script execution, - /// typed binding, database insert, and indexing path as PostTableData - /// * Stops at the first failing row and returns that row's gRPC error code - /// * Commits only when every row succeeds; a failure rolls the entire request back - async fn post_table_data_bulk( - &self, - request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; /// Starts a durable, profile-scoped import staging session. Staging never changes /// profile data; CommitTableDataImport applies every staged chunk atomically. async fn begin_table_data_import( @@ -1511,52 +1428,6 @@ pub mod tables_data_server { }; Box::pin(fut) } - "/komp_ac.tables_data.TablesData/PostTableDataBulk" => { - #[allow(non_camel_case_types)] - struct PostTableDataBulkSvc(pub Arc); - impl< - T: TablesData, - > tonic::server::UnaryService - for PostTableDataBulkSvc { - type Response = super::PostTableDataBulkResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { - let inner = Arc::clone(&self.0); - let fut = async move { - ::post_table_data_bulk(&inner, request) - .await - }; - Box::pin(fut) - } - } - let accept_compression_encodings = self.accept_compression_encodings; - let send_compression_encodings = self.send_compression_encodings; - let max_decoding_message_size = self.max_decoding_message_size; - let max_encoding_message_size = self.max_encoding_message_size; - let inner = self.inner.clone(); - let fut = async move { - let method = PostTableDataBulkSvc(inner); - let codec = tonic_prost::ProstCodec::default(); - let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); - let res = grpc.unary(method, req).await; - Ok(res) - }; - Box::pin(fut) - } "/komp_ac.tables_data.TablesData/BeginTableDataImport" => { #[allow(non_camel_case_types)] struct BeginTableDataImportSvc(pub Arc); diff --git a/server b/server index d4731d51..65f54df5 160000 --- a/server +++ b/server @@ -1 +1 @@ -Subproject commit d4731d51946481f76d812f275f595ea9300eaeff +Subproject commit 65f54df500786ff52797140b63975fa01b05e79f diff --git a/test_luna/README.md b/test_luna/README.md new file mode 100644 index 00000000..344c9230 --- /dev/null +++ b/test_luna/README.md @@ -0,0 +1,388 @@ +# Chromium ERP import test + +Test the web application as a real ERP workflow in Chromium. Do not begin with +a toy CSV. First build and inspect the complete table ecosystem through the web +admin. Create each CSV only after its destination schema and all referenced +parent rows exist. + +Use a disposable database. Log in as `superadmin` with an empty password. + +## Start the application and browser + +From the repository root: + +```sh +cargo run -p server -- server +``` + +This starts gRPC on `[::1]:50051` and the embedded `web` crate on +`http://127.0.0.1:3000`. + +Open an isolated Chromium profile: + +```sh +chromium \ + --user-data-dir=/tmp/komp-ac-test-luna-chromium \ + --remote-debugging-port=9222 \ + --no-first-run \ + http://127.0.0.1:3000/login +``` + +The DevTools targets are at `http://127.0.0.1:9222/json`. Capture console and +Network failures throughout the run. + +## Build the ERP schema first + +Create the tables through `/admin/tables/new`, in the order below. Use the +builder controls for required, indexed, currency, rounding, quantity ledger, +column ordering, and row-display columns. After every table creation, verify +the redirect selects the new table and that the admin workspace reports the +expected types and flags. + +Use a unique suffix if these names already exist. + +### 1. Shared units catalogue + +Create global table `units`: + +| Column | Type | Options | +| --- | --- | --- | +| `code` | text | required, indexed, row display | +| `description` | text | required | +| `active` | boolean | required | + +This exercises global scope. Confirm account, accounting, accounting-transfer, +and quantity-ledger choices cannot be applied to a global table. + +### 2. Profile and chart of accounts + +Create profile `luna_erp` with currency `EUR` by creating `chart_accounts`: + +| Column | Type | Options | +| --- | --- | --- | +| `account_path` | account | required, row display; automatically unique/indexed | +| `label` | text | required, row display | +| `active` | boolean | required | + +The account catalogue must be created before any table using `accounting` or +`accounting_transfer`. + +### 3. Business partners + +Create profile table `partners`: + +| Column | Type | Options | +| --- | --- | --- | +| `partner_code` | text | required, indexed, row display | +| `legal_name` | text | required, indexed, row display | +| `active` | boolean | required | +| `email` | email_address | optional | +| `phone` | phone | optional; inspect generated companion columns | +| `iban` | iban | optional; inspect generated companion columns | +| `payment_card` | credit_card | optional | +| `credit_limit` | money | EUR, half-up rounding | +| `registered_on` | temporal → date | optional | +| `preferred_call_time` | temporal → time | optional | +| `synced_at` | temporal → instant | optional | +| `source_timestamp` | temporal → raw_datetime | optional | +| `payment_window` | duration | optional | +| `contract_period` | period | optional | +| `external_number` | bigint | indexed | +| `risk_score` | numeric | optional | + +Rename at least one generated PHONE or IBAN companion using the generated-name +panel, apply it, and confirm the preview and final table use the alias. + +### 4. Warehouses + +Create profile table `warehouses`: + +| Column | Type | Options | +| --- | --- | --- | +| `warehouse_code` | text | required, indexed, row display | +| `name` | text | required, row display | +| `manager` | link(partners) | required, automatically indexed | +| `unit` | link(units) | required, automatically indexed; cross-scope link | +| `capacity` | int | optional | + +Confirm links cannot target the table currently being created and that the UI +does not offer a redundant manual-index control for a link. + +### 5. Products and quantity ledger + +Create profile table `products`: + +| Column | Type | Options | +| --- | --- | --- | +| `sku` | text | required, indexed, row display | +| `name` | text | required, row display | +| `unit_gtin` | gtin → 8 | optional | +| `case_gtin` | gtin → 12 | optional | +| `retail_gtin` | gtin → 13 | optional | +| `pallet_gtin` | gtin → 14 | optional | +| `supplier` | link(partners) | required, auto-indexed | +| `home_warehouse` | link(warehouses) | required, auto-indexed | +| `unit` | link(units) | required, auto-indexed | +| `sale_price` | money | EUR, exact rounding | +| `purchase_price` | money | USD, half-up rounding | +| `vat_rate` | decimal(5,2) | required | +| `package_count` | int | optional | +| `legacy_id` | bigint | indexed | +| `measured_weight` | numeric | optional | +| `stock` | decimal(18,3) | quantity ledger enabled | +| `active` | boolean | required | + +Confirm `stock` is displayed as quantity-ledger/read-only and starts at zero. +The import mapper must not offer it as a writable destination. + +### 6. Stock transactions with multiple links + +Create `stock_receipts` with: + +- `receipt_no`: required, indexed text and row display; +- `product`: required link(products); +- `warehouse`: required link(warehouses); +- `supplier`: required link(partners); +- `quantity`: required decimal(18,3); +- `received_on`: required date; +- `processed`: int. + +Create `stock_issues` with the same shape, replacing `supplier` with required +`customer` link(partners), and `received_on` with required `issued_on`. + +Through `/admin/logic/new`, attach these scripts: + +- `stock_receipts.processed`: + `(quantity-add "product" "stock" (steel_get_column "stock_receipts" "quantity"))` +- `stock_issues.processed`: + `(quantity-sub "product" "stock" (steel_get_column "stock_issues" "quantity"))` + +These tables test three foreign keys per imported row and indirect updates to a +read-only quantity-ledger column. + +### 7. Sales documents and link chains + +Create `sales_orders`: + +- `order_no`: required/indexed text and row display; +- `customer`: required link(partners); +- `warehouse`: required link(warehouses); +- `ordered_on`: required date; +- `status`: required text; +- `total`: EUR money with half-up rounding; +- `paid`: required boolean. + +Create `sales_order_lines`: + +- `sales_order`: required link(sales_orders); +- `product`: required link(products); +- `quantity`: required decimal(18,3); +- `unit_price`: required EUR money, exact rounding; +- `discount_rate`: decimal(5,2); +- `line_position`: required int. + +This supplies a parent → child → product link chain and composite business +rows for import ordering and FK rollback tests. + +### 8. Accounting-only table-definition coverage + +Create `sales_journal` with an `accounting` definition row, EUR currency, and +half-up rounding. Inspect its generated name, tax-point date, debit, credit, +and account fields; rename at least one alias and apply it. Add an invoice +number and a link to `partners`. + +Create `period_transfers` with an `accounting_transfer` definition row and a +required text reference. + +Confirm each compound definition expands into its managed columns, cannot be +indexed, and can only occur once. These tables cover creation-only accounting +options. Do not attempt a positive accounting import unless the required +accounting periods and accounts have also been configured; otherwise use the +expected refusal as an atomic-failure case. + +## Confirm option/type coverage + +Before making CSVs, verify the ecosystem covers: + +- text, boolean, int, bigint, numeric, decimal and money; +- date, time, instant, raw datetime, duration and period; +- phone, IBAN, email address and credit card; +- GTIN-8, GTIN-12, GTIN-13 and GTIN-14; +- account, accounting and accounting transfer; +- profile-local, global, and cross-scope link targets; +- required and optional fields; +- manual indexes and automatic FK/account indexes; +- exact and half-up money rounding, with EUR and USD metadata; +- row-display selection and ordering; +- generated-column aliases; +- quantity-ledger and ordinary numeric columns. + +If the live backend offers another declarable type, add it to a suitable table +and record it. The backend's live type catalogue is authoritative. + +## Create CSVs last, in dependency order + +Only now create CSV files under a temporary directory such as +`/tmp/komp-ac-test-luna/`. Derive every header from the verified table shown in +the admin UI. Do not commit fixed parent IDs into this directory. + +Import in this order: + +1. `units`; +2. `chart_accounts`; +3. `partners`; +4. `warehouses` using real partner and unit IDs; +5. `products` using real partner, warehouse, and unit IDs; +6. `sales_orders` using real partner and warehouse IDs; +7. `sales_order_lines` using real order and product IDs; +8. `stock_receipts` and `stock_issues` using real linked IDs. + +After each parent import, obtain its inserted IDs from the Analytics page or a +read-only database query, then generate the child CSV. Include quoted commas, +quotes, Unicode, optional blanks, booleans, money values, dates, and decimal +quantities. Use valid GTIN checksums and valid PHONE/IBAN values so a failure is +attributable to the feature being tested rather than bad fixture data. + +For every import, verify upload → mapping → preview → asynchronous progress → +final result. Map by destination name, not blindly with **Map in order**, +because `deleted` may be the first offered destination and read-only/generated +fields change positional alignment. + +## Required import situations + +### Successful relational import + +Import at least two parents and multiple children. Verify persisted joins: + +- warehouse → manager and unit; +- product → supplier, warehouse, and unit; +- order line → order → customer; +- order line → product. + +The displayed link values must resolve to the configured row-display columns, +not merely expose numeric IDs. + +### Quantity-ledger effects + +Import receipts for the same product in multiple rows, then import issues. +Verify: + +`products.stock = sum(receipts.quantity) - sum(issues.quantity)` + +Also confirm `stock` was never writable in the products import mapping. A +failed receipt/issue import must add no quantity contribution. + +### Mapping and type conversion + +Reorder source columns, intentionally ignore one source field, leave optional +destinations blank, and use each offered date format. Confirm preview and the +canonical prepared CSV agree exactly. + +### Foreign-key atomic failure + +Create a child CSV containing valid rows plus one nonexistent product, partner, +warehouse, or order ID. The entire import must fail and commit zero rows and +zero quantity-ledger effects. + +### Validation/type atomic failure + +Repeat with one invalid boolean, date, decimal, money, GTIN, IBAN, or required +value. Assert the reported CSV row is correct and no partial rows remain. + +### Multi-chunk rollback + +Generate more than 1,000 stock or order-line rows, with an invalid row after +the first 1,000. This crosses the import chunk boundary. The final commit must +still be atomic: zero imported rows and no ledger contribution. + +### Schema drift + +Prepare a mapping in one tab. In another tab rename, append, or otherwise +change a destination column before Preview or Import. The stable column IDs +must preserve a rename safely, while removed/read-only destinations must be +refused rather than silently retargeted. + +### Session and concurrency + +Try to start a second import in the same session while a large one is running; +it must be refused. Confirm a different session cannot poll the first session's +job and that logout/session loss is handled without swapping a login page into +the progress card. + +## Pass evidence + +Record: + +- Chromium version and console/network failures; +- every created table and its final column flags; +- generated aliases and link targets; +- each CSV header, row count, mapping, preview, and final alert; +- before/after row counts for successful and failed imports; +- joined FK results; +- quantity-ledger balances and the arithmetic that produced them; +- screenshots of complex mappings, previews, progress, success, and expected + failures. + +An HTTP 200 or an “Import complete” alert alone is not a pass. The database +state, link resolution, atomic rollback, and quantity-ledger effects must agree +with the imported ERP transactions. + +## Executed run and evidence + +The staged run was completed against a freshly built server and a disposable +database. `cargo build -p server` passed before the server was started. Chromium +logged in as `superadmin` with an empty password. + +The live schema created through the browser was: + +- global `units`; +- profile `luna_erp`: `chart_accounts`, `partners`, `warehouses`, `products`, + `stock_receipts`, `stock_issues`, `sales_orders`, `order_lines`, + `journal_entries`, and `account_transfers`; +- the profile's managed accounting tables (`general_ledger`, `journal_lines`, + `ledger_accounts`, and `quantity_ledger`). + +The live tables deliberately cover text, boolean, int, bigint, numeric, +decimal, EUR/USD money with both rounding modes, date/time/instant, duration, +period, phone, IBAN, email, credit card, GTIN-8/12/13/14, account, FK links, +accounting, accounting transfer, generated companions, and a +`products.stock` `decimal(12,3)` quantity ledger. The stock transaction tables +have `stock_effect decimal(12,3)` computed targets, because a script cannot +target its own source `quantity` column. + +The saved Steel scripts are: + +```scheme +(quantity-add "product" "stock" + (steel_get_column "stock_receipts" "quantity")) +(quantity-subtract "product" "stock" + (steel_get_column "stock_issues" "quantity")) +``` + +The following imports completed and were checked with read-only database +queries: partners, warehouses, products, stock receipts, stock issues, sales +orders, order lines, chart accounts, and journal entries. Receipt quantity +`25.500` followed by issue quantity `20.500` left `products.stock = 5.000` and +created two quantity-ledger entries. The journal import created one journal +row and one general-ledger row. The order-line links resolved through the +sales-order and customer/product chain. + +The mixed order-line failure (one valid row followed by product ID `999999`) +reported `Link 'product' points to a missing row`, inserted zero rows, and left +the existing order-line count and quantity effects unchanged. This is the +atomic FK rollback checkpoint for the ladder. + +Observed failures are part of the test evidence: missing required values are +rejected atomically; duration requires an elapsed form such as `PT720H`, period +requires a calendar form such as `P3M`, and `instant` requires a timezone-free +civil datetime. A missing computed `stock_effect` is rejected, and the +subtraction script expects the positive computed amount while applying a +negative ledger contribution. Accounting account values must be paths such as +`4000/SALES`, not numeric ledger IDs. + +Two product issues are also reproducible. Required user columns are displayed +as nullable and generated DDL permits NULL even though backend validation still +enforces `required`. The progress card remained at `0%` while the backend job +had already completed; repeated submissions therefore created duplicate test +rows. The final database counts and ledger arithmetic, rather than the stale +card, were used as the pass criteria. diff --git a/web/src/pages/import_export/import/logic.rs b/web/src/pages/import_export/import/logic.rs index 944bb603..00ac8e98 100644 --- a/web/src/pages/import_export/import/logic.rs +++ b/web/src/pages/import_export/import/logic.rs @@ -15,7 +15,7 @@ use crate::{ table_structure::GetTableStructureRequest, tables_data::{ AbortTableDataImportRequest, BeginTableDataImportRequest, CommitTableDataImportRequest, - PostTableDataBulkRow, StageTableDataImportRequest, + StageTableDataImportRequest, TableDataImportRow, }, }, services::{authenticated_request, reject_cross_site}, @@ -40,7 +40,7 @@ use super::{ /// How many prepared rows the preview shows. const PREVIEW_ROWS: usize = 20; -/// How many rows one bulk insert carries. Also the granularity of the progress +/// How many rows one staged import chunk carries. Also the granularity of the progress /// the page shows, since a chunk is what the running total counts. const CHUNK_ROWS: usize = 1_000; @@ -221,7 +221,7 @@ pub(crate) async fn import_csv( .iter() .enumerate() .map(|(index, row)| { - bulk_row(locale, &destination, &prepared.columns, row).map_err(|error| { + import_row(locale, &destination, &prepared.columns, row).map_err(|error| { tr!( locale, "import-err-csv-row", @@ -350,7 +350,7 @@ struct Running { id: String, profile_name: String, table_name: String, - rows: Vec, + rows: Vec, } /// The insert, chunk by chunk, reporting after each one. @@ -610,14 +610,14 @@ fn source_options(locale: Locale, source: &Source) -> Vec { .collect() } -/// One prepared row as the bulk insert takes it: values under their destination +/// One prepared row as the staged import takes it: values under their destination /// names, converted to the column's type. -fn bulk_row( +fn import_row( locale: Locale, destination: &Destination, columns: &[String], row: &[String], -) -> Result { +) -> Result { let mut data = HashMap::new(); for (index, column) in columns.iter().enumerate() { let data_type = destination.types.get(column).ok_or_else(|| { @@ -634,7 +634,7 @@ fn bulk_row( )?; data.insert(column.clone(), value); } - Ok(PostTableDataBulkRow { data }) + Ok(TableDataImportRow { data }) } async fn render_step( @@ -679,8 +679,8 @@ fn import_failure( return (0, Outcome::SessionLost); } let status = failure.status_code(); - match bulk_failure(error) { - Some((failed_row_index, _inserted_in_chunk)) => ( + match import_row_failure(error) { + Some(failed_row_index) => ( 0, Outcome::RowFailed { status, @@ -700,22 +700,14 @@ fn import_failure( } } -fn bulk_failure(error: &tonic::Status) -> Option<(usize, usize)> { - let failed_row_index = error +fn import_row_failure(error: &tonic::Status) -> Option { + error .metadata() - .get(crate::grpc_error::BULK_FAILED_ROW_INDEX_METADATA_KEY)? + .get(crate::grpc_error::IMPORT_FAILED_ROW_INDEX_METADATA_KEY)? .to_str() .ok()? .parse() - .ok()?; - let inserted_rows = error - .metadata() - .get(crate::grpc_error::BULK_INSERTED_ROWS_METADATA_KEY)? - .to_str() - .ok()? - .parse() - .ok()?; - Some((failed_row_index, inserted_rows)) + .ok() } fn load_error(headers: &HeaderMap, error: LoadError) -> Response {