From 4506643dff7eb2ae0dc2c663a296c02bbee43721 Mon Sep 17 00:00:00 2001 From: Priec Date: Fri, 4 Sep 2026 15:55:38 +0200 Subject: [PATCH] fixing search indexing2 --- client-gui2 | 2 +- common/proto/search.proto | 26 + common/src/proto/descriptor.bin | Bin 245665 -> 247049 bytes common/src/proto/komp_ac.search.rs | 477 ++++++++++++++++++ search/src/lib.rs | 8 + server | 2 +- web/locales/cs/main.ftl | 1 + web/locales/en/main.ftl | 1 + web/locales/sk/main.ftl | 1 + .../pages/import_export/import/destination.rs | 127 ++++- web/src/pages/import_export/import/logic.rs | 36 +- web/src/pages/import_export/import/prepare.rs | 9 +- web/src/pages/import_export/import/state.rs | 2 + web/src/pages/import_export/import/ui.rs | 9 +- .../pages/import_export/import/step.html | 1 + 15 files changed, 663 insertions(+), 39 deletions(-) diff --git a/client-gui2 b/client-gui2 index 08a9b2e0..0a536189 160000 --- a/client-gui2 +++ b/client-gui2 @@ -1 +1 @@ -Subproject commit 08a9b2e01e129dcec7541d0b4f850758f96c7836 +Subproject commit 0a536189bbc017f9174cbc5d276828a84c16f2e0 diff --git a/common/proto/search.proto b/common/proto/search.proto index bc11e840..94cbc107 100644 --- a/common/proto/search.proto +++ b/common/proto/search.proto @@ -13,6 +13,32 @@ service SearchHealth { rpc GetHealth(komp_ac.common.Empty) returns (SearchHealthResponse); } +service SearchAdmin { + rpc ReindexTableChunk(ReindexTableChunkRequest) returns (ReindexResponse); + rpc ReindexTable(ReindexTableRequest) returns (ReindexResponse); + rpc ReindexProfile(ReindexProfileRequest) returns (ReindexResponse); +} + +message ReindexTableChunkRequest { + string profile_name = 1; + string table_name = 2; + repeated int64 row_ids = 3; +} + +message ReindexTableRequest { + string profile_name = 1; + string table_name = 2; +} + +message ReindexProfileRequest { + string profile_name = 1; +} + +message ReindexResponse { + uint64 row_count = 1; + uint64 affected_tables = 2; +} + message SearchIndexError { int64 job_id = 1; string profile_name = 2; diff --git a/common/src/proto/descriptor.bin b/common/src/proto/descriptor.bin index 9239615e6e3ee34903419b30be7454f6750c85a3..adaa6b437bd40e291f2c71e5320633d88cf83fbd 100644 GIT binary patch delta 4227 zcma)9O;a0J7VUohy47mg2(=IrAZ!T?7>s3-7-N!nVw;q09DjgF4H{;Ms*=r&H}hj=HGgAr?t8bNs7kq#1&gD5&w2Orz2>|B_x$Cr z*_S`quf$JtN8*)zzVCTZsz=++jp)md8*4k!!q)TVv%mjsp?0*i;gm;d?{_+f!?I1ywUo14-4ezfsqpMvU?gmURgepaD&Ue{jP zbAJ5xHzLTaM2+_P*0qh@?PmCCko#3i;n~`=*6tTi8tb)=a;^Q!r5^1)k2?EC;piI}D$L(4`$^ElvZ{DHQ8@MnUAAlG>EDj{_J68@7fRdu>UHaDd%%u8 z5z4HUw}T*XJYk1U*3So-Kzc%0q0=kw1SbNQ08G7JF+l>TG?O3)=)K!agJVuyXOm6H z3ljudE`)F_r*txO){sA>)NM+p%*g;iugpOK!Lh89kwB0u zb1pU&ahP)f0BzXNNPjq;k5yDKf;Wt0sHT{RiZ!CcAc=}Utin(eQROfIpjTx*Js(vL z0|;_egh6snD9>0ETF!yo8DH1Tp{(LDp=L?~L2r!pfZ#cnR)a$}=#3TnRTvJ%G|sgK z0BzjRD6R3dt}bk#jdO8jC?~DAbQls|lYR+uv6PbHm=e0J0MMHfDODFBcux5}34-2K zE>vMi3gaAC0|2yh90m~CxEes9o#UoRfk~O>rbrPB@TLuqG^csO;0fL|PnaZd-bCqu zcHTruWj$}AbjZ_0iBYU^jir#F)y!!qL+sSJ$S{Ie<04Cn>w-`_f(p3`B}H*@+y!3` zG;E+<$oFwkywI63*?FMNnBzRqW=wJ(Xfyq>9yvFZ?^y5Z=8*b3ep%_c)SHW(H30N3 z3Qu(d2#yz1x;H@3yU1C)RL5D7V@K-Etf5hFW>Y7$EmqB$&GzzfXwD^b9M#~G;gRMg za~!qfk~t1wjz_^2)SEfZ8VIyG9t9xsOO~lS+F~ng~b1oYm&FJNHe_{jg^1vWR z3CDRc!6p|Sp6AQP6)qYupGhRCmh2ZG)e)Bs!V;v&9keW*I3U>G>ceVn8gY z_Jr-a9#z<=P1l9G00^Nm*Y#`y0W26t>dS(0R5m!$&65h{4ePd^sua9#sFjy8MJe7C z^t!|@A>??Ia|D9t&2(1P3i4AqkePnd3TX#haGWoQ)dEiMggpxxr}NKs6Syh{M+ z(4xp_{aBFZ;t9P>FoL&;Wuox?P~NdV(c$~>y5kpA|9uMoE{_`k!nn&}0KxGtj~fu= z?((?#bkaRhRR;rLZ0-s4pI%5nsNBowjR7Lpu$$7Y! zp!Hb}_1ogBt8IBuzotILT<7;A27*eR_XmiKMuqmL9_r@=HiS~w(~62A<`q*-47h84AT;h_bwur&`ZRbX&ajCz9>wBx0^#{+ zWRyJt9vc&N_^}>696+=1c=(i>)^sR8v)1*rrs4IO{*IJsn$`v<1%UcCcwm6w*x+4@ z0g+>amn}_4t?|SFpfqbjmlTL>MLi-wpsis<=pMqyOJr^9(vU*rst-F?eY|XnNtL4h zT-oIOfsn^$JaJZKtTytg?ks3@)Kfijgy8v9uL2OjmQW2M0LLx9=zz#psjAWhq5RzX zN|z>p*XKUvA!Xp;)_f+ubSNkp1H$+$I2nut9vP`}J2E$tuPn7X`%l}^%JQ8G{x_;D z;^Vg`NNFDwbVsJU7#ceeGOikGhhpN!b6uWlNo7lXB4%m`|#!PQzOVcM+I<)+)Orh&Q1l(ZD96t7J@92Nu z+aFQ61l<68vYRa30edI&hE2D?Uf*~ZLv3$z%0vVAwwyL*x(eFIOu8F7n68&LXN$|>g&ZE-tNbx) zceY#2%1mW9YBW2Q{jH-5|FG9p5Gkm>I5q)vZ@dUfCJMSaUf|kFTHuGyf$TSCAM^w1 zcQNP(>0B3se$c0%o`6R3kZ*s08WKF@YY!0VA7=E|0T6T!v--0d2r7qv{@_O&f8zBA JKRRmzzW`8#j$;4- delta 2944 zcmYjTOHU+K7_GW>tGl`yXsCVw-3<*i!)th$;l02NFd5K+ad@R60hzcku5|CF*cn}o z(S*dEiDuMsp(}oZzrmm2-o2i4?^m~b!Q$|J=iKjI{q;AcAO9P;`^kT!{#v|OZ~Uc) z@4xjoqWHtV8*whtzW4g!-yi%@Kh<8MOWv>_$8n%NKM6|FNL-Ay_LP?dgQ^_&#UVjR zl+O?x2cv-u1wccI&|n%;busy%4XG$Yv=you0_|XEWK?)62@*9THU(bdcnZ9vkcFba zOKQVH11PKVcYprVNV9kX{?Emulc1u8MOQ<&;&e51D}^MpfmW&3L>G=#m4zA_lw}`+ zR*eRQ7J^nSSK?x7s+LQFnoA=GUd=t11Fz=N$bnZw8gZ-;haUANY!iHVjYdrqhR-I{ zr3nC#tE-%O7zmE_yiFP?NC72lpA~DUA@O7v8ji;Y8u`A=hAD+YEEUYP-#7yF(p@tp7N$`$(h}Hi_UkXJ-=#VBR^BHm~4_{Ar^Ro*b(n(Tm=DHb=H$QFc3Ej!UXy$^w8K zmn26ZI4%`RnGHEEC1q2ZfKM$;X#hZ5b~N9|{4GmuU<7Yj<|ANCE3!-g=+KHP$^+C~ z>9flO8)z$7CPuFl{lt6UM$c(@5>-tYn$fSyi2*c3fwwM)N)v3T zM>V(*^jCr^Z`p}N!MHzLPKB3l%ZYG(wi0`k;R%r~JFVa$=51HZ97MLaol1_zqFr8iLS=h6 zi<)7$Gy}lp9DMi>CgV}C|@459T`g?&~~bJcY#3LvAc_?^i%Jd-8HuAX=uI% zp*feks?E|7z_hEzGla(NPEN6qu)(yaro}{!y;QX_2+zH#ws->UI}>^PcJv^^x!xZe zH`5v>`nmUkomQ6Zd1RL+rX*@^V9%xAjd%-ry^9y${{ zi%VTQ=w~Q?Mn?kYx@pZRg5lPj>)81--}B;d7lVI}70dX57rO4u{3u@Z(7ELo#cJ11 ze_W~)i5?5p(U0qU6Z>C${^@<^SAWkGfji&P-K6=^?DUtN4A;I>ndxH4cIFpcRB*r4 zi_XmDzwD9Y`d{|Qagr}9v&R23Hi0+ba^bPl!&qMFNvFp, +} +#[derive(serde::Serialize, serde::Deserialize)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReindexTableRequest { + #[prost(string, tag = "1")] + pub profile_name: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub table_name: ::prost::alloc::string::String, +} +#[derive(serde::Serialize, serde::Deserialize)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReindexProfileRequest { + #[prost(string, tag = "1")] + pub profile_name: ::prost::alloc::string::String, +} +#[derive(serde::Serialize, serde::Deserialize)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReindexResponse { + #[prost(uint64, tag = "1")] + pub row_count: u64, + #[prost(uint64, tag = "2")] + pub affected_tables: u64, +} +#[derive(serde::Serialize, serde::Deserialize)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct SearchIndexError { #[prost(int64, tag = "1")] pub job_id: i64, @@ -871,3 +903,448 @@ pub mod search_health_server { const NAME: &'static str = SERVICE_NAME; } } +/// Generated client implementations. +pub mod search_admin_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct SearchAdminClient { + inner: tonic::client::Grpc, + } + impl SearchAdminClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl SearchAdminClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> SearchAdminClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + std::marker::Send + std::marker::Sync, + { + SearchAdminClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn reindex_table_chunk( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/komp_ac.search.SearchAdmin/ReindexTableChunk", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert( + GrpcMethod::new("komp_ac.search.SearchAdmin", "ReindexTableChunk"), + ); + self.inner.unary(req, path, codec).await + } + pub async fn reindex_table( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/komp_ac.search.SearchAdmin/ReindexTable", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("komp_ac.search.SearchAdmin", "ReindexTable")); + self.inner.unary(req, path, codec).await + } + pub async fn reindex_profile( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::unknown( + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/komp_ac.search.SearchAdmin/ReindexProfile", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("komp_ac.search.SearchAdmin", "ReindexProfile")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod search_admin_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with SearchAdminServer. + #[async_trait] + pub trait SearchAdmin: std::marker::Send + std::marker::Sync + 'static { + async fn reindex_table_chunk( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn reindex_table( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn reindex_profile( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct SearchAdminServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl SearchAdminServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for SearchAdminServer + where + T: SearchAdmin, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/komp_ac.search.SearchAdmin/ReindexTableChunk" => { + #[allow(non_camel_case_types)] + struct ReindexTableChunkSvc(pub Arc); + impl< + T: SearchAdmin, + > tonic::server::UnaryService + for ReindexTableChunkSvc { + type Response = super::ReindexResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::reindex_table_chunk(&inner, request) + .await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReindexTableChunkSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/komp_ac.search.SearchAdmin/ReindexTable" => { + #[allow(non_camel_case_types)] + struct ReindexTableSvc(pub Arc); + impl< + T: SearchAdmin, + > tonic::server::UnaryService + for ReindexTableSvc { + type Response = super::ReindexResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::reindex_table(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReindexTableSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/komp_ac.search.SearchAdmin/ReindexProfile" => { + #[allow(non_camel_case_types)] + struct ReindexProfileSvc(pub Arc); + impl< + T: SearchAdmin, + > tonic::server::UnaryService + for ReindexProfileSvc { + type Response = super::ReindexResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::reindex_profile(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReindexProfileSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + let mut response = http::Response::new( + tonic::body::Body::default(), + ); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }) + } + } + } + } + impl Clone for SearchAdminServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "komp_ac.search.SearchAdmin"; + impl tonic::server::NamedService for SearchAdminServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/search/src/lib.rs b/search/src/lib.rs index a3825fb5..a29ce542 100644 --- a/search/src/lib.rs +++ b/search/src/lib.rs @@ -160,6 +160,14 @@ impl SearcherService { } } + pub fn invalidate_profile(&self, profile_name: &str) -> Result<(), Status> { + self.profiles + .lock() + .map_err(|_| Status::internal("Profile index cache lock poisoned"))? + .remove(profile_name); + Ok(()) + } + async fn run_rpc( &self, request: Request, diff --git a/server b/server index 8ab030c7..66ce2877 160000 --- a/server +++ b/server @@ -1 +1 @@ -Subproject commit 8ab030c7710a696f25c2f3aaf5383a7053d8d78f +Subproject commit 66ce2877296a032349140ad524c555edca457cd0 diff --git a/web/locales/cs/main.ftl b/web/locales/cs/main.ftl index 47554207..1f559100 100644 --- a/web/locales/cs/main.ftl +++ b/web/locales/cs/main.ftl @@ -659,6 +659,7 @@ import-th-status = Stav import-source-none = Přeskočit tento sloupec souboru import-source-column = Sloupec { $position } import-required-short = povinný +import-verification-short = ověřit FROM import-status-mapped = Namapováno import-status-duplicate = Stejný název je použit dvakrát import-status-unmapped = Nenamapováno diff --git a/web/locales/en/main.ftl b/web/locales/en/main.ftl index b7f90f32..baf5effe 100644 --- a/web/locales/en/main.ftl +++ b/web/locales/en/main.ftl @@ -648,6 +648,7 @@ import-th-status = Status import-source-none = Skip this file column import-source-column = Column { $position } import-required-short = required +import-verification-short = verify FROM import-status-mapped = Mapped import-status-duplicate = Same name appears twice import-status-unmapped = Not mapped diff --git a/web/locales/sk/main.ftl b/web/locales/sk/main.ftl index eec10768..c969cddc 100644 --- a/web/locales/sk/main.ftl +++ b/web/locales/sk/main.ftl @@ -657,6 +657,7 @@ import-th-status = Stav import-source-none = Preskočiť tento stĺpec súboru import-source-column = Stĺpec { $position } import-required-short = povinný +import-verification-short = overiť FROM import-status-mapped = Namapované import-status-duplicate = Rovnaký názov je použitý dvakrát import-status-unmapped = Nenamapované diff --git a/web/src/pages/import_export/import/destination.rs b/web/src/pages/import_export/import/destination.rs index 2799f803..7459fe71 100644 --- a/web/src/pages/import_export/import/destination.rs +++ b/web/src/pages/import_export/import/destination.rs @@ -1,10 +1,14 @@ -//! The columns an import may write into, and how the form names one. +//! The columns an import may write into or verify, and how the form names one. //! //! The review is source-first: every file position gets one row and chooses a //! destination by stable identity. Duplicate destination choices are prevented //! in the browser and refused again by the Rust mapping validation. -use crate::definitions::table_structure::{TableColumn, TableStructureResponse}; +use std::collections::HashSet; + +use crate::definitions::table_structure::{ + ImportFieldDescriptor, TableColumn, TableStructureResponse, +}; use super::super::common::schema::{is_importable_system_column, is_system_column}; @@ -50,7 +54,7 @@ impl DestinationKey { } } -/// One column an import may write into. +/// One column an import may write into or use to verify a linked projection. #[derive(Clone, Debug)] pub(crate) struct DestinationColumn { pub key: DestinationKey, @@ -61,27 +65,38 @@ pub(crate) struct DestinationColumn { /// column may have a default, and `information_schema` does not say which /// do, so refusing here would block imports the server would accept. pub required: bool, + /// A linked projection is not written by the import. When mapped, its CSV + /// value is carried to the server as an assertion against the value copied + /// through its LINK column. + pub verification: bool, } -/// The columns of `schema` an import may write into, in the order the table -/// declares them. +/// The columns of `schema` an import may write into or verify, in the order the +/// table declares them. /// /// The exclusions are the server's own flags rather than a guess: /// /// * `is_primary_key` — `id` comes from a sequence. -/// * `read_only` — set for a quantity-ledger column and for a link projection, -/// and an insert naming either is refused. Accounting columns are marked -/// `generated` but *not* read-only, so they stay: they are a user's to fill -/// in. +/// * `read_only` — ordinary read-only columns remain excluded. A linked +/// projection is the exception: it is offered as an optional verification +/// destination, although the import still never writes it. Accounting +/// columns are marked `generated` but *not* read-only, so they stay: they are +/// a user's to fill in. /// * system columns, except the ones an insert actually takes. `deleted` is /// offered, because writing it is how a file that recorded deleted rows loads /// back as deleted rows; `row_revision` and `created_at` are not, because the /// server assigns them and answers `Invalid column` to anything else. -pub(crate) fn destination_columns(schema: &TableStructureResponse) -> Vec { +pub(crate) fn destination_columns( + schema: &TableStructureResponse, + projection_column_ids: &HashSet, +) -> Vec { schema .columns .iter() - .filter(|column| !column.is_primary_key && !column.read_only) + .filter(|column| { + !column.is_primary_key + && (!column.read_only || projection_column_ids.contains(&column.column_id)) + }) .filter(|column| { !is_system_column(&column.name) || is_importable_system_column(&column.name) }) @@ -92,10 +107,35 @@ pub(crate) fn destination_columns(schema: &TableStructureResponse) -> Vec HashSet { + let link_prefixes = fields + .iter() + .filter(|field| field.link.is_some()) + .map(|field| format!("{}.", field.name)) + .collect::>(); + fields + .iter() + .filter(|field| { + !field.writable + && field.generated + && link_prefixes + .iter() + .any(|prefix| field.generated_from.starts_with(prefix)) + }) + .map(|field| field.column_id) + .collect() +} + /// The column a posted key names, in the table as it stands right now. /// /// `None` means the table no longer has it — dropped, or made read-only, while @@ -113,6 +153,10 @@ pub(crate) fn resolve<'a>( mod tests { use super::*; + fn destinations(schema: &TableStructureResponse) -> Vec { + destination_columns(schema, &HashSet::from([92])) + } + fn column(name: &str, column_id: i64) -> TableColumn { TableColumn { name: name.to_string(), @@ -149,21 +193,62 @@ mod tests { generated_from: "accounting".to_string(), ..column("debit", 81) }, + // A linked projection is visible for verification, but remains + // distinguished from columns the import writes. + TableColumn { + generated: true, + read_only: true, + generated_from: "customer.name".to_string(), + ..column("customer_name", 92) + }, column("created_at", 0), ], } } + #[test] + fn only_outputs_reached_through_real_links_are_verifications() { + let fields = vec![ + ImportFieldDescriptor { + column_id: 10, + name: "customer.link".to_string(), + link: Some(Default::default()), + ..Default::default() + }, + ImportFieldDescriptor { + column_id: 92, + name: "customer_name".to_string(), + generated: true, + generated_from: "customer.link.name".to_string(), + ..Default::default() + }, + ImportFieldDescriptor { + column_id: 93, + name: "phone_country".to_string(), + generated: true, + generated_from: "phone.with.dot".to_string(), + ..Default::default() + }, + ]; + + assert_eq!( + projection_verification_column_ids(&fields), + HashSet::from([92]) + ); + } + #[test] fn only_the_columns_an_insert_takes_are_offered() { - let columns = destination_columns(&schema()); + let columns = destinations(&schema()); assert_eq!( columns .iter() .map(|column| column.name.as_str()) .collect::>(), - vec!["deleted", "number", "debit"] + vec!["deleted", "number", "debit", "customer_name"] ); + assert!(!columns[2].verification); + assert!(columns[3].verification); } /// A user column is identified by the id the server gave it, so a rename @@ -171,7 +256,7 @@ mod tests { /// id and no rename, so its name is identity enough. #[test] fn a_user_column_is_named_by_its_stable_id_and_a_system_one_by_its_name() { - let columns = destination_columns(&schema()); + let columns = destinations(&schema()); assert_eq!(columns[0].key.encode(), "system:deleted"); assert_eq!(columns[1].key.encode(), "id:42"); @@ -187,22 +272,22 @@ mod tests { /// column rather than the name. #[test] fn a_renamed_column_is_still_the_same_destination() { - let posted = destination_columns(&schema())[1].key.encode(); + let posted = destinations(&schema())[1].key.encode(); let mut renamed = schema(); renamed.columns[3].name = "invoice_number".to_string(); - let columns = destination_columns(&renamed); + let columns = destinations(&renamed); let resolved = resolve(&columns, &posted).expect("column 42 is column 42 whatever it is called"); assert_eq!(resolved.name, "invoice_number"); } - /// A destination that is gone, or that the table has since made read-only, - /// resolves to nothing — which the caller has to refuse rather than guess - /// past. + /// A destination that is gone, or that the table has since become an + /// ordinary read-only column, resolves to nothing — which the caller has + /// to refuse rather than guess past. #[test] fn a_destination_the_table_no_longer_offers_resolves_to_nothing() { - let columns = destination_columns(&schema()); + let columns = destinations(&schema()); assert!(resolve(&columns, "id:57").is_none()); assert!(resolve(&columns, "id:999").is_none()); assert!(resolve(&columns, "system:row_revision").is_none()); @@ -213,7 +298,7 @@ mod tests { /// Only the user's own `NOT NULL` columns are reported as required. #[test] fn required_is_the_users_own_not_null_columns() { - let columns = destination_columns(&schema()); + let columns = destinations(&schema()); assert_eq!( columns .iter() diff --git a/web/src/pages/import_export/import/logic.rs b/web/src/pages/import_export/import/logic.rs index a0c41ff0..b652b5b1 100644 --- a/web/src/pages/import_export/import/logic.rs +++ b/web/src/pages/import_export/import/logic.rs @@ -27,10 +27,13 @@ use super::{ loader::LoadError, schema::{column_types, csv_value}, }, - destination::{DestinationColumn, destination_columns, resolve}, + destination::{ + DestinationColumn, destination_columns, projection_verification_column_ids, resolve, + }, loader::load_page, prepare::{ - Prepared, Source, canonical_csv, normalize_dates, prepare, read_mapping, read_source, + Assignment, Prepared, Source, canonical_csv, normalize_dates, prepare, read_mapping, + read_source, }, progress::Outcome, state::{ImportForm, MappingRow, MappingStep, PreviewStep, SourceOption, Step}, @@ -58,11 +61,16 @@ struct Destination { } impl Destination { - /// The destination names, for the parts of the preparation that only need - /// to know which columns exist. - fn names(&self) -> Vec { + /// Writable columns are always prepared so an unmapped form field keeps + /// its existing NULL/default behavior. Projection verifications only + /// travel when the user mapped a CSV value to them. + fn prepared_names(&self, assignments: &[Assignment]) -> Vec { self.columns .iter() + .filter(|column| { + !column.verification + || assignments.iter().any(|assignment| assignment.column == column.name) + }) .map(|column| column.name.clone()) .collect() } @@ -132,6 +140,7 @@ pub(crate) async fn prepare_step( key, name: column.name.clone(), required: column.required, + verification: column.verification, chosen: chosen_index .map(|index| (index + 1).to_string()) .unwrap_or_default(), @@ -142,7 +151,10 @@ pub(crate) async fn prepare_step( }) .collect::>(); let mapped = rows.iter().filter(|row| !row.chosen.is_empty()).count(); - let attention = rows.len().saturating_sub(mapped); + let attention = rows + .iter() + .filter(|row| row.chosen.is_empty() && !row.verification) + .count(); let step = Step::Mapping(MappingStep { table_name: destination.table_name.clone(), @@ -196,9 +208,9 @@ pub(crate) async fn preview_step( /// POST /admin/import — the prepared rows, converted and inserted. /// -/// From here on nothing about mapping exists any more: what is sent is the -/// canonical import, and the server decides types, validations, scripts, links -/// and permissions exactly as it does for any other insert. +/// From here on the prepared column names carry the mapping. The server applies +/// ordinary insert rules and treats any mapped linked projection as an +/// assertion that must match the value reached through its link. pub(crate) async fn import_csv( State(state): State, headers: HeaderMap, @@ -557,7 +569,8 @@ async fn destination( .remove(&table_name) .ok_or_else(|| unavailable(headers, tr!(locale, "import-err-missing-structure")))?; - let columns = destination_columns(&structure); + let projection_column_ids = projection_verification_column_ids(&descriptor.fields); + let columns = destination_columns(&structure, &projection_column_ids); if columns.is_empty() { return Err(reject( headers, @@ -610,7 +623,8 @@ async fn prepared( let assignments = read_mapping(locale, &chosen, &source).map_err(|message| reject(headers, message))?; - let mut prepared = prepare(&assignments, &source, &destination.names()); + let prepared_names = destination.prepared_names(&assignments); + let mut prepared = prepare(&assignments, &source, &prepared_names); normalize_dates(locale, &mut prepared, &destination.types, form.date_format) .map_err(|message| reject(headers, message))?; Ok((destination, source, prepared)) diff --git a/web/src/pages/import_export/import/prepare.rs b/web/src/pages/import_export/import/prepare.rs index 483db505..8bcffcd7 100644 --- a/web/src/pages/import_export/import/prepare.rs +++ b/web/src/pages/import_export/import/prepare.rs @@ -181,10 +181,11 @@ impl Prepared { /// Applies the mapping: the values that were asked for, under the names they /// were asked for. /// -/// `writable` is every destination column the table offers. Normal form fields -/// are always included: an unmapped one becomes NULL, exactly as when the user -/// leaves that field empty in the client form. An unmapped system column is -/// omitted so its server-owned default still applies (`deleted = false`). +/// `writable` contains every writable destination plus any mapped projection +/// verification. Normal form fields are always included: an unmapped one +/// becomes NULL, exactly as when the user leaves that field empty in the +/// client form. An unmapped system column is omitted so its server-owned +/// default still applies (`deleted = false`). pub(crate) fn prepare( assignments: &[Assignment], source: &Source, diff --git a/web/src/pages/import_export/import/state.rs b/web/src/pages/import_export/import/state.rs index 1330ad52..601816d8 100644 --- a/web/src/pages/import_export/import/state.rs +++ b/web/src/pages/import_export/import/state.rs @@ -188,6 +188,8 @@ pub(crate) struct MappingRow { pub key: String, pub name: String, pub required: bool, + /// This destination asserts a linked FROM value instead of writing it. + pub verification: bool, /// One-based source position, empty when intentionally not mapped. pub chosen: String, pub example: String, diff --git a/web/src/pages/import_export/import/ui.rs b/web/src/pages/import_export/import/ui.rs index aaba081a..984de8ce 100644 --- a/web/src/pages/import_export/import/ui.rs +++ b/web/src/pages/import_export/import/ui.rs @@ -235,6 +235,7 @@ mod tests { key: "id:42".to_string(), name: "a".to_string(), required: true, + verification: false, example: "value-a".to_string(), chosen: "1".to_string(), }, @@ -242,6 +243,7 @@ mod tests { key: "id:57".to_string(), name: "b".to_string(), required: false, + verification: false, example: String::new(), chosen: String::new(), }, @@ -299,7 +301,11 @@ mod tests { /// to them or deliberately left unused. #[test] fn the_mapping_step_reviews_every_source_column_once() { - let html = render_step(&page(mapping())); + let mut step = mapping(); + if let Step::Mapping(mapping) = &mut step { + mapping.rows[1].verification = true; + } + let html = render_step(&page(step)); assert_eq!(html.matches(r#"name="source_position""#).count(), 2); assert_eq!(html.matches(r#"name="destination""#).count(), 2); @@ -319,6 +325,7 @@ mod tests { html.contains(">Not mapped — leave empty"), "{html}" ); + assert!(html.contains("verify FROM"), "{html}"); } /// The two sides are two lists, not one zipped table: a source chip carries diff --git a/web/templates/pages/import_export/import/step.html b/web/templates/pages/import_export/import/step.html index ec38e5da..d67310ab 100644 --- a/web/templates/pages/import_export/import/step.html +++ b/web/templates/pages/import_export/import/step.html @@ -118,6 +118,7 @@
{{ row.name }} {% if row.required %}{{ nav.tr("import-required-short") }}{% endif %} + {% if row.verification %}{{ nav.tr("import-verification-short") }}{% endif %}