filipriec/komp_ac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bulk import

Browse Source
This commit is contained in:
Filipriec
2026-08-21 22:36:41 +02:00
parent 239d43ac1a
commit 0756e99959
9 changed files with 600 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
common/proto/tables_data.proto
View File

@@ -29,7 +29,7 @@ service TablesData {
rpc PostAccountingTableData(PostAccountingTableDataRequest)
returns (PostTableDataResponse);
// Insert multiple rows by applying PostTableData behavior to each row.
// Insert multiple rows atomically by applying PostTableData behavior to each row.
//
// Behavior:
// - Accepts 1..10,000 rows in one gRPC request
@@ -37,9 +37,27 @@ service TablesData {
// - Each row is inserted through the same validation, script execution,
// typed binding, database insert, and indexing path as PostTableData
// - Stops at the first failing row and returns that row's gRPC error code
// with row index context; rows inserted before the failure remain inserted
// - Commits only when every row succeeds; a failure rolls the entire request back
rpc PostTableDataBulk(PostTableDataBulkRequest) returns (PostTableDataBulkResponse);
// Starts a durable, profile-scoped import staging session. Staging never changes
// profile data; CommitTableDataImport applies every staged chunk atomically.
rpc BeginTableDataImport(BeginTableDataImportRequest)
returns (BeginTableDataImportResponse);
// Adds one table chunk to an import session. Rows retain chunk and table order.
rpc StageTableDataImport(StageTableDataImportRequest)
returns (StageTableDataImportResponse);
// Applies every staged row through the ordinary validated insert machinery in
// one PostgreSQL transaction. Any failure rolls back every imported side effect.
rpc CommitTableDataImport(CommitTableDataImportRequest)
returns (CommitTableDataImportResponse);
// Discards a staged import. Already committed imports cannot be aborted.
rpc AbortTableDataImport(AbortTableDataImportRequest)
returns (AbortTableDataImportResponse);
// Update existing row data with strict type binding and script validation.
//
// Behavior:
@@ -201,6 +219,42 @@ message PostTableDataBulkResponse {
repeated PostTableDataResponse responses = 3;
}
message BeginTableDataImportRequest {
string profile_name = 1;
}
message BeginTableDataImportResponse {
string import_id = 1;
}
message StageTableDataImportRequest {
string import_id = 1;
string table_name = 2;
repeated PostTableDataBulkRow rows = 3;
}
message StageTableDataImportResponse {
int64 staged_rows = 1;
int64 total_staged_rows = 2;
}
message CommitTableDataImportRequest {
string import_id = 1;
}
message CommitTableDataImportResponse {
bool success = 1;
int64 inserted_rows = 2;
}
message AbortTableDataImportRequest {
string import_id = 1;
}
message AbortTableDataImportResponse {
bool success = 1;
}
// Update an existing row.
message PutTableDataRequest {
// Required. Profile (schema) name.

BIN
common/src/proto/descriptor.bin
View File

Binary file not shown.

396
common/src/proto/komp_ac.tables_data.rs
View File

@@ -113,6 +113,54 @@ pub struct PostTableDataBulkResponse {
#[prost(message, repeated, tag = "3")]
pub responses: ::prost::alloc::vec::Vec<PostTableDataResponse>,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BeginTableDataImportRequest {
#[prost(string, tag = "1")]
pub profile_name: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct BeginTableDataImportResponse {
#[prost(string, tag = "1")]
pub import_id: ::prost::alloc::string::String,
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct StageTableDataImportRequest {
#[prost(string, tag = "1")]
pub import_id: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub table_name: ::prost::alloc::string::String,
#[prost(message, repeated, tag = "3")]
pub rows: ::prost::alloc::vec::Vec<PostTableDataBulkRow>,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct StageTableDataImportResponse {
#[prost(int64, tag = "1")]
pub staged_rows: i64,
#[prost(int64, tag = "2")]
pub total_staged_rows: i64,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CommitTableDataImportRequest {
#[prost(string, tag = "1")]
pub import_id: ::prost::alloc::string::String,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct CommitTableDataImportResponse {
#[prost(bool, tag = "1")]
pub success: bool,
#[prost(int64, tag = "2")]
pub inserted_rows: i64,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct AbortTableDataImportRequest {
#[prost(string, tag = "1")]
pub import_id: ::prost::alloc::string::String,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)]
pub struct AbortTableDataImportResponse {
#[prost(bool, tag = "1")]
pub success: bool,
}
/// Update an existing row.
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct PutTableDataRequest {
@@ -583,7 +631,7 @@ pub mod tables_data_client {
);
self.inner.unary(req, path, codec).await
}
/// Insert multiple rows by applying PostTableData behavior to each row.
/// Insert multiple rows atomically by applying PostTableData behavior to each row.
///
/// Behavior:
///
@@ -592,7 +640,7 @@ pub mod tables_data_client {
/// * Each row is inserted through the same validation, script execution,
/// typed binding, database insert, and indexing path as PostTableData
/// * Stops at the first failing row and returns that row's gRPC error code
/// with row index context; rows inserted before the failure remain inserted
/// * Commits only when every row succeeds; a failure rolls the entire request back
pub async fn post_table_data_bulk(
&mut self,
request: impl tonic::IntoRequest<super::PostTableDataBulkRequest>,
@@ -622,6 +670,128 @@ pub mod tables_data_client {
);
self.inner.unary(req, path, codec).await
}
/// Starts a durable, profile-scoped import staging session. Staging never changes
/// profile data; CommitTableDataImport applies every staged chunk atomically.
pub async fn begin_table_data_import(
&mut self,
request: impl tonic::IntoRequest<super::BeginTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::BeginTableDataImportResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::unknown(
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/komp_ac.tables_data.TablesData/BeginTableDataImport",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(
GrpcMethod::new(
"komp_ac.tables_data.TablesData",
"BeginTableDataImport",
),
);
self.inner.unary(req, path, codec).await
}
/// Adds one table chunk to an import session. Rows retain chunk and table order.
pub async fn stage_table_data_import(
&mut self,
request: impl tonic::IntoRequest<super::StageTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::StageTableDataImportResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::unknown(
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/komp_ac.tables_data.TablesData/StageTableDataImport",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(
GrpcMethod::new(
"komp_ac.tables_data.TablesData",
"StageTableDataImport",
),
);
self.inner.unary(req, path, codec).await
}
/// Applies every staged row through the ordinary validated insert machinery in
/// one PostgreSQL transaction. Any failure rolls back every imported side effect.
pub async fn commit_table_data_import(
&mut self,
request: impl tonic::IntoRequest<super::CommitTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::CommitTableDataImportResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::unknown(
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/komp_ac.tables_data.TablesData/CommitTableDataImport",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(
GrpcMethod::new(
"komp_ac.tables_data.TablesData",
"CommitTableDataImport",
),
);
self.inner.unary(req, path, codec).await
}
/// Discards a staged import. Already committed imports cannot be aborted.
pub async fn abort_table_data_import(
&mut self,
request: impl tonic::IntoRequest<super::AbortTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::AbortTableDataImportResponse>,
tonic::Status,
> {
self.inner
.ready()
.await
.map_err(|e| {
tonic::Status::unknown(
format!("Service was not ready: {}", e.into()),
)
})?;
let codec = tonic_prost::ProstCodec::default();
let path = http::uri::PathAndQuery::from_static(
"/komp_ac.tables_data.TablesData/AbortTableDataImport",
);
let mut req = request.into_request();
req.extensions_mut()
.insert(
GrpcMethod::new(
"komp_ac.tables_data.TablesData",
"AbortTableDataImport",
),
);
self.inner.unary(req, path, codec).await
}
/// Update existing row data with strict type binding and script validation.
///
/// Behavior:
@@ -992,7 +1162,7 @@ pub mod tables_data_server {
tonic::Response<super::PostTableDataResponse>,
tonic::Status,
>;
/// Insert multiple rows by applying PostTableData behavior to each row.
/// Insert multiple rows atomically by applying PostTableData behavior to each row.
///
/// Behavior:
///
@@ -1001,7 +1171,7 @@ pub mod tables_data_server {
/// * Each row is inserted through the same validation, script execution,
/// typed binding, database insert, and indexing path as PostTableData
/// * Stops at the first failing row and returns that row's gRPC error code
/// with row index context; rows inserted before the failure remain inserted
/// * Commits only when every row succeeds; a failure rolls the entire request back
async fn post_table_data_bulk(
&self,
request: tonic::Request<super::PostTableDataBulkRequest>,
@@ -1009,6 +1179,40 @@ pub mod tables_data_server {
tonic::Response<super::PostTableDataBulkResponse>,
tonic::Status,
>;
/// Starts a durable, profile-scoped import staging session. Staging never changes
/// profile data; CommitTableDataImport applies every staged chunk atomically.
async fn begin_table_data_import(
&self,
request: tonic::Request<super::BeginTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::BeginTableDataImportResponse>,
tonic::Status,
>;
/// Adds one table chunk to an import session. Rows retain chunk and table order.
async fn stage_table_data_import(
&self,
request: tonic::Request<super::StageTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::StageTableDataImportResponse>,
tonic::Status,
>;
/// Applies every staged row through the ordinary validated insert machinery in
/// one PostgreSQL transaction. Any failure rolls back every imported side effect.
async fn commit_table_data_import(
&self,
request: tonic::Request<super::CommitTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::CommitTableDataImportResponse>,
tonic::Status,
>;
/// Discards a staged import. Already committed imports cannot be aborted.
async fn abort_table_data_import(
&self,
request: tonic::Request<super::AbortTableDataImportRequest>,
) -> std::result::Result<
tonic::Response<super::AbortTableDataImportResponse>,
tonic::Status,
>;
/// Update existing row data with strict type binding and script validation.
///
/// Behavior:
@@ -1353,6 +1557,190 @@ pub mod tables_data_server {
};
Box::pin(fut)
}
"/komp_ac.tables_data.TablesData/BeginTableDataImport" => {
#[allow(non_camel_case_types)]
struct BeginTableDataImportSvc<T: TablesData>(pub Arc<T>);
impl<
T: TablesData,
> tonic::server::UnaryService<super::BeginTableDataImportRequest>
for BeginTableDataImportSvc<T> {
type Response = super::BeginTableDataImportResponse;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::BeginTableDataImportRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as TablesData>::begin_table_data_import(&inner, request)
.await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = BeginTableDataImportSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/komp_ac.tables_data.TablesData/StageTableDataImport" => {
#[allow(non_camel_case_types)]
struct StageTableDataImportSvc<T: TablesData>(pub Arc<T>);
impl<
T: TablesData,
> tonic::server::UnaryService<super::StageTableDataImportRequest>
for StageTableDataImportSvc<T> {
type Response = super::StageTableDataImportResponse;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::StageTableDataImportRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as TablesData>::stage_table_data_import(&inner, request)
.await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = StageTableDataImportSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/komp_ac.tables_data.TablesData/CommitTableDataImport" => {
#[allow(non_camel_case_types)]
struct CommitTableDataImportSvc<T: TablesData>(pub Arc<T>);
impl<
T: TablesData,
> tonic::server::UnaryService<super::CommitTableDataImportRequest>
for CommitTableDataImportSvc<T> {
type Response = super::CommitTableDataImportResponse;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::CommitTableDataImportRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as TablesData>::commit_table_data_import(&inner, request)
.await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = CommitTableDataImportSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/komp_ac.tables_data.TablesData/AbortTableDataImport" => {
#[allow(non_camel_case_types)]
struct AbortTableDataImportSvc<T: TablesData>(pub Arc<T>);
impl<
T: TablesData,
> tonic::server::UnaryService<super::AbortTableDataImportRequest>
for AbortTableDataImportSvc<T> {
type Response = super::AbortTableDataImportResponse;
type Future = BoxFuture<
tonic::Response<Self::Response>,
tonic::Status,
>;
fn call(
&mut self,
request: tonic::Request<super::AbortTableDataImportRequest>,
) -> Self::Future {
let inner = Arc::clone(&self.0);
let fut = async move {
<T as TablesData>::abort_table_data_import(&inner, request)
.await
};
Box::pin(fut)
}
}
let accept_compression_encodings = self.accept_compression_encodings;
let send_compression_encodings = self.send_compression_encodings;
let max_decoding_message_size = self.max_decoding_message_size;
let max_encoding_message_size = self.max_encoding_message_size;
let inner = self.inner.clone();
let fut = async move {
let method = AbortTableDataImportSvc(inner);
let codec = tonic_prost::ProstCodec::default();
let mut grpc = tonic::server::Grpc::new(codec)
.apply_compression_config(
accept_compression_encodings,
send_compression_encodings,
)
.apply_max_message_size_config(
max_decoding_message_size,
max_encoding_message_size,
);
let res = grpc.unary(method, req).await;
Ok(res)
};
Box::pin(fut)
}
"/komp_ac.tables_data.TablesData/PutTableData" => {
#[allow(non_camel_case_types)]
struct PutTableDataSvc<T: TablesData>(pub Arc<T>);