bulk import2

This commit is contained in:
Filipriec
2026-08-21 22:37:30 +02:00
parent 0756e99959
commit adaa2c4ac6
8 changed files with 421 additions and 207 deletions

View File

@@ -15,7 +15,7 @@ use crate::{
table_structure::GetTableStructureRequest,
tables_data::{
AbortTableDataImportRequest, BeginTableDataImportRequest, CommitTableDataImportRequest,
PostTableDataBulkRow, StageTableDataImportRequest,
StageTableDataImportRequest, TableDataImportRow,
},
},
services::{authenticated_request, reject_cross_site},
@@ -40,7 +40,7 @@ use super::{
/// How many prepared rows the preview shows.
const PREVIEW_ROWS: usize = 20;
/// How many rows one bulk insert carries. Also the granularity of the progress
/// How many rows one staged import chunk carries. Also the granularity of the progress
/// the page shows, since a chunk is what the running total counts.
const CHUNK_ROWS: usize = 1_000;
@@ -221,7 +221,7 @@ pub(crate) async fn import_csv(
.iter()
.enumerate()
.map(|(index, row)| {
bulk_row(locale, &destination, &prepared.columns, row).map_err(|error| {
import_row(locale, &destination, &prepared.columns, row).map_err(|error| {
tr!(
locale,
"import-err-csv-row",
@@ -350,7 +350,7 @@ struct Running {
id: String,
profile_name: String,
table_name: String,
rows: Vec<PostTableDataBulkRow>,
rows: Vec<TableDataImportRow>,
}
/// The insert, chunk by chunk, reporting after each one.
@@ -610,14 +610,14 @@ fn source_options(locale: Locale, source: &Source) -> Vec<SourceOption> {
.collect()
}
/// One prepared row as the bulk insert takes it: values under their destination
/// One prepared row as the staged import takes it: values under their destination
/// names, converted to the column's type.
fn bulk_row(
fn import_row(
locale: Locale,
destination: &Destination,
columns: &[String],
row: &[String],
) -> Result<PostTableDataBulkRow, String> {
) -> Result<TableDataImportRow, String> {
let mut data = HashMap::new();
for (index, column) in columns.iter().enumerate() {
let data_type = destination.types.get(column).ok_or_else(|| {
@@ -634,7 +634,7 @@ fn bulk_row(
)?;
data.insert(column.clone(), value);
}
Ok(PostTableDataBulkRow { data })
Ok(TableDataImportRow { data })
}
async fn render_step(
@@ -679,8 +679,8 @@ fn import_failure(
return (0, Outcome::SessionLost);
}
let status = failure.status_code();
match bulk_failure(error) {
Some((failed_row_index, _inserted_in_chunk)) => (
match import_row_failure(error) {
Some(failed_row_index) => (
0,
Outcome::RowFailed {
status,
@@ -700,22 +700,14 @@ fn import_failure(
}
}
fn bulk_failure(error: &tonic::Status) -> Option<(usize, usize)> {
let failed_row_index = error
fn import_row_failure(error: &tonic::Status) -> Option<usize> {
error
.metadata()
.get(crate::grpc_error::BULK_FAILED_ROW_INDEX_METADATA_KEY)?
.get(crate::grpc_error::IMPORT_FAILED_ROW_INDEX_METADATA_KEY)?
.to_str()
.ok()?
.parse()
.ok()?;
let inserted_rows = error
.metadata()
.get(crate::grpc_error::BULK_INSERTED_ROWS_METADATA_KEY)?
.to_str()
.ok()?
.parse()
.ok()?;
Some((failed_row_index, inserted_rows))
.ok()
}
fn load_error(headers: &HeaderMap, error: LoadError) -> Response {