filipriec/komp_ac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

robust decimal solution to push tables data to the backend

Browse Source
This commit is contained in:
filipriec
2025-06-22 22:08:22 +02:00
parent 1b8f19f1ce
commit 1b1e7b7205
7 changed files with 1902 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
server/src/tables_data/handlers/post_table_data.rs
View File

@@ -8,6 +8,8 @@ use common::proto::multieko2::tables_data::{PostTableDataRequest, PostTableDataR
use std::collections::HashMap;
use std::sync::Arc;
use prost_types::value::Kind;
use rust_decimal::Decimal;
use std::str::FromStr;
use crate::steel::server::execution::{self, Value};
use crate::steel::server::functions::SteelContext;
@@ -24,7 +26,6 @@ pub async fn post_table_data(
let profile_name = request.profile_name;
let table_name = request.table_name;
// Lookup profile
let schema = sqlx::query!(
"SELECT id FROM schemas WHERE name = $1",
profile_name
@@ -35,7 +36,6 @@ pub async fn post_table_data(
let schema_id = schema.ok_or_else(|| Status::not_found("Profile not found"))?.id;
// Lookup table_definition
let table_def = sqlx::query!(
r#"SELECT id, columns FROM table_definitions
WHERE schema_id = $1 AND table_name = $2"#,
@@ -48,7 +48,6 @@ pub async fn post_table_data(
let table_def = table_def.ok_or_else(|| Status::not_found("Table not found"))?;
// Parse columns from JSON
let columns_json: Vec<String> = serde_json::from_value(table_def.columns.clone())
.map_err(|e| Status::internal(format!("Column parsing error: {}", e)))?;
@@ -63,7 +62,6 @@ pub async fn post_table_data(
columns.push((name, sql_type));
}
// Get all foreign key columns for this table
let fk_columns = sqlx::query!(
r#"SELECT ltd.table_name
FROM table_definition_links tdl
@@ -75,17 +73,13 @@ pub async fn post_table_data(
.await
.map_err(|e| Status::internal(format!("Foreign key lookup error: {}", e)))?;
// Build system columns with foreign keys
let mut system_columns = vec!["deleted".to_string()];
for fk in fk_columns {
let base_name = fk.table_name.split_once('_').map_or(fk.table_name.as_str(), |(_, rest)| rest);
system_columns.push(format!("{}_id", base_name));
system_columns.push(format!("{}_id", fk.table_name));
}
// Convert to HashSet for faster lookups
let system_columns_set: std::collections::HashSet<_> = system_columns.iter().map(|s| s.as_str()).collect();
// Validate all data columns
let user_columns: Vec<&String> = columns.iter().map(|(name, _)| name).collect();
for key in request.data.keys() {
if !system_columns_set.contains(key.as_str()) &&
@@ -94,17 +88,18 @@ pub async fn post_table_data(
}
}
// ========================================================================
// FIX #1: SCRIPT VALIDATION LOOP
// This loop now correctly handles JSON `null` (which becomes `None`).
// ========================================================================
let mut string_data_for_scripts = HashMap::new();
for (key, proto_value) in &request.data {
let str_val = match &proto_value.kind {
Some(Kind::StringValue(s)) => s.clone(),
Some(Kind::StringValue(s)) => {
let trimmed = s.trim();
if trimmed.is_empty() {
continue;
}
trimmed.to_string()
},
Some(Kind::NumberValue(n)) => n.to_string(),
Some(Kind::BoolValue(b)) => b.to_string(),
// This now correctly skips both protobuf `NULL` and JSON `null`.
Some(Kind::NullValue(_)) | None => continue,
Some(Kind::StructValue(_)) | Some(Kind::ListValue(_)) => {
return Err(Status::invalid_argument(format!("Unsupported type for script validation in column '{}'", key)));
@@ -113,7 +108,6 @@ pub async fn post_table_data(
string_data_for_scripts.insert(key.clone(), str_val);
}
// Validate Steel scripts
let scripts = sqlx::query!(
"SELECT target_column, script FROM table_scripts WHERE table_definitions_id = $1",
table_def.id
@@ -163,17 +157,11 @@ pub async fn post_table_data(
}
}
// Prepare SQL parameters
let mut params = PgArguments::default();
let mut columns_list = Vec::new();
let mut placeholders = Vec::new();
let mut param_idx = 1;
// ========================================================================
// FIX #2: DATABASE INSERTION LOOP
// This loop now correctly handles JSON `null` (which becomes `None`)
// without crashing and correctly inserts a SQL NULL.
// ========================================================================
for (col, proto_value) in request.data {
let sql_type = if system_columns_set.contains(col.as_str()) {
match col.as_str() {
@@ -188,67 +176,96 @@ pub async fn post_table_data(
.ok_or_else(|| Status::invalid_argument(format!("Column not found: {}", col)))?
};
// Check for `None` (from JSON null) or `Some(NullValue)` first.
let kind = match &proto_value.kind {
None | Some(Kind::NullValue(_)) => {
// It's a null value. Add the correct SQL NULL type and continue.
match sql_type {
"BOOLEAN" => params.add(None::<bool>),
"TEXT" | "VARCHAR(15)" | "VARCHAR(255)" => params.add(None::<String>),
"TEXT" => params.add(None::<String>),
"TIMESTAMPTZ" => params.add(None::<DateTime<Utc>>),
"BIGINT" => params.add(None::<i64>),
s if s.starts_with("NUMERIC") => params.add(None::<Decimal>),
_ => return Err(Status::invalid_argument(format!("Unsupported type for null value: {}", sql_type))),
}.map_err(|e| Status::internal(format!("Failed to add null parameter for {}: {}", col, e)))?;
columns_list.push(format!("\"{}\"", col));
placeholders.push(format!("${}", param_idx));
param_idx += 1;
continue; // Skip to the next column in the loop
continue;
}
// If it's not null, just pass the inner `Kind` through.
Some(k) => k,
};
// From here, we know `kind` is not a null type.
match sql_type {
"TEXT" | "VARCHAR(15)" | "VARCHAR(255)" => {
if let Kind::StringValue(value) = kind {
if let Some(max_len) = sql_type.strip_prefix("VARCHAR(").and_then(|s| s.strip_suffix(')')).and_then(|s| s.parse::<usize>().ok()) {
if value.len() > max_len {
return Err(Status::internal(format!("Value too long for {}", col)));
}
if sql_type == "TEXT" {
if let Kind::StringValue(value) = kind {
let trimmed_value = value.trim();
if trimmed_value.is_empty() {
params.add(None::<String>).map_err(|e| Status::internal(format!("Failed to add null parameter for {}: {}", col, e)))?;
} else {
if col == "telefon" && trimmed_value.len() > 15 {
return Err(Status::internal(format!("Value too long for {}", col)));
}
params.add(value).map_err(|e| Status::invalid_argument(format!("Failed to add text parameter for {}: {}", col, e)))?;
} else {
return Err(Status::invalid_argument(format!("Expected string for column '{}'", col)));
params.add(trimmed_value).map_err(|e| Status::invalid_argument(format!("Failed to add text parameter for {}: {}", col, e)))?;
}
},
"BOOLEAN" => {
if let Kind::BoolValue(val) = kind {
params.add(val).map_err(|e| Status::invalid_argument(format!("Failed to add boolean parameter for {}: {}", col, e)))?;
} else {
return Err(Status::invalid_argument(format!("Expected boolean for column '{}'", col)));
} else {
return Err(Status::invalid_argument(format!("Expected string for column '{}'", col)));
}
} else if sql_type == "BOOLEAN" {
if let Kind::BoolValue(val) = kind {
params.add(val).map_err(|e| Status::invalid_argument(format!("Failed to add boolean parameter for {}: {}", col, e)))?;
} else {
return Err(Status::invalid_argument(format!("Expected boolean for column '{}'", col)));
}
} else if sql_type == "TIMESTAMPTZ" {
if let Kind::StringValue(value) = kind {
let dt = DateTime::parse_from_rfc3339(value).map_err(|_| Status::invalid_argument(format!("Invalid timestamp for {}", col)))?;
params.add(dt.with_timezone(&Utc)).map_err(|e| Status::invalid_argument(format!("Failed to add timestamp parameter for {}: {}", col, e)))?;
} else {
return Err(Status::invalid_argument(format!("Expected ISO 8601 string for column '{}'", col)));
}
} else if sql_type == "BIGINT" {
if let Kind::NumberValue(val) = kind {
if val.fract() != 0.0 {
return Err(Status::invalid_argument(format!("Expected integer for column '{}', but got a float", col)));
}
},
"TIMESTAMPTZ" => {
if let Kind::StringValue(value) = kind {
let dt = DateTime::parse_from_rfc3339(value).map_err(|_| Status::invalid_argument(format!("Invalid timestamp for {}", col)))?;
params.add(dt.with_timezone(&Utc)).map_err(|e| Status::invalid_argument(format!("Failed to add timestamp parameter for {}: {}", col, e)))?;
} else {
return Err(Status::invalid_argument(format!("Expected ISO 8601 string for column '{}'", col)));
}
},
"BIGINT" => {
if let Kind::NumberValue(val) = kind {
if val.fract() != 0.0 {
return Err(Status::invalid_argument(format!("Expected integer for column '{}', but got a float", col)));
params.add(*val as i64).map_err(|e| Status::invalid_argument(format!("Failed to add integer parameter for {}: {}", col, e)))?;
} else {
return Err(Status::invalid_argument(format!("Expected number for column '{}'", col)));
}
} else if sql_type.starts_with("NUMERIC") {
// MODIFIED: This block is now stricter.
let decimal_val = match kind {
Kind::StringValue(s) => {
let trimmed = s.trim();
if trimmed.is_empty() {
None // Treat empty string as NULL
} else {
// This is the only valid path: parse from a string.
Some(Decimal::from_str(trimmed).map_err(|_| {
Status::invalid_argument(format!(
"Invalid decimal string format for column '{}': {}",
col, s
))
})?)
}
params.add(*val as i64).map_err(|e| Status::invalid_argument(format!("Failed to add integer parameter for {}: {}", col, e)))?;
} else {
return Err(Status::invalid_argument(format!("Expected number for column '{}'", col)));
}
},
_ => return Err(Status::invalid_argument(format!("Unsupported type {}", sql_type))),
// CATCH-ALL: Reject NumberValue, BoolValue, etc. for NUMERIC fields.
_ => {
return Err(Status::invalid_argument(format!(
"Expected a string representation for decimal column '{}', but received a different type.",
col
)));
}
};
params.add(decimal_val).map_err(|e| {
Status::invalid_argument(format!(
"Failed to add decimal parameter for {}: {}",
col, e
))
})?;
} else {
return Err(Status::invalid_argument(format!("Unsupported type {}", sql_type)));
}
columns_list.push(format!("\"{}\"", col));
@@ -260,7 +277,6 @@ pub async fn post_table_data(
return Err(Status::invalid_argument("No valid columns to insert"));
}
// Qualify table name with schema
let qualified_table = crate::shared::schema_qualifier::qualify_table_name_for_data(
db_pool,
&profile_name,
@@ -283,6 +299,12 @@ pub async fn post_table_data(
Ok(id) => id,
Err(e) => {
if let Some(db_err) = e.as_database_error() {
if db_err.code() == Some(std::borrow::Cow::Borrowed("22P02")) ||
db_err.code() == Some(std::borrow::Cow::Borrowed("22003")) {
return Err(Status::invalid_argument(format!(
"Numeric field overflow or invalid format. Check precision and scale. Details: {}", db_err.message()
)));
}
if db_err.code() == Some(std::borrow::Cow::Borrowed("42P01")) {
return Err(Status::internal(format!(
"Table '{}' is defined but does not physically exist in the database as {}",