From b01eeddfe02ad0cd747c444b50b205fde85a1319 Mon Sep 17 00:00:00 2001 From: filipriec Date: Fri, 7 Mar 2025 10:45:35 +0100 Subject: [PATCH] steel scripts --- Cargo.lock | 146 ------------------ client/config.toml | 2 +- common/build.rs | 1 + common/proto/table_script.proto | 6 +- common/src/lib.rs | 3 + common/src/proto/descriptor.bin | Bin 17738 -> 18439 bytes common/src/proto/multieko2.table_script.rs | 19 +++ server/Cargo.toml | 2 - .../20250306132638_create_table_scripts.sql | 8 +- server/src/table_script/handlers.rs | 4 +- .../handlers/post_table_script.rs | 144 ++++++++--------- 11 files changed, 102 insertions(+), 233 deletions(-) create mode 100644 common/src/proto/multieko2.table_script.rs diff --git a/Cargo.lock b/Cargo.lock index dc3ff96..cc485b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,20 +17,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "const-random", - "getrandom 0.2.15", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.3" @@ -195,26 +181,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bincode" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad1fa75f77bbd06f187540aa1d70ca50b80b27ce85e7f41c0ce7ff42b34ed3b" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cef5dd4a4457dd11529e743d18ba4fabbd5f20b6895f4c865cb257337dcf9f" -dependencies = [ - "virtue", -] - [[package]] name = "bitflags" version = "2.8.0" @@ -352,26 +318,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.15", - "once_cell", - "tiny-keccak", -] - [[package]] name = "core-foundation" version = "0.9.4" @@ -452,12 +398,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crunchy" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" - [[package]] name = "crypto-common" version = "0.1.6" @@ -1204,15 +1144,6 @@ dependencies = [ "syn", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "itertools" version = "0.13.0" @@ -1473,9 +1404,6 @@ name = "once_cell" version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" -dependencies = [ - "portable-atomic", -] [[package]] name = "openssl" @@ -1646,12 +1574,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" -[[package]] -name = "portable-atomic" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" - [[package]] name = "powerfmt" version = "0.2.0" @@ -1862,34 +1784,6 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" -[[package]] -name = "rhai" -version = "1.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce4d759a4729a655ddfdbb3ff6e77fb9eadd902dae12319455557796e435d2a6" -dependencies = [ - "ahash", - "bitflags", - "instant", - "num-traits", - "once_cell", - "rhai_codegen", - "smallvec", - "smartstring", - "thin-vec", -] - -[[package]] -name = "rhai_codegen" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5a11a05ee1ce44058fa3d5961d05194fdbe3ad6b40f904af764d81b86450e6b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "rsa" version = "0.9.7" @@ -2081,13 +1975,11 @@ dependencies = [ name = "server" version = "0.1.0" dependencies = [ - "bincode", "chrono", "common", "dotenvy", "lazy_static", "prost", - "rhai", "rstest", "serde", "serde_json", @@ -2185,17 +2077,6 @@ dependencies = [ "serde", ] -[[package]] -name = "smartstring" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" -dependencies = [ - "autocfg", - "static_assertions", - "version_check", -] - [[package]] name = "socket2" version = "0.5.8" @@ -2520,12 +2401,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "thin-vec" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38c90d48152c236a3ab59271da4f4ae63d678c5d7ad6b7714d7cb9760be5e4b" - [[package]] name = "thiserror" version = "2.0.11" @@ -2579,15 +2454,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -2903,12 +2769,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" -[[package]] -name = "unty" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129" - [[package]] name = "url" version = "2.5.4" @@ -2944,12 +2804,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - [[package]] name = "want" version = "0.3.1" diff --git a/client/config.toml b/client/config.toml index a10e1e7..be88752 100644 --- a/client/config.toml +++ b/client/config.toml @@ -54,5 +54,5 @@ save_and_quit = ["wq"] revert = ["r"] [colors] -theme = "dark" +theme = "light" # Options: "light", "dark", "high_contrast" diff --git a/common/build.rs b/common/build.rs index 5bcf5f9..e2d4acb 100644 --- a/common/build.rs +++ b/common/build.rs @@ -12,6 +12,7 @@ fn main() -> Result<(), Box> { "proto/table_structure.proto", "proto/table_definition.proto", "proto/tables_data.proto", + "proto/table_script.proto", ], &["proto"], )?; diff --git a/common/proto/table_script.proto b/common/proto/table_script.proto index 021b3fc..b7eeb8f 100644 --- a/common/proto/table_script.proto +++ b/common/proto/table_script.proto @@ -4,11 +4,11 @@ package multieko2.table_script; message PostTableScriptRequest { int64 table_definition_id = 1; string target_column = 2; - string rhai_script = 3; + string script = 3; + string description = 4; } message TableScriptResponse { int64 id = 1; - string created_at = 2; - string warnings = 3; + string warnings = 2; } diff --git a/common/src/lib.rs b/common/src/lib.rs index 1605e69..f353400 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -19,6 +19,9 @@ pub mod proto { pub mod tables_data { include!("proto/multieko2.tables_data.rs"); } + pub mod table_script { + include!("proto/multieko2.table_script.rs"); + } pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!("proto/descriptor.bin"); } diff --git a/common/src/proto/descriptor.bin b/common/src/proto/descriptor.bin index 042b396feb710f95fb06504bf44016c3bf0df301..f697648aaccb38aa5b38fb977888af53a734e184 100644 GIT binary patch delta 695 zcmYk4zfQw25QpvflBVY%s0$%b{*e>x!`>5)%>-;te2(z^dZYLofUaE>nFU)4@9SC(b=3_^wmBt>8v+; zOm+9Z7)+k?#G}$`C(`!t)SjsrgL1G$Sn85$qU@(7p+*TQaf6pD028VMtpET3 diff --git a/common/src/proto/multieko2.table_script.rs b/common/src/proto/multieko2.table_script.rs new file mode 100644 index 0000000..76d52bb --- /dev/null +++ b/common/src/proto/multieko2.table_script.rs @@ -0,0 +1,19 @@ +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PostTableScriptRequest { + #[prost(int64, tag = "1")] + pub table_definition_id: i64, + #[prost(string, tag = "2")] + pub target_column: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub script: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub description: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TableScriptResponse { + #[prost(int64, tag = "1")] + pub id: i64, + #[prost(string, tag = "2")] + pub warnings: ::prost::alloc::string::String, +} diff --git a/server/Cargo.toml b/server/Cargo.toml index 7dc5c21..6fc65c2 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -18,8 +18,6 @@ tonic = "0.12.3" tonic-reflection = "0.12.3" tracing = "0.1.41" time = { version = "0.3.37", features = ["local-offset"] } -rhai = "1.21.0" -bincode = "2.0.0" [lib] name = "server" diff --git a/server/migrations/20250306132638_create_table_scripts.sql b/server/migrations/20250306132638_create_table_scripts.sql index 822b5d4..662ea0b 100644 --- a/server/migrations/20250306132638_create_table_scripts.sql +++ b/server/migrations/20250306132638_create_table_scripts.sql @@ -3,9 +3,11 @@ CREATE TABLE table_scripts ( id BIGSERIAL PRIMARY KEY, table_definitions_id BIGINT NOT NULL REFERENCES table_definitions(id), target_column TEXT NOT NULL, - rhai_script TEXT NOT NULL, - compiled_script BYTEA NOT NULL, - created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + script TEXT NOT NULL, + source_tables TEXT[] NOT NULL, -- Added to track which tables are used in calculation + source_columns TEXT[] NOT NULL, -- Added to track which columns are used in calculation + description TEXT, -- Optional description of what the script does + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, UNIQUE(table_definitions_id, target_column) ); diff --git a/server/src/table_script/handlers.rs b/server/src/table_script/handlers.rs index da0a988..4d31919 100644 --- a/server/src/table_script/handlers.rs +++ b/server/src/table_script/handlers.rs @@ -1,4 +1,4 @@ // src/table_script/handlers.rs -pub mod post_table_script; +// pub mod post_table_script; -pub use post_table_script::post_table_script; +// pub use post_table_script::post_table_script; diff --git a/server/src/table_script/handlers/post_table_script.rs b/server/src/table_script/handlers/post_table_script.rs index ec19631..8598ee9 100644 --- a/server/src/table_script/handlers/post_table_script.rs +++ b/server/src/table_script/handlers/post_table_script.rs @@ -1,10 +1,9 @@ // src/table_script/handlers/post_table_script.rs use tonic::Status; use sqlx::{PgPool, Error as SqlxError}; -use rhai::Engine; -use time::OffsetDateTime; -use bincode::serialize; use common::proto::multieko2::table_script::{PostTableScriptRequest, TableScriptResponse}; +use regex::Regex; +use std::collections::HashSet; pub async fn post_table_script( db_pool: &PgPool, @@ -12,7 +11,7 @@ pub async fn post_table_script( ) -> Result { // Fetch table definition let table_def = sqlx::query!( - r#"SELECT table_name, columns, linked_table_id + r#"SELECT table_name, columns, linked_table_id FROM table_definitions WHERE id = $1"#, request.table_definition_id ) @@ -21,48 +20,22 @@ pub async fn post_table_script( .map_err(|e| Status::internal(format!("Database error: {}", e)))? .ok_or_else(|| Status::not_found("Table definition not found"))?; - // Validate target column exists - let columns: Vec = serde_json::from_value(table_def.columns.clone()) - .map_err(|e| Status::invalid_argument(format!("Invalid columns format: {}", e)))?; - - let column_names: Vec<&str> = columns.iter() - .filter_map(|col| col.split_whitespace().next()) - .map(|name| name.trim_matches('"')) - .collect(); - - if !column_names.contains(&request.target_column.as_str()) { - return Err(Status::invalid_argument("Target column not found in table definition")); - } - - // Check for existing data - let row_count: i64 = sqlx::query_scalar(&format!( - "SELECT COUNT(*) FROM {}", - sanitize_identifier(&table_def.table_name) - )) - .fetch_one(db_pool) - .await - .map_err(|e| Status::internal(format!("Data check failed: {}", e)))?; - - if row_count > 0 { - return Err(Status::failed_precondition( - "Cannot add scripts to tables with existing data" - )); - } - - // Compile Rhai script - let (ast, warnings) = compile_rhai_script(&request.rhai_script) - .map_err(|e| Status::invalid_argument(format!("Script error: {}", e)))?; + // Extract source tables and columns from the script + let (source_tables, source_columns) = extract_sources(&request.script, &db_pool).await + .map_err(|e| Status::invalid_argument(format!("Source extraction error: {}", e)))?; // Store script in database let script_record = sqlx::query!( - r#"INSERT INTO table_scripts - (table_definitions_id, target_column, rhai_script, compiled_script) - VALUES ($1, $2, $3, $4) - RETURNING id, created_at"#, + r#"INSERT INTO table_scripts + (table_definitions_id, target_column, script, source_tables, source_columns, description) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id"#, request.table_definition_id, request.target_column, - request.rhai_script, - bincode::serialize(&ast).map_err(|e| Status::internal(format!("Serialization failed: {}", e)))?, + request.script, + &source_tables as &[String], + &source_columns as &[String], + request.description.unwrap_or_default() ) .fetch_one(db_pool) .await @@ -75,45 +48,64 @@ pub async fn post_table_script( Ok(TableScriptResponse { id: script_record.id, - created_at: Some(script_record.created_at.into()), - warnings, - }) - Ok(TableScriptResponse { - id: script_record.id, - created_at: script_record.created_at.to_string(), - warnings, + warnings: String::new(), }) } -fn sanitize_identifier(s: &str) -> String { - s.replace(|c: char| !c.is_ascii_alphanumeric() && c != '_', "") - .trim() - .to_lowercase() -} - -fn compile_rhai_script(script: &str) -> Result<(AST, String), String> { - let mut engine = Engine::new(); - let mut scope = Scope::new(); +// Extract source tables and columns from the script +async fn extract_sources(script: &str, pool: &PgPool) -> Result<(Vec, Vec), String> { + let mut tables = HashSet::new(); + let mut columns = HashSet::new(); - // Add custom API bindings - engine.register_fn("get_column", |_: &str, _: &str| Ok(())); - engine.register_fn("set_result", |_: &mut Scope, _: rhai::Dynamic| Ok(())); - - let ast = engine.compile(script).map_err(|e| e.to_string())?; + // Regular expression to find table/column references + // In Steel, we're looking for patterns like: + // - (get-table-column "table_name" "column_name") + // - column8 (for the current table) - // Validate script structure - if !script.contains("set_result") { - return Err("Script must call set_result()".into()); + // Pattern for explicit table.column references + let table_column_pattern = Regex::new(r#"\(get-table-column\s+"([^"]+)"\s+"([^"]+)"\)"#) + .map_err(|e| format!("Regex error: {}", e))?; + + // Add current table's columns (simple column references) + let column_pattern = Regex::new(r#"column\d+"#) + .map_err(|e| format!("Regex error: {}", e))?; + + // Find all table.column references + for cap in table_column_pattern.captures_iter(script) { + if let (Some(table_match), Some(column_match)) = (cap.get(1), cap.get(2)) { + let table_name = table_match.as_str().to_string(); + let column_name = column_match.as_str().to_string(); + + // Verify table exists in the database + let table_exists = sqlx::query!( + "SELECT EXISTS(SELECT 1 FROM pg_tables WHERE tablename = $1) as exists", + table_name + ) + .fetch_one(pool) + .await + .map_err(|e| format!("Database error: {}", e))? + .exists + .unwrap_or(false); + + if !table_exists { + return Err(format!("Referenced table '{}' does not exist", table_name)); + } + + tables.insert(table_name); + columns.insert(format!("{}.{}", table_name, column_name)); + } } - - let warnings = engine.gen_ast_clear_comments(&ast) - .iter() - .filter_map(|(_, err)| match err { - rhai::ParseError(warn, _) => Some(warn.to_string()), - _ => None - }) - .collect::>() - .join(", "); - - Ok((ast, warnings)) + + // Find simple column references (assumed to be from the current table) + for column_match in column_pattern.find_iter(script) { + columns.insert(column_match.as_str().to_string()); + } + + // Ensure at least one table is identified + if tables.is_empty() { + // If no explicit tables found, assume it's using the current table + tables.insert("current".to_string()); + } + + Ok((tables.into_iter().collect(), columns.into_iter().collect())) }