unimportant change

Cargo.lock

@@ -449,6 +449,7 @@ dependencies = [
  "dotenvy",
  "lazy_static",
  "prost",
+ "prost-types",
  "ratatui",
  "serde",
  "serde_json",
@@ -487,6 +488,7 @@ name = "common"
 version = "0.3.13"
 dependencies = [
  "prost",
+ "prost-types",
  "serde",
  "tantivy",
  "tonic",
@@ -2843,6 +2845,7 @@ dependencies = [
  "jsonwebtoken",
  "lazy_static",
  "prost",
+ "prost-types",
  "regex",
  "rstest",
  "rust-stemmers",

Cargo.toml

@@ -24,6 +24,7 @@ tokio = { version = "1.44.2", features = ["full"] }
 tonic = "0.13.0"
 prost = "0.13.5"
 async-trait = "0.1.88"
+prost-types = "0.13.0"
 
 # Data Handling & Serialization
 serde = { version = "1.0.219", features = ["derive"] }

client/Cargo.toml

@@ -9,6 +9,7 @@ anyhow = "1.0.98"
 async-trait = "0.1.88"
 common = { path = "../common" }
 
+prost-types = { workspace = true }
 crossterm = "0.28.1"
 dirs = "6.0.0"
 dotenvy = "0.15.7"

client/src/components/common/autocomplete.rs

@@ -1,30 +1,18 @@
 // src/components/common/autocomplete.rs
 
-use common::proto::multieko2::search::search_response::Hit;
 use crate::config::colors::themes::Theme;
+use crate::state::pages::form::FormState;
+use common::proto::multieko2::search::search_response::Hit;
 use ratatui::{
     layout::Rect,
     style::{Color, Modifier, Style},
     widgets::{Block, List, ListItem, ListState},
     Frame,
 };
-use std::collections::HashMap;
 use unicode_width::UnicodeWidthStr;
 
-/// Converts a serde_json::Value into a displayable String.
-/// Handles String, Number, and Bool variants. Returns an empty string for Null and others.
-fn json_value_to_string(value: &serde_json::Value) -> String {
-    match value {
-        serde_json::Value::String(s) => s.clone(),
-        serde_json::Value::Number(n) => n.to_string(),
-        serde_json::Value::Bool(b) => b.to_string(),
-        // Return an empty string for Null, Array, or Object so we can filter them out.
-        _ => String::new(),
-    }
-}
-
 /// Renders an opaque dropdown list for simple string-based suggestions.
-/// This function remains unchanged.
+/// THIS IS THE RESTORED FUNCTION.
 pub fn render_autocomplete_dropdown(
     f: &mut Frame,
     input_rect: Rect,
@@ -84,22 +72,22 @@ pub fn render_autocomplete_dropdown(
         .collect();
 
     let list = List::new(items);
-    let mut profile_list_state = ListState::default();
-    profile_list_state.select(selected_index);
+    let mut list_state = ListState::default();
+    list_state.select(selected_index);
 
-    f.render_stateful_widget(list, dropdown_area, &mut profile_list_state);
+    f.render_stateful_widget(list, dropdown_area, &mut list_state);
 }
 
-// --- MODIFIED FUNCTION FOR RICH SUGGESTIONS ---
 /// Renders an opaque dropdown list for rich `Hit`-based suggestions.
-/// Displays the value of the first meaningful column, followed by the Hit ID.
-pub fn render_rich_autocomplete_dropdown(
+/// RENAMED from render_rich_autocomplete_dropdown
+pub fn render_hit_autocomplete_dropdown(
     f: &mut Frame,
     input_rect: Rect,
     frame_area: Rect,
     theme: &Theme,
     suggestions: &[Hit],
     selected_index: Option<usize>,
+    form_state: &FormState,
 ) {
     if suggestions.is_empty() {
         return;
@@ -107,50 +95,9 @@ pub fn render_rich_autocomplete_dropdown(
 
     let display_names: Vec<String> = suggestions
         .iter()
-        .map(|hit| {
-            // Use serde_json::Value to handle mixed types (string, null, etc.)
-            if let Ok(content_map) =
-                serde_json::from_str::<HashMap<String, serde_json::Value>>(
-                    &hit.content_json,
-                )
-            {
-                // Define keys to ignore for a cleaner display
-                const IGNORED_KEYS: &[&str] = &["id", "deleted", "created_at"];
-
-                // Get keys, filter out ignored ones, and sort for consistency
-                let mut keys: Vec<_> = content_map
-                    .keys()
-                    .filter(|k| !IGNORED_KEYS.contains(&k.as_str()))
-                    .cloned()
-                    .collect();
-                keys.sort();
-
-                // Get only the first non-empty value from the sorted keys
-                let values: Vec<_> = keys
-                    .iter()
-                    .map(|key| {
-                        content_map
-                            .get(key)
-                            .map(json_value_to_string)
-                            .unwrap_or_default()
-                    })
-                    .filter(|s| !s.is_empty()) // Filter out null/empty values
-                    .take(1) // Changed from take(2) to take(1)
-                    .collect();
-
-                let display_part = values.first().cloned().unwrap_or_default(); // Get the first value
-                if display_part.is_empty() {
-                    format!("ID: {}", hit.id)
-                } else {
-                    format!("{} | ID: {}", display_part, hit.id) // ID at the end
-                }
-            } else {
-                format!("ID: {} (parse error)", hit.id)
-            }
-        })
+        .map(|hit| form_state.get_display_name_for_hit(hit))
         .collect();
 
-    // --- Calculate Dropdown Size & Position ---
     let max_suggestion_width =
         display_names.iter().map(|s| s.width()).max().unwrap_or(0) as u16;
     let horizontal_padding: u16 = 2;
@@ -164,7 +111,6 @@ pub fn render_rich_autocomplete_dropdown(
         height: dropdown_height,
     };
 
-    // --- Clamping Logic ---
     if dropdown_area.bottom() > frame_area.height {
         dropdown_area.y = input_rect.y.saturating_sub(dropdown_height);
     }
@@ -174,7 +120,6 @@ pub fn render_rich_autocomplete_dropdown(
     dropdown_area.x = dropdown_area.x.max(0);
     dropdown_area.y = dropdown_area.y.max(0);
 
-    // --- Rendering Logic ---
     let background_block =
         Block::default().style(Style::default().bg(Color::DarkGray));
     f.render_widget(background_block, dropdown_area);

client/src/components/form/form.rs

@@ -78,25 +78,25 @@ pub fn render_form(
 
     // --- NEW: RENDER AUTOCOMPLETE ---
     if form_state.autocomplete_active {
-        // Use the Rect of the active field that render_canvas found for us.
         if let Some(active_rect) = active_field_rect {
             let selected_index = form_state.get_selected_suggestion_index();
 
-            // THE DECIDER LOGIC:
-            // 1. Check for rich suggestions first.
             if let Some(rich_suggestions) = form_state.get_rich_suggestions() {
                 if !rich_suggestions.is_empty() {
-                    autocomplete::render_rich_autocomplete_dropdown(
+                    // CHANGE THIS to call the renamed function
+                    autocomplete::render_hit_autocomplete_dropdown(
                         f,
                         active_rect,
-                        f.area(), // Use f.area() for clamping, not f.size()
+                        f.area(),
                         theme,
                         rich_suggestions,
                         selected_index,
+                        form_state,
                     );
                 }
             }
-            // 2. Fallback to simple suggestions if rich ones aren't available.
+            // The fallback to simple suggestions is now correctly handled
+            // because the original render_autocomplete_dropdown exists again.
             else if let Some(simple_suggestions) = form_state.get_suggestions() {
                 if !simple_suggestions.is_empty() {
                     autocomplete::render_autocomplete_dropdown(
@@ -112,3 +112,4 @@ pub fn render_form(
         }
     }
 }
+

client/src/components/handlers/canvas.rs

@@ -1,16 +1,16 @@
 // src/components/handlers/canvas.rs
+
 use ratatui::{
-    widgets::{Paragraph, Block, Borders},
-    layout::{Layout, Constraint, Direction, Rect},
-    style::{Style, Modifier},
+    layout::{Alignment, Constraint, Direction, Layout, Rect},
+    style::{Modifier, Style},
     text::{Line, Span},
+    widgets::{Block, Borders, Paragraph},
     Frame,
-    prelude::Alignment,
 };
 use crate::config::colors::themes::Theme;
+use crate::state::app::highlight::HighlightState;
 use crate::state::pages::canvas_state::CanvasState;
-use crate::state::app::highlight::HighlightState; // Ensure correct import path
-use std::cmp::{min, max};
+use std::cmp::{max, min};
 
 pub fn render_canvas(
     f: &mut Frame,
@@ -21,9 +21,8 @@ pub fn render_canvas(
     inputs: &[&String],
     theme: &Theme,
     is_edit_mode: bool,
-    highlight_state: &HighlightState, // Using the enum state
+    highlight_state: &HighlightState,
 ) -> Option<Rect> {
-    // ... (setup code remains the same) ...
     let columns = Layout::default()
         .direction(Direction::Horizontal)
         .constraints([Constraint::Percentage(30), Constraint::Percentage(70)])
@@ -58,46 +57,47 @@ pub fn render_canvas(
 
     let mut active_field_input_rect = None;
 
-    // Render labels
     for (i, field) in fields.iter().enumerate() {
         let label = Paragraph::new(Line::from(Span::styled(
             format!("{}:", field),
-            Style::default().fg(theme.fg)),
-        ));
-        f.render_widget(label, Rect {
-            x: columns[0].x,
-            y: input_block.y + 1 + i as u16,
-            width: columns[0].width,
-            height: 1,
-        });
+            Style::default().fg(theme.fg),
+        )));
+        f.render_widget(
+            label,
+            Rect {
+                x: columns[0].x,
+                y: input_block.y + 1 + i as u16,
+                width: columns[0].width,
+                height: 1,
+            },
+        );
     }
 
-
-    // Render inputs and cursor
-    for (i, input) in inputs.iter().enumerate() {
+    for (i, _input) in inputs.iter().enumerate() {
         let is_active = i == *current_field_idx;
         let current_cursor_pos = form_state.current_cursor_pos();
-        let text = input.as_str();
-        let text_len = text.chars().count();
 
+        // Use the trait method to get display value
+        let text = form_state.get_display_value_for_field(i);
+        let text_len = text.chars().count();
         let line: Line;
 
-        // --- Use match on the highlight_state enum ---
         match highlight_state {
             HighlightState::Off => {
-                // Not in highlight mode, render normally
                 line = Line::from(Span::styled(
                     text,
-                    if is_active { Style::default().fg(theme.highlight) } else { Style::default().fg(theme.fg) }
+                    if is_active {
+                        Style::default().fg(theme.highlight)
+                    } else {
+                        Style::default().fg(theme.fg)
+                    },
                 ));
             }
             HighlightState::Characterwise { anchor } => {
-                // --- Character-wise Highlight Logic ---
                 let (anchor_field, anchor_char) = *anchor;
                 let start_field = min(anchor_field, *current_field_idx);
                 let end_field = max(anchor_field, *current_field_idx);
 
-                // Use start_char and end_char consistently
                 let (start_char, end_char) = if anchor_field == *current_field_idx {
                     (min(anchor_char, current_cursor_pos), max(anchor_char, current_cursor_pos))
                 } else if anchor_field < *current_field_idx {
@@ -111,24 +111,20 @@ pub fn render_canvas(
                 let normal_style_outside = Style::default().fg(theme.fg);
 
                 if i >= start_field && i <= end_field {
-                    // This line is within the character-wise highlight range
-                    if start_field == end_field { // Case 1: Single Line Highlight
-                        // Use start_char and end_char here
+                    if start_field == end_field {
                         let clamped_start = start_char.min(text_len);
-                        let clamped_end = end_char.min(text_len); // Use text_len for slicing logic
+                        let clamped_end = end_char.min(text_len);
 
                         let before: String = text.chars().take(clamped_start).collect();
                         let highlighted: String = text.chars().skip(clamped_start).take(clamped_end.saturating_sub(clamped_start) + 1).collect();
-                        // Define 'after' here
                         let after: String = text.chars().skip(clamped_end + 1).collect();
 
                         line = Line::from(vec![
                             Span::styled(before, normal_style_in_highlight),
                             Span::styled(highlighted, highlight_style),
-                            Span::styled(after, normal_style_in_highlight), // Use defined 'after'
+                            Span::styled(after, normal_style_in_highlight),
                         ]);
-                    } else if i == start_field { // Case 2: Multi-Line Highlight - Start Line
-                        // Use start_char here
+                    } else if i == start_field {
                         let safe_start = start_char.min(text_len);
                         let before: String = text.chars().take(safe_start).collect();
                         let highlighted: String = text.chars().skip(safe_start).collect();
@@ -136,8 +132,7 @@ pub fn render_canvas(
                             Span::styled(before, normal_style_in_highlight),
                             Span::styled(highlighted, highlight_style),
                         ]);
-                    } else if i == end_field { // Case 3: Multi-Line Highlight - End Line (Corrected index)
-                        // Use end_char here
+                    } else if i == end_field {
                         let safe_end_inclusive = if text_len > 0 { end_char.min(text_len - 1) } else { 0 };
                         let highlighted: String = text.chars().take(safe_end_inclusive + 1).collect();
                         let after: String = text.chars().skip(safe_end_inclusive + 1).collect();
@@ -145,19 +140,17 @@ pub fn render_canvas(
                             Span::styled(highlighted, highlight_style),
                             Span::styled(after, normal_style_in_highlight),
                         ]);
-                    } else { // Case 4: Multi-Line Highlight - Middle Line (Corrected index)
-                        line = Line::from(Span::styled(text, highlight_style)); // Highlight whole line
+                    } else {
+                        line = Line::from(Span::styled(text, highlight_style));
                     }
-                } else { // Case 5: Line Outside Character-wise Highlight Range
+                } else {
                     line = Line::from(Span::styled(
                         text,
-                        // Use normal styling (active or inactive)
                         if is_active { normal_style_in_highlight } else { normal_style_outside }
                     ));
                 }
             }
             HighlightState::Linewise { anchor_line } => {
-                // --- Linewise Highlight Logic ---
                 let start_field = min(*anchor_line, *current_field_idx);
                 let end_field = max(*anchor_line, *current_field_idx);
                 let highlight_style = Style::default().fg(theme.highlight).bg(theme.highlight_bg).add_modifier(Modifier::BOLD);
@@ -165,25 +158,31 @@ pub fn render_canvas(
                 let normal_style_outside = Style::default().fg(theme.fg);
 
                 if i >= start_field && i <= end_field {
-                    // Highlight the entire line
                     line = Line::from(Span::styled(text, highlight_style));
                 } else {
-                    // Line outside linewise highlight range
                     line = Line::from(Span::styled(
                         text,
-                        // Use normal styling (active or inactive)
                         if is_active { normal_style_in_highlight } else { normal_style_outside }
                     ));
                 }
             }
-        } // End match highlight_state
+        }
 
         let input_display = Paragraph::new(line).alignment(Alignment::Left);
         f.render_widget(input_display, input_rows[i]);
 
         if is_active {
             active_field_input_rect = Some(input_rows[i]);
-            let cursor_x = input_rows[i].x + form_state.current_cursor_pos() as u16;
+
+            // --- CORRECTED CURSOR POSITIONING LOGIC ---
+            // Use the new generic trait method to check for an override.
+            let cursor_x = if form_state.has_display_override(i) {
+                // If an override exists, place the cursor at the end.
+                input_rows[i].x + text.chars().count() as u16
+            } else {
+                // Otherwise, use the real cursor position.
+                input_rows[i].x + form_state.current_cursor_pos() as u16
+            };
             let cursor_y = input_rows[i].y;
             f.set_cursor_position((cursor_x, cursor_y));
         }
@@ -191,4 +190,3 @@ pub fn render_canvas(
 
     active_field_input_rect
 }
-

client/src/functions/modes/edit/auth_e.rs

@@ -4,6 +4,7 @@ use crate::services::grpc_client::GrpcClient;
 use crate::state::pages::canvas_state::CanvasState;
 use crate::state::pages::form::FormState;
 use crate::state::pages::auth::RegisterState;
+use crate::state::app::state::AppState;
 use crate::tui::functions::common::form::{revert, save};
 use crossterm::event::{KeyCode, KeyEvent};
 use std::any::Any;
@@ -13,6 +14,7 @@ pub async fn execute_common_action<S: CanvasState + Any>(
     action: &str,
     state: &mut S,
     grpc_client: &mut GrpcClient,
+    app_state: &AppState,
     current_position: &mut u64,
     total_count: u64,
 ) -> Result<String> {
@@ -27,6 +29,7 @@ pub async fn execute_common_action<S: CanvasState + Any>(
                 match action {
                     "save" => {
                         let outcome = save(
+                            app_state,
                             form_state,
                             grpc_client,
                         )

client/src/functions/modes/edit/form_e.rs

@@ -3,6 +3,7 @@
 use crate::services::grpc_client::GrpcClient;
 use crate::state::pages::canvas_state::CanvasState;
 use crate::state::pages::form::FormState;
+use crate::state::app::state::AppState;
 use crate::tui::functions::common::form::{revert, save};
 use crate::tui::functions::common::form::SaveOutcome;
 use crate::modes::handlers::event::EventOutcome;
@@ -14,6 +15,7 @@ pub async fn execute_common_action<S: CanvasState + Any>(
     action: &str,
     state: &mut S,
     grpc_client: &mut GrpcClient,
+    app_state: &AppState,
 ) -> Result<EventOutcome> {
     match action {
         "save" | "revert" => {
@@ -26,6 +28,7 @@ pub async fn execute_common_action<S: CanvasState + Any>(
                 match action {
                     "save" => {
                         let save_result = save(
+                            app_state,
                             form_state,
                             grpc_client,
                         ).await;

client/src/modes/canvas/common_mode.rs

@@ -32,6 +32,7 @@ pub async fn handle_core_action(
                 Ok(EventOutcome::Ok(message))
             } else {
                 let save_outcome = form_save(
+                    app_state,
                     form_state,
                     grpc_client,
                 ).await.context("Register save action failed")?;
@@ -52,6 +53,7 @@ pub async fn handle_core_action(
                 login_save(auth_state, login_state, auth_client, app_state).await.context("Login save n quit action failed")?
             } else {
                 let save_outcome = form_save(
+                    app_state,
                     form_state,
                     grpc_client,
                 ).await?;

client/src/modes/canvas/edit.rs

@@ -132,13 +132,24 @@ pub async fn handle_edit_event(
                             .get(selected_idx)
                             .cloned()
                         {
+                            // --- THIS IS THE CORE LOGIC CHANGE ---
+
+                            // 1. Get the friendly display name for the UI
+                            let display_name =
+                                form_state.get_display_name_for_hit(&selection);
+
+                            // 2. Store the REAL ID in the form's values
                             let current_input =
                                 form_state.get_current_input_mut();
                             *current_input = selection.id.to_string();
-                            let new_cursor_pos = current_input.len();
-                            form_state.set_current_cursor_pos(new_cursor_pos);
-                            // FIX: Access ideal_cursor_column through event_handler
-                            event_handler.ideal_cursor_column = new_cursor_pos;
+
+                            // 3. Set the persistent display override in the map
+                            form_state.link_display_map.insert(
+                                form_state.current_field,
+                                display_name,
+                            );
+
+                            // 4. Finalize state
                             form_state.deactivate_autocomplete();
                             form_state.set_has_unsaved_changes(true);
                             return Ok(EditEventOutcome::Message(

client/src/modes/common/command_mode.rs

@@ -15,7 +15,7 @@ use anyhow::Result;
 pub async fn handle_command_event(
     key: KeyEvent,
     config: &Config,
-    app_state: &AppState,
+    app_state: &mut AppState,
     login_state: &LoginState,
     register_state: &RegisterState,
     form_state: &mut FormState,
@@ -74,7 +74,7 @@ pub async fn handle_command_event(
 async fn process_command(
     config: &Config,
     form_state: &mut FormState,
-    app_state: &AppState,
+    app_state: &mut AppState,
     login_state: &LoginState,
     register_state: &RegisterState,
     command_input: &mut String,
@@ -117,6 +117,7 @@ async fn process_command(
         },
         "save" => {
             let outcome = save(
+                app_state,
                 form_state,
                 grpc_client,
             ).await?;

client/src/services/grpc_client.rs

@@ -1,7 +1,6 @@
 // src/services/grpc_client.rs
 
-use tonic::transport::Channel;
-use common::proto::multieko2::common::{CountResponse, Empty};
+use common::proto::multieko2::common::Empty;
 use common::proto::multieko2::table_structure::table_structure_service_client::TableStructureServiceClient;
 use common::proto::multieko2::table_structure::{GetTableStructureRequest, TableStructureResponse};
 use common::proto::multieko2::table_definition::{
@@ -23,8 +22,10 @@ use common::proto::multieko2::tables_data::{
 use common::proto::multieko2::search::{
     searcher_client::SearcherClient, SearchRequest, SearchResponse,
 };
-use anyhow::{Context, Result}; // Added Context
-use std::collections::HashMap; // NEW
+use anyhow::{Context, Result};
+use std::collections::HashMap;
+use tonic::transport::Channel;
+use prost_types::Value;
 
 #[derive(Clone)]
 pub struct GrpcClient {
@@ -48,7 +49,6 @@ impl GrpcClient {
             TableDefinitionClient::new(channel.clone());
         let table_script_client = TableScriptClient::new(channel.clone());
         let tables_data_client = TablesDataClient::new(channel.clone());
-        // NEW: Instantiate the search client
         let search_client = SearcherClient::new(channel.clone());
 
         Ok(Self {
@@ -56,7 +56,7 @@ impl GrpcClient {
             table_definition_client,
             table_script_client,
             tables_data_client,
-            search_client, // NEW
+            search_client,
         })
     }
 
@@ -135,7 +135,7 @@ impl GrpcClient {
         Ok(response.into_inner().count as u64)
     }
 
-    pub async fn get_table_data_by_position(
+pub async fn get_table_data_by_position(
         &mut self,
         profile_name: String,
         table_name: String,
@@ -159,12 +159,14 @@ impl GrpcClient {
         &mut self,
         profile_name: String,
         table_name: String,
-        data: HashMap<String, String>,
+        // CHANGE THIS: Accept the pre-converted data
+        data: HashMap<String, Value>,
     ) -> Result<PostTableDataResponse> {
+        // The conversion logic is now gone from here.
         let grpc_request = PostTableDataRequest {
             profile_name,
             table_name,
-            data,
+            data, // This is now the correct type
         };
         let request = tonic::Request::new(grpc_request);
         let response = self
@@ -180,13 +182,15 @@ impl GrpcClient {
         profile_name: String,
         table_name: String,
         id: i64,
-        data: HashMap<String, String>,
+        // CHANGE THIS: Accept the pre-converted data
+        data: HashMap<String, Value>,
     ) -> Result<PutTableDataResponse> {
+        // The conversion logic is now gone from here.
         let grpc_request = PutTableDataRequest {
             profile_name,
             table_name,
             id,
-            data,
+            data, // This is now the correct type
         };
         let request = tonic::Request::new(grpc_request);
         let response = self

client/src/services/ui_service.rs

@@ -1,16 +1,100 @@
 // src/services/ui_service.rs
 
 use crate::services::grpc_client::GrpcClient;
-use crate::state::pages::form::FormState;
-use crate::tui::functions::common::form::SaveOutcome;
-use crate::state::pages::add_logic::AddLogicState;
 use crate::state::app::state::AppState;
+use crate::state::pages::add_logic::AddLogicState;
+use crate::state::pages::form::{FieldDefinition, FormState};
+use crate::tui::functions::common::form::SaveOutcome;
 use crate::utils::columns::filter_user_columns;
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
+use std::sync::Arc;
 
 pub struct UiService;
 
 impl UiService {
+    pub async fn load_table_view(
+        grpc_client: &mut GrpcClient,
+        app_state: &mut AppState,
+        profile_name: &str,
+        table_name: &str,
+    ) -> Result<FormState> {
+        // 1. & 2. Fetch and Cache Schema - UNCHANGED
+        let table_structure = grpc_client
+            .get_table_structure(profile_name.to_string(), table_name.to_string())
+            .await
+            .context(format!(
+                "Failed to get table structure for {}.{}",
+                profile_name, table_name
+            ))?;
+        let cache_key = format!("{}.{}", profile_name, table_name);
+        app_state
+            .schema_cache
+            .insert(cache_key, Arc::new(table_structure.clone()));
+        tracing::info!("Schema for '{}.{}' cached.", profile_name, table_name);
+
+        // --- START: FINAL, SIMPLIFIED, CORRECT LOGIC ---
+
+        // 3a. Create definitions for REGULAR fields first.
+        let mut fields: Vec<FieldDefinition> = table_structure
+            .columns
+            .iter()
+            .filter(|col| {
+                !col.is_primary_key
+                    && col.name != "deleted"
+                    && col.name != "created_at"
+                    && !col.name.ends_with("_id") // Filter out ALL potential links
+            })
+            .map(|col| FieldDefinition {
+                display_name: col.name.clone(),
+                data_key: col.name.clone(),
+                is_link: false,
+                link_target_table: None,
+            })
+            .collect();
+
+        // 3b. Now, find and APPEND definitions for LINK fields based on the `_id` convention.
+        let link_fields: Vec<FieldDefinition> = table_structure
+            .columns
+            .iter()
+            .filter(|col| col.name.ends_with("_id")) // Find all foreign key columns
+            .map(|col| {
+                // The table we link to is derived from the column name.
+                // e.g., "test_diacritics_id" -> "test_diacritics"
+                let target_table_base = col
+                    .name
+                    .strip_suffix("_id")
+                    .unwrap_or(&col.name);
+
+                // Find the full table name from the profile tree for display.
+                // e.g., "test_diacritics" -> "2025_test_diacritics"
+                let full_target_table_name = app_state
+                    .profile_tree
+                    .profiles
+                    .iter()
+                    .find(|p| p.name == profile_name)
+                    .and_then(|p| p.tables.iter().find(|t| t.name.ends_with(target_table_base)))
+                    .map_or(target_table_base.to_string(), |t| t.name.clone());
+
+                FieldDefinition {
+                    display_name: full_target_table_name.clone(),
+                    data_key: col.name.clone(), // The actual FK column name
+                    is_link: true,
+                    link_target_table: Some(full_target_table_name),
+                }
+            })
+            .collect();
+
+        fields.extend(link_fields); // Append the link fields to the end
+
+        // --- END: FINAL, SIMPLIFIED, CORRECT LOGIC ---
+
+        Ok(FormState::new(
+            profile_name.to_string(),
+            table_name.to_string(),
+            fields,
+        ))
+    }
+
     pub async fn initialize_add_logic_table_data(
         grpc_client: &mut GrpcClient,
         add_logic_state: &mut AddLogicState,
@@ -92,6 +176,7 @@ impl UiService {
         }
     }
 
+    // REFACTOR THIS FUNCTION
     pub async fn initialize_app_state_and_form(
         grpc_client: &mut GrpcClient,
         app_state: &mut AppState,
@@ -102,7 +187,6 @@ impl UiService {
             .context("Failed to get profile tree")?;
         app_state.profile_tree = profile_tree;
 
-        // Determine initial table to load (e.g., first table of first profile, or a default)
         let initial_profile_name = app_state
             .profile_tree
             .profiles
@@ -115,33 +199,26 @@ impl UiService {
             .profiles
             .first()
             .and_then(|p| p.tables.first().map(|t| t.name.clone()))
-            .unwrap_or_else(|| "2025_company_data1".to_string()); // Fallback if no tables
+            .unwrap_or_else(|| "2025_company_data1".to_string());
 
         app_state.set_current_view_table(
             initial_profile_name.clone(),
             initial_table_name.clone(),
         );
 
-        let table_structure = grpc_client
-            .get_table_structure(
-                initial_profile_name.clone(),
-                initial_table_name.clone(),
-            )
-            .await
-            .context(format!(
-                "Failed to get initial table structure for {}.{}",
-                initial_profile_name, initial_table_name
-            ))?;
+        // NOW, just call our new central function. This avoids code duplication.
+        let form_state = Self::load_table_view(
+            grpc_client,
+            app_state,
+            &initial_profile_name,
+            &initial_table_name,
+        )
+        .await?;
 
-        let column_names: Vec<String> = table_structure
-            .columns
-            .iter()
-            .map(|col| col.name.clone())
-            .collect();
+        // The field names for the UI are derived from the new form_state
+        let field_names = form_state.fields.iter().map(|f| f.display_name.clone()).collect();
 
-        let filtered_columns = filter_user_columns(column_names);
-
-        Ok((initial_profile_name, initial_table_name, filtered_columns))
+        Ok((initial_profile_name, initial_table_name, field_names))
     }
 
     pub async fn fetch_and_set_table_count(

client/src/state/app/state.rs

@@ -1,15 +1,19 @@
 // src/state/app/state.rs
 
-use std::env;
-use common::proto::multieko2::table_definition::ProfileTreeResponse;
-use crate::modes::handlers::mode_manager::AppMode;
-use crate::ui::handlers::context::DialogPurpose;
-use crate::state::app::search::SearchState; // ADDED
 use anyhow::Result;
+use common::proto::multieko2::table_definition::ProfileTreeResponse;
+// NEW: Import the types we need for the cache
+use common::proto::multieko2::table_structure::TableStructureResponse;
+use crate::modes::handlers::mode_manager::AppMode;
+use crate::state::app::search::SearchState;
+use crate::ui::handlers::context::DialogPurpose;
+use std::collections::HashMap;
+use std::env;
+use std::sync::Arc;
 #[cfg(feature = "ui-debug")]
 use std::time::Instant;
 
-// --- YOUR EXISTING DIALOGSTATE IS UNTOUCHED ---
+// --- DialogState and UiState are unchanged ---
 pub struct DialogState {
     pub dialog_show: bool,
     pub dialog_title: String,
@@ -30,7 +34,7 @@ pub struct UiState {
     pub show_form: bool,
     pub show_login: bool,
     pub show_register: bool,
-    pub show_search_palette: bool, // ADDED
+    pub show_search_palette: bool,
     pub focus_outside_canvas: bool,
     pub dialog: DialogState,
 }
@@ -52,10 +56,12 @@ pub struct AppState {
     pub current_view_profile_name: Option<String>,
     pub current_view_table_name: Option<String>,
 
+    // NEW: The "Rulebook" cache. We use Arc for efficient sharing.
+    pub schema_cache: HashMap<String, Arc<TableStructureResponse>>,
+
     pub focused_button_index: usize,
     pub pending_table_structure_fetch: Option<(String, String)>,
 
-    // ADDED: State for the search palette
     pub search_state: Option<SearchState>,
 
     // UI preferences
@@ -67,9 +73,7 @@ pub struct AppState {
 
 impl AppState {
     pub fn new() -> Result<Self> {
-        let current_dir = env::current_dir()?
-            .to_string_lossy()
-            .to_string();
+        let current_dir = env::current_dir()?.to_string_lossy().to_string();
         Ok(AppState {
             current_dir,
             profile_tree: ProfileTreeResponse::default(),
@@ -77,9 +81,10 @@ impl AppState {
             current_view_profile_name: None,
             current_view_table_name: None,
             current_mode: AppMode::General,
+            schema_cache: HashMap::new(), // NEW: Initialize the cache
             focused_button_index: 0,
             pending_table_structure_fetch: None,
-            search_state: None, // ADDED
+            search_state: None,
             ui: UiState::default(),
 
             #[cfg(feature = "ui-debug")]

client/src/state/pages/canvas_state.rs

@@ -3,6 +3,7 @@
 use common::proto::multieko2::search::search_response::Hit;
 
 pub trait CanvasState {
+    // --- Existing methods (unchanged) ---
     fn current_field(&self) -> usize;
     fn current_cursor_pos(&self) -> usize;
     fn has_unsaved_changes(&self) -> bool;
@@ -10,15 +11,22 @@ pub trait CanvasState {
     fn get_current_input(&self) -> &str;
     fn get_current_input_mut(&mut self) -> &mut String;
     fn fields(&self) -> Vec<&str>;
-
     fn set_current_field(&mut self, index: usize);
     fn set_current_cursor_pos(&mut self, pos: usize);
     fn set_has_unsaved_changes(&mut self, changed: bool);
-
-    // --- Autocomplete Support ---
     fn get_suggestions(&self) -> Option<&[String]>;
     fn get_selected_suggestion_index(&self) -> Option<usize>;
     fn get_rich_suggestions(&self) -> Option<&[Hit]> {
         None
     }
+
+    fn get_display_value_for_field(&self, index: usize) -> &str {
+        self.inputs()
+            .get(index)
+            .map(|s| s.as_str())
+            .unwrap_or("")
+    }
+    fn has_display_override(&self, _index: usize) -> bool {
+        false
+    }
 }

client/src/state/pages/form.rs

@@ -3,18 +3,26 @@
 use crate::config::colors::themes::Theme;
 use crate::state::app::highlight::HighlightState;
 use crate::state::pages::canvas_state::CanvasState;
-use common::proto::multieko2::search::search_response::Hit; // Import Hit
+use common::proto::multieko2::search::search_response::Hit;
 use ratatui::layout::Rect;
 use ratatui::Frame;
 use std::collections::HashMap;
 
-// A struct to bridge the display name (label) to the data key from the server.
+fn json_value_to_string(value: &serde_json::Value) -> String {
+    match value {
+        serde_json::Value::String(s) => s.clone(),
+        serde_json::Value::Number(n) => n.to_string(),
+        serde_json::Value::Bool(b) => b.to_string(),
+        _ => String::new(),
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct FieldDefinition {
     pub display_name: String,
     pub data_key: String,
     pub is_link: bool,
-    pub link_target_table: Option<String>,                       
+    pub link_target_table: Option<String>,
 }
 
 #[derive(Clone)]
@@ -29,12 +37,11 @@ pub struct FormState {
     pub current_field: usize,
     pub has_unsaved_changes: bool,
     pub current_cursor_pos: usize,
-
-    // --- MODIFIED AUTOCOMPLETE STATE ---
     pub autocomplete_active: bool,
-    pub autocomplete_suggestions: Vec<Hit>, // Changed to use the Hit struct
+    pub autocomplete_suggestions: Vec<Hit>,
     pub selected_suggestion_index: Option<usize>,
-    pub autocomplete_loading: bool, // To show a loading indicator
+    pub autocomplete_loading: bool,
+    pub link_display_map: HashMap<usize, String>,
 }
 
 impl FormState {
@@ -55,11 +62,48 @@ impl FormState {
             current_field: 0,
             has_unsaved_changes: false,
             current_cursor_pos: 0,
-            // --- INITIALIZE NEW STATE ---
             autocomplete_active: false,
             autocomplete_suggestions: Vec::new(),
             selected_suggestion_index: None,
-            autocomplete_loading: false, // Initialize loading state
+            autocomplete_loading: false,
+            link_display_map: HashMap::new(),
+        }
+    }
+
+    pub fn get_display_name_for_hit(&self, hit: &Hit) -> String {
+        if let Ok(content_map) =
+            serde_json::from_str::<HashMap<String, serde_json::Value>>(
+                &hit.content_json,
+            )
+        {
+            const IGNORED_KEYS: &[&str] = &["id", "deleted", "created_at"];
+            let mut keys: Vec<_> = content_map
+                .keys()
+                .filter(|k| !IGNORED_KEYS.contains(&k.as_str()))
+                .cloned()
+                .collect();
+            keys.sort();
+
+            let values: Vec<_> = keys
+                .iter()
+                .map(|key| {
+                    content_map
+                        .get(key)
+                        .map(json_value_to_string)
+                        .unwrap_or_default()
+                })
+                .filter(|s| !s.is_empty())
+                .take(1)
+                .collect();
+
+            let display_part = values.first().cloned().unwrap_or_default();
+            if display_part.is_empty() {
+                format!("ID: {}", hit.id)
+            } else {
+                format!("{} | ID: {}", display_part, hit.id)
+            }
+        } else {
+            format!("ID: {} (parse error)", hit.id)
         }
     }
 
@@ -78,7 +122,7 @@ impl FormState {
         crate::components::form::form::render_form(
             f,
             area,
-            self, // <--- This now correctly passes the concrete &FormState
+            self,
             &fields_str_slice,
             &self.current_field,
             &values_str_slice,
@@ -102,7 +146,8 @@ impl FormState {
         } else {
             self.current_position = 1;
         }
-        self.deactivate_autocomplete(); // Deactivate on reset
+        self.deactivate_autocomplete();
+        self.link_display_map.clear();
     }
 
     pub fn get_current_input(&self) -> &str {
@@ -113,6 +158,7 @@ impl FormState {
     }
 
     pub fn get_current_input_mut(&mut self) -> &mut String {
+        self.link_display_map.remove(&self.current_field);
         self.values
             .get_mut(self.current_field)
             .expect("Invalid current_field index")
@@ -159,11 +205,10 @@ impl FormState {
         self.has_unsaved_changes = false;
         self.current_field = 0;
         self.current_cursor_pos = 0;
-        self.deactivate_autocomplete(); // Deactivate on update
+        self.deactivate_autocomplete();
+        self.link_display_map.clear();
     }
 
-    // --- NEW HELPER METHOD ---
-    /// Deactivates autocomplete and clears its state.
     pub fn deactivate_autocomplete(&mut self) {
         self.autocomplete_active = false;
         self.autocomplete_suggestions.clear();
@@ -176,58 +221,42 @@ impl CanvasState for FormState {
     fn current_field(&self) -> usize {
         self.current_field
     }
-
     fn current_cursor_pos(&self) -> usize {
         self.current_cursor_pos
     }
-
     fn has_unsaved_changes(&self) -> bool {
         self.has_unsaved_changes
     }
-
     fn inputs(&self) -> Vec<&String> {
         self.values.iter().collect()
     }
-
     fn get_current_input(&self) -> &str {
         FormState::get_current_input(self)
     }
-
     fn get_current_input_mut(&mut self) -> &mut String {
         FormState::get_current_input_mut(self)
     }
-
     fn fields(&self) -> Vec<&str> {
         self.fields
             .iter()
             .map(|f| f.display_name.as_str())
             .collect()
     }
-
     fn set_current_field(&mut self, index: usize) {
         if index < self.fields.len() {
             self.current_field = index;
         }
-        // Deactivate autocomplete when changing fields
         self.deactivate_autocomplete();
     }
-
     fn set_current_cursor_pos(&mut self, pos: usize) {
         self.current_cursor_pos = pos;
     }
-
     fn set_has_unsaved_changes(&mut self, changed: bool) {
         self.has_unsaved_changes = changed;
     }
-
-    // --- MODIFIED: Implement autocomplete trait methods ---
-
-    /// Returns None because this state uses rich suggestions.
     fn get_suggestions(&self) -> Option<&[String]> {
         None
     }
-
-    /// Returns rich suggestions.
     fn get_rich_suggestions(&self) -> Option<&[Hit]> {
         if self.autocomplete_active {
             Some(&self.autocomplete_suggestions)
@@ -235,7 +264,6 @@ impl CanvasState for FormState {
             None
         }
     }
-
     fn get_selected_suggestion_index(&self) -> Option<usize> {
         if self.autocomplete_active {
             self.selected_suggestion_index
@@ -243,4 +271,19 @@ impl CanvasState for FormState {
             None
         }
     }
+
+    fn get_display_value_for_field(&self, index: usize) -> &str {
+        if let Some(display_text) = self.link_display_map.get(&index) {
+            return display_text.as_str();
+        }
+        self.inputs()
+            .get(index)
+            .map(|s| s.as_str())
+            .unwrap_or("")
+    }
+
+    // --- IMPLEMENT THE NEW TRAIT METHOD ---
+    fn has_display_override(&self, index: usize) -> bool {
+        self.link_display_map.contains_key(&index)
+    }
 }

client/src/tui/functions/common/form.rs

@@ -1,19 +1,22 @@
 // src/tui/functions/common/form.rs
 
 use crate::services::grpc_client::GrpcClient;
+use crate::state::app::state::AppState; // NEW: Import AppState
 use crate::state::pages::form::FormState;
-use anyhow::{Context, Result}; // Added Context
-use std::collections::HashMap; // NEW
+use crate::utils::data_converter; // NEW: Import our translator
+use anyhow::{anyhow, Context, Result};
+use std::collections::HashMap;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum SaveOutcome {
     NoChange,
     UpdatedExisting,
-    CreatedNew(i64), // Keep the ID
+    CreatedNew(i64),
 }
 
-// MODIFIED save function
+// MODIFIED save function signature and logic
 pub async fn save(
+    app_state: &AppState, // NEW: Pass in AppState
     form_state: &mut FormState,
     grpc_client: &mut GrpcClient,
 ) -> Result<SaveOutcome> {
@@ -21,42 +24,64 @@ pub async fn save(
         return Ok(SaveOutcome::NoChange);
     }
 
-    let data_map: HashMap<String, String> = form_state.fields.iter()
+    // --- NEW: VALIDATION & CONVERSION STEP ---
+    let cache_key =
+        format!("{}.{}", form_state.profile_name, form_state.table_name);
+    let schema = match app_state.schema_cache.get(&cache_key) {
+        Some(s) => s,
+        None => {
+            return Err(anyhow!(
+                "Schema for table '{}' not found in cache. Cannot save.",
+                form_state.table_name
+            ));
+        }
+    };
+
+    let data_map: HashMap<String, String> = form_state
+        .fields
+        .iter()
         .zip(form_state.values.iter())
         .map(|(field_def, value)| (field_def.data_key.clone(), value.clone()))
         .collect();
 
+    // Use our new translator. It returns a user-friendly error on failure.
+    let converted_data =
+        match data_converter::convert_and_validate_data(&data_map, schema) {
+            Ok(data) => data,
+            Err(user_error) => return Err(anyhow!(user_error)),
+        };
+    // --- END OF NEW STEP ---
+
     let outcome: SaveOutcome;
-
-    let is_new_entry = form_state.id == 0 || (form_state.total_count > 0 && form_state.current_position > form_state.total_count) || (form_state.total_count == 0 && form_state.current_position == 1) ;
-
+    let is_new_entry = form_state.id == 0
+        || (form_state.total_count > 0
+            && form_state.current_position > form_state.total_count)
+        || (form_state.total_count == 0 && form_state.current_position == 1);
 
     if is_new_entry {
         let response = grpc_client
             .post_table_data(
                 form_state.profile_name.clone(),
                 form_state.table_name.clone(),
-                data_map,
+                converted_data, // Use the validated & converted data
             )
             .await
             .context("Failed to post new table data")?;
 
         if response.success {
             form_state.id = response.inserted_id;
-            // After creating a new entry, total_count increases, and current_position becomes this new total_count
             form_state.total_count += 1;
             form_state.current_position = form_state.total_count;
             outcome = SaveOutcome::CreatedNew(response.inserted_id);
         } else {
-            return Err(anyhow::anyhow!(
+            return Err(anyhow!(
                 "Server failed to insert data: {}",
                 response.message
             ));
         }
     } else {
-        // This assumes form_state.id is valid for an existing record
         if form_state.id == 0 {
-            return Err(anyhow::anyhow!(
+            return Err(anyhow!(
                 "Cannot update record: ID is 0, but not classified as new entry."
             ));
         }
@@ -65,7 +90,7 @@ pub async fn save(
                 form_state.profile_name.clone(),
                 form_state.table_name.clone(),
                 form_state.id,
-                data_map,
+                converted_data, // Use the validated & converted data
             )
             .await
             .context("Failed to put (update) table data")?;
@@ -73,7 +98,7 @@ pub async fn save(
         if response.success {
             outcome = SaveOutcome::UpdatedExisting;
         } else {
-            return Err(anyhow::anyhow!(
+            return Err(anyhow!(
                 "Server failed to update data: {}",
                 response.message
             ));

client/src/ui/handlers/ui.rs

@@ -350,123 +350,91 @@ pub async fn run_ui() -> Result<()> {
             let current_view_profile = app_state.current_view_profile_name.clone();
             let current_view_table = app_state.current_view_table_name.clone();
 
+            // This condition correctly detects a table switch.
             if prev_view_profile_name != current_view_profile
                 || prev_view_table_name != current_view_table
             {
                 if let (Some(prof_name), Some(tbl_name)) =
                     (current_view_profile.as_ref(), current_view_table.as_ref())
                 {
+                    // --- START OF REFACTORED LOGIC ---
                     app_state.show_loading_dialog(
                         "Loading Table",
                         &format!("Fetching data for {}.{}...", prof_name, tbl_name),
                     );
                     needs_redraw = true;
 
-                    match grpc_client
-                        .get_table_structure(prof_name.clone(), tbl_name.clone())
-                        .await
+                    // 1. Call our new, central function. It handles fetching AND caching.
+                    match UiService::load_table_view(
+                        &mut grpc_client,
+                        &mut app_state,
+                        prof_name,
+                        tbl_name,
+                    )
+                    .await
                     {
-                        Ok(structure_response) => {
-                            // --- START OF MODIFIED LOGIC ---
-                            let all_columns: Vec<String> = structure_response
-                                .columns
-                                .iter()
-                                .map(|c| c.name.clone())
-                                .collect();
-
-                            let mut field_definitions: Vec<FieldDefinition> =
-                                filter_user_columns(all_columns)
-                                    .into_iter()
-                                    .filter(|col_name| !col_name.ends_with("_id"))
-                                    .map(|col_name| FieldDefinition {
-                                        display_name: col_name.clone(),
-                                        data_key: col_name,
-                                        is_link: false,
-                                        link_target_table: None, // Regular fields have no target
-                                    })
-                                    .collect();
-
-                            let linked_tables: Vec<String> = app_state
-                                .profile_tree
-                                .profiles
-                                .iter()
-                                .find(|p| p.name == *prof_name)
-                                .and_then(|profile| {
-                                    profile.tables.iter().find(|t| t.name == *tbl_name)
-                                })
-                                .map_or(vec![], |table| table.depends_on.clone());
-
-                            for linked_table_name in linked_tables {
-                                let base_name = linked_table_name
-                                    .split_once('_')
-                                    .map_or(linked_table_name.as_str(), |(_, rest)| rest);
-                                let data_key = format!("{}_id", base_name);
-                                let display_name = linked_table_name.clone(); // Clone for use below
-
-                                field_definitions.push(FieldDefinition {
-                                    display_name,
-                                    data_key,
-                                    is_link: true,
-                                    // --- POPULATE THE NEW FIELD ---
-                                    link_target_table: Some(linked_table_name),
-                                });
-                            }
-                            // --- END OF MODIFIED LOGIC ---
-
-                            form_state = FormState::new(
-                                prof_name.clone(),
-                                tbl_name.clone(),
-                                field_definitions, // This now contains the complete definitions
-                            );
-
+                        Ok(mut new_form_state) => {
+                            // 2. The function succeeded, we have a new FormState.
+                            //    Now, fetch its data.
                             if let Err(e) = UiService::fetch_and_set_table_count(
                                 &mut grpc_client,
-                                &mut form_state,
+                                &mut new_form_state,
                             )
                             .await
                             {
+                                // Handle count fetching error
                                 app_state.update_dialog_content(
                                     &format!("Error fetching count: {}", e),
                                     vec!["OK".to_string()],
-                                    DialogPurpose::LoginFailed,
+                                    DialogPurpose::LoginFailed, // Or a more appropriate purpose
                                 );
-                            } else if form_state.total_count > 0 {
+                            } else if new_form_state.total_count > 0 {
+                                // If there are records, load the first/last one
                                 if let Err(e) = UiService::load_table_data_by_position(
                                     &mut grpc_client,
-                                    &mut form_state,
+                                    &mut new_form_state,
                                 )
                                 .await
                                 {
+                                    // Handle data loading error
                                     app_state.update_dialog_content(
                                         &format!("Error loading data: {}", e),
                                         vec!["OK".to_string()],
-                                        DialogPurpose::LoginFailed,
+                                        DialogPurpose::LoginFailed, // Or a more appropriate purpose
                                     );
                                 } else {
+                                    // Success! Hide the loading dialog.
                                     app_state.hide_dialog();
                                 }
                             } else {
-                                form_state.reset_to_empty();
+                                // No records, so just reset to an empty form.
+                                new_form_state.reset_to_empty();
                                 app_state.hide_dialog();
                             }
 
+                            // 3. CRITICAL: Replace the old form_state with the new one.
+                            form_state = new_form_state;
+
+                            // 4. Update our tracking variables.
                             prev_view_profile_name = current_view_profile;
                             prev_view_table_name = current_view_table;
                             table_just_switched = true;
                         }
-
                         Err(e) => {
+                            // This handles errors from load_table_view (e.g., schema fetch failed)
                             app_state.update_dialog_content(
-                                &format!("Error fetching table structure: {}", e),
+                                &format!("Error loading table: {}", e),
                                 vec!["OK".to_string()],
-                                DialogPurpose::LoginFailed,
+                                DialogPurpose::LoginFailed, // Or a more appropriate purpose
                             );
+                            // Revert the view change in app_state to avoid a loop
                             app_state.current_view_profile_name =
                                 prev_view_profile_name.clone();
                             app_state.current_view_table_name =
                                 prev_view_table_name.clone();
                         }
                     }
+                    // --- END OF REFACTORED LOGIC ---
                 }
                 needs_redraw = true;
             }

client/src/utils/data_converter.rs

@@ -0,0 +1,50 @@
+// src/utils/data_converter.rs
+
+use common::proto::multieko2::table_structure::TableStructureResponse;
+use prost_types::{value::Kind, NullValue, Value};
+use std::collections::HashMap;
+
+pub fn convert_and_validate_data(
+    data: &HashMap<String, String>,
+    schema: &TableStructureResponse,
+) -> Result<HashMap<String, Value>, String> {
+    let type_map: HashMap<_, _> = schema
+        .columns
+        .iter()
+        .map(|col| (col.name.as_str(), col.data_type.as_str()))
+        .collect();
+
+    data.iter()
+        .map(|(key, str_value)| {
+            let expected_type = type_map.get(key.as_str()).unwrap_or(&"TEXT");
+
+            let kind = if str_value.is_empty() {
+                // TODO: Use the correct enum variant
+                Kind::NullValue(NullValue::NullValue.into())
+            } else {
+                // Attempt to parse the string based on the expected type
+                match *expected_type {
+                    "BOOL" => match str_value.to_lowercase().parse::<bool>() {
+                        Ok(v) => Kind::BoolValue(v),
+                        Err(_) => return Err(format!("Invalid boolean for '{}': must be 'true' or 'false'", key)),
+                    },
+                    "INT8" | "INT4" | "INT2" | "SERIAL" | "BIGSERIAL" => {
+                        match str_value.parse::<f64>() {
+                            Ok(v) => Kind::NumberValue(v),
+                            Err(_) => return Err(format!("Invalid number for '{}': must be a whole number", key)),
+                        }
+                    }
+                    "NUMERIC" | "FLOAT4" | "FLOAT8" => match str_value.parse::<f64>() {
+                        Ok(v) => Kind::NumberValue(v),
+                        Err(_) => return Err(format!("Invalid decimal for '{}': must be a number", key)),
+                    },
+                    "TIMESTAMPTZ" | "DATE" | "TIME" | "TEXT" | "VARCHAR" | "UUID" => {
+                        Kind::StringValue(str_value.clone())
+                    }
+                    _ => Kind::StringValue(str_value.clone()),
+                }
+            };
+            Ok((key.clone(), Value { kind: Some(kind) }))
+        })
+        .collect()
+}

client/src/utils/mod.rs

@@ -2,5 +2,8 @@
 
 pub mod columns;
 pub mod debug_logger;
+pub mod data_converter;
+
 pub use columns::*;
 pub use debug_logger::*;
+pub use data_converter::*;

common/Cargo.toml

@@ -5,6 +5,8 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
+prost-types = { workspace = true }
+
 tonic = "0.13.0"
 prost = "0.13.5"
 serde = { version = "1.0.219", features = ["derive"] }

common/proto/tables_data.proto

@@ -3,6 +3,7 @@ syntax = "proto3";
 package multieko2.tables_data;
 
 import "common.proto";
+import "google/protobuf/struct.proto";
 
 service TablesData {
   rpc PostTableData (PostTableDataRequest) returns (PostTableDataResponse);
@@ -16,7 +17,7 @@ service TablesData {
 message PostTableDataRequest {
   string profile_name = 1;
   string table_name = 2;
-  map<string, string> data = 3;
+  map<string, google.protobuf.Value> data = 3;
 }
 
 message PostTableDataResponse {
@@ -29,7 +30,7 @@ message PutTableDataRequest {
   string profile_name = 1;
   string table_name = 2;
   int64 id = 3;
-  map<string, string> data = 4;
+  map<string, google.protobuf.Value> data = 4;
 }
 
 message PutTableDataResponse {

common/src/proto/multieko2.tables_data.rs

@@ -5,10 +5,10 @@ pub struct PostTableDataRequest {
     pub profile_name: ::prost::alloc::string::String,
     #[prost(string, tag = "2")]
     pub table_name: ::prost::alloc::string::String,
-    #[prost(map = "string, string", tag = "3")]
+    #[prost(map = "string, message", tag = "3")]
     pub data: ::std::collections::HashMap<
         ::prost::alloc::string::String,
-        ::prost::alloc::string::String,
+        ::prost_types::Value,
     >,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -28,10 +28,10 @@ pub struct PutTableDataRequest {
     pub table_name: ::prost::alloc::string::String,
     #[prost(int64, tag = "3")]
     pub id: i64,
-    #[prost(map = "string, string", tag = "4")]
+    #[prost(map = "string, message", tag = "4")]
     pub data: ::std::collections::HashMap<
         ::prost::alloc::string::String,
-        ::prost::alloc::string::String,
+        ::prost_types::Value,
     >,
 }
 #[derive(Clone, PartialEq, ::prost::Message)]

server/Cargo.toml

@@ -10,6 +10,7 @@ search = { path = "../search" }
 
 anyhow = { workspace = true }
 tantivy = { workspace = true }
+prost-types = { workspace = true }
 chrono = { version = "0.4.40", features = ["serde"] }
 dotenvy = "0.15.7"
 prost = "0.13.5"

server/src/tables_data/handlers/post_table_data.rs

@@ -8,16 +8,15 @@ use common::proto::multieko2::tables_data::{PostTableDataRequest, PostTableDataR
 use std::collections::HashMap;
 use std::sync::Arc;
 use crate::shared::schema_qualifier::qualify_table_name_for_data;
+use prost_types::value::Kind; // NEW: Import the Kind enum
 
 use crate::steel::server::execution::{self, Value};
 use crate::steel::server::functions::SteelContext;
 
-// Add these imports
 use crate::indexer::{IndexCommand, IndexCommandData};
 use tokio::sync::mpsc;
 use tracing::error;
 
-// MODIFIED: Function signature now accepts the indexer sender
 pub async fn post_table_data(
     db_pool: &PgPool,
     request: PostTableDataRequest,
@@ -25,11 +24,6 @@ pub async fn post_table_data(
 ) -> Result<PostTableDataResponse, Status> {
     let profile_name = request.profile_name;
     let table_name = request.table_name;
-    let mut data = HashMap::new();
-
-    for (key, value) in request.data {
-        data.insert(key, value.trim().to_string());
-    }
 
     // Lookup profile
     let profile = sqlx::query!(
@@ -94,13 +88,32 @@ pub async fn post_table_data(
 
     // Validate all data columns
     let user_columns: Vec<&String> = columns.iter().map(|(name, _)| name).collect();
-    for key in data.keys() {
+    for key in request.data.keys() {
         if !system_columns_set.contains(key.as_str()) &&
            !user_columns.contains(&&key.to_string()) {
             return Err(Status::invalid_argument(format!("Invalid column: {}", key)));
         }
     }
 
+    // ========================================================================
+    // FIX #1: SCRIPT VALIDATION LOOP
+    // This loop now correctly handles JSON `null` (which becomes `None`).
+    // ========================================================================
+    let mut string_data_for_scripts = HashMap::new();
+    for (key, proto_value) in &request.data {
+        let str_val = match &proto_value.kind {
+            Some(Kind::StringValue(s)) => s.clone(),
+            Some(Kind::NumberValue(n)) => n.to_string(),
+            Some(Kind::BoolValue(b)) => b.to_string(),
+            // This now correctly skips both protobuf `NULL` and JSON `null`.
+            Some(Kind::NullValue(_)) | None => continue,
+            Some(Kind::StructValue(_)) | Some(Kind::ListValue(_)) => {
+                return Err(Status::invalid_argument(format!("Unsupported type for script validation in column '{}'", key)));
+            }
+        };
+        string_data_for_scripts.insert(key.clone(), str_val);
+    }
+
     // Validate Steel scripts
     let scripts = sqlx::query!(
         "SELECT target_column, script FROM table_scripts WHERE table_definitions_id = $1",
@@ -113,21 +126,18 @@ pub async fn post_table_data(
     for script_record in scripts {
         let target_column = script_record.target_column;
 
-        // Ensure target column exists in submitted data
-        let user_value = data.get(&target_column)
+        let user_value = string_data_for_scripts.get(&target_column)
             .ok_or_else(|| Status::invalid_argument(
                 format!("Script target column '{}' is required", target_column)
             ))?;
 
-        // Create execution context
         let context = SteelContext {
-            current_table: table_name.clone(), // Keep base name for scripts
+            current_table: table_name.clone(),
             profile_id,
-            row_data: data.clone(),
+            row_data: string_data_for_scripts.clone(),
             db_pool: Arc::new(db_pool.clone()),
         };
 
-        // Execute validation script
         let script_result = execution::execute_script(
             script_record.script,
             "STRINGS",
@@ -138,7 +148,6 @@ pub async fn post_table_data(
             format!("Script execution failed for '{}': {}", target_column, e)
         ))?;
 
-        // Validate script output
         let Value::Strings(mut script_output) = script_result else {
             return Err(Status::internal("Script must return string values"));
         };
@@ -160,11 +169,16 @@ pub async fn post_table_data(
     let mut placeholders = Vec::new();
     let mut param_idx = 1;
 
-    for (col, value) in data {
+    // ========================================================================
+    // FIX #2: DATABASE INSERTION LOOP
+    // This loop now correctly handles JSON `null` (which becomes `None`)
+    // without crashing and correctly inserts a SQL NULL.
+    // ========================================================================
+    for (col, proto_value) in request.data {
         let sql_type = if system_columns_set.contains(col.as_str()) {
             match col.as_str() {
                 "deleted" => "BOOLEAN",
-                _ if col.ends_with("_id") => "BIGINT", // Handle foreign keys
+                _ if col.ends_with("_id") => "BIGINT",
                 _ => return Err(Status::invalid_argument("Invalid system column")),
             }
         } else {
@@ -174,36 +188,65 @@ pub async fn post_table_data(
                 .ok_or_else(|| Status::invalid_argument(format!("Column not found: {}", col)))?
         };
 
+        // Check for `None` (from JSON null) or `Some(NullValue)` first.
+        let kind = match &proto_value.kind {
+            None | Some(Kind::NullValue(_)) => {
+                // It's a null value. Add the correct SQL NULL type and continue.
+                match sql_type {
+                    "BOOLEAN" => params.add(None::<bool>),
+                    "TEXT" | "VARCHAR(15)" | "VARCHAR(255)" => params.add(None::<String>),
+                    "TIMESTAMPTZ" => params.add(None::<DateTime<Utc>>),
+                    "BIGINT" => params.add(None::<i64>),
+                    _ => return Err(Status::invalid_argument(format!("Unsupported type for null value: {}", sql_type))),
+                }.map_err(|e| Status::internal(format!("Failed to add null parameter for {}: {}", col, e)))?;
+
+                columns_list.push(format!("\"{}\"", col));
+                placeholders.push(format!("${}", param_idx));
+                param_idx += 1;
+                continue; // Skip to the next column in the loop
+            }
+            // If it's not null, just pass the inner `Kind` through.
+            Some(k) => k,
+        };
+
+        // From here, we know `kind` is not a null type.
         match sql_type {
             "TEXT" | "VARCHAR(15)" | "VARCHAR(255)" => {
-                if let Some(max_len) = sql_type.strip_prefix("VARCHAR(")
-                    .and_then(|s| s.strip_suffix(')'))
-                    .and_then(|s| s.parse::<usize>().ok())
-                {
-                    if value.len() > max_len {
-                        return Err(Status::internal(format!("Value too long for {}", col)));
+                if let Kind::StringValue(value) = kind {
+                    if let Some(max_len) = sql_type.strip_prefix("VARCHAR(").and_then(|s| s.strip_suffix(')')).and_then(|s| s.parse::<usize>().ok()) {
+                        if value.len() > max_len {
+                            return Err(Status::internal(format!("Value too long for {}", col)));
+                        }
                     }
+                    params.add(value).map_err(|e| Status::invalid_argument(format!("Failed to add text parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected string for column '{}'", col)));
                 }
-                params.add(value)
-                    .map_err(|e| Status::invalid_argument(format!("Failed to add text parameter for {}: {}", col, e)))?;
             },
             "BOOLEAN" => {
-                let val = value.parse::<bool>()
-                    .map_err(|_| Status::invalid_argument(format!("Invalid boolean for {}", col)))?;
-                params.add(val)
-                    .map_err(|e| Status::invalid_argument(format!("Failed to add boolean parameter for {}: {}", col, e)))?;
+                if let Kind::BoolValue(val) = kind {
+                    params.add(val).map_err(|e| Status::invalid_argument(format!("Failed to add boolean parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected boolean for column '{}'", col)));
+                }
             },
             "TIMESTAMPTZ" => {
-                let dt = DateTime::parse_from_rfc3339(&value)
-                    .map_err(|_| Status::invalid_argument(format!("Invalid timestamp for {}", col)))?;
-                params.add(dt.with_timezone(&Utc))
-                    .map_err(|e| Status::invalid_argument(format!("Failed to add timestamp parameter for {}: {}", col, e)))?;
+                if let Kind::StringValue(value) = kind {
+                    let dt = DateTime::parse_from_rfc3339(value).map_err(|_| Status::invalid_argument(format!("Invalid timestamp for {}", col)))?;
+                    params.add(dt.with_timezone(&Utc)).map_err(|e| Status::invalid_argument(format!("Failed to add timestamp parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected ISO 8601 string for column '{}'", col)));
+                }
             },
             "BIGINT" => {
-                let val = value.parse::<i64>()
-                    .map_err(|_| Status::invalid_argument(format!("Invalid integer for {}", col)))?;
-                params.add(val)
-                    .map_err(|e| Status::invalid_argument(format!("Failed to add integer parameter for {}: {}", col, e)))?;
+                if let Kind::NumberValue(val) = kind {
+                    if val.fract() != 0.0 {
+                        return Err(Status::invalid_argument(format!("Expected integer for column '{}', but got a float", col)));
+                    }
+                    params.add(*val as i64).map_err(|e| Status::invalid_argument(format!("Failed to add integer parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected number for column '{}'", col)));
+                }
             },
             _ => return Err(Status::invalid_argument(format!("Unsupported type {}", sql_type))),
         }
@@ -227,7 +270,6 @@ pub async fn post_table_data(
         placeholders.join(", ")
     );
 
-    // Execute query with enhanced error handling
     let result = sqlx::query_scalar_with::<_, i64, _>(&sql, params)
         .fetch_one(db_pool)
         .await;
@@ -235,7 +277,6 @@ pub async fn post_table_data(
     let inserted_id = match result {
         Ok(id) => id,
         Err(e) => {
-            // Handle "relation does not exist" error specifically
             if let Some(db_err) = e.as_database_error() {
                 if db_err.code() == Some(std::borrow::Cow::Borrowed("42P01")) {
                     return Err(Status::internal(format!(
@@ -248,15 +289,12 @@ pub async fn post_table_data(
         }
     };
 
-    // After a successful insert, send a command to the indexer.
     let command = IndexCommand::AddOrUpdate(IndexCommandData {
         table_name: table_name.clone(),
         row_id: inserted_id,
     });
 
     if let Err(e) = indexer_tx.send(command).await {
-        // If sending fails, the DB is updated but the index will be stale.
-        // This is a critical situation to log and monitor.
         error!(
             "CRITICAL: DB insert for table '{}' (id: {}) succeeded but failed to queue for indexing: {}. Search index is now inconsistent.",
             table_name, inserted_id, e

server/src/tables_data/handlers/put_table_data.rs

@@ -4,8 +4,8 @@ use sqlx::{PgPool, Arguments, Postgres};
 use sqlx::postgres::PgArguments;
 use chrono::{DateTime, Utc};
 use common::proto::multieko2::tables_data::{PutTableDataRequest, PutTableDataResponse};
-use std::collections::HashMap;
-use crate::shared::schema_qualifier::qualify_table_name_for_data; // Import schema qualifier
+use crate::shared::schema_qualifier::qualify_table_name_for_data;
+use prost_types::value::Kind;
 
 pub async fn put_table_data(
     db_pool: &PgPool,
@@ -15,20 +15,9 @@ pub async fn put_table_data(
     let table_name = request.table_name;
     let record_id = request.id;
 
-    // Preprocess and validate data
-    let mut processed_data = HashMap::new();
-    let mut null_fields = Vec::new();
-
-    // CORRECTED: Generic handling for all fields.
-    // Any field with an empty string will be added to the null_fields list.
-    // The special, hardcoded logic for "firma" has been removed.
-    for (key, value) in request.data {
-        let trimmed = value.trim().to_string();
-        if trimmed.is_empty() {
-            null_fields.push(key);
-        } else {
-            processed_data.insert(key, trimmed);
-        }
+    // If no data is provided to update, it's an invalid request.
+    if request.data.is_empty() {
+        return Err(Status::invalid_argument("No fields provided to update."));
     }
 
     // Lookup profile
@@ -70,14 +59,29 @@ pub async fn put_table_data(
         columns.push((name, sql_type));
     }
 
-    // CORRECTED: "firma" is not a system column.
-    // It should be treated as a user-defined column.
-    let system_columns = ["deleted"];
+    // Get all foreign key columns for this table (needed for validation)
+    let fk_columns = sqlx::query!(
+        r#"SELECT ltd.table_name
+        FROM table_definition_links tdl
+        JOIN table_definitions ltd ON tdl.linked_table_id = ltd.id
+        WHERE tdl.source_table_id = $1"#,
+        table_def.id
+    )
+    .fetch_all(db_pool)
+    .await
+    .map_err(|e| Status::internal(format!("Foreign key lookup error: {}", e)))?;
+
+    let mut system_columns = vec!["deleted".to_string()];
+    for fk in fk_columns {
+        let base_name = fk.table_name.split_once('_').map_or(fk.table_name.as_str(), |(_, rest)| rest);
+        system_columns.push(format!("{}_id", base_name));
+    }
+    let system_columns_set: std::collections::HashSet<_> = system_columns.iter().map(|s| s.as_str()).collect();
     let user_columns: Vec<&String> = columns.iter().map(|(name, _)| name).collect();
 
     // Validate input columns
-    for key in processed_data.keys() {
-        if !system_columns.contains(&key.as_str()) && !user_columns.contains(&key) {
+    for key in request.data.keys() {
+        if !system_columns_set.contains(key.as_str()) && !user_columns.contains(&key) {
             return Err(Status::invalid_argument(format!("Invalid column: {}", key)));
         }
     }
@@ -87,54 +91,65 @@ pub async fn put_table_data(
     let mut set_clauses = Vec::new();
     let mut param_idx = 1;
 
-    // Add data parameters for non-empty fields
-    for (col, value) in &processed_data {
-        // CORRECTED: The logic for "firma" is removed from this match.
-        // It will now fall through to the `else` block and have its type
-        // correctly looked up from the `columns` vector.
-        let sql_type = if system_columns.contains(&col.as_str()) {
+    for (col, proto_value) in request.data {
+        let sql_type = if system_columns_set.contains(col.as_str()) {
             match col.as_str() {
                 "deleted" => "BOOLEAN",
+                _ if col.ends_with("_id") => "BIGINT",
                 _ => return Err(Status::invalid_argument("Invalid system column")),
             }
         } else {
             columns.iter()
-                .find(|(name, _)| name == col)
+                .find(|(name, _)| name == &col)
                 .map(|(_, sql_type)| sql_type.as_str())
                 .ok_or_else(|| Status::invalid_argument(format!("Column not found: {}", col)))?
         };
 
+        // A provided value cannot be null or empty in a PUT request.
+        // To clear a field, it should be set to an empty string "" for text,
+        // or a specific value for other types if needed (though typically not done).
+        // For now, we reject nulls.
+        let kind = proto_value.kind.ok_or_else(|| {
+            Status::invalid_argument(format!("Value for column '{}' cannot be empty in a PUT request. To clear a text field, send an empty string.", col))
+        })?;
+
         match sql_type {
             "TEXT" | "VARCHAR(15)" | "VARCHAR(255)" => {
-                if let Some(max_len) = sql_type.strip_prefix("VARCHAR(")
-                    .and_then(|s| s.strip_suffix(')'))
-                    .and_then(|s| s.parse::<usize>().ok())
-                {
-                    if value.len() > max_len {
-                        return Err(Status::internal(format!("Value too long for {}", col)));
-                    }
+                if let Kind::StringValue(value) = kind {
+                    params.add(value)
+                        .map_err(|e| Status::internal(format!("Failed to add text parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected string for column '{}'", col)));
                 }
-                params.add(value)
-                    .map_err(|e| Status::internal(format!("Failed to add text parameter for {}: {}", col, e)))?;
             },
             "BOOLEAN" => {
-                let val = value.parse::<bool>()
-                    .map_err(|_| Status::invalid_argument(format!("Invalid boolean for {}", col)))?;
-                params.add(val)
-                    .map_err(|e| Status::internal(format!("Failed to add boolean parameter for {}: {}", col, e)))?;
+                if let Kind::BoolValue(val) = kind {
+                    params.add(val)
+                        .map_err(|e| Status::internal(format!("Failed to add boolean parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected boolean for column '{}'", col)));
+                }
             },
             "TIMESTAMPTZ" => {
-                let dt = DateTime::parse_from_rfc3339(value)
-                    .map_err(|_| Status::invalid_argument(format!("Invalid timestamp for {}", col)))?;
-                params.add(dt.with_timezone(&Utc))
-                    .map_err(|e| Status::internal(format!("Failed to add timestamp parameter for {}: {}", col, e)))?;
+                if let Kind::StringValue(value) = kind {
+                    let dt = DateTime::parse_from_rfc3339(&value)
+                        .map_err(|_| Status::invalid_argument(format!("Invalid timestamp for {}", col)))?;
+                    params.add(dt.with_timezone(&Utc))
+                        .map_err(|e| Status::internal(format!("Failed to add timestamp parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected ISO 8601 string for column '{}'", col)));
+                }
             },
-            // ADDED: BIGINT handling for completeness, if needed for other columns.
             "BIGINT" => {
-                let val = value.parse::<i64>()
-                    .map_err(|_| Status::invalid_argument(format!("Invalid integer for {}", col)))?;
-                params.add(val)
-                    .map_err(|e| Status::internal(format!("Failed to add integer parameter for {}: {}", col, e)))?;
+                if let Kind::NumberValue(val) = kind {
+                    if val.fract() != 0.0 {
+                        return Err(Status::invalid_argument(format!("Expected integer for column '{}', but got a float", col)));
+                    }
+                    params.add(val as i64)
+                        .map_err(|e| Status::internal(format!("Failed to add integer parameter for {}: {}", col, e)))?;
+                } else {
+                    return Err(Status::invalid_argument(format!("Expected number for column '{}'", col)));
+                }
             },
             _ => return Err(Status::invalid_argument(format!("Unsupported type {}", sql_type))),
         }
@@ -143,27 +158,10 @@ pub async fn put_table_data(
         param_idx += 1;
     }
 
-    // Add NULL clauses for empty fields
-    for field in null_fields {
-        // Make sure the field is valid
-        if !system_columns.contains(&field.as_str()) && !user_columns.contains(&&field) {
-            return Err(Status::invalid_argument(format!("Invalid column to set NULL: {}", field)));
-        }
-        set_clauses.push(format!("\"{}\" = NULL", field));
-    }
-
-    // Ensure we have at least one field to update
-    if set_clauses.is_empty() {
-        return Err(Status::invalid_argument("No valid fields to update"));
-    }
-
-    // Add ID parameter at the end
     params.add(record_id)
         .map_err(|e| Status::internal(format!("Failed to add record_id parameter: {}", e)))?;
 
-    // Qualify table name with schema
     let qualified_table = qualify_table_name_for_data(&table_name)?;
-
     let set_clause = set_clauses.join(", ");
     let sql = format!(
         "UPDATE {} SET {} WHERE id = ${} AND deleted = FALSE RETURNING id",
@@ -184,7 +182,6 @@ pub async fn put_table_data(
         }),
         Ok(None) => Err(Status::not_found("Record not found or already deleted")),
         Err(e) => {
-            // Handle "relation does not exist" error specifically
             if let Some(db_err) = e.as_database_error() {
                 if db_err.code() == Some(std::borrow::Cow::Borrowed("42P01")) {
                     return Err(Status::internal(format!(