diff --git a/Cargo.lock b/Cargo.lock index e5460370..e5e073c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1284,6 +1284,7 @@ dependencies = [ name = "common" version = "0.8.48" dependencies = [ + "icu_casemap", "prost", "prost-build", "prost-types", @@ -1296,6 +1297,8 @@ dependencies = [ "tonic-build", "tonic-prost", "tonic-prost-build", + "unicode-normalization", + "unicode-properties", ] [[package]] @@ -9106,6 +9109,7 @@ dependencies = [ "askama", "axum", "axum-extra", + "common", "fluent-templates", "isocountry", "jiff", diff --git a/Cargo.toml b/Cargo.toml index 3a9e923c..56db47d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,9 @@ crossterm = "0.29.0" toml = "1.1.2" toml_edit = "0.25.12" unicode-width = "0.2.2" +unicode-normalization = "0.1.25" +unicode-properties = { version = "0.1.4", features = ["emoji", "general-category"] } +icu_casemap = "2.2.0" # Fuzzy matching (picker) nucleo = "0.5.0" diff --git a/common/Cargo.toml b/common/Cargo.toml index 75d49a76..07c97fe5 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -21,6 +21,9 @@ serde = { version = "1.0.228", features = ["derive"] } tantivy = { workspace = true, optional = true } serde_json.workspace = true tonic-prost = "0.14.6" +unicode-normalization.workspace = true +unicode-properties.workspace = true +icu_casemap.workspace = true [build-dependencies] tonic-build = { version = "0.14.6" } diff --git a/common/proto/table_definition.proto b/common/proto/table_definition.proto index a53446e7..8dc9ecfe 100644 --- a/common/proto/table_definition.proto +++ b/common/proto/table_definition.proto @@ -147,7 +147,12 @@ message GeneratedColumnAlias { // ignored, so a typo cannot pass silently. string generated_name = 1; - // What the column should be called instead. Same rules as any column name. + // What the column should be called instead. Preserved exactly for display. + // Aliases may contain Unicode letters, numbers, spaces, punctuation and + // symbols, but not leading/trailing whitespace or control characters, and + // are limited to 63 UTF-8 bytes. Uniqueness and lookup use a compatibility- + // normalized, Unicode-case-folded key that collapses whitespace and ignores + // emoji while keeping punctuation significant. string alias = 2; } diff --git a/common/src/alias.rs b/common/src/alias.rs new file mode 100644 index 00000000..361e9478 --- /dev/null +++ b/common/src/alias.rs @@ -0,0 +1,92 @@ +use icu_casemap::CaseMapper; +use unicode_normalization::UnicodeNormalization; +use unicode_properties::emoji::{ + is_emoji_presentation_selector, is_tag_character, is_text_presentation_selector, is_zwj, +}; +use unicode_properties::{GeneralCategoryGroup, UnicodeEmoji, UnicodeGeneralCategory}; + +/// PostgreSQL's identifier limit. Display aliases are used as result-column +/// labels in a few query paths, so the limit is measured in UTF-8 bytes. +pub const MAX_ALIAS_BYTES: usize = 63; + +/// Whether a display alias contains control, formatting, private-use, or +/// unassigned characters that should never be persisted as visible naming. +pub fn has_disallowed_alias_characters(alias: &str) -> bool { + alias + .chars() + .any(|character| { + character.general_category_group() == GeneralCategoryGroup::Other + && !is_zwj(character) + && !is_emoji_presentation_selector(character) + && !is_text_presentation_selector(character) + && !is_tag_character(character) + }) +} + +/// The hidden key used to compare and resolve public column aliases. +/// +/// Display spelling is never changed. Compatibility-equivalent spelling and +/// case compare alike, whitespace runs compare as one space, and emoji are +/// ignored. Letters, marks, numbers, punctuation and useful non-emoji symbols +/// remain significant. +pub fn canonical_alias(alias: &str) -> String { + let compatible = alias.nfkc().collect::(); + let folded = CaseMapper::new().fold_string(&compatible).into_owned(); + let mut canonical = String::new(); + let mut pending_space = false; + + for character in folded.chars() { + match character.general_category_group() { + GeneralCategoryGroup::Letter + | GeneralCategoryGroup::Mark + | GeneralCategoryGroup::Number + | GeneralCategoryGroup::Punctuation => { + if pending_space && !canonical.is_empty() { + canonical.push(' '); + } + pending_space = false; + canonical.push(character); + } + GeneralCategoryGroup::Symbol if !character.is_emoji_char() => { + if pending_space && !canonical.is_empty() { + canonical.push(' '); + } + pending_space = false; + canonical.push(character); + } + GeneralCategoryGroup::Separator => { + pending_space = !canonical.is_empty(); + } + GeneralCategoryGroup::Symbol | GeneralCategoryGroup::Other => {} + } + } + + canonical.nfc().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aliases_compare_by_compatibility_case_and_without_emoji() { + assert_eq!(canonical_alias("Apple"), "apple"); + assert_eq!(canonical_alias("APPLE🙂"), "apple"); + assert_eq!(canonical_alias("Apple"), "apple"); + assert_eq!(canonical_alias("Straße"), canonical_alias("STRASSE")); + assert_eq!(canonical_alias("Apple👨‍👩‍👧"), "apple"); + assert!(!has_disallowed_alias_characters("Apple👨‍👩‍👧")); + } + + #[test] + fn punctuation_and_useful_symbols_remain_significant() { + assert_eq!(canonical_alias("Apple."), "apple."); + assert_eq!(canonical_alias("Apple!"), "apple!"); + assert_eq!(canonical_alias("Price €"), "price €"); + } + + #[test] + fn whitespace_runs_compare_as_one_space() { + assert_eq!(canonical_alias("First Name"), "first name"); + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index 5eda2fa7..7aa0124e 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -2,6 +2,7 @@ #[cfg_attr(not(feature = "tantivy"), path = "search_light.rs")] pub mod search; +pub mod alias; pub mod decimal; pub mod grpc_error; pub mod money; diff --git a/common/src/proto/descriptor.bin b/common/src/proto/descriptor.bin index fe56b7c8..bdf7cb64 100644 Binary files a/common/src/proto/descriptor.bin and b/common/src/proto/descriptor.bin differ diff --git a/common/src/proto/komp_ac.table_definition.rs b/common/src/proto/komp_ac.table_definition.rs index e47565e4..3b8cb7f6 100644 --- a/common/src/proto/komp_ac.table_definition.rs +++ b/common/src/proto/komp_ac.table_definition.rs @@ -90,7 +90,12 @@ pub struct GeneratedColumnAlias { /// ignored, so a typo cannot pass silently. #[prost(string, tag = "1")] pub generated_name: ::prost::alloc::string::String, - /// What the column should be called instead. Same rules as any column name. + /// What the column should be called instead. Preserved exactly for display. + /// Aliases may contain Unicode letters, numbers, spaces, punctuation and + /// symbols, but not leading/trailing whitespace or control characters, and + /// are limited to 63 UTF-8 bytes. Uniqueness and lookup use a compatibility- + /// normalized, Unicode-case-folded key that collapses whitespace and ignores + /// emoji while keeping punctuation significant. #[prost(string, tag = "2")] pub alias: ::prost::alloc::string::String, } diff --git a/common/src/search.rs b/common/src/search.rs index 358d358c..3dcf4b9b 100644 --- a/common/src/search.rs +++ b/common/src/search.rs @@ -120,7 +120,7 @@ pub fn canonical_search_number(input: &str) -> String { /// Normalizes a column name to the JSON-key form used at index time. pub fn normalize_column_name(column: &str) -> String { - column.to_ascii_lowercase() + crate::alias::canonical_alias(column) } /// Creates the column-aware search schema. diff --git a/common/src/search_light.rs b/common/src/search_light.rs index 7242de63..133735e2 100644 --- a/common/src/search_light.rs +++ b/common/src/search_light.rs @@ -65,7 +65,7 @@ pub fn canonical_search_number(input: &str) -> String { } pub fn normalize_column_name(column: &str) -> String { - column.to_ascii_lowercase() + crate::alias::canonical_alias(column) } #[cfg(test)] diff --git a/search/src/lib.rs b/search/src/lib.rs index f95383e5..7a042fb6 100644 --- a/search/src/lib.rs +++ b/search/src/lib.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, Mutex}; +use common::alias::canonical_alias; use common::proto::komp_ac::search::searcher_server::Searcher; pub use common::proto::komp_ac::search::searcher_server::SearcherServer; use common::proto::komp_ac::search::{ @@ -951,7 +952,7 @@ async fn resolve_constraint_targets( let rows = if let Some(table_name) = table_filter { sqlx::query( r#" - SELECT td.table_name, tdc.physical_name, tdc.field_type + SELECT td.table_name, tdc.display_name, tdc.physical_name, tdc.field_type FROM schemas s JOIN table_definitions td ON td.schema_id = s.id JOIN table_definition_columns tdc ON tdc.table_definition_id = td.id @@ -965,35 +966,42 @@ async fn resolve_constraint_targets( ORDER BY visible.is_global ASC LIMIT 1 ) - AND tdc.display_name = $3 "#, ) .bind(profile_name) .bind(table_name) - .bind(column) .fetch_all(pool) .await } else { sqlx::query( r#" - SELECT td.table_name, tdc.physical_name, tdc.field_type + SELECT td.table_name, tdc.display_name, tdc.physical_name, tdc.field_type FROM schemas s JOIN table_definitions td ON td.schema_id = s.id JOIN table_definition_columns tdc ON tdc.table_definition_id = td.id WHERE (s.name = $1 OR td.is_global = TRUE) AND td.deleted = FALSE - AND tdc.display_name = $2 "#, ) .bind(profile_name) - .bind(column) .fetch_all(pool) .await } .map_err(|e| Status::internal(format!("Column mapping lookup failed: {}", e)))?; - let rows = if rows.is_empty() { - let Some(field_type) = public_search_system_type(column) else { + let requested_key = canonical_alias(column); + let mut matching_rows = Vec::new(); + for row in rows { + let display_name: String = row + .try_get("display_name") + .map_err(|e| Status::internal(format!("Column mapping alias read failed: {e}")))?; + if canonical_alias(&display_name) == requested_key { + matching_rows.push(row); + } + } + + let rows = if matching_rows.is_empty() { + let Some(field_type) = public_search_system_type(&requested_key) else { return Err(Status::invalid_argument(format!( "Column alias '{}' was not found{}", column, @@ -1028,12 +1036,12 @@ async fn resolve_constraint_targets( .into_iter() .map(|table_name| SearchConstraintTarget { table_name: table_filter.is_none().then_some(table_name), - column: column.to_string(), + column: requested_key.clone(), query: query.clone(), }) .collect()); } else { - rows + matching_rows }; let mut seen = HashSet::new(); @@ -1239,13 +1247,14 @@ async fn resolve_order_column( table_name: &str, requested_column: &str, ) -> Result { - if requested_column.eq_ignore_ascii_case("position") { + let requested_key = canonical_alias(requested_column); + if requested_key == "position" { return Ok(ResolvedOrderColumn::Position); } // Sorting by "the display column" means the first one: it is the part // callers read left to right. A table with none sorts by id instead. - let requested_column = if requested_column.eq_ignore_ascii_case("row_display_columns") { + let requested_column = if requested_key == "row_display_columns" { table_row_display_columns(pool, profile_name, table_name) .await? .into_iter() @@ -1254,17 +1263,18 @@ async fn resolve_order_column( } else { requested_column.to_string() }; + let requested_key = canonical_alias(&requested_column); let physical_to_display = table_physical_to_display_map(pool, profile_name, table_name).await?; let physical_column = physical_to_display .iter() - .find(|(_, display)| display.eq_ignore_ascii_case(&requested_column)) + .find(|(_, display)| canonical_alias(display) == requested_key) .map(|(physical, _)| physical.clone()) .or_else(|| { - (is_system_column(&requested_column) - && !is_internal_column(&requested_column) - && !physical_to_display.contains_key(&requested_column)) - .then(|| requested_column.clone()) + (is_system_column(&requested_key) + && !is_internal_column(&requested_key) + && !physical_to_display.contains_key(&requested_key)) + .then(|| requested_key.clone()) }) .ok_or_else(|| { Status::invalid_argument(format!( diff --git a/server b/server index 49537094..dddd59fd 160000 --- a/server +++ b/server @@ -1 +1 @@ -Subproject commit 49537094d7de692f5f1e72856833e9549b39c36a +Subproject commit dddd59fd6167ec5a0126b26bcda6ddc48539cd99 diff --git a/web/Cargo.toml b/web/Cargo.toml index bd3bfdc7..7ce0cd24 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -22,6 +22,7 @@ serde_json = "1" tokio = { version = "1", features = ["macros", "rt-multi-thread", "net"] } tonic = "0.14.6" tonic-prost = "0.14.6" +common = { path = "../common" } [dev-dependencies] tower = { version = "0.5.3", features = ["util"] } diff --git a/web/locales/cs/main.ftl b/web/locales/cs/main.ftl index 76753864..1873094a 100644 --- a/web/locales/cs/main.ftl +++ b/web/locales/cs/main.ftl @@ -853,6 +853,8 @@ error-identifier-underscore = { $label } nesmí začínat podtržítkem. error-identifier-number = { $label } nesmí začínat číslicí. error-identifier-too-long = { $label } nesmí být delší než { $limit } znaků. error-identifier-charset = { $label } může obsahovat jen malá písmena, číslice a podtržítko. +error-alias-charset = { $label } musí obsahovat písmeno, číslo, interpunkci nebo symbol mimo emoji a nesmí obsahovat řídicí znaky. +error-alias-too-long = { $label } nesmí být delší než { $limit } bajtů UTF-8. error-identifier-reserved = { $label } používá vyhrazený název. # --- Přidat validaci ---------------------------------------------------------------- diff --git a/web/locales/en/main.ftl b/web/locales/en/main.ftl index a6a402a9..c1e72766 100644 --- a/web/locales/en/main.ftl +++ b/web/locales/en/main.ftl @@ -838,6 +838,8 @@ error-identifier-underscore = { $label } cannot start with an underscore. error-identifier-number = { $label } cannot start with a number. error-identifier-too-long = { $label } cannot be longer than { $limit } characters. error-identifier-charset = { $label } may only use lowercase letters, digits and underscores. +error-alias-charset = { $label } must contain a letter, number, punctuation mark or non-emoji symbol, and cannot contain control characters. +error-alias-too-long = { $label } cannot be longer than { $limit } UTF-8 bytes. error-identifier-reserved = { $label } uses a reserved name. # --- Add validation -------------------------------------------------------- diff --git a/web/locales/sk/main.ftl b/web/locales/sk/main.ftl index 28ea08c2..cd48b799 100644 --- a/web/locales/sk/main.ftl +++ b/web/locales/sk/main.ftl @@ -851,6 +851,8 @@ error-identifier-underscore = { $label } nesmie začínať podčiarkovníkom. error-identifier-number = { $label } nesmie začínať číslom. error-identifier-too-long = { $label } nesmie byť dlhšie ako { $limit } znakov. error-identifier-charset = { $label } môže obsahovať len malé písmená, číslice a podčiarkovník. +error-alias-charset = { $label } musí obsahovať písmeno, číslo, interpunkciu alebo symbol mimo emoji a nesmie obsahovať riadiace znaky. +error-alias-too-long = { $label } nesmie byť dlhšie ako { $limit } bajtov UTF-8. error-identifier-reserved = { $label } používa vyhradený názov. # --- Pridať validáciu ---------------------------------------------------------- diff --git a/web/src/pages/add_table/draft.rs b/web/src/pages/add_table/draft.rs index f67e49a4..7c21546f 100644 --- a/web/src/pages/add_table/draft.rs +++ b/web/src/pages/add_table/draft.rs @@ -13,7 +13,7 @@ use crate::{ definitions::table_definition::{GeneratedColumnAlias, PostTableDefinitionRequest}, - schema::{ColumnCatalog, ColumnDraft, proto_columns, validate_identifier, validate_table_name}, + schema::{ColumnCatalog, ColumnDraft, proto_columns, validate_column_alias, validate_identifier, validate_table_name}, {i18n::Locale, tr}, }; @@ -361,10 +361,16 @@ impl TableDraft { ); for (source, alias) in &renames { - if let Some(error) = validate_identifier(locale, alias, "label-column-alias", true) { + if let Some(error) = validate_column_alias(locale, alias) { return Err(error); } - if taken.iter().filter(|name| *name == alias).count() > 1 { + let canonical = common::alias::canonical_alias(alias); + if taken + .iter() + .filter(|name| common::alias::canonical_alias(name) == canonical) + .count() + > 1 + { return Err(tr!( locale, "draft-err-alias-taken", @@ -756,12 +762,9 @@ mod tests { source: "debit".to_string(), alias: "Md".to_string(), }]; - assert!( - draft.validate(crate::i18n::Locale::default()).is_err(), - "an alias is a column name" - ); + assert!(draft.validate(crate::i18n::Locale::default()).is_ok()); - draft.generated_aliases[0].alias = "note".to_string(); + draft.generated_aliases[0].alias = "NOTE🙂".to_string(); assert!( draft.validate(crate::i18n::Locale::default()).is_err(), "a declared column holds the name" diff --git a/web/src/pages/admin/table_definition/logic.rs b/web/src/pages/admin/table_definition/logic.rs index ef3a33c3..70b63ef0 100644 --- a/web/src/pages/admin/table_definition/logic.rs +++ b/web/src/pages/admin/table_definition/logic.rs @@ -27,7 +27,7 @@ use crate::{ PutTableDefinitionRequest, }, {i18n::Locale, tr}, - schema::{ColumnForm, proto_columns, validate_table_name}, + schema::{ColumnForm, proto_columns, validate_column_alias, validate_table_name}, services::{authenticated_request, reject_cross_site}, }; @@ -445,9 +445,8 @@ pub(crate) async fn set_column_alias( table: form.table.clone(), }); - let alias = form.alias.trim().to_string(); - if alias.is_empty() { - let message = tr!(Locale::from_headers(&headers), "td-err-choose-rename"); + let alias = form.alias; + if let Some(message) = validate_column_alias(Locale::from_headers(&headers), &alias) { return refuse(state, headers, inputs, Page::Presentation, message).await; } diff --git a/web/src/schema/mod.rs b/web/src/schema/mod.rs index 4498f85a..7771ab47 100644 --- a/web/src/schema/mod.rs +++ b/web/src/schema/mod.rs @@ -1138,6 +1138,38 @@ pub(crate) fn validate_identifier( None } +pub(crate) fn validate_column_alias( + locale: crate::i18n::Locale, + value: &str, +) -> Option { + let label = crate::tr!(locale, "label-column-alias"); + if value.is_empty() { + return Some(crate::tr!(locale, "error-identifier-empty", "label" => label)); + } + if value != value.trim() { + return Some(crate::tr!(locale, "error-identifier-whitespace", "label" => label)); + } + if value.len() > common::alias::MAX_ALIAS_BYTES { + return Some(crate::tr!( + locale, + "error-alias-too-long", + "label" => label, + "limit" => common::alias::MAX_ALIAS_BYTES as i64, + )); + } + if common::alias::has_disallowed_alias_characters(value) + || value.chars().any(|character| matches!(character, '\u{2028}' | '\u{2029}')) + || common::alias::canonical_alias(value).is_empty() + { + return Some(crate::tr!(locale, "error-alias-charset", "label" => label)); + } + let canonical = common::alias::canonical_alias(value); + if canonical != "account" && crate::system_column::is_system_column(&canonical) { + return Some(crate::tr!(locale, "error-identifier-reserved", "label" => label)); + } + None +} + /// Postgres's `NAMEDATALEN - 1`: a longer identifier is truncated, not refused. const MAX_IDENTIFIER_LENGTH: usize = 63;