From e6840f6d9bf7aede6da31c248f09bbe9faabeb96 Mon Sep 17 00:00:00 2001 From: Filipriec Date: Tue, 25 Aug 2026 10:22:27 +0200 Subject: [PATCH] aliases are normalized and used in normalized state, but nonnormalized aliases contain everything --- Cargo.lock | 4 + Cargo.toml | 3 + common/Cargo.toml | 3 + common/proto/table_definition.proto | 7 +- common/src/alias.rs | 92 ++++++++++++++++++ common/src/lib.rs | 1 + common/src/proto/descriptor.bin | Bin 194114 -> 194456 bytes common/src/proto/komp_ac.table_definition.rs | 7 +- common/src/search.rs | 2 +- common/src/search_light.rs | 2 +- search/src/lib.rs | 44 +++++---- server | 2 +- web/Cargo.toml | 1 + web/locales/cs/main.ftl | 2 + web/locales/en/main.ftl | 2 + web/locales/sk/main.ftl | 2 + web/src/pages/add_table/draft.rs | 19 ++-- web/src/pages/admin/table_definition/logic.rs | 7 +- web/src/schema/mod.rs | 32 ++++++ 19 files changed, 198 insertions(+), 34 deletions(-) create mode 100644 common/src/alias.rs diff --git a/Cargo.lock b/Cargo.lock index e5460370..e5e073c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1284,6 +1284,7 @@ dependencies = [ name = "common" version = "0.8.48" dependencies = [ + "icu_casemap", "prost", "prost-build", "prost-types", @@ -1296,6 +1297,8 @@ dependencies = [ "tonic-build", "tonic-prost", "tonic-prost-build", + "unicode-normalization", + "unicode-properties", ] [[package]] @@ -9106,6 +9109,7 @@ dependencies = [ "askama", "axum", "axum-extra", + "common", "fluent-templates", "isocountry", "jiff", diff --git a/Cargo.toml b/Cargo.toml index 3a9e923c..56db47d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,9 @@ crossterm = "0.29.0" toml = "1.1.2" toml_edit = "0.25.12" unicode-width = "0.2.2" +unicode-normalization = "0.1.25" +unicode-properties = { version = "0.1.4", features = ["emoji", "general-category"] } +icu_casemap = "2.2.0" # Fuzzy matching (picker) nucleo = "0.5.0" diff --git a/common/Cargo.toml b/common/Cargo.toml index 75d49a76..07c97fe5 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -21,6 +21,9 @@ serde = { version = "1.0.228", features = ["derive"] } tantivy = { workspace = true, optional = true } serde_json.workspace = true tonic-prost = "0.14.6" +unicode-normalization.workspace = true +unicode-properties.workspace = true +icu_casemap.workspace = true [build-dependencies] tonic-build = { version = "0.14.6" } diff --git a/common/proto/table_definition.proto b/common/proto/table_definition.proto index a53446e7..8dc9ecfe 100644 --- a/common/proto/table_definition.proto +++ b/common/proto/table_definition.proto @@ -147,7 +147,12 @@ message GeneratedColumnAlias { // ignored, so a typo cannot pass silently. string generated_name = 1; - // What the column should be called instead. Same rules as any column name. + // What the column should be called instead. Preserved exactly for display. + // Aliases may contain Unicode letters, numbers, spaces, punctuation and + // symbols, but not leading/trailing whitespace or control characters, and + // are limited to 63 UTF-8 bytes. Uniqueness and lookup use a compatibility- + // normalized, Unicode-case-folded key that collapses whitespace and ignores + // emoji while keeping punctuation significant. string alias = 2; } diff --git a/common/src/alias.rs b/common/src/alias.rs new file mode 100644 index 00000000..361e9478 --- /dev/null +++ b/common/src/alias.rs @@ -0,0 +1,92 @@ +use icu_casemap::CaseMapper; +use unicode_normalization::UnicodeNormalization; +use unicode_properties::emoji::{ + is_emoji_presentation_selector, is_tag_character, is_text_presentation_selector, is_zwj, +}; +use unicode_properties::{GeneralCategoryGroup, UnicodeEmoji, UnicodeGeneralCategory}; + +/// PostgreSQL's identifier limit. Display aliases are used as result-column +/// labels in a few query paths, so the limit is measured in UTF-8 bytes. +pub const MAX_ALIAS_BYTES: usize = 63; + +/// Whether a display alias contains control, formatting, private-use, or +/// unassigned characters that should never be persisted as visible naming. +pub fn has_disallowed_alias_characters(alias: &str) -> bool { + alias + .chars() + .any(|character| { + character.general_category_group() == GeneralCategoryGroup::Other + && !is_zwj(character) + && !is_emoji_presentation_selector(character) + && !is_text_presentation_selector(character) + && !is_tag_character(character) + }) +} + +/// The hidden key used to compare and resolve public column aliases. +/// +/// Display spelling is never changed. Compatibility-equivalent spelling and +/// case compare alike, whitespace runs compare as one space, and emoji are +/// ignored. Letters, marks, numbers, punctuation and useful non-emoji symbols +/// remain significant. +pub fn canonical_alias(alias: &str) -> String { + let compatible = alias.nfkc().collect::(); + let folded = CaseMapper::new().fold_string(&compatible).into_owned(); + let mut canonical = String::new(); + let mut pending_space = false; + + for character in folded.chars() { + match character.general_category_group() { + GeneralCategoryGroup::Letter + | GeneralCategoryGroup::Mark + | GeneralCategoryGroup::Number + | GeneralCategoryGroup::Punctuation => { + if pending_space && !canonical.is_empty() { + canonical.push(' '); + } + pending_space = false; + canonical.push(character); + } + GeneralCategoryGroup::Symbol if !character.is_emoji_char() => { + if pending_space && !canonical.is_empty() { + canonical.push(' '); + } + pending_space = false; + canonical.push(character); + } + GeneralCategoryGroup::Separator => { + pending_space = !canonical.is_empty(); + } + GeneralCategoryGroup::Symbol | GeneralCategoryGroup::Other => {} + } + } + + canonical.nfc().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn aliases_compare_by_compatibility_case_and_without_emoji() { + assert_eq!(canonical_alias("Apple"), "apple"); + assert_eq!(canonical_alias("APPLE🙂"), "apple"); + assert_eq!(canonical_alias("Apple"), "apple"); + assert_eq!(canonical_alias("Straße"), canonical_alias("STRASSE")); + assert_eq!(canonical_alias("Apple👨‍👩‍👧"), "apple"); + assert!(!has_disallowed_alias_characters("Apple👨‍👩‍👧")); + } + + #[test] + fn punctuation_and_useful_symbols_remain_significant() { + assert_eq!(canonical_alias("Apple."), "apple."); + assert_eq!(canonical_alias("Apple!"), "apple!"); + assert_eq!(canonical_alias("Price €"), "price €"); + } + + #[test] + fn whitespace_runs_compare_as_one_space() { + assert_eq!(canonical_alias("First Name"), "first name"); + } +} diff --git a/common/src/lib.rs b/common/src/lib.rs index 5eda2fa7..7aa0124e 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -2,6 +2,7 @@ #[cfg_attr(not(feature = "tantivy"), path = "search_light.rs")] pub mod search; +pub mod alias; pub mod decimal; pub mod grpc_error; pub mod money; diff --git a/common/src/proto/descriptor.bin b/common/src/proto/descriptor.bin index fe56b7c8a34d8933115389d95983418771fd8255..bdf7cb640342622a010a9486e7f075adc4d55e91 100644 GIT binary patch delta 8803 zcmY*edvugVmhW5j^_T9i2_YY$6A~UR;{(JrYNE#n&baE#qK-3;im#coI|1UDosb|2 z>tx4cFYJQ4C_95_?I`Pd1_fOpFk?s~oWLALHV9-RMqpu02 zO)q(ct7YJMQc&oFpO%pTs&aQ5#p8faLGg8&Rs+!(d+mw2&es*d`O~b6( zs-|?1`m5TSs)p)@)Xb`;)b#qg#;TgS)YQ6~>Gd=s+$p{8k%NKtFJ}Rv^kBby81?Nt7g>H{US4|aduTrEqH>9RDft_aC zo|sczSKZLS6Q^qH>wh_CR%%W|b*c)QGiSl3X_&9EsXTxkvokZRYHR+Xdd5|5QRUOI z-15ijYiB_Jm(@+F#>cA~!K|&Vn#Fz?WF_m@`~o7?4MD1UX8qsQaBpojXw|cLMT5Q2 zfRQzi)l9FdYfJ}0eh}qYM8m5&8Zn1;aNBD0cW z6M`X#s3#(!w~9omCILk?G?IXjTt)E_t{0N4sAQz2qH{Hg659!L7&e9@6uV4Y7iFKcx`hEj63`e22+3Bm3j>5?t6Nx*&*70^-HCs~RckD(XTER~HiK8dnV){_`(x$F}_$7SX96gPpwSa8?V=rPt}pm#}_ zRY$cG03B6=`Yy$XSYfFL7Yqv?fD8Q4Z==Ych}5ii=qH&rie0HCeRjeoeCnD61Zx0& z2O+tM@=UIw7m}Me*EpwqHgz+JN;{SiAwUx$bovYv<6W;FJYkZCNpwSMGhJH|+zspU zLjTVc*{vRIp}PwHOtCRRIVQ+s+qSATEi|cot6L@-p|sUy4IqTJa@JI63Ocu`#5%ez zy)8%KfNXQI%+qsk<5(^u%|SqRs=9Ub=H#8u{yZUI|4vGjX@dbFv6Du9$3e`xlPS+FPmVyIna2gw5-s zgh`b=;X$E`QrB1o4+>pWd0ntjz=yHW|Cl02vrlI!pv=bZpCAy70yH@T z2+3~eOh8C>yH7C2YwmF$2oR`%j>=B$aZW`qs6Flj$;3Fz50JP*+X{sB0h;6hgwz2_ z40FAZIzWZv9E8*Xy6keh0H9tHSJ_V92mqaugo}Dzm}1(=y)-K0=4CJU(v=gf6u1XT zxOuQ!GIUbdCjjl964Ty3NTK=00ikq|QqHRwdx*sK)~G0y0G(3w8d!D61uiC{Sl=NR zI6z1p;=n!6`4{6%JFKqXK#S6cN#t2cADqJ!8|5H4hiPmo_$R~33%!1IdIQbR^yfm7 z2Tni5i!CP)oPH`T!{UV(5B=j5IZYHK0_nvu@;JrDX}`odM?QC(01y@jXf^>rjQ!kg z0)SBc+-(AJkwNbXw5L#L;!$I001F%%H z(vu1G+Y?lj{n2LnELx5x6NNzA^185R86fzt3+#WlYC0@irlxJB+sc;-QDEob)hrXk z%o!GF?=KTDzg|-yutM$HN>7xpz(#2)?Bx|AFXgCBsN2>m~b$a+Dg zH>j)oDXIR@P7@3MC}LyL?Rg3IZ;LwGPE*QToFW>b*dpv92MEO$fkTdsNU$HW>gU_& zzI4{vjYe)2Hmw0c$qJ$pXl+7lRoA^wKPqe! zK~XRv@VT9>+N_qmPZKLQBR7JAAkQ>olFcG9BG-UvHVfq2YO+j>cu$>qpVm~qH<-!h zy(bdMi%d4~J&wghZ3>0{HX#nGJ=^KZ!fgV>?$Uz9rfgS*JLrcM+p%R>KAIu8UErfI zwxI!n+j+$|S|SE?sNd|Mn=3kUmXLN-he#A$B(fzP9IvM=5!ts(eY1m}zGD}BcX1qV zin~O9ajpR~>=Gs8at%=0C2&eSXqA9JQh(Y>58v_8U?tuQJ`(xEFA_OVKN3Y(o>i+Iip zoVEMe3#)}?@-dd3_TMy#Dtiad$;UviNaXzBgW4;wsZY?M6rumA5dCUvCyg)sR0M@6 zs5q+xhw`xcrjzceIGhtiGXxLkB7z189%fr^w?qu+Q%`?D-@mOd=SOVEFtJbMhjvwH z;DKWJ$Xo-I`UFZEon>smaTVP~<+mIcc4rkk>nmL-UNZr4$g>3ux90i1j`ru>tWF z7Eli^Hl$1Wy^y1F-3sWaY|INeD%{{w>i zl8bc6=w&I)Ckg~Apri6;`m&6fNTXK|#zFHSTrfQJUy+gjkff^i(*1Y6BJHJfI6L80 zXAKY}0nJJPA^EDyJ3vUj%GOMQ|Aw=vuc>eL(p|T_=Hh}zD7+@^eFBgN$wKqF0Yc$5 zdBu1gm!i=By^Q>)n*TqD%kQOq3@BogmPuRw06`GYtP&7{%Vf-?0U!jI$^4<(xkYT! z8*Vi~paPoJ0P0@*SptIkhVyNvh~x666e*qMK;QzpX<7MAP96S?&jw8- zdCj%f3Jl)h@-HcTNCCQtaQba<>4#oWH*ory1>nBDF&E4{Pyx-hofLfU-YD~MKP%Br z=Zp77j^Q@`t`hth4iD8|5P<76@Zi-FQ6lO@4=u6&Xr9(r5Um>$}Y-jXws zZ|z%LLSqa#TVzRT&|^43=(Q{V0QIKZbG8NGv`c$mN{RrSb{Q504i>{ms- z^kBgb-o+omZsQiN*g){Yo z9ECIW1ExHnDM-{VHUA(znC@~*EY=y>CG8UiAShiD)$Vde_^N zn~<|`kIXaoN%ZRRxKA1zk#+l|NS6M!`re(QL@hr=H>J^FzN=!Lcl%__kvZ@7Nsw=c zr%FPK?Fit02)C+SkvPK0U_AKf}Bw$?6CvxSOo$V z&?F~kIjgl3=f;7DF)zbmDHU=E-$ z2T=FgzexZ=?dMX8U&Bj7|8p7nLUsOy?$3NKW0&d$ma;V`+-U*?K|mu22*DHX;{$}? z33r;5vMndw)&c}7prf)aC*9V9UQkcEt>xZQj^-)1IRe4WfM#<9#HMvhCd>{82&q$Y zjQOVn5K^ajb9@x;EM;^?ojOcE%bameL?b9?qp68V@yr8BYp^Eec=MOZ3bHVMrfINt! zaVs)o6fR#fvOu*Sq3;!a%XxYSghsI|=6bk$U2pDQWBX`g8VzPa=!WE6&lY)rkeutG zWc_E}m`AaZ^VP5VXhC{@j>Arv@7W3sW5Aj3!HhJ_7#;f0dAJi@)lcOG&v~c-ABNCq zHe;cByq_K|U+CFLpbYgwR6Vs`RaKG|#}q(u>^OXyjJU1^^J0MIJV0 z+!ay5`HQ{CzpF=&(%l7%y=XCis!{zkv_!QWrF$M;;+fk3HzHU|Jd~t(RE}~D^|B}O z`D+4zu`hcG{dW{2OgR{WsNq?-2tw#xsvbE;=Tb{^>ilTBl-1vmaLSm_U+&=_>nD!W zG76V_xO($JGKOE)-tt7By8Hy)AHL<`Wv!hb$k4$KTcsABp!X|Q;eA!VROn_*x5~56 zQn~>iSmj}-d(sjyV6A%QBsEp89ZY1$t@RS-F^P%ny0siO{L^Trgw5+zr-jX+Y#ba{~Aa3%;eFtx{hKO%^i<@Lo583UOiSMvo^?X4GD%uA#xr%D{aBw*$*HP^rj;{Odl&GY(tCL^S)XMFHiCjZ% z_u@HQxro}%w(f9KqK@iRO=qd&`p&^I;YzC0!{yIS$+c7`r}JbpB`3&-p4X$E8lant zKJ{bwGEwmtQ!z3Cg?K@jt&v$Dn=c^>Z}s z#(n>v&Z)A`!wY6kk&|VghiazbA|2V%t!_R~b8hJN{9w>n&X;a4F=}uV=-nP(IohQ4 zC91?D*(2v^Gu?Is?`0P~5Jz;5c=@Ap4NyDcjZWtppmoH%rb0iAk6|YqQ+*d`O8S^* g_gXYU<(Ownd_ZU%^OEM@Ge9UD%l_@x^vU`E1I4SAbpQYW delta 8460 zcmY*edvMj&oxk66&i&=y-z9|n5^ll`kKl`!iDDw7h}Pm`5!7u9O07<(?rvvicdD-2 zu^neSd+|{etpe`KsgI%p1p}o(fF#fb9GFN95FQ)WKmZ#F0!erU4V8u6{d~_OH#>9x z`J8h;pYwhF&g=g6TxxqyYR(4wpMMgk7u7vWvqp!nw^EoF#VK~G{>N|8ku>T0J8I}{ zRq(@5DJdvYi%1k@rt6N?G~uzCPd@e2@_&BrXFvUU`IGoP{e1b~KJ&BZo_f0cX>?SE zVSX5AQrxf-9vYG4U=Jnt7DnN_DilgZ!E%bNs^7GRhKcNQN{kB!VP+*_ky=3_V@N=d z1hkTXkX%8@p}rTAE2wmsqoVU;5~Z$_tpW5@GQG(n-wW!;lrAyUtP&zpHMwa8xPYF^ z;i$tS2$baXBV{p3BPXDQOdhvEZ(!5hI<&KpRL{uj3{NfSxKr-9X7fPFR}3 z1w+DF2tg194HWwmk^Xc8eLvekiSb4<;2>;5Ng5ImtN{!>gybg5v$cj^NN(a<6WT+2CU1Af~^$Wr~j~#rWb6b#K^D$ z6XdaNpXhiU-SlraEVwqTGHuY=p`WayTPN-CuICAk_zs^eK#1?)WGyAbLBw~G7@PTb zDt}mivWQA`XC3`|3fio-c|yS2os=pw<^e))Ck?;WL+I_K8^(l_jT}(3{(U`7nbMq7 z;EHVa6`3bE8O;<8a|)b{W*R;s{H0Mqs9N-h273OkmYf2Tq*^Ft3nNb`D6~-dM5myj z&_dO>hOY}mED;5tQ|zny)&>eG`#B{>o17##WNrR~1AM*5-_+CgIrowAHgw$cW z{s!j&P&^14A61M!N@9{TDh?$;uN0pMtB(4_B}5$l9rcL=gw#<^+;6xT6I^K> z`Zs@~H!C|xQp|gSU7{}$j`*s+i z6VPmm3}OCxnrwGPVDw&aI{+gK&63X$gD>!se;0BQM!`1}Td7~(N_VEep+spoPa+^8 z&vz9gGna*?52(1ldn-Llm1wf@!~m#XrDVRzFd#g+N<#|6|4kV_df%&m?GvJ);yuw% zVpHA|ZsQ6~65bP73-2}@7`9k9d`eR*77J0}gxQ(JVu;z5fz2^gPw|l)grcDpJ}4|3Ra86D6}e-;;63C-~EiH zRjlzsXoS!j;Wjcr2(1y=$awA)yI-s8KEvF#-dr?tt8m2(2uiI$G2e^12Sq`hh@I7w zn(3aXP9)00zlP8tiSlX?;uHOPGd*0`Aj0BsTo`aWTeVqtHPeLZ&8Ud5Ak4GPm}Ij^ z4b3%Rn#}@LwwfFhBev-WcF~&ZZ3CHX-Zqg+_cPhRZJdG$#uSQz9YP$@H}0nKg*ya> zO*ewXrZnom-%StR)`*3|@zD&yMuAVk+J*)QHnPXJJ0b=&300gKr~l9)M(X4qDy>3; z_l8W!nnbFgUxICF;)MOmNud08>sfo~Umw_wDE5!zMRB*tFUd7vuHB;anp^{vb_?th zk2)pby+YlXxnD0>N@+c@h3>o;4LMC-750k!qJ9Z3*}bB8Os>IbV7qwCX`*Zo=-L)~ zyz0O}x43W*vM-9B(m-VoYFDi@Ie)OLa8eVj?Jh=MPKIHAk-)7Zik3r1f^F*BaMB45bqVEdB+rLFcxqTyX3?NW8^oy-0-e8n8o{qxY_W4P(pxqX99%R zeIj+8>%ER==@Zva@S`zHpQyYEtA|hpQE*wrt_r=kl^z;$StQECJ1|25&u~TP8xPWy z$}2t^Xob`j;qGjJkh;Pp`5>f*M8O;xdr2=kNHel?q&plAVWk(Oy95Fu0)Te20fgWS z(g^}W@C9kNJ2iwe@}hKSDImBBXx#*an=eYNMSNieG=od*St<<-sTU;uw)xVYKOn?z0a~{JO>fek zsR2QqFN=!Jtl5a&THrGc1TLUGYXO40Kql;U9}wIHKGPASg;Llr69`m5Pvtdqp-k9J zqt^_^p7SU|P!t7k%h;gFvVQe zOQaZ)b;5l00rp(p-j+!BT!F#hE|KoJqL@uw3a+EZE%a30;Fjj7yuB^WQL!p4%UK-6 zZ#LL-d52q;Gnx0dWfCUy*=}eQd@N(DbVEDcm;G2K?2T?HXRXGE9tg$&T0uYv*7!mK zgkX&i{ZNkWN-1tNegGjPfHuQ`Fm#e z=z;7XC9W)-pETRHMhd&Z0l_vvYa1XG*GPAg2ZZ1niIcpgqI0b;#lC zAgF77F=o?TQR^gMjZiZ{@C(rT1qi>^N%yP)2&r|_J!=3$YMnGUBX$6&)-Q!XPy+Ny zeDkiAIH=lrxZY}II_uVWQ*gBsYoT=r-1XAE-KF_elA)L4@>}m$RrEq?y>xGPfG}gd z9Bbb#(_DTVq^NS!#$g4ZpEb_c(G7BBrSArJgPeGavjT%R`uavGA=0~OFN+iAfU?>d=>YgwrYI^VqOIGc}{LP$iaL7zKDv#T3&SxkV_ zz{O(YmjIt33uSA9V*RiedP%QyHfzP2$O^ewV96CShl zFjCv~6Mv>ZS8mUl8G^H2y312qgy3wKQE@mMMO+dEpUT)l{f`~=Xu+qvpg#sXO1Oxd z^xh77yrRhqp%Fq&GSBWcfDmfpy@uT_;hNf|XMahvD|Y25T(rA#6fW9bOqpdUsM!|X z{v|zH(c)b!F~!*;-E#&YC@m7_-5(eV(0={n<20*ce{L48!u>MO-Y82<5$>0`QCh=L zb*=h~<21Xn)#staR9mY|cnVirD^u=4P)ehqO~#Jtzw4x_**2N5@0_I^ltWV3Jr4-Q z0BAo?K-h4|3j#v$5DRi0m2z+n`^^*xR6v_lK+~J_!W^W-5}PSgv!(oHxBI;U2wXtV zW#x96w0i}5!EN_@MJb~rQWTqF0|FJ$Q#oo!ByJ^Kis&_iaSyN-BWg#b7&h7U^2ZA7 zx%{ynl}WG6;X5izO8K2is9{m?rHp;0kDZ_w3%=x|-3)j#jN{j-pZSU&daBdM4UJIh z^l<}(Qm2pGF!t_*KQ;lu2taEDpy_r0fB=Gef{#u7(4C2b(=v8W5-x^6d7p;ryH3*N z>}i>}&iI#MW4inn0t971s|*O`F8`SULb=OtAsIHR+pi};paOa-8`bUC6ZC@G?bnm< zWjL8VenkXAlmYFE2#A%cN2ctu1_-GhInw@{0SKucUJ-wUXl59l)qi!0p2(i{(L^IC zXQg{T1mr>Xlq*zO6kL$8OVZwr$}sYRKL?d@IC_1cf#4IMUFZNYw%4Ce0ioLK&!=VP zE%2g0pq80<0NQu}f_qWA11cc67ySXXjKlG@4+jv!0cgVkh;V%E!vP4TuYEWGq4YI} z1Lr%X${6}4VJ*f@5$Cty4Hf&XUfV-63f@rh z68>c4=8b5+4$sg-Kb)`ZU4R>rwfPDMQoJe0`2@94iG2R70ATDwl`{W5F~XDsA&3U{ zAyyDVuhwhM&|k`RPMzOOHLEX^h|b6;_(_S>6}sa*eO9#s&#UHn!Zc&L70SIznFd5)g~E#WOGm_jRr=@!dcJzq zKq3cjl}gzcBqnm`R&oCDx1yaAHm}vM_0pN@wRqd_pOQUX%X#OwG-h&@u4U$Oe)!=Q zG}P6vx=3TGs%{{aE4NPJmDE$YcIy;&6n?*BCIY`n<4O6!+cx3o&_9ls=1r<(RIUNU zP3oF!@g{4D__((yH7fHDdd4Nn=w;u~6O+-9GnxcWOu?j6Y5k6RLSJ>6O2p_GsE9`4CogFw_HR}5>(IeH31BraBYE;Rb z1AMM(WCwQn*>SXL)_X6}o=MFE6UB$CW`+BppOTMP&0OPC?37$Jdz3n)5B!DhDBhz6 zm4)w8nBZ_Cps1UDdi1yS{_XozFbGd+N#6n8eJa0jU=x)0aW;O^PsgBE-S;j1&F!uK zpU(Bus_?3rQ{)P2RXElByq}H|YSRlZ({s1AsURFMmW!xOrG^h|0=-S);bV(*vBbgg zc>P`fP1|VdalD-MM<9tr9as6oa}7{Cu0~Ym8lZJtO{_A{<|8=>C-rx((zMEx%B{p` cgvv?fPW^z;IH}V1A2mQIoUGq?m5zV=f0yMRs{jB1 diff --git a/common/src/proto/komp_ac.table_definition.rs b/common/src/proto/komp_ac.table_definition.rs index e47565e4..3b8cb7f6 100644 --- a/common/src/proto/komp_ac.table_definition.rs +++ b/common/src/proto/komp_ac.table_definition.rs @@ -90,7 +90,12 @@ pub struct GeneratedColumnAlias { /// ignored, so a typo cannot pass silently. #[prost(string, tag = "1")] pub generated_name: ::prost::alloc::string::String, - /// What the column should be called instead. Same rules as any column name. + /// What the column should be called instead. Preserved exactly for display. + /// Aliases may contain Unicode letters, numbers, spaces, punctuation and + /// symbols, but not leading/trailing whitespace or control characters, and + /// are limited to 63 UTF-8 bytes. Uniqueness and lookup use a compatibility- + /// normalized, Unicode-case-folded key that collapses whitespace and ignores + /// emoji while keeping punctuation significant. #[prost(string, tag = "2")] pub alias: ::prost::alloc::string::String, } diff --git a/common/src/search.rs b/common/src/search.rs index 358d358c..3dcf4b9b 100644 --- a/common/src/search.rs +++ b/common/src/search.rs @@ -120,7 +120,7 @@ pub fn canonical_search_number(input: &str) -> String { /// Normalizes a column name to the JSON-key form used at index time. pub fn normalize_column_name(column: &str) -> String { - column.to_ascii_lowercase() + crate::alias::canonical_alias(column) } /// Creates the column-aware search schema. diff --git a/common/src/search_light.rs b/common/src/search_light.rs index 7242de63..133735e2 100644 --- a/common/src/search_light.rs +++ b/common/src/search_light.rs @@ -65,7 +65,7 @@ pub fn canonical_search_number(input: &str) -> String { } pub fn normalize_column_name(column: &str) -> String { - column.to_ascii_lowercase() + crate::alias::canonical_alias(column) } #[cfg(test)] diff --git a/search/src/lib.rs b/search/src/lib.rs index f95383e5..7a042fb6 100644 --- a/search/src/lib.rs +++ b/search/src/lib.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::Path; use std::sync::{Arc, Mutex}; +use common::alias::canonical_alias; use common::proto::komp_ac::search::searcher_server::Searcher; pub use common::proto::komp_ac::search::searcher_server::SearcherServer; use common::proto::komp_ac::search::{ @@ -951,7 +952,7 @@ async fn resolve_constraint_targets( let rows = if let Some(table_name) = table_filter { sqlx::query( r#" - SELECT td.table_name, tdc.physical_name, tdc.field_type + SELECT td.table_name, tdc.display_name, tdc.physical_name, tdc.field_type FROM schemas s JOIN table_definitions td ON td.schema_id = s.id JOIN table_definition_columns tdc ON tdc.table_definition_id = td.id @@ -965,35 +966,42 @@ async fn resolve_constraint_targets( ORDER BY visible.is_global ASC LIMIT 1 ) - AND tdc.display_name = $3 "#, ) .bind(profile_name) .bind(table_name) - .bind(column) .fetch_all(pool) .await } else { sqlx::query( r#" - SELECT td.table_name, tdc.physical_name, tdc.field_type + SELECT td.table_name, tdc.display_name, tdc.physical_name, tdc.field_type FROM schemas s JOIN table_definitions td ON td.schema_id = s.id JOIN table_definition_columns tdc ON tdc.table_definition_id = td.id WHERE (s.name = $1 OR td.is_global = TRUE) AND td.deleted = FALSE - AND tdc.display_name = $2 "#, ) .bind(profile_name) - .bind(column) .fetch_all(pool) .await } .map_err(|e| Status::internal(format!("Column mapping lookup failed: {}", e)))?; - let rows = if rows.is_empty() { - let Some(field_type) = public_search_system_type(column) else { + let requested_key = canonical_alias(column); + let mut matching_rows = Vec::new(); + for row in rows { + let display_name: String = row + .try_get("display_name") + .map_err(|e| Status::internal(format!("Column mapping alias read failed: {e}")))?; + if canonical_alias(&display_name) == requested_key { + matching_rows.push(row); + } + } + + let rows = if matching_rows.is_empty() { + let Some(field_type) = public_search_system_type(&requested_key) else { return Err(Status::invalid_argument(format!( "Column alias '{}' was not found{}", column, @@ -1028,12 +1036,12 @@ async fn resolve_constraint_targets( .into_iter() .map(|table_name| SearchConstraintTarget { table_name: table_filter.is_none().then_some(table_name), - column: column.to_string(), + column: requested_key.clone(), query: query.clone(), }) .collect()); } else { - rows + matching_rows }; let mut seen = HashSet::new(); @@ -1239,13 +1247,14 @@ async fn resolve_order_column( table_name: &str, requested_column: &str, ) -> Result { - if requested_column.eq_ignore_ascii_case("position") { + let requested_key = canonical_alias(requested_column); + if requested_key == "position" { return Ok(ResolvedOrderColumn::Position); } // Sorting by "the display column" means the first one: it is the part // callers read left to right. A table with none sorts by id instead. - let requested_column = if requested_column.eq_ignore_ascii_case("row_display_columns") { + let requested_column = if requested_key == "row_display_columns" { table_row_display_columns(pool, profile_name, table_name) .await? .into_iter() @@ -1254,17 +1263,18 @@ async fn resolve_order_column( } else { requested_column.to_string() }; + let requested_key = canonical_alias(&requested_column); let physical_to_display = table_physical_to_display_map(pool, profile_name, table_name).await?; let physical_column = physical_to_display .iter() - .find(|(_, display)| display.eq_ignore_ascii_case(&requested_column)) + .find(|(_, display)| canonical_alias(display) == requested_key) .map(|(physical, _)| physical.clone()) .or_else(|| { - (is_system_column(&requested_column) - && !is_internal_column(&requested_column) - && !physical_to_display.contains_key(&requested_column)) - .then(|| requested_column.clone()) + (is_system_column(&requested_key) + && !is_internal_column(&requested_key) + && !physical_to_display.contains_key(&requested_key)) + .then(|| requested_key.clone()) }) .ok_or_else(|| { Status::invalid_argument(format!( diff --git a/server b/server index 49537094..dddd59fd 160000 --- a/server +++ b/server @@ -1 +1 @@ -Subproject commit 49537094d7de692f5f1e72856833e9549b39c36a +Subproject commit dddd59fd6167ec5a0126b26bcda6ddc48539cd99 diff --git a/web/Cargo.toml b/web/Cargo.toml index bd3bfdc7..7ce0cd24 100644 --- a/web/Cargo.toml +++ b/web/Cargo.toml @@ -22,6 +22,7 @@ serde_json = "1" tokio = { version = "1", features = ["macros", "rt-multi-thread", "net"] } tonic = "0.14.6" tonic-prost = "0.14.6" +common = { path = "../common" } [dev-dependencies] tower = { version = "0.5.3", features = ["util"] } diff --git a/web/locales/cs/main.ftl b/web/locales/cs/main.ftl index 76753864..1873094a 100644 --- a/web/locales/cs/main.ftl +++ b/web/locales/cs/main.ftl @@ -853,6 +853,8 @@ error-identifier-underscore = { $label } nesmí začínat podtržítkem. error-identifier-number = { $label } nesmí začínat číslicí. error-identifier-too-long = { $label } nesmí být delší než { $limit } znaků. error-identifier-charset = { $label } může obsahovat jen malá písmena, číslice a podtržítko. +error-alias-charset = { $label } musí obsahovat písmeno, číslo, interpunkci nebo symbol mimo emoji a nesmí obsahovat řídicí znaky. +error-alias-too-long = { $label } nesmí být delší než { $limit } bajtů UTF-8. error-identifier-reserved = { $label } používá vyhrazený název. # --- Přidat validaci ---------------------------------------------------------------- diff --git a/web/locales/en/main.ftl b/web/locales/en/main.ftl index a6a402a9..c1e72766 100644 --- a/web/locales/en/main.ftl +++ b/web/locales/en/main.ftl @@ -838,6 +838,8 @@ error-identifier-underscore = { $label } cannot start with an underscore. error-identifier-number = { $label } cannot start with a number. error-identifier-too-long = { $label } cannot be longer than { $limit } characters. error-identifier-charset = { $label } may only use lowercase letters, digits and underscores. +error-alias-charset = { $label } must contain a letter, number, punctuation mark or non-emoji symbol, and cannot contain control characters. +error-alias-too-long = { $label } cannot be longer than { $limit } UTF-8 bytes. error-identifier-reserved = { $label } uses a reserved name. # --- Add validation -------------------------------------------------------- diff --git a/web/locales/sk/main.ftl b/web/locales/sk/main.ftl index 28ea08c2..cd48b799 100644 --- a/web/locales/sk/main.ftl +++ b/web/locales/sk/main.ftl @@ -851,6 +851,8 @@ error-identifier-underscore = { $label } nesmie začínať podčiarkovníkom. error-identifier-number = { $label } nesmie začínať číslom. error-identifier-too-long = { $label } nesmie byť dlhšie ako { $limit } znakov. error-identifier-charset = { $label } môže obsahovať len malé písmená, číslice a podčiarkovník. +error-alias-charset = { $label } musí obsahovať písmeno, číslo, interpunkciu alebo symbol mimo emoji a nesmie obsahovať riadiace znaky. +error-alias-too-long = { $label } nesmie byť dlhšie ako { $limit } bajtov UTF-8. error-identifier-reserved = { $label } používa vyhradený názov. # --- Pridať validáciu ---------------------------------------------------------- diff --git a/web/src/pages/add_table/draft.rs b/web/src/pages/add_table/draft.rs index f67e49a4..7c21546f 100644 --- a/web/src/pages/add_table/draft.rs +++ b/web/src/pages/add_table/draft.rs @@ -13,7 +13,7 @@ use crate::{ definitions::table_definition::{GeneratedColumnAlias, PostTableDefinitionRequest}, - schema::{ColumnCatalog, ColumnDraft, proto_columns, validate_identifier, validate_table_name}, + schema::{ColumnCatalog, ColumnDraft, proto_columns, validate_column_alias, validate_identifier, validate_table_name}, {i18n::Locale, tr}, }; @@ -361,10 +361,16 @@ impl TableDraft { ); for (source, alias) in &renames { - if let Some(error) = validate_identifier(locale, alias, "label-column-alias", true) { + if let Some(error) = validate_column_alias(locale, alias) { return Err(error); } - if taken.iter().filter(|name| *name == alias).count() > 1 { + let canonical = common::alias::canonical_alias(alias); + if taken + .iter() + .filter(|name| common::alias::canonical_alias(name) == canonical) + .count() + > 1 + { return Err(tr!( locale, "draft-err-alias-taken", @@ -756,12 +762,9 @@ mod tests { source: "debit".to_string(), alias: "Md".to_string(), }]; - assert!( - draft.validate(crate::i18n::Locale::default()).is_err(), - "an alias is a column name" - ); + assert!(draft.validate(crate::i18n::Locale::default()).is_ok()); - draft.generated_aliases[0].alias = "note".to_string(); + draft.generated_aliases[0].alias = "NOTE🙂".to_string(); assert!( draft.validate(crate::i18n::Locale::default()).is_err(), "a declared column holds the name" diff --git a/web/src/pages/admin/table_definition/logic.rs b/web/src/pages/admin/table_definition/logic.rs index ef3a33c3..70b63ef0 100644 --- a/web/src/pages/admin/table_definition/logic.rs +++ b/web/src/pages/admin/table_definition/logic.rs @@ -27,7 +27,7 @@ use crate::{ PutTableDefinitionRequest, }, {i18n::Locale, tr}, - schema::{ColumnForm, proto_columns, validate_table_name}, + schema::{ColumnForm, proto_columns, validate_column_alias, validate_table_name}, services::{authenticated_request, reject_cross_site}, }; @@ -445,9 +445,8 @@ pub(crate) async fn set_column_alias( table: form.table.clone(), }); - let alias = form.alias.trim().to_string(); - if alias.is_empty() { - let message = tr!(Locale::from_headers(&headers), "td-err-choose-rename"); + let alias = form.alias; + if let Some(message) = validate_column_alias(Locale::from_headers(&headers), &alias) { return refuse(state, headers, inputs, Page::Presentation, message).await; } diff --git a/web/src/schema/mod.rs b/web/src/schema/mod.rs index 4498f85a..7771ab47 100644 --- a/web/src/schema/mod.rs +++ b/web/src/schema/mod.rs @@ -1138,6 +1138,38 @@ pub(crate) fn validate_identifier( None } +pub(crate) fn validate_column_alias( + locale: crate::i18n::Locale, + value: &str, +) -> Option { + let label = crate::tr!(locale, "label-column-alias"); + if value.is_empty() { + return Some(crate::tr!(locale, "error-identifier-empty", "label" => label)); + } + if value != value.trim() { + return Some(crate::tr!(locale, "error-identifier-whitespace", "label" => label)); + } + if value.len() > common::alias::MAX_ALIAS_BYTES { + return Some(crate::tr!( + locale, + "error-alias-too-long", + "label" => label, + "limit" => common::alias::MAX_ALIAS_BYTES as i64, + )); + } + if common::alias::has_disallowed_alias_characters(value) + || value.chars().any(|character| matches!(character, '\u{2028}' | '\u{2029}')) + || common::alias::canonical_alias(value).is_empty() + { + return Some(crate::tr!(locale, "error-alias-charset", "label" => label)); + } + let canonical = common::alias::canonical_alias(value); + if canonical != "account" && crate::system_column::is_system_column(&canonical) { + return Some(crate::tr!(locale, "error-identifier-reserved", "label" => label)); + } + None +} + /// Postgres's `NAMEDATALEN - 1`: a longer identifier is truncated, not refused. const MAX_IDENTIFIER_LENGTH: usize = 63;