filipriec/komp_ac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

search fix to new architecture

Browse Source
This commit is contained in:
Priec
2026-08-13 21:23:49 +02:00
parent 8662b00ccf
commit 810c8ffd08
8 changed files with 1069 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

282
search/src/query_builder.rs
View File

@@ -1,10 +1,11 @@
use common::search::{
json_path_term, normalize_column_name, normalize_exact, tokenize_ngram, tokenize_word,
SchemaFields,
ARCHIVED_ROW_KEY_PREFIX, SchemaFields, json_path_term, normalize_column_name,
normalize_exact, tokenize_ngram, tokenize_word,
};
use common::proto::komp_ac::search::SearchVersionScope;
use tantivy::query::{
BooleanQuery, BoostQuery, EmptyQuery, FuzzyTermQuery, Occur, PhraseQuery, Query, QueryParser,
TermQuery,
RegexQuery, TermQuery,
};
use tantivy::schema::{IndexRecordOption, Term};
use tantivy::Index;
@@ -19,7 +20,6 @@ pub enum ConstraintMode {
#[derive(Clone, Debug)]
pub struct SearchConstraint {
pub targets: Vec<SearchConstraintTarget>,
pub query: String,
pub mode: ConstraintMode,
}
@@ -27,6 +27,7 @@ pub struct SearchConstraint {
pub struct SearchConstraintTarget {
pub table_name: Option<String>,
pub column: String,
pub query: String,
}
pub fn build_master_query(
@@ -35,6 +36,7 @@ pub fn build_master_query(
free_query: &str,
must: &[SearchConstraint],
table_filter: Option<&str>,
version_scope: SearchVersionScope,
) -> Result<Box<dyn Query>, Status> {
let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new();
let mut has_search_clause = false;
@@ -60,6 +62,17 @@ pub fn build_master_query(
));
}
let archived_rows = RegexQuery::from_pattern(
&format!("{ARCHIVED_ROW_KEY_PREFIX}.*"),
fields.row_key,
)
.map_err(|error| Status::internal(format!("Archived-row query build failed: {error}")))?;
match version_scope {
SearchVersionScope::Current => clauses.push((Occur::MustNot, Box::new(archived_rows))),
SearchVersionScope::Archived => clauses.push((Occur::Must, Box::new(archived_rows))),
SearchVersionScope::All => {}
}
if !has_search_clause {
return Ok(Box::new(EmptyQuery));
}
@@ -75,9 +88,9 @@ fn constraint_predicate(
for target in &constraint.targets {
let column_predicate = match constraint.mode {
ConstraintMode::Exact => exact_predicate(fields, &target.column, &constraint.query)?,
ConstraintMode::Exact => exact_predicate(fields, &target.column, &target.query)?,
ConstraintMode::Fuzzy => {
fuzzy_predicate_scoped(fields, &target.column, &constraint.query)?
fuzzy_predicate_scoped(fields, &target.column, &target.query)?
}
};
@@ -289,3 +302,260 @@ fn fuzzy_distance(word_len: usize) -> Option<u8> {
_ => Some(2),
}
}
#[cfg(test)]
mod tests {
use super::*;
use common::search::{archived_search_row_key, create_search_schema, register_tokenizers};
use tantivy::collector::Count;
use tantivy::schema::OwnedValue;
use tantivy::TantivyDocument;
struct TestDocument<'a> {
row_key: String,
row_id: u64,
table_name: &'a str,
values: &'a [(&'a str, &'a str)],
}
fn index_documents(documents: &[TestDocument<'_>]) -> (Index, SchemaFields) {
let schema = create_search_schema();
let index = Index::create_in_ram(schema.clone());
register_tokenizers(&index).expect("tokenizers should register");
let fields = SchemaFields::from(&schema).expect("schema should match");
let mut writer = index.writer(50_000_000).expect("writer should open");
for source in documents {
let mut document = TantivyDocument::default();
document.add_u64(fields.pg_id, source.row_id);
document.add_text(fields.table_name, source.table_name);
document.add_text(fields.row_key, &source.row_key);
let mut object = std::collections::BTreeMap::new();
for (column, value) in source.values {
document.add_text(fields.all_text, value);
object.insert((*column).to_string(), OwnedValue::from(*value));
}
document.add_object(fields.data_word, object.clone());
document.add_object(fields.data_ngram, object.clone());
document.add_object(fields.data_exact, object);
writer.add_document(document).expect("document should index");
}
writer.commit().expect("documents should commit");
(index, fields)
}
fn index_versions() -> (Index, SchemaFields) {
index_documents(&[
TestDocument {
row_key: "customers:7".to_string(),
row_id: 7,
table_name: "customers",
values: &[("1", "new")],
},
TestDocument {
row_key: archived_search_row_key(40, 7, 1),
row_id: 7,
table_name: "customers",
values: &[("1", "old")],
},
])
}
fn count(index: &Index, query: &dyn Query) -> usize {
index
.reader()
.expect("reader should open")
.searcher()
.search(query, &Count)
.expect("query should run")
}
#[test]
fn version_scope_separates_current_and_archived_documents() {
let (index, fields) = index_versions();
let constraint = SearchConstraint {
targets: vec![SearchConstraintTarget {
table_name: None,
column: "1".to_string(),
query: "old".to_string(),
}],
mode: ConstraintMode::Exact,
};
let current = build_master_query(
&index,
&fields,
"",
std::slice::from_ref(&constraint),
Some("customers"),
SearchVersionScope::Current,
)
.expect("current query should build");
let archived = build_master_query(
&index,
&fields,
"",
&[constraint],
Some("customers"),
SearchVersionScope::Archived,
)
.expect("archive query should build");
assert_eq!(count(&index, &*current), 0);
assert_eq!(count(&index, &*archived), 1);
}
#[test]
fn physical_constraint_survives_any_display_alias_rename() {
let (index, fields) = index_versions();
for public_alias in ["adresar", "customer", "renamed_again"] {
let resolved_physical_column = if !public_alias.is_empty() { "1" } else { unreachable!() };
let query = build_master_query(
&index,
&fields,
"",
&[SearchConstraint {
targets: vec![SearchConstraintTarget {
table_name: None,
column: resolved_physical_column.to_string(),
query: "new".to_string(),
}],
mode: ConstraintMode::Exact,
}],
Some("customers"),
SearchVersionScope::Current,
)
.expect("renamed alias should resolve to the same physical query");
assert_eq!(count(&index, &*query), 1, "alias {public_alias}");
}
}
#[test]
fn exact_constraints_are_anded_and_table_scoped() {
let (index, fields) = index_documents(&[
TestDocument {
row_key: "customers:1".to_string(),
row_id: 1,
table_name: "customers",
values: &[("1", "Alice Example"), ("2", "Bratislava")],
},
TestDocument {
row_key: "customers:2".to_string(),
row_id: 2,
table_name: "customers",
values: &[("1", "Alice Example"), ("2", "Kosice")],
},
TestDocument {
row_key: "suppliers:3".to_string(),
row_id: 3,
table_name: "suppliers",
values: &[("1", "Alice Example"), ("2", "Bratislava")],
},
]);
let constraints = [
SearchConstraint {
targets: vec![SearchConstraintTarget {
table_name: None,
column: "1".to_string(),
query: "Alice Example".to_string(),
}],
mode: ConstraintMode::Exact,
},
SearchConstraint {
targets: vec![SearchConstraintTarget {
table_name: None,
column: "2".to_string(),
query: "Bratislava".to_string(),
}],
mode: ConstraintMode::Exact,
},
];
let query = build_master_query(
&index,
&fields,
"",
&constraints,
Some("customers"),
SearchVersionScope::Current,
)
.expect("exact query should build");
assert_eq!(count(&index, &*query), 1);
}
#[test]
fn fuzzy_constraints_stay_within_the_resolved_physical_column() {
let (index, fields) = index_documents(&[
TestDocument {
row_key: "customers:1".to_string(),
row_id: 1,
table_name: "customers",
values: &[("1", "Alice"), ("2", "Bratislava")],
},
TestDocument {
row_key: "customers:2".to_string(),
row_id: 2,
table_name: "customers",
values: &[("1", "Bratislava"), ("2", "Kosice")],
},
]);
let constraint = |column: &str| SearchConstraint {
targets: vec![SearchConstraintTarget {
table_name: None,
column: column.to_string(),
query: "Bratislva".to_string(),
}],
mode: ConstraintMode::Fuzzy,
};
let address_query = build_master_query(
&index,
&fields,
"",
&[constraint("2")],
Some("customers"),
SearchVersionScope::Current,
)
.expect("fuzzy address query should build");
let name_query = build_master_query(
&index,
&fields,
"",
&[constraint("1")],
Some("customers"),
SearchVersionScope::Current,
)
.expect("fuzzy name query should build");
assert_eq!(count(&index, &*address_query), 1);
assert_eq!(count(&index, &*name_query), 1);
}
#[test]
fn free_text_searches_all_public_values_but_respects_table_filter() {
let (index, fields) = index_documents(&[
TestDocument {
row_key: "customers:1".to_string(),
row_id: 1,
table_name: "customers",
values: &[("1", "Alice Example")],
},
TestDocument {
row_key: "suppliers:2".to_string(),
row_id: 2,
table_name: "suppliers",
values: &[("8", "Alice Example")],
},
]);
let query = build_master_query(
&index,
&fields,
"Alice Example",
&[],
Some("customers"),
SearchVersionScope::Current,
)
.expect("free-text query should build");
assert_eq!(count(&index, &*query), 1);
}
}