search crate compiled, lets get to fixing all the other errors

This commit is contained in:
filipriec
2025-06-08 20:10:57 +02:00
parent 2c7bda3ff1
commit b60e03eb70
7 changed files with 633 additions and 11 deletions

View File

@@ -6,6 +6,7 @@ license = "AGPL-3.0-or-later"
[dependencies]
common = { path = "../common" }
search = { path = "../search" }
chrono = { version = "0.4.40", features = ["serde"] }
dotenvy = "0.15.7"

View File

@@ -0,0 +1,83 @@
// In server/src/bin/manual_indexer.rs
use sqlx::{PgPool, Row};
use tantivy::schema::*;
use tantivy::{doc, Index};
use std::path::Path;
// --- CONFIGURATION ---
// IMPORTANT: Change this to a table name that actually exists and has data in your test DB.
// From your grpcurl output, "2025_test_post" is a good candidate.
const TABLE_TO_INDEX: &str = "2025_test_post2";
const INDEX_DIR: &str = "./tantivy_indexes";
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// --- Database Connection ---
// This assumes you have a .env file with DATABASE_URL
dotenvy::dotenv().ok();
let database_url = std::env::var("DATABASE_URL")
.expect("DATABASE_URL must be set in your .env file");
let pool = PgPool::connect(&database_url).await?;
println!("Connected to database.");
// --- Tantivy Schema Definition ---
let mut schema_builder = Schema::builder();
// This field will store the original Postgres row ID. It's crucial.
schema_builder.add_u64_field("pg_id", INDEXED | STORED);
// This field will contain ALL text data from the row, concatenated.
schema_builder.add_text_field("all_text", TEXT | STORED);
let schema = schema_builder.build();
// --- Index Creation ---
let index_path = Path::new(INDEX_DIR).join(TABLE_TO_INDEX);
if index_path.exists() {
println!("Removing existing index at: {}", index_path.display());
std::fs::remove_dir_all(&index_path)?;
}
std::fs::create_dir_all(&index_path)?;
let index = Index::create_in_dir(&index_path, schema.clone())?;
let mut index_writer = index.writer(100_000_000)?; // 100MB heap
println!("Indexing table: {}", TABLE_TO_INDEX);
// --- Data Fetching and Indexing ---
let qualified_table = format!("gen.\"{}\"", TABLE_TO_INDEX);
let query_str = format!("SELECT id, to_jsonb(t) AS data FROM {} t", qualified_table);
let rows = sqlx::query(&query_str).fetch_all(&pool).await?;
if rows.is_empty() {
println!("Warning: No rows found in table '{}'. Index will be empty.", TABLE_TO_INDEX);
}
let pg_id_field = schema.get_field("pg_id").unwrap();
let all_text_field = schema.get_field("all_text").unwrap();
for row in &rows {
let id: i64 = row.try_get("id")?;
let data: serde_json::Value = row.try_get("data")?;
// Concatenate all text values from the JSON into one big string.
let mut full_text = String::new();
if let Some(obj) = data.as_object() {
for value in obj.values() {
if let Some(s) = value.as_str() {
full_text.push_str(s);
full_text.push(' ');
}
}
}
// Add the document to Tantivy
index_writer.add_document(doc!(
pg_id_field => id as u64,
all_text_field => full_text
))?;
}
// --- Finalize ---
index_writer.commit()?;
println!("Successfully indexed {} documents into '{}'", rows.len(), index_path.display());
Ok(())
}

3
server/src/bin/mod.rs Normal file
View File

@@ -0,0 +1,3 @@
// src/bin/mod.rs
pub mod manual_indexer;

View File

@@ -21,10 +21,22 @@ use common::proto::multieko2::{
table_script::table_script_server::TableScriptServer,
auth::auth_service_server::AuthServiceServer
};
use search::run_search_service;
pub async fn run_server(db_pool: sqlx::PgPool) -> Result<(), Box<dyn std::error::Error>> {
// Initialize JWT for authentication
crate::auth::logic::jwt::init_jwt()?;
// ==================== SEARCH SERVER SETUP ==================
let search_addr = "[::1]:50052".to_string();
println!("Spawning Search Service on {}", search_addr);
tokio::spawn(async move {
if let Err(e) = run_search_service(&search_addr).await {
eprintln!("[Error] Search service failed to start: {}", e);
}
});
// ============================================================
let addr = "[::1]:50051".parse()?;