diff --git a/assets/i18n/sk/main.ftl b/assets/i18n/sk/main.ftl index d851a6b..1f15ca0 100644 --- a/assets/i18n/sk/main.ftl +++ b/assets/i18n/sk/main.ftl @@ -219,7 +219,7 @@ option-label = Označenie možnosti optional = voliteľné stock-untracked-hint = Nechajte prázdne = dostupné bez sledovania zásob available = Dostupné -choose-option = Options +choose-option = Možnosť from-price = od { $price } admin-discounts = Zľavy admin-discounts-desc = Nastavte zľavnené ceny produktov. Zľava sa v obchode zobrazí ako akcia. diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 0c2ad66..749f829 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -53,6 +53,7 @@ mod m20260625_000001_add_avatar_to_users; mod m20260627_000001_order_residence_address; mod m20260627_000002_payment_settings; mod m20260627_000003_account_cart_items; +mod m20260729_000001_add_legacy_id; pub struct Migrator; #[async_trait::async_trait] @@ -110,6 +111,7 @@ impl MigratorTrait for Migrator { Box::new(m20260627_000001_order_residence_address::Migration), Box::new(m20260627_000002_payment_settings::Migration), Box::new(m20260627_000003_account_cart_items::Migration), + Box::new(m20260729_000001_add_legacy_id::Migration), // inject-above (do not remove this comment) ] } diff --git a/migration/src/m20260729_000001_add_legacy_id.rs b/migration/src/m20260729_000001_add_legacy_id.rs new file mode 100644 index 0000000..6335ed5 --- /dev/null +++ b/migration/src/m20260729_000001_add_legacy_id.rs @@ -0,0 +1,49 @@ +//! `legacy_id` — the row's id in the old PrestaShop e-shop +//! (), recorded during the catalog import. +//! +//! Its only job is to make the import repeatable: the task looks a row up by +//! `legacy_id` to decide "already imported, skip" instead of inserting a second +//! copy. SKU cannot serve that purpose — the old shop reuses codes both across +//! products and across sizes of one product (e.g. `2181` on two sizes of the +//! same item), so it identifies nothing. +//! +//! NULL for everything created in this shop; only imported rows carry a value. +//! The uniqueness is enforced by a partial unique index rather than in Rust +//! because it is exactly the guarantee the import relies on to stay idempotent, +//! and it has to hold even if two imports were ever run concurrently. + +use loco_rs::schema::*; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +const TABLES: [&str; 3] = ["categories", "products", "product_variants"]; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, m: &SchemaManager) -> Result<(), DbErr> { + for table in TABLES { + add_column(m, table, "legacy_id", ColType::IntegerNull).await?; + // Partial index: many rows legitimately have no legacy id, and + // NULLs must not collide with each other. + m.get_connection() + .execute_unprepared(&format!( + "CREATE UNIQUE INDEX idx_{table}_legacy_id + ON {table} (legacy_id) WHERE legacy_id IS NOT NULL;" + )) + .await?; + } + Ok(()) + } + + async fn down(&self, m: &SchemaManager) -> Result<(), DbErr> { + for table in TABLES { + m.get_connection() + .execute_unprepared(&format!("DROP INDEX IF EXISTS idx_{table}_legacy_id;")) + .await?; + remove_column(m, table, "legacy_id").await?; + } + Ok(()) + } +} diff --git a/src/app.rs b/src/app.rs index 8b523cc..6237aab 100644 --- a/src/app.rs +++ b/src/app.rs @@ -140,6 +140,7 @@ impl Hooks for App { #[allow(unused_variables)] fn register_tasks(tasks: &mut Tasks) { + tasks.register(tasks::import_catalog::ImportCatalog); // tasks-inject (do not remove) } async fn truncate(ctx: &AppContext) -> Result<()> { diff --git a/src/models/_entities/categories.rs b/src/models/_entities/categories.rs index 469bbcf..18d4264 100644 --- a/src/models/_entities/categories.rs +++ b/src/models/_entities/categories.rs @@ -19,6 +19,7 @@ pub struct Model { pub position: i32, pub published: bool, pub parent_id: Option, + pub legacy_id: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/models/_entities/product_variants.rs b/src/models/_entities/product_variants.rs index 8ea75bc..6d05b91 100644 --- a/src/models/_entities/product_variants.rs +++ b/src/models/_entities/product_variants.rs @@ -18,6 +18,7 @@ pub struct Model { pub price_cents: i64, pub sale_price_cents: Option, pub business_sale_price_cents: Option, + pub legacy_id: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/models/_entities/products.rs b/src/models/_entities/products.rs index 5ce2853..3c181d5 100644 --- a/src/models/_entities/products.rs +++ b/src/models/_entities/products.rs @@ -21,6 +21,7 @@ pub struct Model { pub published: bool, pub published_at: Option, pub category_id: Option, + pub legacy_id: Option, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/src/models/products.rs b/src/models/products.rs index e4e86ba..8b62025 100644 --- a/src/models/products.rs +++ b/src/models/products.rs @@ -57,7 +57,8 @@ impl Entity { let sql = format!( r#" SELECT p.created_at, p.updated_at, p.id, p.name, p.slug, p.description, - p.short_description, p.view_count, p.published, p.published_at, p.category_id + p.short_description, p.view_count, p.published, p.published_at, p.category_id, + p.legacy_id FROM products p WHERE {published_clause} ( p.search_vector @@ websearch_to_tsquery('sk_unaccent', $1) diff --git a/src/tasks/import_catalog.rs b/src/tasks/import_catalog.rs new file mode 100644 index 0000000..7db4619 --- /dev/null +++ b/src/tasks/import_catalog.rs @@ -0,0 +1,442 @@ +//! One-off import of the old PrestaShop catalog () +//! scraped into `data_scrape/`. +//! +//! Run with: +//! +//! ```sh +//! cargo loco task import_catalog path:data_scrape +//! ``` +//! +//! Arguments (all optional): +//! - `path:` — folder holding `categories.json`, `products.json` and `images/` +//! (default `data_scrape`) +//! - `draft:true` — import products unpublished so they can be reviewed in +//! admin before going live (default: published) +//! +//! The data lives in the JSON files, never in this file — point `path:` at a +//! fresh scrape and re-run. +//! +//! **Repeatable.** Every row records its old-shop id in `legacy_id`, and +//! anything already imported is skipped, so a second run tops up what is missing +//! instead of duplicating the catalog. That also means edits made in admin after +//! an import are never clobbered by a later run. +//! +//! Prices import as-is, tax included, into `price_cents` — the old shop's +//! displayed price is what the customer pays, matching how this shop treats +//! `price_cents` today. The old VAT rates are deliberately not imported; there +//! is no tax concept here yet. + +use std::{ + collections::HashMap, + fs, + path::{Path, PathBuf}, +}; + +use loco_rs::prelude::*; +use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait}; +use serde::Deserialize; + +use crate::{ + controllers::media::{detect_image_extension, store_upload, IMAGE_STORAGE_DIR}, + models::_entities::{categories, product_images, product_variants, products}, + shared::slug::{slugify, unique_slug}, +}; + +// -- JSON shapes ---------------------------------------------------------- +// Only the fields the shop can store are declared; serde ignores the rest of +// the scrape (source urls, tax rates, net prices, attribute groups). + +#[derive(Deserialize)] +struct CategoryJson { + id: i32, + name: String, + /// Plain text: the category page renders this escaped, not as HTML. + description: Option, + parent_id: Option, + depth: i32, + image: Option, +} + +#[derive(Deserialize)] +struct FileRefJson { + file: String, +} + +#[derive(Deserialize)] +struct ProductJson { + id: i32, + name: String, + reference: String, + price: f64, + quantity_available: Option, + /// HTML: the product page renders these through `| safe`. + short_description_html: String, + description_html: String, + breadcrumb: Vec, + categories: Vec, + default_image_id: Option, + images: Vec, + variants: Vec, +} + +#[derive(Deserialize)] +struct CrumbJson { + id: i32, +} + +#[derive(Deserialize)] +struct ImageJson { + id: i32, + file: String, +} + +#[derive(Deserialize)] +struct VariantJson { + id: i32, + label: String, + reference: String, + quantity_available: i32, + price: f64, + /// Only read so the report can say how much is being dropped: this shop has + /// no minimum-order-quantity rule and variants cannot carry their own image. + minimal_quantity: i32, + image_id: Option, +} + +// -- helpers -------------------------------------------------------------- + +/// Euros as scraped → minor units. The scrape carries at most two decimals, so +/// rounding here is exact rather than a policy decision. +fn cents(eur: f64) -> i64 { + (eur * 100.0).round() as i64 +} + +/// `None` for a blank scraped string, so empty codes and descriptions land as +/// NULL rather than `""` — the same normalisation the admin forms apply. +fn non_empty(value: &str) -> Option { + let trimmed = value.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) +} + +fn read_json Deserialize<'de>>(path: &Path) -> Result { + let raw = fs::read_to_string(path) + .map_err(|e| Error::string(&format!("cannot read {}: {e}", path.display())))?; + serde_json::from_str(&raw) + .map_err(|e| Error::string(&format!("cannot parse {}: {e}", path.display()))) +} + +/// Copy one scraped image into the app's own upload storage, returning the +/// stored filename. Goes through `store_upload` so imported images are +/// indistinguishable from ones uploaded through admin. +async fn ingest_image(ctx: &AppContext, root: &Path, rel: &str) -> Result> { + let path = root.join(rel); + let Ok(data) = fs::read(&path) else { + tracing::warn!(file = %path.display(), "image missing, skipping"); + return Ok(None); + }; + let extension = detect_image_extension(&data)?; + Ok(Some( + store_upload(ctx, IMAGE_STORAGE_DIR, extension, data).await?, + )) +} + +// -- categories ----------------------------------------------------------- + +/// Insert the category tree, parents before children so `parent_id` always +/// resolves. Returns old-shop id → new id for the product phase. +async fn import_categories( + ctx: &AppContext, + root: &Path, + report: &mut Report, +) -> Result> { + let mut list: Vec = read_json(&root.join("categories.json"))?; + list.sort_by_key(|c| c.depth); + + let mut ids: HashMap = HashMap::new(); + + for (position, cat) in list.iter().enumerate() { + if let Some(existing) = categories::Entity::find() + .filter(categories::Column::LegacyId.eq(cat.id)) + .one(&ctx.db) + .await? + { + ids.insert(cat.id, existing.id); + report.categories_skipped += 1; + continue; + } + + let slug = unique_slug(&slugify(&cat.name), |candidate| { + let db = &ctx.db; + async move { + Ok(categories::Entity::find() + .filter(categories::Column::Slug.eq(candidate)) + .one(db) + .await? + .is_some()) + } + }) + .await?; + + let image_id = match &cat.image { + Some(image) => ingest_image(ctx, root, &image.file).await?, + None => None, + }; + if image_id.is_some() { + report.images += 1; + } + + let inserted = categories::ActiveModel { + name: Set(cat.name.clone()), + slug: Set(slug), + description: Set(cat.description.as_deref().and_then(non_empty)), + parent_id: Set(cat.parent_id.and_then(|old| ids.get(&old).copied())), + image_id: Set(image_id), + position: Set(position as i32), + published: Set(true), + legacy_id: Set(Some(cat.id)), + ..Default::default() + } + .insert(&ctx.db) + .await?; + + ids.insert(cat.id, inserted.id); + report.categories += 1; + } + + Ok(ids) +} + +// -- products ------------------------------------------------------------- + +async fn import_products( + ctx: &AppContext, + root: &Path, + category_ids: &HashMap, + published: bool, + report: &mut Report, +) -> Result<()> { + let list: Vec = read_json(&root.join("products.json"))?; + + for item in &list { + if products::Entity::find() + .filter(products::Column::LegacyId.eq(item.id)) + .one(&ctx.db) + .await? + .is_some() + { + report.products_skipped += 1; + continue; + } + + let slug = unique_slug(&slugify(&item.name), |candidate| { + let db = &ctx.db; + async move { + Ok(products::Entity::find() + .filter(products::Column::Slug.eq(candidate)) + .one(db) + .await? + .is_some()) + } + }) + .await?; + + // The breadcrumb's last node is the product's primary category. A few + // products have no breadcrumb; fall back to the first category they are + // listed in. Extra memberships are dropped — a product holds one + // category here. + let legacy_category = item + .breadcrumb + .last() + .or_else(|| item.categories.first()) + .map(|c| c.id); + let category_id = legacy_category.and_then(|old| category_ids.get(&old).copied()); + if item.categories.len() > 1 { + report.dropped_categories += item.categories.len() - 1; + } + + // Upload files before opening the transaction: storage is not + // transactional, and a failed insert leaving an unreferenced file behind + // is harmless, while holding a transaction open across I/O is not. + let mut uploaded: Vec<(i32, String)> = Vec::new(); + for image in &item.images { + if let Some(filename) = ingest_image(ctx, root, &image.file).await? { + uploaded.push((image.id, filename)); + } + } + // Default image first so it becomes position 0 — the one shown on cards. + uploaded.sort_by_key(|(id, _)| Some(*id) != item.default_image_id); + + let txn = ctx.db.begin().await?; + let now = chrono::Utc::now(); + + let product = products::ActiveModel { + name: Set(item.name.clone()), + slug: Set(slug), + short_description: Set(non_empty(&item.short_description_html)), + description: Set(non_empty(&item.description_html)), + category_id: Set(category_id), + published: Set(published), + published_at: Set(published.then(|| now.into())), + legacy_id: Set(Some(item.id)), + ..Default::default() + } + .insert(&txn) + .await?; + + // Price, stock and code live on variants — a product row holds none of + // them. Simple products therefore get exactly one unlabelled variant, + // the same shape the admin form produces for a single-option product. + if item.variants.is_empty() { + product_variants::ActiveModel { + product_id: Set(product.id), + label: Set(String::new()), + position: Set(0), + sku: Set(non_empty(&item.reference)), + stock: Set(item.quantity_available), + price_cents: Set(cents(item.price)), + ..Default::default() + } + .insert(&txn) + .await?; + report.variants += 1; + } else { + for (position, variant) in item.variants.iter().enumerate() { + if variant.minimal_quantity > 1 { + report.dropped_min_quantities += 1; + } + // A variant pointing at an image other than the product's default + // loses that association: images belong to the product here. + if variant.image_id.is_some() && variant.image_id != item.default_image_id { + report.dropped_variant_images += 1; + } + product_variants::ActiveModel { + product_id: Set(product.id), + label: Set(variant.label.clone()), + position: Set(position as i32), + sku: Set(non_empty(&variant.reference)), + stock: Set(Some(variant.quantity_available)), + price_cents: Set(cents(variant.price)), + legacy_id: Set(Some(variant.id)), + ..Default::default() + } + .insert(&txn) + .await?; + report.variants += 1; + } + } + + for (position, (_, filename)) in uploaded.iter().enumerate() { + product_images::ActiveModel { + product_id: Set(product.id), + image_id: Set(filename.clone()), + position: Set(position as i32), + alt: Set(Some(item.name.clone())), + ..Default::default() + } + .insert(&txn) + .await?; + report.images += 1; + } + + txn.commit().await?; + report.products += 1; + } + + Ok(()) +} + +// -- report --------------------------------------------------------------- + +#[derive(Default)] +struct Report { + categories: usize, + categories_skipped: usize, + products: usize, + products_skipped: usize, + variants: usize, + images: usize, + /// Scraped facts with no column to land in. Each is counted as it is + /// encountered, so the summary reports what this run actually dropped rather + /// than a fixed list of what the schema cannot hold in general. + dropped_categories: usize, + dropped_min_quantities: usize, + dropped_variant_images: usize, +} + +impl Report { + fn print(&self, published: bool) { + println!("\nImport finished."); + println!( + " categories : {} created, {} already present", + self.categories, self.categories_skipped + ); + println!( + " products : {} created, {} already present ({})", + self.products, + self.products_skipped, + if published { "published" } else { "draft" } + ); + println!(" options : {} created", self.variants); + println!(" images : {} stored", self.images); + let dropped = [ + ( + self.dropped_categories, + "extra category memberships — each product keeps one category", + ), + ( + self.dropped_min_quantities, + "minimum order quantities — no such rule in this shop", + ), + ( + self.dropped_variant_images, + "per-variant images — images belong to the product here", + ), + ]; + if dropped.iter().any(|(count, _)| *count > 0) { + println!("\nNot imported (no home in this schema):"); + // Count trails the noun so the line reads correctly whether it is 1 + // or 14. + for (count, what) in dropped.iter().filter(|(count, _)| *count > 0) { + let (noun, tail) = what.split_once(" — ").unwrap_or((what, "")); + println!(" {noun} ({count}) — {tail}"); + } + } + } +} + +// -- task ----------------------------------------------------------------- + +pub struct ImportCatalog; + +#[async_trait] +impl Task for ImportCatalog { + fn task(&self) -> TaskInfo { + TaskInfo { + name: "import_catalog".to_string(), + detail: "Import the scraped kompress catalog (path:DIR draft:true)".to_string(), + } + } + + async fn run(&self, ctx: &AppContext, vars: &task::Vars) -> Result<()> { + let root = vars + .cli_arg("path") + .map_or_else(|_| PathBuf::from("data_scrape"), PathBuf::from); + let published = vars.cli_arg("draft").map_or(true, |v| v != "true"); + + if !root.join("products.json").exists() { + return Err(Error::string(&format!( + "no products.json in {} — pass path:DIR", + root.display() + ))); + } + + println!("Importing from {} …", root.display()); + let mut report = Report::default(); + + let category_ids = import_categories(ctx, &root, &mut report).await?; + import_products(ctx, &root, &category_ids, published, &mut report).await?; + + report.print(published); + Ok(()) + } +} diff --git a/src/tasks/mod.rs b/src/tasks/mod.rs index 8b13789..214eb89 100644 --- a/src/tasks/mod.rs +++ b/src/tasks/mod.rs @@ -1 +1 @@ - +pub mod import_catalog;