From f0d7d5b9061e3e341718043c7c53362cb5af3647 Mon Sep 17 00:00:00 2001 From: Priec Date: Wed, 29 Jul 2026 21:57:19 +0200 Subject: [PATCH] fix to import data --- data_scrape/packs.json | 23 +++++ src/tasks/import_catalog.rs | 175 +++++++++++++++++++++++++++++++++--- 2 files changed, 186 insertions(+), 12 deletions(-) create mode 100644 data_scrape/packs.json diff --git a/data_scrape/packs.json b/data_scrape/packs.json new file mode 100644 index 0000000..dd6ab61 --- /dev/null +++ b/data_scrape/packs.json @@ -0,0 +1,23 @@ +[ + { + "legacy_id": 73, + "pack_size": 100, + "name_suffix": "balenie 100 ks", + "replace": [ + { + "from": "Minimálna objednávka 100 ks.", + "to": "Predáva sa v balení po 100 ks (0,04 €/ks)." + } + ] + }, + { + "legacy_id": 268, + "pack_size": 50, + "name_suffix": "balenie 50 párov" + }, + { + "legacy_id": 269, + "pack_size": 50, + "name_suffix": "balenie 50 párov" + } +] diff --git a/src/tasks/import_catalog.rs b/src/tasks/import_catalog.rs index f86f9d2..24d5e81 100644 --- a/src/tasks/import_catalog.rs +++ b/src/tasks/import_catalog.rs @@ -8,8 +8,8 @@ //! ``` //! //! Arguments (all optional): -//! - `path:` — folder holding `categories.json`, `products.json` and `images/` -//! (default `data_scrape`) +//! - `path:` — folder holding `categories.json`, `products.json`, `images/` +//! and the optional `packs.json` (default `data_scrape`) //! - `draft:true` — import products unpublished so they can be reviewed in //! admin before going live (default: published) //! @@ -19,7 +19,9 @@ //! **Repeatable.** Every row records its old-shop id in `legacy_id`, and //! anything already imported is skipped, so a second run tops up what is missing //! instead of duplicating the catalog. That also means edits made in admin after -//! an import are never clobbered by a later run. +//! an import are never clobbered by a later run — with one deliberate +//! exception, the pack rules in `packs.json` (see [`PackJson`]), which a re-run +//! does re-apply to products it imported earlier. //! //! Descriptions are stored exactly as `products.json` holds them: the scraper //! owns extraction and cleanup, so this task never rewrites content. @@ -36,7 +38,10 @@ use std::{ }; use loco_rs::prelude::*; -use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait}; +use sea_orm::{ + ActiveModelTrait, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, Set, + TransactionTrait, +}; use serde::Deserialize; use crate::{ @@ -82,6 +87,45 @@ struct ProductJson { variants: Vec, } +/// A product the old shop priced per piece but only ever sold by the box, with +/// the minimum stated in prose (and, for the gloves, in `minimal_quantity`). +/// This shop has no minimum-order rule, so the pack becomes the purchasable +/// unit instead: quantity 1 *is* the minimum. That is also how the old shop +/// already listed 23 other variants ("balené po 100ks"), and its stock counts +/// were whole multiples of the pack, so nothing is being invented here. +/// +/// Editorial data, kept in `packs.json` rather than in this file so the pack +/// sizes stay reviewable and changeable without a rebuild. +#[derive(Deserialize)] +struct PackJson { + legacy_id: i32, + pack_size: i32, + /// Appended to the product name in parentheses, e.g. "(balenie 100 ks)". + name_suffix: String, + /// Prose fix-ups for text that describes the old per-piece rule and would + /// read as wrong once the pack is the unit. + #[serde(default)] + replace: Vec, +} + +#[derive(Deserialize)] +struct ReplaceJson { + from: String, + to: String, +} + +impl PackJson { + fn name(&self, source: &str) -> String { + format!("{source} ({})", self.name_suffix) + } + + fn text(&self, source: &str) -> String { + self.replace + .iter() + .fold(source.to_string(), |acc, r| acc.replace(&r.from, &r.to)) + } +} + #[derive(Deserialize)] struct CrumbJson { id: i32, @@ -215,6 +259,83 @@ async fn import_categories( // -- products ------------------------------------------------------------- +/// Pack rules keyed by old-shop product id. The file is optional: a scrape +/// without it simply imports everything at its per-piece price. +fn read_packs(root: &Path) -> Result> { + let path = root.join("packs.json"); + if !path.exists() { + return Ok(HashMap::new()); + } + let list: Vec = read_json(&path)?; + for pack in &list { + if pack.pack_size < 1 { + return Err(Error::string(&format!( + "packs.json: pack_size for legacy_id {} must be at least 1", + pack.legacy_id + ))); + } + } + Ok(list.into_iter().map(|p| (p.legacy_id, p)).collect()) +} + +/// Apply a pack rule to a product an earlier run already imported. +/// +/// Every value is recomputed from `products.json`, never from what is in the +/// database, so this lands on the same numbers however often it runs. It does +/// overwrite admin edits to the name, descriptions, price and stock of these +/// few products — the one deliberate exception to the rule that a re-run never +/// touches existing rows. +async fn repack_existing( + ctx: &AppContext, + item: &ProductJson, + pack: &PackJson, + report: &mut Report, +) -> Result<()> { + let Some(product) = products::Entity::find() + .filter(products::Column::LegacyId.eq(item.id)) + .one(&ctx.db) + .await? + else { + return Ok(()); + }; + + let txn = ctx.db.begin().await?; + + let mut active = product.clone().into_active_model(); + active.name = Set(pack.name(&item.name)); + active.short_description = Set(non_empty(&pack.text(&item.short_description_html))); + active.description = Set(non_empty(&pack.text(&item.description_html))); + active.update(&txn).await?; + + let variants = product_variants::Entity::find() + .filter(product_variants::Column::ProductId.eq(product.id)) + .all(&txn) + .await?; + for variant in variants { + // A simple product carries one synthetic variant with no legacy id, so + // its price and stock come from the product itself. + let source = if item.variants.is_empty() { + Some((item.price, item.quantity_available)) + } else { + item.variants + .iter() + .find(|v| Some(v.id) == variant.legacy_id) + .map(|v| (v.price, Some(v.quantity_available))) + }; + let Some((price, stock)) = source else { + continue; + }; + let mut active = variant.into_active_model(); + active.price_cents = Set(cents(price * f64::from(pack.pack_size))); + active.stock = Set(stock.map(|q| q / pack.pack_size)); + active.update(&txn).await?; + } + + txn.commit().await?; + report.repacked += 1; + Ok(()) +} + async fn import_products( ctx: &AppContext, root: &Path, @@ -223,8 +344,14 @@ async fn import_products( report: &mut Report, ) -> Result<()> { let list: Vec = read_json(&root.join("products.json"))?; + let packs = read_packs(root)?; for item in &list { + let pack = packs.get(&item.id); + // Repacking is the one thing a re-run does apply to products it already + // imported: the values are recomputed from `products.json`, so it is + // idempotent, and it is the only way a shop that was imported before + // these pack rules existed can pick them up without a wipe. if products::Entity::find() .filter(products::Column::LegacyId.eq(item.id)) .one(&ctx.db) @@ -232,9 +359,14 @@ async fn import_products( .is_some() { report.products_skipped += 1; + if let Some(pack) = pack { + repack_existing(ctx, item, pack, report).await?; + } continue; } + // Slugged from the bare name so the URL does not carry the pack suffix + // and stays identical whether or not a pack rule applies. let slug = unique_slug(&slugify(&item.name), |candidate| { let db = &ctx.db; async move { @@ -279,10 +411,18 @@ async fn import_products( let now = chrono::Utc::now(); let product = products::ActiveModel { - name: Set(item.name.clone()), + name: Set(pack.map_or_else(|| item.name.clone(), |p| p.name(&item.name))), slug: Set(slug), - short_description: Set(non_empty(&item.short_description_html)), - description: Set(non_empty(&item.description_html)), + short_description: Set(non_empty( + &pack.map_or_else( + || item.short_description_html.clone(), + |p| p.text(&item.short_description_html), + ), + )), + description: Set(non_empty(&pack.map_or_else( + || item.description_html.clone(), + |p| p.text(&item.description_html), + ))), category_id: Set(category_id), published: Set(published), published_at: Set(published.then(|| now.into())), @@ -295,14 +435,15 @@ async fn import_products( // Price, stock and code live on variants — a product row holds none of // them. Simple products therefore get exactly one unlabelled variant, // the same shape the admin form produces for a single-option product. + let size = pack.map_or(1, |p| p.pack_size); if item.variants.is_empty() { product_variants::ActiveModel { product_id: Set(product.id), label: Set(String::new()), position: Set(0), sku: Set(non_empty(&item.reference)), - stock: Set(item.quantity_available), - price_cents: Set(cents(item.price)), + stock: Set(item.quantity_available.map(|q| q / size)), + price_cents: Set(cents(item.price * f64::from(size))), ..Default::default() } .insert(&txn) @@ -310,7 +451,9 @@ async fn import_products( report.variants += 1; } else { for (position, variant) in item.variants.iter().enumerate() { - if variant.minimal_quantity > 1 { + // A pack rule absorbs the minimum instead of dropping it: one + // pack is the old minimum, so there is nothing left to enforce. + if variant.minimal_quantity > 1 && pack.is_none() { report.dropped_min_quantities += 1; } // A variant pointing at an image other than the product's default @@ -323,8 +466,8 @@ async fn import_products( label: Set(variant.label.clone()), position: Set(position as i32), sku: Set(non_empty(&variant.reference)), - stock: Set(Some(variant.quantity_available)), - price_cents: Set(cents(variant.price)), + stock: Set(Some(variant.quantity_available / size)), + price_cents: Set(cents(variant.price * f64::from(size))), legacy_id: Set(Some(variant.id)), ..Default::default() } @@ -364,6 +507,8 @@ struct Report { products_skipped: usize, variants: usize, images: usize, + /// Products already present that a `packs.json` rule brought up to date. + repacked: usize, /// Scraped facts with no column to land in. Each is counted as it is /// encountered, so the summary reports what this run actually dropped rather /// than a fixed list of what the schema cannot hold in general. @@ -387,6 +532,12 @@ impl Report { ); println!(" options : {} created", self.variants); println!(" images : {} stored", self.images); + if self.repacked > 0 { + println!( + " repacked : {} existing product(s) updated from packs.json", + self.repacked + ); + } let dropped = [ ( self.dropped_categories,