fix to import data

data_scrape/packs.json

@@ -0,0 +1,23 @@
+[
+  {
+    "legacy_id": 73,
+    "pack_size": 100,
+    "name_suffix": "balenie 100 ks",
+    "replace": [
+      {
+        "from": "Minimálna objednávka 100 ks.",
+        "to": "Predáva sa v balení po 100 ks (0,04 €/ks)."
+      }
+    ]
+  },
+  {
+    "legacy_id": 268,
+    "pack_size": 50,
+    "name_suffix": "balenie 50 párov"
+  },
+  {
+    "legacy_id": 269,
+    "pack_size": 50,
+    "name_suffix": "balenie 50 párov"
+  }
+]

src/tasks/import_catalog.rs

@@ -8,8 +8,8 @@
 //! ```
 //!
 //! Arguments (all optional):
-//!   - `path:` — folder holding `categories.json`, `products.json` and `images/`
+//!   - `path:` — folder holding `categories.json`, `products.json`, `images/`
-//!     (default `data_scrape`)
+//!     and the optional `packs.json` (default `data_scrape`)
 //!   - `draft:true` — import products unpublished so they can be reviewed in
 //!     admin before going live (default: published)
 //!
@@ -19,7 +19,9 @@
 //! **Repeatable.** Every row records its old-shop id in `legacy_id`, and
 //! anything already imported is skipped, so a second run tops up what is missing
 //! instead of duplicating the catalog. That also means edits made in admin after
-//! an import are never clobbered by a later run.
+//! an import are never clobbered by a later run — with one deliberate
+//! exception, the pack rules in `packs.json` (see [`PackJson`]), which a re-run
+//! does re-apply to products it imported earlier.
 //!
 //! Descriptions are stored exactly as `products.json` holds them: the scraper
 //! owns extraction and cleanup, so this task never rewrites content.
@@ -36,7 +38,10 @@ use std::{
 };
 
 use loco_rs::prelude::*;
-use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait};
+use sea_orm::{
+    ActiveModelTrait, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, Set,
+    TransactionTrait,
+};
 use serde::Deserialize;
 
 use crate::{
@@ -82,6 +87,45 @@ struct ProductJson {
     variants: Vec<VariantJson>,
 }
 
+/// A product the old shop priced per piece but only ever sold by the box, with
+/// the minimum stated in prose (and, for the gloves, in `minimal_quantity`).
+/// This shop has no minimum-order rule, so the pack becomes the purchasable
+/// unit instead: quantity 1 *is* the minimum. That is also how the old shop
+/// already listed 23 other variants ("balené po 100ks"), and its stock counts
+/// were whole multiples of the pack, so nothing is being invented here.
+///
+/// Editorial data, kept in `packs.json` rather than in this file so the pack
+/// sizes stay reviewable and changeable without a rebuild.
+#[derive(Deserialize)]
+struct PackJson {
+    legacy_id: i32,
+    pack_size: i32,
+    /// Appended to the product name in parentheses, e.g. "(balenie 100 ks)".
+    name_suffix: String,
+    /// Prose fix-ups for text that describes the old per-piece rule and would
+    /// read as wrong once the pack is the unit.
+    #[serde(default)]
+    replace: Vec<ReplaceJson>,
+}
+
+#[derive(Deserialize)]
+struct ReplaceJson {
+    from: String,
+    to: String,
+}
+
+impl PackJson {
+    fn name(&self, source: &str) -> String {
+        format!("{source} ({})", self.name_suffix)
+    }
+
+    fn text(&self, source: &str) -> String {
+        self.replace
+            .iter()
+            .fold(source.to_string(), |acc, r| acc.replace(&r.from, &r.to))
+    }
+}
+
 #[derive(Deserialize)]
 struct CrumbJson {
     id: i32,
@@ -215,6 +259,83 @@ async fn import_categories(
 
 // -- products -------------------------------------------------------------
 
+/// Pack rules keyed by old-shop product id. The file is optional: a scrape
+/// without it simply imports everything at its per-piece price.
+fn read_packs(root: &Path) -> Result<HashMap<i32, PackJson>> {
+    let path = root.join("packs.json");
+    if !path.exists() {
+        return Ok(HashMap::new());
+    }
+    let list: Vec<PackJson> = read_json(&path)?;
+    for pack in &list {
+        if pack.pack_size < 1 {
+            return Err(Error::string(&format!(
+                "packs.json: pack_size for legacy_id {} must be at least 1",
+                pack.legacy_id
+            )));
+        }
+    }
+    Ok(list.into_iter().map(|p| (p.legacy_id, p)).collect())
+}
+
+/// Apply a pack rule to a product an earlier run already imported.
+///
+/// Every value is recomputed from `products.json`, never from what is in the
+/// database, so this lands on the same numbers however often it runs. It does
+/// overwrite admin edits to the name, descriptions, price and stock of these
+/// few products — the one deliberate exception to the rule that a re-run never
+/// touches existing rows.
+async fn repack_existing(
+    ctx: &AppContext,
+    item: &ProductJson,
+    pack: &PackJson,
+    report: &mut Report,
+) -> Result<()> {
+    let Some(product) = products::Entity::find()
+        .filter(products::Column::LegacyId.eq(item.id))
+        .one(&ctx.db)
+        .await?
+    else {
+        return Ok(());
+    };
+
+    let txn = ctx.db.begin().await?;
+
+    let mut active = product.clone().into_active_model();
+    active.name = Set(pack.name(&item.name));
+    active.short_description = Set(non_empty(&pack.text(&item.short_description_html)));
+    active.description = Set(non_empty(&pack.text(&item.description_html)));
+    active.update(&txn).await?;
+
+    let variants = product_variants::Entity::find()
+        .filter(product_variants::Column::ProductId.eq(product.id))
+        .all(&txn)
+        .await?;
+    for variant in variants {
+        // A simple product carries one synthetic variant with no legacy id, so
+        // its price and stock come from the product itself.
+        let source = if item.variants.is_empty() {
+            Some((item.price, item.quantity_available))
+        } else {
+            item.variants
+                .iter()
+                .find(|v| Some(v.id) == variant.legacy_id)
+                .map(|v| (v.price, Some(v.quantity_available)))
+        };
+        let Some((price, stock)) = source else {
+            continue;
+        };
+        let mut active = variant.into_active_model();
+        active.price_cents = Set(cents(price * f64::from(pack.pack_size)));
+        active.stock = Set(stock.map(|q| q / pack.pack_size));
+        active.update(&txn).await?;
+    }
+
+    txn.commit().await?;
+    report.repacked += 1;
+    Ok(())
+}
+
 async fn import_products(
     ctx: &AppContext,
     root: &Path,
@@ -223,8 +344,14 @@ async fn import_products(
     report: &mut Report,
 ) -> Result<()> {
     let list: Vec<ProductJson> = read_json(&root.join("products.json"))?;
+    let packs = read_packs(root)?;
 
     for item in &list {
+        let pack = packs.get(&item.id);
+        // Repacking is the one thing a re-run does apply to products it already
+        // imported: the values are recomputed from `products.json`, so it is
+        // idempotent, and it is the only way a shop that was imported before
+        // these pack rules existed can pick them up without a wipe.
         if products::Entity::find()
             .filter(products::Column::LegacyId.eq(item.id))
             .one(&ctx.db)
@@ -232,9 +359,14 @@ async fn import_products(
             .is_some()
         {
             report.products_skipped += 1;
+            if let Some(pack) = pack {
+                repack_existing(ctx, item, pack, report).await?;
+            }
             continue;
         }
 
+        // Slugged from the bare name so the URL does not carry the pack suffix
+        // and stays identical whether or not a pack rule applies.
         let slug = unique_slug(&slugify(&item.name), |candidate| {
             let db = &ctx.db;
             async move {
@@ -279,10 +411,18 @@ async fn import_products(
         let now = chrono::Utc::now();
 
         let product = products::ActiveModel {
-            name: Set(item.name.clone()),
+            name: Set(pack.map_or_else(|| item.name.clone(), |p| p.name(&item.name))),
             slug: Set(slug),
-            short_description: Set(non_empty(&item.short_description_html)),
+            short_description: Set(non_empty(
-            description: Set(non_empty(&item.description_html)),
+                &pack.map_or_else(
+                    || item.short_description_html.clone(),
+                    |p| p.text(&item.short_description_html),
+                ),
+            )),
+            description: Set(non_empty(&pack.map_or_else(
+                || item.description_html.clone(),
+                |p| p.text(&item.description_html),
+            ))),
             category_id: Set(category_id),
             published: Set(published),
             published_at: Set(published.then(|| now.into())),
@@ -295,14 +435,15 @@ async fn import_products(
         // Price, stock and code live on variants — a product row holds none of
         // them. Simple products therefore get exactly one unlabelled variant,
         // the same shape the admin form produces for a single-option product.
+        let size = pack.map_or(1, |p| p.pack_size);
         if item.variants.is_empty() {
             product_variants::ActiveModel {
                 product_id: Set(product.id),
                 label: Set(String::new()),
                 position: Set(0),
                 sku: Set(non_empty(&item.reference)),
-                stock: Set(item.quantity_available),
+                stock: Set(item.quantity_available.map(|q| q / size)),
-                price_cents: Set(cents(item.price)),
+                price_cents: Set(cents(item.price * f64::from(size))),
                 ..Default::default()
             }
             .insert(&txn)
@@ -310,7 +451,9 @@ async fn import_products(
             report.variants += 1;
         } else {
             for (position, variant) in item.variants.iter().enumerate() {
-                if variant.minimal_quantity > 1 {
+                // A pack rule absorbs the minimum instead of dropping it: one
+                // pack is the old minimum, so there is nothing left to enforce.
+                if variant.minimal_quantity > 1 && pack.is_none() {
                     report.dropped_min_quantities += 1;
                 }
                 // A variant pointing at an image other than the product's default
@@ -323,8 +466,8 @@ async fn import_products(
                     label: Set(variant.label.clone()),
                     position: Set(position as i32),
                     sku: Set(non_empty(&variant.reference)),
-                    stock: Set(Some(variant.quantity_available)),
+                    stock: Set(Some(variant.quantity_available / size)),
-                    price_cents: Set(cents(variant.price)),
+                    price_cents: Set(cents(variant.price * f64::from(size))),
                     legacy_id: Set(Some(variant.id)),
                     ..Default::default()
                 }
@@ -364,6 +507,8 @@ struct Report {
     products_skipped: usize,
     variants: usize,
     images: usize,
+    /// Products already present that a `packs.json` rule brought up to date.
+    repacked: usize,
     /// Scraped facts with no column to land in. Each is counted as it is
     /// encountered, so the summary reports what this run actually dropped rather
     /// than a fixed list of what the schema cannot hold in general.
@@ -387,6 +532,12 @@ impl Report {
         );
         println!("  options    : {} created", self.variants);
         println!("  images     : {} stored", self.images);
+        if self.repacked > 0 {
+            println!(
+                "  repacked   : {} existing product(s) updated from packs.json",
+                self.repacked
+            );
+        }
         let dropped = [
             (
                 self.dropped_categories,