filipriec/kompress_eshop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix to import data

Browse Source
This commit is contained in:
Priec
2026-07-29 21:57:19 +02:00
parent aac19831e3
commit f0d7d5b906
2 changed files with 186 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
data_scrape/packs.json Normal file
View File

@@ -0,0 +1,23 @@
[
{
"legacy_id": 73,
"pack_size": 100,
"name_suffix": "balenie 100 ks",
"replace": [
{
"from": "Minimálna objednávka 100 ks.",
"to": "Predáva sa v balení po 100 ks (0,04 €/ks)."
}
]
},
{
"legacy_id": 268,
"pack_size": 50,
"name_suffix": "balenie 50 párov"
},
{
"legacy_id": 269,
"pack_size": 50,
"name_suffix": "balenie 50 párov"
}
]

175
src/tasks/import_catalog.rs
View File

@@ -8,8 +8,8 @@
//! ```
//!
//! Arguments (all optional):
//! - `path:` — folder holding `categories.json`, `products.json` and `images/`
//! (default `data_scrape`)
//! - `path:` — folder holding `categories.json`, `products.json`, `images/`
//! and the optional `packs.json` (default `data_scrape`)
//! - `draft:true` — import products unpublished so they can be reviewed in
//! admin before going live (default: published)
//!
@@ -19,7 +19,9 @@
//! **Repeatable.** Every row records its old-shop id in `legacy_id`, and
//! anything already imported is skipped, so a second run tops up what is missing
//! instead of duplicating the catalog. That also means edits made in admin after
//! an import are never clobbered by a later run.
//! an import are never clobbered by a later run — with one deliberate
//! exception, the pack rules in `packs.json` (see [`PackJson`]), which a re-run
//! does re-apply to products it imported earlier.
//!
//! Descriptions are stored exactly as `products.json` holds them: the scraper
//! owns extraction and cleanup, so this task never rewrites content.
@@ -36,7 +38,10 @@ use std::{
};
use loco_rs::prelude::*;
use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait};
use sea_orm::{
ActiveModelTrait, ColumnTrait, EntityTrait, IntoActiveModel, QueryFilter, Set,
TransactionTrait,
};
use serde::Deserialize;
use crate::{
@@ -82,6 +87,45 @@ struct ProductJson {
variants: Vec<VariantJson>,
}
/// A product the old shop priced per piece but only ever sold by the box, with
/// the minimum stated in prose (and, for the gloves, in `minimal_quantity`).
/// This shop has no minimum-order rule, so the pack becomes the purchasable
/// unit instead: quantity 1 *is* the minimum. That is also how the old shop
/// already listed 23 other variants ("balené po 100ks"), and its stock counts
/// were whole multiples of the pack, so nothing is being invented here.
///
/// Editorial data, kept in `packs.json` rather than in this file so the pack
/// sizes stay reviewable and changeable without a rebuild.
#[derive(Deserialize)]
struct PackJson {
legacy_id: i32,
pack_size: i32,
/// Appended to the product name in parentheses, e.g. "(balenie 100 ks)".
name_suffix: String,
/// Prose fix-ups for text that describes the old per-piece rule and would
/// read as wrong once the pack is the unit.
#[serde(default)]
replace: Vec<ReplaceJson>,
}
#[derive(Deserialize)]
struct ReplaceJson {
from: String,
to: String,
}
impl PackJson {
fn name(&self, source: &str) -> String {
format!("{source} ({})", self.name_suffix)
}
fn text(&self, source: &str) -> String {
self.replace
.iter()
.fold(source.to_string(), |acc, r| acc.replace(&r.from, &r.to))
}
}
#[derive(Deserialize)]
struct CrumbJson {
id: i32,
@@ -215,6 +259,83 @@ async fn import_categories(
// -- products -------------------------------------------------------------
/// Pack rules keyed by old-shop product id. The file is optional: a scrape
/// without it simply imports everything at its per-piece price.
fn read_packs(root: &Path) -> Result<HashMap<i32, PackJson>> {
let path = root.join("packs.json");
if !path.exists() {
return Ok(HashMap::new());
}
let list: Vec<PackJson> = read_json(&path)?;
for pack in &list {
if pack.pack_size < 1 {
return Err(Error::string(&format!(
"packs.json: pack_size for legacy_id {} must be at least 1",
pack.legacy_id
)));
}
}
Ok(list.into_iter().map(|p| (p.legacy_id, p)).collect())
}
/// Apply a pack rule to a product an earlier run already imported.
///
/// Every value is recomputed from `products.json`, never from what is in the
/// database, so this lands on the same numbers however often it runs. It does
/// overwrite admin edits to the name, descriptions, price and stock of these
/// few products — the one deliberate exception to the rule that a re-run never
/// touches existing rows.
async fn repack_existing(
ctx: &AppContext,
item: &ProductJson,
pack: &PackJson,
report: &mut Report,
) -> Result<()> {
let Some(product) = products::Entity::find()
.filter(products::Column::LegacyId.eq(item.id))
.one(&ctx.db)
.await?
else {
return Ok(());
};
let txn = ctx.db.begin().await?;
let mut active = product.clone().into_active_model();
active.name = Set(pack.name(&item.name));
active.short_description = Set(non_empty(&pack.text(&item.short_description_html)));
active.description = Set(non_empty(&pack.text(&item.description_html)));
active.update(&txn).await?;
let variants = product_variants::Entity::find()
.filter(product_variants::Column::ProductId.eq(product.id))
.all(&txn)
.await?;
for variant in variants {
// A simple product carries one synthetic variant with no legacy id, so
// its price and stock come from the product itself.
let source = if item.variants.is_empty() {
Some((item.price, item.quantity_available))
} else {
item.variants
.iter()
.find(|v| Some(v.id) == variant.legacy_id)
.map(|v| (v.price, Some(v.quantity_available)))
};
let Some((price, stock)) = source else {
continue;
};
let mut active = variant.into_active_model();
active.price_cents = Set(cents(price * f64::from(pack.pack_size)));
active.stock = Set(stock.map(|q| q / pack.pack_size));
active.update(&txn).await?;
}
txn.commit().await?;
report.repacked += 1;
Ok(())
}
async fn import_products(
ctx: &AppContext,
root: &Path,
@@ -223,8 +344,14 @@ async fn import_products(
report: &mut Report,
) -> Result<()> {
let list: Vec<ProductJson> = read_json(&root.join("products.json"))?;
let packs = read_packs(root)?;
for item in &list {
let pack = packs.get(&item.id);
// Repacking is the one thing a re-run does apply to products it already
// imported: the values are recomputed from `products.json`, so it is
// idempotent, and it is the only way a shop that was imported before
// these pack rules existed can pick them up without a wipe.
if products::Entity::find()
.filter(products::Column::LegacyId.eq(item.id))
.one(&ctx.db)
@@ -232,9 +359,14 @@ async fn import_products(
.is_some()
{
report.products_skipped += 1;
if let Some(pack) = pack {
repack_existing(ctx, item, pack, report).await?;
}
continue;
}
// Slugged from the bare name so the URL does not carry the pack suffix
// and stays identical whether or not a pack rule applies.
let slug = unique_slug(&slugify(&item.name), |candidate| {
let db = &ctx.db;
async move {
@@ -279,10 +411,18 @@ async fn import_products(
let now = chrono::Utc::now();
let product = products::ActiveModel {
name: Set(item.name.clone()),
name: Set(pack.map_or_else(|| item.name.clone(), |p| p.name(&item.name))),
slug: Set(slug),
short_description: Set(non_empty(&item.short_description_html)),
description: Set(non_empty(&item.description_html)),
short_description: Set(non_empty(
&pack.map_or_else(
|| item.short_description_html.clone(),
|p| p.text(&item.short_description_html),
),
)),
description: Set(non_empty(&pack.map_or_else(
|| item.description_html.clone(),
|p| p.text(&item.description_html),
))),
category_id: Set(category_id),
published: Set(published),
published_at: Set(published.then(|| now.into())),
@@ -295,14 +435,15 @@ async fn import_products(
// Price, stock and code live on variants — a product row holds none of
// them. Simple products therefore get exactly one unlabelled variant,
// the same shape the admin form produces for a single-option product.
let size = pack.map_or(1, |p| p.pack_size);
if item.variants.is_empty() {
product_variants::ActiveModel {
product_id: Set(product.id),
label: Set(String::new()),
position: Set(0),
sku: Set(non_empty(&item.reference)),
stock: Set(item.quantity_available),
price_cents: Set(cents(item.price)),
stock: Set(item.quantity_available.map(|q| q / size)),
price_cents: Set(cents(item.price * f64::from(size))),
..Default::default()
}
.insert(&txn)
@@ -310,7 +451,9 @@ async fn import_products(
report.variants += 1;
} else {
for (position, variant) in item.variants.iter().enumerate() {
if variant.minimal_quantity > 1 {
// A pack rule absorbs the minimum instead of dropping it: one
// pack is the old minimum, so there is nothing left to enforce.
if variant.minimal_quantity > 1 && pack.is_none() {
report.dropped_min_quantities += 1;
}
// A variant pointing at an image other than the product's default
@@ -323,8 +466,8 @@ async fn import_products(
label: Set(variant.label.clone()),
position: Set(position as i32),
sku: Set(non_empty(&variant.reference)),
stock: Set(Some(variant.quantity_available)),
price_cents: Set(cents(variant.price)),
stock: Set(Some(variant.quantity_available / size)),
price_cents: Set(cents(variant.price * f64::from(size))),
legacy_id: Set(Some(variant.id)),
..Default::default()
}
@@ -364,6 +507,8 @@ struct Report {
products_skipped: usize,
variants: usize,
images: usize,
/// Products already present that a `packs.json` rule brought up to date.
repacked: usize,
/// Scraped facts with no column to land in. Each is counted as it is
/// encountered, so the summary reports what this run actually dropped rather
/// than a fixed list of what the schema cannot hold in general.
@@ -387,6 +532,12 @@ impl Report {
);
println!(" options : {} created", self.variants);
println!(" images : {} stored", self.images);
if self.repacked > 0 {
println!(
" repacked : {} existing product(s) updated from packs.json",
self.repacked
);
}
let dropped = [
(
self.dropped_categories,