import data and slugify

data_scrape/scrape.py

@@ -78,6 +78,52 @@ def find(pattern, text, group=1, flags=re.S):
     return m.group(group) if m else None
 
 
+def clean_content(html):
+    """Strip the old shop's wrapper markup, keeping the authored rich text.
+
+    A description is genuinely rich text — `<p>`, `<ul>`, `<li>`, `<strong>` are
+    content and are kept verbatim. What is not content is the layout scaffolding
+    that accumulated in these fields over the years: `<div>`s someone pasted in
+    (`<div id="product_images" class="hlavni_obrazek">`) and stray `id`
+    attributes left over from the PrestaShop theme (`tabs-1`, `idTab-yotpo`).
+    Those ids would be injected into every page that renders the description and
+    can collide with the host page's own ids; the `<div>`s carry no formatting
+    here. Only the tags are dropped, never their contents.
+    """
+    if not html:
+        return ""
+    h = re.sub(r"(?i)</?div\b[^>]*>", "", html)
+    h = re.sub(r'(?i)\s+id\s*=\s*"[^"]*"', "", h)
+    return h.strip()
+
+
+def extract_balanced(page, open_pattern, tag="div"):
+    """Inner HTML of the element opened by `open_pattern`, nesting-aware.
+
+    The description fields contain nested `<div>`s (the shop's own content has
+    leftovers like `<div id="product_images">` pasted into them). A non-greedy
+    `(.*?)</div>` stops at the *first* closing tag, which truncates the field
+    mid-element and yields markup with an unclosed `<div>` — that then breaks the
+    layout of whatever page renders it. Counting depth is the only correct way to
+    find the matching close.
+    """
+    m = re.search(open_pattern, page, re.S | re.I)
+    if not m:
+        return None
+    start, depth = m.end(), 1
+    token = re.compile(r"<(/?)%s\b[^>]*>" % tag, re.S | re.I)
+    pos = start
+    while True:
+        t = token.search(page, pos)
+        if not t:
+            # Source itself is unbalanced; return what we have rather than lose it.
+            return page[start:]
+        depth += -1 if t.group(1) else 1
+        if depth == 0:
+            return page[start:t.start()]
+        pos = t.end()
+
+
 def cat_id_from_url(url):
     m = re.match(r"^%s/(\d+)-" % re.escape(BASE), url)
     return int(m.group(1)) if m else None
@@ -312,8 +358,8 @@ def parse_product(url, page):
     name = strip_tags(find(r'<div id="pb-left-column">\s*<h1[^>]*>(.*?)</h1>', page)
                       or find(r'<h1[^>]*>(.*?)</h1>', page) or "")
 
-    short_html = find(r'id="short_description_content"[^>]*>(.*?)</div>\s*(?:<p class="buttons_bottom_block"|</div>)', page)
-    desc_html = find(r'<div id="idTab1"[^>]*>(.*?)</div>\s*(?:<!--|<div id="idTab)', page)
+    short_html = clean_content(extract_balanced(page, r'<div id="short_description_content"[^>]*>'))
+    desc_html = clean_content(extract_balanced(page, r'<div id="idTab1"[^>]*>'))
 
     # breadcrumb -> category trail (last node is the product name, not a link)
     crumb = find(r'<div class="breadcrumb">(.*?)</div>', page) or ""