Cleaning Scraped Data: Remove Noise & Normalize Output

import re
from datetime import datetime
def clean_text(text: str) -> str:
    """Remove whitespace, newlines, and normalize spaces"""
    return " ".join(text.split()).strip()
def clean_price(text: str) -> float | None:
    """Extract numeric price from any format"""
    match = re.search(r'[\d,.]+', text.replace(',', ''))
    return float(match.group()) if match else None
def clean_date(text: str) -> str | None:
    """Parse various date formats to ISO format"""
    formats = ["%B %d, %Y", "%Y-%m-%d", "%m/%d/%Y", "%d %b %Y"]
    for fmt in formats:
        try:
            return datetime.strptime(text.strip(), fmt).date().isoformat()
        except ValueError:
            continue
    return None
def clean_url(url: str, base: str) -> str:
    """Convert relative URLs to absolute"""
    from urllib.parse import urljoin
    return urljoin(base, url.strip())

def clean_field(value: str | None, default: str = "") -> str:
    if value is None:
        return default
    cleaned = clean_text(value)
    if cleaned.lower() in ("n/a", "null", "none", "-", ""):
        return default
    return cleaned

# Simple deduplication by URL
seen_urls = set()
unique_products = []
for product in products:
    if product["url"] not in seen_urls:
        seen_urls.add(product["url"])
        unique_products.append(product)

def validate_product(product: dict) -> list[str]:
    errors = []
    if not product.get("name"):
        errors.append("Missing name")
    if not product.get("price") or product["price"] <= 0:
        errors.append("Invalid price")
    if not product.get("url", "").startswith("http"):
        errors.append("Invalid URL")
    return errors

name = clean_text(element.select_one(".title").text)
price = clean_price(element.select_one(".price").text)