import re from urllib.parse import urlparse def extract_domain_from_url(url: str) -> str: """ Extracts the root domain / hostname without 'www.' from a URL. Examples: 'https://www.digikala.com/product/123' -> 'digikala.com' 'http://dorar.net/hadith/3420' -> 'dorar.net' 'www.al-islam.org/path' -> 'al-islam.org' 'digikala.com/item' -> 'digikala.com' """ if not url or not isinstance(url, str): return "" trimmed = url.strip() if not trimmed: return "" candidate = trimmed if not re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", candidate): candidate = "https://" + candidate try: parsed = urlparse(candidate) hostname = (parsed.hostname or "").lower().strip() if hostname.startswith("www."): hostname = hostname[4:] if hostname: return hostname except Exception: pass # Regex fallback if urlparse failed match = re.match(r"^(?:https?://)?(?:www\.)?([^/\s:?#]+)", trimmed, re.IGNORECASE) if match: return match.group(1).lower().strip() return "" def normalize_links_for_storage(links_data): """ Ensures any link without a title (or with an empty / generic placeholder title) has its title automatically set to the domain of the URL. Returns the normalized structure ready to be stored in JSONField. """ if not links_data: return {} if isinstance(links_data, str): trimmed = links_data.strip() if trimmed.startswith(("{", "[")): try: import json parsed = json.loads(trimmed) return normalize_links_for_storage(parsed) except (ValueError, TypeError, json.JSONDecodeError): pass return {} # Case 1: Dict format { title: url } if isinstance(links_data, dict): normalized = {} for title, url_or_obj in links_data.items(): if isinstance(url_or_obj, str) and url_or_obj.strip(): url = url_or_obj.strip() t = str(title).strip() if title else "" if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" # Deduplicate key if multiple links share the same domain/title key = t counter = 2 while key in normalized and normalized[key] != url: key = f"{t} ({counter})" counter += 1 normalized[key] = url elif isinstance(url_or_obj, dict): url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") if url and isinstance(url, str) and url.strip(): url = url.strip() t = str(url_or_obj.get("title") or title or "").strip() if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" key = t counter = 2 while key in normalized and normalized[key] != url: key = f"{t} ({counter})" counter += 1 normalized[key] = url return normalized # Case 2: List format [ {"title": "...", "url": "..."}, ... ] if isinstance(links_data, list): normalized = [] for item in links_data: if isinstance(item, str) and item.strip(): url = item.strip() title = extract_domain_from_url(url) or "Link" normalized.append({"title": title, "url": url}) elif isinstance(item, dict): url = item.get("url") or item.get("link") or item.get("href") or item.get("value") if url and isinstance(url, str) and url.strip(): url = url.strip() title = str(item.get("title") or item.get("name") or "").strip() if not title or title.lower() in ("link", "untitled", "url", "default"): title = extract_domain_from_url(url) or "Link" normalized.append({"title": title, "url": url}) return normalized return links_data def format_links_output(links_data): """ Standardizes links into a list of structured objects: [ {"title": "dorar.net", "url": "https://dorar.net/hadith/3420"}, {"title": "islamweb.net", "url": "https://www.islamweb.net/ar/fatwa/1909"} ] Handles dict, list of dicts, list of strings, and raw JSON strings cleanly. Always ensures title defaults to domain if title is empty or generic 'Link'. """ if not links_data: return [] if isinstance(links_data, str): trimmed = links_data.strip() if trimmed.startswith(("[", "{")): try: import json parsed = json.loads(trimmed) return format_links_output(parsed) except (ValueError, TypeError, json.JSONDecodeError): pass if trimmed: return [{"title": extract_domain_from_url(trimmed) or "Link", "url": trimmed}] return [] if isinstance(links_data, dict): out = [] for title, url_or_obj in links_data.items(): if isinstance(url_or_obj, str) and url_or_obj.strip(): url = url_or_obj.strip() t = str(title).strip() if title else "" if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" out.append({"title": t, "url": url}) elif isinstance(url_or_obj, dict): url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") if url and isinstance(url, str) and url.strip(): url = url.strip() t = str(url_or_obj.get("title") or title or "").strip() if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" out.append({"title": t, "url": url}) elif isinstance(url_or_obj, list): out.extend(format_links_output(url_or_obj)) return out if isinstance(links_data, list): out = [] for item in links_data: if isinstance(item, str) and item.strip(): url = item.strip() out.append({ "title": extract_domain_from_url(url) or "Link", "url": url }) elif isinstance(item, dict): url = item.get("url") or item.get("link") or item.get("href") or item.get("value") if url and isinstance(url, str) and url.strip(): url = url.strip() t = str(item.get("title") or item.get("name") or "").strip() if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" out.append({"title": t, "url": url}) elif isinstance(item, list): out.extend(format_links_output(item)) return out return []