import re from urllib.parse import urlparse def ensure_external_url(url: str) -> str: """ Ensures a URL string has an explicit http:// or https:// scheme so browsers never treat domain-only links (e.g. 'mamad.com') as relative paths. """ if not url or not isinstance(url, str): return "" trimmed = url.strip() if not trimmed: return "" if re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", trimmed): return trimmed if trimmed.startswith("//"): return "https:" + trimmed if trimmed.startswith("/"): return trimmed return "https://" + trimmed.lstrip("/") def is_dummy_placeholder_link(title: str, url: str) -> bool: """ Detects legacy seeded dummy links ('Первая ссылка...' -> 'google.com') so they are never stored or returned. """ t = (title or "").strip() u = (url or "").strip().lower() if "Первая ссылка" in t: return True if u in ( "google.com", "https://google.com", "http://google.com", "https://www.google.com", "http://www.google.com", ) and ("Первая" in t or t in ("google.com", "Link", "")): return True return False def extract_domain_from_url(url: str) -> str: """ Extracts the root domain / hostname without 'www.' from a URL. Examples: 'https://www.digikala.com/product/123' -> 'digikala.com' 'http://dorar.net/hadith/3420' -> 'dorar.net' 'www.al-islam.org/path' -> 'al-islam.org' 'digikala.com/item' -> 'digikala.com' """ if not url or not isinstance(url, str): return "" trimmed = url.strip() if not trimmed: return "" candidate = ensure_external_url(trimmed) try: parsed = urlparse(candidate) hostname = (parsed.hostname or "").lower().strip() if hostname.startswith("www."): hostname = hostname[4:] if hostname: return hostname except Exception: pass # Regex fallback if urlparse failed match = re.match(r"^(?:https?://)?(?:www\.)?([^/\s:?#]+)", trimmed, re.IGNORECASE) if match: return match.group(1).lower().strip() return "" def normalize_links_for_storage(links_data): """ Ensures any link without a title (or with an empty / generic placeholder title) has its title automatically set to the domain of the URL, and ensures the URL starts with http:// or https://. Returns the normalized structure ready to be stored in JSONField. """ if not links_data: return {} if isinstance(links_data, str): trimmed = links_data.strip() if trimmed.startswith(("{", "[")): try: import json parsed = json.loads(trimmed) return normalize_links_for_storage(parsed) except (ValueError, TypeError, json.JSONDecodeError): pass return {} # Case 1: Dict format { title: url } if isinstance(links_data, dict): normalized = {} for title, url_or_obj in links_data.items(): if isinstance(url_or_obj, str) and url_or_obj.strip(): raw_url = url_or_obj.strip() t = str(title).strip() if title else "" if is_dummy_placeholder_link(t, raw_url): continue url = ensure_external_url(raw_url) if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" # Deduplicate key if multiple links share the same domain/title key = t counter = 2 while key in normalized and normalized[key] != url: key = f"{t} ({counter})" counter += 1 normalized[key] = url elif isinstance(url_or_obj, dict): raw_url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") if raw_url and isinstance(raw_url, str) and raw_url.strip(): raw_url = raw_url.strip() t = str(url_or_obj.get("title") or title or "").strip() if is_dummy_placeholder_link(t, raw_url): continue url = ensure_external_url(raw_url) if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" key = t counter = 2 while key in normalized and normalized[key] != url: key = f"{t} ({counter})" counter += 1 normalized[key] = url return normalized # Case 2: List format [ {"title": "...", "url": "..."}, ... ] if isinstance(links_data, list): normalized = [] for item in links_data: if isinstance(item, str) and item.strip(): raw_url = item.strip() if is_dummy_placeholder_link("", raw_url): continue url = ensure_external_url(raw_url) title = extract_domain_from_url(url) or "Link" normalized.append({"title": title, "url": url}) elif isinstance(item, dict): raw_url = item.get("url") or item.get("link") or item.get("href") or item.get("value") if raw_url and isinstance(raw_url, str) and raw_url.strip(): raw_url = raw_url.strip() title = str(item.get("title") or item.get("name") or "").strip() if is_dummy_placeholder_link(title, raw_url): continue url = ensure_external_url(raw_url) if not title or title.lower() in ("link", "untitled", "url", "default"): title = extract_domain_from_url(url) or "Link" normalized.append({"title": title, "url": url}) return normalized return links_data def format_links_output(links_data): """ Standardizes links into a list of structured objects: [ {"title": "dorar.net", "url": "https://dorar.net/hadith/3420"}, {"title": "islamweb.net", "url": "https://www.islamweb.net/ar/fatwa/1909"} ] Handles dict, list of dicts, list of strings, and raw JSON strings cleanly. Always ensures title defaults to domain if title is empty or generic 'Link', and ensures url has an explicit http(s):// scheme. """ if not links_data: return [] if isinstance(links_data, str): trimmed = links_data.strip() if trimmed.startswith(("[", "{")): try: import json parsed = json.loads(trimmed) return format_links_output(parsed) except (ValueError, TypeError, json.JSONDecodeError): pass if trimmed and not is_dummy_placeholder_link("", trimmed): url = ensure_external_url(trimmed) return [{"title": extract_domain_from_url(url) or "Link", "url": url}] return [] if isinstance(links_data, dict): out = [] for title, url_or_obj in links_data.items(): if isinstance(url_or_obj, str) and url_or_obj.strip(): raw_url = url_or_obj.strip() t = str(title).strip() if title else "" if is_dummy_placeholder_link(t, raw_url): continue url = ensure_external_url(raw_url) if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" out.append({"title": t, "url": url}) elif isinstance(url_or_obj, dict): raw_url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") if raw_url and isinstance(raw_url, str) and raw_url.strip(): raw_url = raw_url.strip() t = str(url_or_obj.get("title") or title or "").strip() if is_dummy_placeholder_link(t, raw_url): continue url = ensure_external_url(raw_url) if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" out.append({"title": t, "url": url}) elif isinstance(url_or_obj, list): out.extend(format_links_output(url_or_obj)) return out if isinstance(links_data, list): out = [] for item in links_data: if isinstance(item, str) and item.strip(): raw_url = item.strip() if is_dummy_placeholder_link("", raw_url): continue url = ensure_external_url(raw_url) out.append({ "title": extract_domain_from_url(url) or "Link", "url": url }) elif isinstance(item, dict): raw_url = item.get("url") or item.get("link") or item.get("href") or item.get("value") if raw_url and isinstance(raw_url, str) and raw_url.strip(): raw_url = raw_url.strip() t = str(item.get("title") or item.get("name") or "").strip() if is_dummy_placeholder_link(t, raw_url): continue url = ensure_external_url(raw_url) if not t or t.lower() in ("link", "untitled", "url", "default"): t = extract_domain_from_url(url) or "Link" out.append({"title": t, "url": url}) elif isinstance(item, list): out.extend(format_links_output(item)) return out return []