|
|
@ -2,6 +2,45 @@ import re |
|
|
from urllib.parse import urlparse |
|
|
from urllib.parse import urlparse |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def ensure_external_url(url: str) -> str: |
|
|
|
|
|
""" |
|
|
|
|
|
Ensures a URL string has an explicit http:// or https:// scheme so browsers |
|
|
|
|
|
never treat domain-only links (e.g. 'mamad.com') as relative paths. |
|
|
|
|
|
""" |
|
|
|
|
|
if not url or not isinstance(url, str): |
|
|
|
|
|
return "" |
|
|
|
|
|
trimmed = url.strip() |
|
|
|
|
|
if not trimmed: |
|
|
|
|
|
return "" |
|
|
|
|
|
if re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", trimmed): |
|
|
|
|
|
return trimmed |
|
|
|
|
|
if trimmed.startswith("//"): |
|
|
|
|
|
return "https:" + trimmed |
|
|
|
|
|
if trimmed.startswith("/"): |
|
|
|
|
|
return trimmed |
|
|
|
|
|
return "https://" + trimmed.lstrip("/") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_dummy_placeholder_link(title: str, url: str) -> bool: |
|
|
|
|
|
""" |
|
|
|
|
|
Detects legacy seeded dummy links ('Первая ссылка...' -> 'google.com') |
|
|
|
|
|
so they are never stored or returned. |
|
|
|
|
|
""" |
|
|
|
|
|
t = (title or "").strip() |
|
|
|
|
|
u = (url or "").strip().lower() |
|
|
|
|
|
if "Первая ссылка" in t: |
|
|
|
|
|
return True |
|
|
|
|
|
if u in ( |
|
|
|
|
|
"google.com", |
|
|
|
|
|
"https://google.com", |
|
|
|
|
|
"http://google.com", |
|
|
|
|
|
"https://www.google.com", |
|
|
|
|
|
"http://www.google.com", |
|
|
|
|
|
) and ("Первая" in t or t in ("google.com", "Link", "")): |
|
|
|
|
|
return True |
|
|
|
|
|
return False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def extract_domain_from_url(url: str) -> str: |
|
|
def extract_domain_from_url(url: str) -> str: |
|
|
""" |
|
|
""" |
|
|
Extracts the root domain / hostname without 'www.' from a URL. |
|
|
Extracts the root domain / hostname without 'www.' from a URL. |
|
|
@ -17,9 +56,7 @@ def extract_domain_from_url(url: str) -> str: |
|
|
if not trimmed: |
|
|
if not trimmed: |
|
|
return "" |
|
|
return "" |
|
|
|
|
|
|
|
|
candidate = trimmed |
|
|
|
|
|
if not re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://", candidate): |
|
|
|
|
|
candidate = "https://" + candidate |
|
|
|
|
|
|
|
|
candidate = ensure_external_url(trimmed) |
|
|
|
|
|
|
|
|
try: |
|
|
try: |
|
|
parsed = urlparse(candidate) |
|
|
parsed = urlparse(candidate) |
|
|
@ -42,7 +79,8 @@ def extract_domain_from_url(url: str) -> str: |
|
|
def normalize_links_for_storage(links_data): |
|
|
def normalize_links_for_storage(links_data): |
|
|
""" |
|
|
""" |
|
|
Ensures any link without a title (or with an empty / generic placeholder title) |
|
|
Ensures any link without a title (or with an empty / generic placeholder title) |
|
|
has its title automatically set to the domain of the URL. |
|
|
|
|
|
|
|
|
has its title automatically set to the domain of the URL, and ensures the URL |
|
|
|
|
|
starts with http:// or https://. |
|
|
Returns the normalized structure ready to be stored in JSONField. |
|
|
Returns the normalized structure ready to be stored in JSONField. |
|
|
""" |
|
|
""" |
|
|
if not links_data: |
|
|
if not links_data: |
|
|
@ -64,8 +102,11 @@ def normalize_links_for_storage(links_data): |
|
|
normalized = {} |
|
|
normalized = {} |
|
|
for title, url_or_obj in links_data.items(): |
|
|
for title, url_or_obj in links_data.items(): |
|
|
if isinstance(url_or_obj, str) and url_or_obj.strip(): |
|
|
if isinstance(url_or_obj, str) and url_or_obj.strip(): |
|
|
url = url_or_obj.strip() |
|
|
|
|
|
|
|
|
raw_url = url_or_obj.strip() |
|
|
t = str(title).strip() if title else "" |
|
|
t = str(title).strip() if title else "" |
|
|
|
|
|
if is_dummy_placeholder_link(t, raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
|
|
|
|
|
|
@ -78,10 +119,13 @@ def normalize_links_for_storage(links_data): |
|
|
normalized[key] = url |
|
|
normalized[key] = url |
|
|
|
|
|
|
|
|
elif isinstance(url_or_obj, dict): |
|
|
elif isinstance(url_or_obj, dict): |
|
|
url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") |
|
|
|
|
|
if url and isinstance(url, str) and url.strip(): |
|
|
|
|
|
url = url.strip() |
|
|
|
|
|
|
|
|
raw_url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") |
|
|
|
|
|
if raw_url and isinstance(raw_url, str) and raw_url.strip(): |
|
|
|
|
|
raw_url = raw_url.strip() |
|
|
t = str(url_or_obj.get("title") or title or "").strip() |
|
|
t = str(url_or_obj.get("title") or title or "").strip() |
|
|
|
|
|
if is_dummy_placeholder_link(t, raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
key = t |
|
|
key = t |
|
|
@ -97,14 +141,20 @@ def normalize_links_for_storage(links_data): |
|
|
normalized = [] |
|
|
normalized = [] |
|
|
for item in links_data: |
|
|
for item in links_data: |
|
|
if isinstance(item, str) and item.strip(): |
|
|
if isinstance(item, str) and item.strip(): |
|
|
url = item.strip() |
|
|
|
|
|
|
|
|
raw_url = item.strip() |
|
|
|
|
|
if is_dummy_placeholder_link("", raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
title = extract_domain_from_url(url) or "Link" |
|
|
title = extract_domain_from_url(url) or "Link" |
|
|
normalized.append({"title": title, "url": url}) |
|
|
normalized.append({"title": title, "url": url}) |
|
|
elif isinstance(item, dict): |
|
|
elif isinstance(item, dict): |
|
|
url = item.get("url") or item.get("link") or item.get("href") or item.get("value") |
|
|
|
|
|
if url and isinstance(url, str) and url.strip(): |
|
|
|
|
|
url = url.strip() |
|
|
|
|
|
|
|
|
raw_url = item.get("url") or item.get("link") or item.get("href") or item.get("value") |
|
|
|
|
|
if raw_url and isinstance(raw_url, str) and raw_url.strip(): |
|
|
|
|
|
raw_url = raw_url.strip() |
|
|
title = str(item.get("title") or item.get("name") or "").strip() |
|
|
title = str(item.get("title") or item.get("name") or "").strip() |
|
|
|
|
|
if is_dummy_placeholder_link(title, raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
if not title or title.lower() in ("link", "untitled", "url", "default"): |
|
|
if not title or title.lower() in ("link", "untitled", "url", "default"): |
|
|
title = extract_domain_from_url(url) or "Link" |
|
|
title = extract_domain_from_url(url) or "Link" |
|
|
normalized.append({"title": title, "url": url}) |
|
|
normalized.append({"title": title, "url": url}) |
|
|
@ -121,7 +171,8 @@ def format_links_output(links_data): |
|
|
{"title": "islamweb.net", "url": "https://www.islamweb.net/ar/fatwa/1909"} |
|
|
{"title": "islamweb.net", "url": "https://www.islamweb.net/ar/fatwa/1909"} |
|
|
] |
|
|
] |
|
|
Handles dict, list of dicts, list of strings, and raw JSON strings cleanly. |
|
|
Handles dict, list of dicts, list of strings, and raw JSON strings cleanly. |
|
|
Always ensures title defaults to domain if title is empty or generic 'Link'. |
|
|
|
|
|
|
|
|
Always ensures title defaults to domain if title is empty or generic 'Link', |
|
|
|
|
|
and ensures url has an explicit http(s):// scheme. |
|
|
""" |
|
|
""" |
|
|
if not links_data: |
|
|
if not links_data: |
|
|
return [] |
|
|
return [] |
|
|
@ -135,24 +186,31 @@ def format_links_output(links_data): |
|
|
return format_links_output(parsed) |
|
|
return format_links_output(parsed) |
|
|
except (ValueError, TypeError, json.JSONDecodeError): |
|
|
except (ValueError, TypeError, json.JSONDecodeError): |
|
|
pass |
|
|
pass |
|
|
if trimmed: |
|
|
|
|
|
return [{"title": extract_domain_from_url(trimmed) or "Link", "url": trimmed}] |
|
|
|
|
|
|
|
|
if trimmed and not is_dummy_placeholder_link("", trimmed): |
|
|
|
|
|
url = ensure_external_url(trimmed) |
|
|
|
|
|
return [{"title": extract_domain_from_url(url) or "Link", "url": url}] |
|
|
return [] |
|
|
return [] |
|
|
|
|
|
|
|
|
if isinstance(links_data, dict): |
|
|
if isinstance(links_data, dict): |
|
|
out = [] |
|
|
out = [] |
|
|
for title, url_or_obj in links_data.items(): |
|
|
for title, url_or_obj in links_data.items(): |
|
|
if isinstance(url_or_obj, str) and url_or_obj.strip(): |
|
|
if isinstance(url_or_obj, str) and url_or_obj.strip(): |
|
|
url = url_or_obj.strip() |
|
|
|
|
|
|
|
|
raw_url = url_or_obj.strip() |
|
|
t = str(title).strip() if title else "" |
|
|
t = str(title).strip() if title else "" |
|
|
|
|
|
if is_dummy_placeholder_link(t, raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
out.append({"title": t, "url": url}) |
|
|
out.append({"title": t, "url": url}) |
|
|
elif isinstance(url_or_obj, dict): |
|
|
elif isinstance(url_or_obj, dict): |
|
|
url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") |
|
|
|
|
|
if url and isinstance(url, str) and url.strip(): |
|
|
|
|
|
url = url.strip() |
|
|
|
|
|
|
|
|
raw_url = url_or_obj.get("url") or url_or_obj.get("link") or url_or_obj.get("href") |
|
|
|
|
|
if raw_url and isinstance(raw_url, str) and raw_url.strip(): |
|
|
|
|
|
raw_url = raw_url.strip() |
|
|
t = str(url_or_obj.get("title") or title or "").strip() |
|
|
t = str(url_or_obj.get("title") or title or "").strip() |
|
|
|
|
|
if is_dummy_placeholder_link(t, raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
out.append({"title": t, "url": url}) |
|
|
out.append({"title": t, "url": url}) |
|
|
@ -164,16 +222,22 @@ def format_links_output(links_data): |
|
|
out = [] |
|
|
out = [] |
|
|
for item in links_data: |
|
|
for item in links_data: |
|
|
if isinstance(item, str) and item.strip(): |
|
|
if isinstance(item, str) and item.strip(): |
|
|
url = item.strip() |
|
|
|
|
|
|
|
|
raw_url = item.strip() |
|
|
|
|
|
if is_dummy_placeholder_link("", raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
out.append({ |
|
|
out.append({ |
|
|
"title": extract_domain_from_url(url) or "Link", |
|
|
"title": extract_domain_from_url(url) or "Link", |
|
|
"url": url |
|
|
"url": url |
|
|
}) |
|
|
}) |
|
|
elif isinstance(item, dict): |
|
|
elif isinstance(item, dict): |
|
|
url = item.get("url") or item.get("link") or item.get("href") or item.get("value") |
|
|
|
|
|
if url and isinstance(url, str) and url.strip(): |
|
|
|
|
|
url = url.strip() |
|
|
|
|
|
|
|
|
raw_url = item.get("url") or item.get("link") or item.get("href") or item.get("value") |
|
|
|
|
|
if raw_url and isinstance(raw_url, str) and raw_url.strip(): |
|
|
|
|
|
raw_url = raw_url.strip() |
|
|
t = str(item.get("title") or item.get("name") or "").strip() |
|
|
t = str(item.get("title") or item.get("name") or "").strip() |
|
|
|
|
|
if is_dummy_placeholder_link(t, raw_url): |
|
|
|
|
|
continue |
|
|
|
|
|
url = ensure_external_url(raw_url) |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
if not t or t.lower() in ("link", "untitled", "url", "default"): |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
t = extract_domain_from_url(url) or "Link" |
|
|
out.append({"title": t, "url": url}) |
|
|
out.append({"title": t, "url": url}) |
|
|
|