You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
289 lines
11 KiB
289 lines
11 KiB
from __future__ import annotations
|
|
|
|
import datetime
|
|
import html
|
|
import logging
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from typing import Optional, Dict, Any, List
|
|
from urllib.parse import urlparse
|
|
|
|
import requests
|
|
from django.conf import settings
|
|
from django.db import transaction, close_old_connections, connection
|
|
from django.utils import timezone
|
|
from django.utils.text import slugify
|
|
|
|
from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem
|
|
from apps.podcast.signals import invalidate_podcast_cache
|
|
from utils.slug import generate_smart_slug
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205"
|
|
|
|
|
|
def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time:
|
|
"""
|
|
Parse itunes:duration into datetime.time.
|
|
Supports formats:
|
|
- Total seconds: '3037'
|
|
- MM:SS: '45:30'
|
|
- HH:MM:SS: '01:15:30'
|
|
"""
|
|
if not duration_str:
|
|
return datetime.time(0, 0, 0)
|
|
|
|
duration_str = str(duration_str).strip()
|
|
try:
|
|
# Pure seconds
|
|
if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str):
|
|
total_seconds = int(float(duration_str))
|
|
hours = (total_seconds // 3600) % 24
|
|
minutes = (total_seconds % 3600) // 60
|
|
seconds = total_seconds % 60
|
|
return datetime.time(hours, minutes, seconds)
|
|
|
|
# HH:MM:SS or MM:SS
|
|
parts = [int(p) for p in duration_str.split(':') if p.isdigit()]
|
|
if len(parts) == 3:
|
|
return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60)
|
|
elif len(parts) == 2:
|
|
return datetime.time(0, parts[0] % 60, parts[1] % 60)
|
|
elif len(parts) == 1:
|
|
return datetime.time(0, 0, parts[0] % 60)
|
|
except Exception as e:
|
|
logger.warning(f"Error parsing duration '{duration_str}': {e}")
|
|
|
|
return datetime.time(0, 0, 0)
|
|
|
|
|
|
def clean_html_text(raw_text: Optional[str]) -> str:
|
|
"""Strips HTML tags and unescapes HTML entities for clean plaintext storage."""
|
|
if not raw_text:
|
|
return ""
|
|
text = re.sub(r'<[^>]+>', ' ', raw_text)
|
|
text = html.unescape(text)
|
|
text = re.sub(r'[ \t]+', ' ', text)
|
|
text = re.sub(r'\n\s*\n', '\n\n', text)
|
|
return text.strip()
|
|
|
|
|
|
class MaveRSSImporter:
|
|
"""
|
|
High-performance bulk synchronization service for podcast RSS feeds.
|
|
Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations.
|
|
"""
|
|
|
|
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None):
|
|
self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL
|
|
self.user_agent = user_agent or (
|
|
"Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)"
|
|
)
|
|
|
|
def fetch_feed_content(self) -> bytes:
|
|
headers = {
|
|
"User-Agent": self.user_agent,
|
|
"Accept": "application/rss+xml, application/xml, text/xml, */*",
|
|
}
|
|
response = requests.get(self.feed_url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.content
|
|
|
|
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]:
|
|
"""
|
|
Main synchronization method using bulk operations.
|
|
"""
|
|
logger.info(f"Starting RSS sync for feed: {self.feed_url}")
|
|
|
|
# Step 1: Fetch and parse feed in-memory (0 DB queries)
|
|
content = self.fetch_feed_content()
|
|
root = ET.fromstring(content)
|
|
|
|
channel = root.find("channel")
|
|
if channel is None:
|
|
raise ValueError("Invalid RSS: <channel> element not found.")
|
|
|
|
channel_title = (channel.findtext("title") or "پادکست").strip()
|
|
channel_desc = clean_html_text(channel.findtext("description") or "")
|
|
channel_img_el = channel.find("image")
|
|
channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None
|
|
if not channel_img_url:
|
|
itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
|
|
if itunes_img is not None:
|
|
channel_img_url = itunes_img.get("href")
|
|
|
|
raw_items = channel.findall("item")
|
|
total_items_in_feed = len(raw_items)
|
|
|
|
# Parse items in-memory (oldest first so priority increments sequentially)
|
|
parsed_items: List[Dict[str, Any]] = []
|
|
for item in reversed(raw_items):
|
|
guid_el = item.find("guid")
|
|
guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None
|
|
enclosure = item.find("enclosure")
|
|
audio_url = enclosure.get("url") if enclosure is not None else None
|
|
|
|
if not audio_url:
|
|
continue
|
|
|
|
if not guid:
|
|
guid = audio_url
|
|
|
|
title = (item.findtext("title") or "قسمت پادکست").strip()
|
|
description = clean_html_text(
|
|
item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary")
|
|
or item.findtext("description")
|
|
or ""
|
|
)
|
|
duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration")
|
|
audio_time = parse_duration_to_time(duration_str)
|
|
|
|
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
|
|
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url
|
|
|
|
parsed_items.append({
|
|
"guid": guid,
|
|
"title": title,
|
|
"description": description,
|
|
"audio_time": audio_time,
|
|
"audio_url": audio_url,
|
|
"thumbnail_url": thumbnail_url,
|
|
})
|
|
|
|
|
|
|
|
# Step 2: Resolve or create playlist (1-2 queries)
|
|
if playlist is None:
|
|
playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first()
|
|
if not playlist:
|
|
playlist = PodcastPlaylist(
|
|
title=channel_title,
|
|
description=channel_desc,
|
|
rss_feed_url=self.feed_url,
|
|
status=True,
|
|
)
|
|
playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist)
|
|
playlist.save()
|
|
else:
|
|
if not playlist.rss_feed_url:
|
|
playlist.rss_feed_url = self.feed_url
|
|
playlist.save(update_fields=['rss_feed_url'])
|
|
|
|
# Step 3: Check which items already exist in DB using a SINGLE IN query!
|
|
all_guids = [p["guid"] for p in parsed_items]
|
|
existing_guid_map = {
|
|
p.rss_guid: p.id
|
|
for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid")
|
|
}
|
|
|
|
# Check existing playlist items (1 query)
|
|
existing_playlist_podcast_ids = set(
|
|
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True)
|
|
)
|
|
|
|
existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first()
|
|
current_priority = existing_max_priority.priority if existing_max_priority else 0
|
|
|
|
# Step 4: Separate new podcasts vs existing ones
|
|
new_items_to_create: List[Dict[str, Any]] = []
|
|
existing_podcasts_to_link: List[int] = []
|
|
|
|
# Pre-fetch existing slugs in DB to avoid collisions
|
|
existing_slugs = set(Podcast.objects.values_list("slug", flat=True))
|
|
|
|
for item in parsed_items:
|
|
guid = item["guid"]
|
|
if guid in existing_guid_map:
|
|
podcast_id = existing_guid_map[guid]
|
|
if podcast_id not in existing_playlist_podcast_ids:
|
|
existing_podcasts_to_link.append(podcast_id)
|
|
else:
|
|
# Generate unique slug in-memory
|
|
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}"
|
|
slug = base_slug[:200]
|
|
counter = 1
|
|
while slug in existing_slugs:
|
|
slug = f"{base_slug[:190]}-{counter}"
|
|
counter += 1
|
|
existing_slugs.add(slug)
|
|
item["generated_slug"] = slug
|
|
new_items_to_create.append(item)
|
|
|
|
created_count = 0
|
|
|
|
# Step 5: Bulk create new podcasts (1 query!)
|
|
with transaction.atomic():
|
|
if new_items_to_create:
|
|
podcasts_to_insert = [
|
|
Podcast(
|
|
title=item["title"],
|
|
slug=item["generated_slug"],
|
|
description=item["description"],
|
|
audio_time=item["audio_time"],
|
|
external_audio_url=item["audio_url"],
|
|
external_thumbnail_url=item["thumbnail_url"],
|
|
rss_guid=item["guid"],
|
|
status=True,
|
|
)
|
|
for item in new_items_to_create
|
|
]
|
|
Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200)
|
|
created_count = len(podcasts_to_insert)
|
|
|
|
# Retrieve newly created podcast IDs (1 query)
|
|
new_guids = [item["guid"] for item in new_items_to_create]
|
|
newly_created_ids = list(
|
|
Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True)
|
|
)
|
|
|
|
# Step 6: Bulk create PlaylistItem links (1 query!)
|
|
all_ids_to_link = existing_podcasts_to_link + newly_created_ids
|
|
if all_ids_to_link:
|
|
playlist_items_to_create = []
|
|
for pid in all_ids_to_link:
|
|
current_priority += 1
|
|
playlist_items_to_create.append(
|
|
PlaylistItem(
|
|
playlist=playlist,
|
|
podcast_id=pid,
|
|
priority=current_priority,
|
|
)
|
|
)
|
|
PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200)
|
|
|
|
# Step 7: Update playlist total time & sync date (1 query)
|
|
playlist.total_time = playlist.calculate_total_time()
|
|
playlist.last_synced_at = timezone.now()
|
|
playlist.save(update_fields=['total_time', 'last_synced_at'])
|
|
|
|
# Invalidate caches
|
|
invalidate_podcast_cache()
|
|
|
|
skipped_count = total_items_in_feed - created_count
|
|
|
|
result = {
|
|
"status": "success",
|
|
"playlist_id": playlist.id,
|
|
"playlist_title": playlist.title,
|
|
"total_feed_items": total_items_in_feed,
|
|
"created": created_count,
|
|
"skipped": skipped_count,
|
|
"errors": [],
|
|
"synced_at": playlist.last_synced_at.isoformat(),
|
|
}
|
|
logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}")
|
|
return result
|
|
|
|
|
|
def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]:
|
|
"""Convenience helper function for manual or automated sync."""
|
|
playlist = None
|
|
if playlist_id:
|
|
playlist = PodcastPlaylist.objects.get(id=playlist_id)
|
|
if not feed_url:
|
|
feed_url = playlist.rss_feed_url
|
|
|
|
url = feed_url or DEFAULT_MAVE_FEED_URL
|
|
importer = MaveRSSImporter(feed_url=url)
|
|
return importer.sync(playlist=playlist)
|