from __future__ import annotations import datetime import html import logging import re import xml.etree.ElementTree as ET from typing import Optional, Dict, Any, List from urllib.parse import urlparse import requests from django.conf import settings from django.db import transaction, close_old_connections, connection from django.utils import timezone from django.utils.text import slugify from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem from apps.podcast.signals import invalidate_podcast_cache from utils.slug import generate_smart_slug logger = logging.getLogger(__name__) DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205" def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time: """ Parse itunes:duration into datetime.time. Supports formats: - Total seconds: '3037' - MM:SS: '45:30' - HH:MM:SS: '01:15:30' """ if not duration_str: return datetime.time(0, 0, 0) duration_str = str(duration_str).strip() try: # Pure seconds if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str): total_seconds = int(float(duration_str)) hours = (total_seconds // 3600) % 24 minutes = (total_seconds % 3600) // 60 seconds = total_seconds % 60 return datetime.time(hours, minutes, seconds) # HH:MM:SS or MM:SS parts = [int(p) for p in duration_str.split(':') if p.isdigit()] if len(parts) == 3: return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60) elif len(parts) == 2: return datetime.time(0, parts[0] % 60, parts[1] % 60) elif len(parts) == 1: return datetime.time(0, 0, parts[0] % 60) except Exception as e: logger.warning(f"Error parsing duration '{duration_str}': {e}") return datetime.time(0, 0, 0) def clean_html_text(raw_text: Optional[str]) -> str: """Strips HTML tags and unescapes HTML entities for clean plaintext storage.""" if not raw_text: return "" text = re.sub(r'<[^>]+>', ' ', raw_text) text = html.unescape(text) text = re.sub(r'[ \t]+', ' ', text) text = re.sub(r'\n\s*\n', '\n\n', text) return text.strip() class MaveRSSImporter: """ High-performance bulk synchronization service for podcast RSS feeds. Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations. """ def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None): self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL self.user_agent = user_agent or ( "Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)" ) def fetch_feed_content(self) -> bytes: headers = { "User-Agent": self.user_agent, "Accept": "application/rss+xml, application/xml, text/xml, */*", } response = requests.get(self.feed_url, headers=headers, timeout=30) response.raise_for_status() return response.content def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]: """ Main synchronization method using bulk operations. """ logger.info(f"Starting RSS sync for feed: {self.feed_url}") # Step 1: Fetch and parse feed in-memory (0 DB queries) content = self.fetch_feed_content() root = ET.fromstring(content) channel = root.find("channel") if channel is None: raise ValueError("Invalid RSS: element not found.") channel_title = (channel.findtext("title") or "پادکست").strip() channel_desc = clean_html_text(channel.findtext("description") or "") channel_img_el = channel.find("image") channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None if not channel_img_url: itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") if itunes_img is not None: channel_img_url = itunes_img.get("href") raw_items = channel.findall("item") total_items_in_feed = len(raw_items) # Parse items in-memory (oldest first so priority increments sequentially) parsed_items: List[Dict[str, Any]] = [] for item in reversed(raw_items): guid_el = item.find("guid") guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None enclosure = item.find("enclosure") audio_url = enclosure.get("url") if enclosure is not None else None if not audio_url: continue if not guid: guid = audio_url title = (item.findtext("title") or "قسمت پادکست").strip() description = clean_html_text( item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary") or item.findtext("description") or "" ) duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration") audio_time = parse_duration_to_time(duration_str) thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url parsed_items.append({ "guid": guid, "title": title, "description": description, "audio_time": audio_time, "audio_url": audio_url, "thumbnail_url": thumbnail_url, }) # Step 2: Resolve or create playlist (1-2 queries) if playlist is None: playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first() if not playlist: playlist = PodcastPlaylist( title=channel_title, description=channel_desc, rss_feed_url=self.feed_url, status=True, ) playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist) playlist.save() else: if not playlist.rss_feed_url: playlist.rss_feed_url = self.feed_url playlist.save(update_fields=['rss_feed_url']) # Step 3: Check which items already exist in DB using a SINGLE IN query! all_guids = [p["guid"] for p in parsed_items] existing_guid_map = { p.rss_guid: p.id for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid") } # Check existing playlist items (1 query) existing_playlist_podcast_ids = set( PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True) ) existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first() current_priority = existing_max_priority.priority if existing_max_priority else 0 # Step 4: Separate new podcasts vs existing ones new_items_to_create: List[Dict[str, Any]] = [] existing_podcasts_to_link: List[int] = [] # Pre-fetch existing slugs in DB to avoid collisions existing_slugs = set(Podcast.objects.values_list("slug", flat=True)) for item in parsed_items: guid = item["guid"] if guid in existing_guid_map: podcast_id = existing_guid_map[guid] if podcast_id not in existing_playlist_podcast_ids: existing_podcasts_to_link.append(podcast_id) else: # Generate unique slug in-memory base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}" slug = base_slug[:200] counter = 1 while slug in existing_slugs: slug = f"{base_slug[:190]}-{counter}" counter += 1 existing_slugs.add(slug) item["generated_slug"] = slug new_items_to_create.append(item) created_count = 0 # Step 5: Bulk create new podcasts (1 query!) with transaction.atomic(): if new_items_to_create: podcasts_to_insert = [ Podcast( title=item["title"], slug=item["generated_slug"], description=item["description"], audio_time=item["audio_time"], external_audio_url=item["audio_url"], external_thumbnail_url=item["thumbnail_url"], rss_guid=item["guid"], status=True, ) for item in new_items_to_create ] Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200) created_count = len(podcasts_to_insert) # Retrieve newly created podcast IDs (1 query) new_guids = [item["guid"] for item in new_items_to_create] newly_created_ids = list( Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True) ) # Step 6: Bulk create PlaylistItem links (1 query!) all_ids_to_link = existing_podcasts_to_link + newly_created_ids if all_ids_to_link: playlist_items_to_create = [] for pid in all_ids_to_link: current_priority += 1 playlist_items_to_create.append( PlaylistItem( playlist=playlist, podcast_id=pid, priority=current_priority, ) ) PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200) # Step 7: Update playlist total time & sync date (1 query) playlist.total_time = playlist.calculate_total_time() playlist.last_synced_at = timezone.now() playlist.save(update_fields=['total_time', 'last_synced_at']) # Invalidate caches invalidate_podcast_cache() skipped_count = total_items_in_feed - created_count result = { "status": "success", "playlist_id": playlist.id, "playlist_title": playlist.title, "total_feed_items": total_items_in_feed, "created": created_count, "skipped": skipped_count, "errors": [], "synced_at": playlist.last_synced_at.isoformat(), } logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}") return result def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: """Convenience helper function for manual or automated sync.""" playlist = None if playlist_id: playlist = PodcastPlaylist.objects.get(id=playlist_id) if not feed_url: feed_url = playlist.rss_feed_url url = feed_url or DEFAULT_MAVE_FEED_URL importer = MaveRSSImporter(feed_url=url) return importer.sync(playlist=playlist)