from __future__ import annotations import datetime import html import logging import re import xml.etree.ElementTree as ET from collections import defaultdict, Counter from typing import Optional, Dict, Any, List import requests from django.conf import settings from django.core.files.base import ContentFile from django.db import transaction from django.utils import timezone from django.utils.text import slugify from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem from apps.podcast.signals import invalidate_podcast_cache from utils.slug import generate_smart_slug logger = logging.getLogger(__name__) DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205" def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time: """ Parse itunes:duration into datetime.time. Supports formats: - Total seconds: '3037' - MM:SS: '45:30' - HH:MM:SS: '01:15:30' """ if not duration_str: return datetime.time(0, 0, 0) duration_str = str(duration_str).strip() try: # Pure seconds if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str): total_seconds = int(float(duration_str)) hours = (total_seconds // 3600) % 24 minutes = (total_seconds % 3600) // 60 seconds = total_seconds % 60 return datetime.time(hours, minutes, seconds) # HH:MM:SS or MM:SS parts = [int(p) for p in duration_str.split(':') if p.isdigit()] if len(parts) == 3: return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60) elif len(parts) == 2: return datetime.time(0, parts[0] % 60, parts[1] % 60) elif len(parts) == 1: return datetime.time(0, 0, parts[0] % 60) except Exception as e: logger.warning(f"Error parsing duration '{duration_str}': {e}") return datetime.time(0, 0, 0) def clean_html_text(raw_text: Optional[str]) -> str: """Strips HTML tags and unescapes HTML entities for clean plaintext storage.""" if not raw_text: return "" text = re.sub(r'<[^>]+>', ' ', raw_text) text = html.unescape(text) text = re.sub(r'[ \t]+', ' ', text) text = re.sub(r'\n\s*\n', '\n\n', text) return text.strip() def extract_season_title(items: List[Dict[str, Any]], season_num: Optional[int], channel_title: str) -> str: """ Dynamically extracts the series/playlist title for a given season. Works dynamically for any season (e.g. season 7, 8, etc.) without hardcoding. """ titles = [it.get("title", "") for it in items if it.get("title")] candidates = [] for t in titles: # Strip fraction patterns like ' 36/36' or ' 1/20' cleaned = re.sub(r'\s+\d+/\d+.*$', '', t) # Strip episode markers cleaned = re.split( r'\s+[—–-]\s*(?:Выпуск|выпуск|Часть|часть|Урок|урок|Серия|серия|Эпизод|эпизод|Беседа|беседа|Episode|episode|Part|part|Лекция|лекция)', cleaned )[0] # Strip trailing dash + number: ' — #1', ' - 12' cleaned = re.split(r'\s+[—–-]\s*#?\d+', cleaned)[0] # Strip trailing '#12' cleaned = re.split(r'\s+#\d+', cleaned)[0] cleaned = cleaned.strip(' —–-:,') if cleaned: candidates.append(cleaned) if candidates: counts = Counter(candidates) most_common, freq = counts.most_common(1)[0] if freq >= max(1, len(titles) * 0.25): return most_common if season_num is not None: return f"{channel_title} - Сезон {season_num}" return channel_title class MaveRSSImporter: """ High-performance bulk synchronization service for podcast RSS feeds. Dynamically groups episodes into season-based playlists. """ def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None): self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL self.user_agent = user_agent or ( "Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)" ) def fetch_feed_content(self) -> bytes: headers = { "User-Agent": self.user_agent, "Accept": "application/rss+xml, application/xml, text/xml, */*", } response = requests.get(self.feed_url, headers=headers, timeout=30) response.raise_for_status() return response.content def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]: """ Main synchronization method using bulk operations. Dynamically handles multi-season feeds: - Automatically identifies all seasons in the feed (including any newly added seasons) - Extracts season titles and season covers dynamically - Resolves or creates season-specific PodcastPlaylist records - Correctly assigns episodes to their respective season playlists """ logger.info(f"Starting RSS sync for feed: {self.feed_url}") content = self.fetch_feed_content() root = ET.fromstring(content) channel = root.find("channel") if channel is None: raise ValueError("Invalid RSS: element not found.") channel_title = (channel.findtext("title") or "پادکست").strip() channel_desc = clean_html_text(channel.findtext("description") or "") channel_img_el = channel.find("image") channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None if not channel_img_url: itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") if itunes_img is not None: channel_img_url = itunes_img.get("href") raw_items = channel.findall("item") total_items_in_feed = len(raw_items) # Parse items in-memory parsed_items: List[Dict[str, Any]] = [] for item in reversed(raw_items): guid_el = item.find("guid") guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None enclosure = item.find("enclosure") audio_url = enclosure.get("url") if enclosure is not None else None if not audio_url: continue if not guid: guid = audio_url title = (item.findtext("title") or "قسمت پادکست").strip() description = clean_html_text( item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary") or item.findtext("description") or "" ) duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration") audio_time = parse_duration_to_time(duration_str) thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url season_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}season") season_num = int(season_str) if (season_str and season_str.isdigit()) else None episode_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}episode") episode_num = int(episode_str) if (episode_str and episode_str.isdigit()) else None parsed_items.append({ "guid": guid, "title": title, "description": description, "audio_time": audio_time, "audio_url": audio_url, "thumbnail_url": thumbnail_url, "season_num": season_num, "episode_num": episode_num, }) # Group items by season seasons_map: Dict[Optional[int], List[Dict[str, Any]]] = defaultdict(list) for it in parsed_items: seasons_map[it["season_num"]].append(it) # Sort items inside each season by episode_num or reverse order for s_num in seasons_map: seasons_map[s_num].sort( key=lambda x: (x["episode_num"] is None, x["episode_num"] or 0) ) # Check existing podcasts in DB across all feed items (1 single query) all_guids = [p["guid"] for p in parsed_items] existing_podcasts_qs = Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid") existing_guid_map = {p.rss_guid: p.id for p in existing_podcasts_qs} # Pre-fetch existing slugs to prevent collisions existing_slugs = set(Podcast.objects.values_list("slug", flat=True)) new_podcasts_to_create: List[Podcast] = [] for item in parsed_items: if item["guid"] not in existing_guid_map: base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{item['guid'][:8]}" slug = base_slug[:200] counter = 1 while slug in existing_slugs: slug = f"{base_slug[:190]}-{counter}" counter += 1 existing_slugs.add(slug) item["generated_slug"] = slug new_podcasts_to_create.append( Podcast( title=item["title"], slug=slug, description=item["description"], audio_time=item["audio_time"], external_audio_url=item["audio_url"], external_thumbnail_url=item["thumbnail_url"], rss_guid=item["guid"], status=True, ) ) created_podcasts_count = len(new_podcasts_to_create) created_guids_set = {p.rss_guid for p in new_podcasts_to_create} if new_podcasts_to_create: Podcast.objects.bulk_create(new_podcasts_to_create, batch_size=200) # Re-fetch the newly created podcasts to get their DB ids new_guid_qs = Podcast.objects.filter( rss_guid__in=[p["guid"] for p in parsed_items if p["guid"] not in existing_guid_map] ).only("id", "rss_guid") for p in new_guid_qs: existing_guid_map[p.rss_guid] = p.id # Now process each season and link to its corresponding PodcastPlaylist synced_playlists_info = [] added_episodes_list: List[Dict[str, Any]] = [] target_playlist = playlist # Sort seasons: 1, 2, 3... (None at end) sorted_season_keys = sorted( seasons_map.keys(), key=lambda x: (x is None, x or 0) ) for s_num in sorted_season_keys: season_items = seasons_map[s_num] season_title = extract_season_title(season_items, s_num, channel_title) season_cover = season_items[0].get("thumbnail_url") or channel_img_url # Find matching playlist for this season season_pl = None if target_playlist and ( target_playlist.rss_season_number == s_num or (target_playlist.rss_season_number is None and s_num == 1) ): season_pl = target_playlist if season_pl.rss_season_number is None: season_pl.rss_season_number = s_num season_pl.save(update_fields=['rss_season_number']) else: if s_num is not None: season_pl = PodcastPlaylist.objects.filter( rss_feed_url=self.feed_url, rss_season_number=s_num ).first() if not season_pl and s_num == 1: # Fallback to playlist with null rss_season_number if matching feed season_pl = PodcastPlaylist.objects.filter( rss_feed_url=self.feed_url, rss_season_number__isnull=True ).first() if season_pl: season_pl.rss_season_number = 1 season_pl.save(update_fields=['rss_season_number']) is_new_playlist = False if not season_pl: # Create dynamically for any season (e.g. season 7, 8, etc.) season_pl = PodcastPlaylist( title=season_title, description=channel_desc, rss_feed_url=self.feed_url, rss_season_number=s_num, status=True, ) season_pl.slug = generate_smart_slug(season_title, PodcastPlaylist) season_pl.save() is_new_playlist = True # Download cover if playlist has no thumbnail if not season_pl.thumbnail and season_cover: try: img_res = requests.get(season_cover, timeout=10) if img_res.status_code == 200 and img_res.content: ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg" season_pl.thumbnail.save(f"playlist_{season_pl.id}.{ext}", ContentFile(img_res.content), save=True) except Exception as img_err: logger.warning(f"Could not download playlist thumbnail: {img_err}") # Collect IDs of podcasts in this season season_podcast_ids = [existing_guid_map[it["guid"]] for it in season_items if it["guid"] in existing_guid_map] # Cross-season cleanup: Remove these podcasts from any OTHER playlists of this feed other_feed_playlists = PodcastPlaylist.objects.filter( rss_feed_url=self.feed_url ).exclude(id=season_pl.id) if other_feed_playlists.exists(): PlaylistItem.objects.filter( playlist__in=other_feed_playlists, podcast_id__in=season_podcast_ids ).delete() # Now link items to season_pl existing_links = dict( PlaylistItem.objects.filter(playlist=season_pl).values_list("podcast_id", "priority") ) items_to_create = [] for idx, it in enumerate(season_items, start=1): p_id = existing_guid_map.get(it["guid"]) if not p_id: continue priority = it["episode_num"] if (it["episode_num"] and it["episode_num"] > 0) else idx if p_id not in existing_links: items_to_create.append( PlaylistItem( playlist=season_pl, podcast_id=p_id, priority=priority, ) ) if items_to_create: PlaylistItem.objects.bulk_create(items_to_create, batch_size=200) newly_linked_pids = {it.podcast_id for it in items_to_create} for it in season_items: p_id = existing_guid_map.get(it["guid"]) if p_id and (it["guid"] in created_guids_set or p_id in newly_linked_pids): added_episodes_list.append({ "id": p_id, "title": it["title"], "season_number": s_num, "episode_number": it["episode_num"], "playlist_id": season_pl.id, "playlist_title": season_pl.title, "audio_time": str(it["audio_time"]) if it["audio_time"] else None, }) # Inherit categories from playlist to podcasts if any playlist_categories = list(season_pl.categories.all()) if playlist_categories: PodcastCategoryThrough = Podcast.categories.through through_records = [ PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id) for p_id in season_podcast_ids for cat in playlist_categories ] PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True) # Update total time & sync date season_pl.total_time = season_pl.calculate_total_time() season_pl.last_synced_at = timezone.now() season_pl.save(update_fields=['total_time', 'last_synced_at']) synced_playlists_info.append({ "id": season_pl.id, "title": season_pl.title, "season_number": s_num, "episodes_count": len(season_items), "is_new": is_new_playlist, }) # Invalidate caches invalidate_podcast_cache() skipped_count = total_items_in_feed - created_podcasts_count primary_title = ( target_playlist.title if target_playlist else (f"{len(synced_playlists_info)} Playlists Synced" if len(synced_playlists_info) > 1 else (synced_playlists_info[0]["title"] if synced_playlists_info else "")) ) created_playlists = [ { "id": p["id"], "title": p["title"], "season_number": p["season_number"], "episodes_count": p["episodes_count"], } for p in synced_playlists_info if p.get("is_new") ] result = { "status": "success", "playlist_id": target_playlist.id if target_playlist else (synced_playlists_info[0]["id"] if synced_playlists_info else None), "playlist_title": primary_title, "total_feed_items": total_items_in_feed, "created": created_podcasts_count, "skipped": skipped_count, "created_playlists": created_playlists, "created_episodes": added_episodes_list, "playlists": synced_playlists_info, "synced_at": timezone.now().isoformat(), } logger.info(f"RSS multi-season bulk sync completed: {len(synced_playlists_info)} playlists processed, created={created_podcasts_count}, skipped={skipped_count}") return result def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: """Convenience helper function for manual or automated sync.""" playlist = None if playlist_id: playlist = PodcastPlaylist.objects.get(id=playlist_id) if not feed_url: feed_url = playlist.rss_feed_url url = feed_url or DEFAULT_MAVE_FEED_URL importer = MaveRSSImporter(feed_url=url) return importer.sync(playlist=playlist)