diff --git a/apps/podcast/migrations/0007_podcastplaylist_rss_season_number.py b/apps/podcast/migrations/0007_podcastplaylist_rss_season_number.py new file mode 100644 index 0000000..7892633 --- /dev/null +++ b/apps/podcast/migrations/0007_podcastplaylist_rss_season_number.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.30 on 2026-11-10 09:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('podcast', '0006_podcast_external_audio_url_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='podcastplaylist', + name='rss_season_number', + field=models.PositiveIntegerField(blank=True, db_index=True, null=True, verbose_name='RSS Season Number'), + ), + ] diff --git a/apps/podcast/models.py b/apps/podcast/models.py index aed58e1..e1a24f9 100644 --- a/apps/podcast/models.py +++ b/apps/podcast/models.py @@ -180,6 +180,7 @@ class PodcastPlaylist(LowercaseSlugMixin, models.Model): # RSS Sync Support rss_feed_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('RSS Feed URL')) + rss_season_number = models.PositiveIntegerField(null=True, blank=True, db_index=True, verbose_name=_('RSS Season Number')) last_synced_at = models.DateTimeField(null=True, blank=True, verbose_name=_('Last Synced At')) created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at')) diff --git a/apps/podcast/serializers_admin.py b/apps/podcast/serializers_admin.py index 986cefa..c70e831 100644 --- a/apps/podcast/serializers_admin.py +++ b/apps/podcast/serializers_admin.py @@ -277,6 +277,7 @@ class AdminPodcastPlaylistListSerializer(serializers.ModelSerializer): "collections_detail", "items_count", "rss_feed_url", + "rss_season_number", "last_synced_at", ] read_only_fields = ["id", "view_count", "total_time", "created_at", "updated_at", "items_count"] @@ -321,6 +322,7 @@ class AdminPodcastPlaylistDetailSerializer(serializers.ModelSerializer): "collections_detail", "playlist_items_detail", "rss_feed_url", + "rss_season_number", "last_synced_at", "remove_thumbnail", ] diff --git a/apps/podcast/services/rss_importer.py b/apps/podcast/services/rss_importer.py index ab6a460..964385c 100644 --- a/apps/podcast/services/rss_importer.py +++ b/apps/podcast/services/rss_importer.py @@ -5,12 +5,13 @@ import html import logging import re import xml.etree.ElementTree as ET +from collections import defaultdict, Counter from typing import Optional, Dict, Any, List -from urllib.parse import urlparse import requests from django.conf import settings -from django.db import transaction, close_old_connections, connection +from django.core.files.base import ContentFile +from django.db import transaction from django.utils import timezone from django.utils.text import slugify @@ -69,10 +70,44 @@ def clean_html_text(raw_text: Optional[str]) -> str: return text.strip() +def extract_season_title(items: List[Dict[str, Any]], season_num: Optional[int], channel_title: str) -> str: + """ + Dynamically extracts the series/playlist title for a given season. + Works dynamically for any season (e.g. season 7, 8, etc.) without hardcoding. + """ + titles = [it.get("title", "") for it in items if it.get("title")] + candidates = [] + for t in titles: + # Strip fraction patterns like ' 36/36' or ' 1/20' + cleaned = re.sub(r'\s+\d+/\d+.*$', '', t) + # Strip episode markers + cleaned = re.split( + r'\s+[—–-]\s*(?:Выпуск|выпуск|Часть|часть|Урок|урок|Серия|серия|Эпизод|эпизод|Беседа|беседа|Episode|episode|Part|part|Лекция|лекция)', + cleaned + )[0] + # Strip trailing dash + number: ' — #1', ' - 12' + cleaned = re.split(r'\s+[—–-]\s*#?\d+', cleaned)[0] + # Strip trailing '#12' + cleaned = re.split(r'\s+#\d+', cleaned)[0] + cleaned = cleaned.strip(' —–-:,') + if cleaned: + candidates.append(cleaned) + + if candidates: + counts = Counter(candidates) + most_common, freq = counts.most_common(1)[0] + if freq >= max(1, len(titles) * 0.25): + return most_common + + if season_num is not None: + return f"{channel_title} - Сезон {season_num}" + return channel_title + + class MaveRSSImporter: """ High-performance bulk synchronization service for podcast RSS feeds. - Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations. + Dynamically groups episodes into season-based playlists. """ def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None): @@ -93,10 +128,14 @@ class MaveRSSImporter: def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]: """ Main synchronization method using bulk operations. + Dynamically handles multi-season feeds: + - Automatically identifies all seasons in the feed (including any newly added seasons) + - Extracts season titles and season covers dynamically + - Resolves or creates season-specific PodcastPlaylist records + - Correctly assigns episodes to their respective season playlists """ logger.info(f"Starting RSS sync for feed: {self.feed_url}") - # Step 1: Fetch and parse feed in-memory (0 DB queries) content = self.fetch_feed_content() root = ET.fromstring(content) @@ -116,7 +155,7 @@ class MaveRSSImporter: raw_items = channel.findall("item") total_items_in_feed = len(raw_items) - # Parse items in-memory (oldest first so priority increments sequentially) + # Parse items in-memory parsed_items: List[Dict[str, Any]] = [] for item in reversed(raw_items): guid_el = item.find("guid") @@ -142,6 +181,12 @@ class MaveRSSImporter: thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url + season_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}season") + season_num = int(season_str) if (season_str and season_str.isdigit()) else None + + episode_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}episode") + episode_num = int(episode_str) if (episode_str and episode_str.isdigit()) else None + parsed_items.append({ "guid": guid, "title": title, @@ -149,69 +194,33 @@ class MaveRSSImporter: "audio_time": audio_time, "audio_url": audio_url, "thumbnail_url": thumbnail_url, + "season_num": season_num, + "episode_num": episode_num, }) + # Group items by season + seasons_map: Dict[Optional[int], List[Dict[str, Any]]] = defaultdict(list) + for it in parsed_items: + seasons_map[it["season_num"]].append(it) + # Sort items inside each season by episode_num or reverse order + for s_num in seasons_map: + seasons_map[s_num].sort( + key=lambda x: (x["episode_num"] is None, x["episode_num"] or 0) + ) - # Step 2: Resolve or create playlist (1-2 queries) - if playlist is None: - playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first() - if not playlist: - playlist = PodcastPlaylist( - title=channel_title, - description=channel_desc, - rss_feed_url=self.feed_url, - status=True, - ) - playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist) - playlist.save() - else: - if not playlist.rss_feed_url: - playlist.rss_feed_url = self.feed_url - playlist.save(update_fields=['rss_feed_url']) - - # Download playlist cover if missing - if not playlist.thumbnail and channel_img_url: - try: - img_res = requests.get(channel_img_url, timeout=10) - if img_res.status_code == 200 and img_res.content: - from django.core.files.base import ContentFile - ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg" - playlist.thumbnail.save(f"playlist_{playlist.id}.{ext}", ContentFile(img_res.content), save=True) - except Exception as img_err: - logger.warning(f"Could not download playlist thumbnail: {img_err}") - - # Step 3: Check which items already exist in DB using a SINGLE IN query! + # Check existing podcasts in DB across all feed items (1 single query) all_guids = [p["guid"] for p in parsed_items] - existing_guid_map = { - p.rss_guid: p.id - for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid") - } - - # Check existing playlist items (1 query) - existing_playlist_podcast_ids = set( - PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True) - ) - - existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first() - current_priority = existing_max_priority.priority if existing_max_priority else 0 + existing_podcasts_qs = Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid") + existing_guid_map = {p.rss_guid: p.id for p in existing_podcasts_qs} - # Step 4: Separate new podcasts vs existing ones - new_items_to_create: List[Dict[str, Any]] = [] - existing_podcasts_to_link: List[int] = [] - - # Pre-fetch existing slugs in DB to avoid collisions + # Pre-fetch existing slugs to prevent collisions existing_slugs = set(Podcast.objects.values_list("slug", flat=True)) + new_podcasts_to_create: List[Podcast] = [] for item in parsed_items: - guid = item["guid"] - if guid in existing_guid_map: - podcast_id = existing_guid_map[guid] - if podcast_id not in existing_playlist_podcast_ids: - existing_podcasts_to_link.append(podcast_id) - else: - # Generate unique slug in-memory - base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}" + if item["guid"] not in existing_guid_map: + base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{item['guid'][:8]}" slug = base_slug[:200] counter = 1 while slug in existing_slugs: @@ -219,17 +228,11 @@ class MaveRSSImporter: counter += 1 existing_slugs.add(slug) item["generated_slug"] = slug - new_items_to_create.append(item) - created_count = 0 - - # Step 5: Bulk create new podcasts (1 query!) - with transaction.atomic(): - if new_items_to_create: - podcasts_to_insert = [ + new_podcasts_to_create.append( Podcast( title=item["title"], - slug=item["generated_slug"], + slug=slug, description=item["description"], audio_time=item["audio_time"], external_audio_url=item["audio_url"], @@ -237,68 +240,166 @@ class MaveRSSImporter: rss_guid=item["guid"], status=True, ) - for item in new_items_to_create - ] - Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200) - created_count = len(podcasts_to_insert) + ) + + created_podcasts_count = len(new_podcasts_to_create) + if new_podcasts_to_create: + Podcast.objects.bulk_create(new_podcasts_to_create, batch_size=200) + # Re-fetch the newly created podcasts to get their DB ids + new_guid_qs = Podcast.objects.filter( + rss_guid__in=[p["guid"] for p in parsed_items if p["guid"] not in existing_guid_map] + ).only("id", "rss_guid") + for p in new_guid_qs: + existing_guid_map[p.rss_guid] = p.id + + # Now process each season and link to its corresponding PodcastPlaylist + synced_playlists_info = [] + target_playlist = playlist + + # Sort seasons: 1, 2, 3... (None at end) + sorted_season_keys = sorted( + seasons_map.keys(), + key=lambda x: (x is None, x or 0) + ) - # Retrieve newly created podcast IDs (1 query) - new_guids = [item["guid"] for item in new_items_to_create] - newly_created_ids = list( - Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True) + for s_num in sorted_season_keys: + season_items = seasons_map[s_num] + season_title = extract_season_title(season_items, s_num, channel_title) + season_cover = season_items[0].get("thumbnail_url") or channel_img_url + + # Find matching playlist for this season + season_pl = None + if target_playlist and ( + target_playlist.rss_season_number == s_num + or (target_playlist.rss_season_number is None and s_num == 1) + ): + season_pl = target_playlist + if season_pl.rss_season_number is None: + season_pl.rss_season_number = s_num + season_pl.save(update_fields=['rss_season_number']) + else: + if s_num is not None: + season_pl = PodcastPlaylist.objects.filter( + rss_feed_url=self.feed_url, + rss_season_number=s_num + ).first() + + if not season_pl and s_num == 1: + # Fallback to playlist with null rss_season_number if matching feed + season_pl = PodcastPlaylist.objects.filter( + rss_feed_url=self.feed_url, + rss_season_number__isnull=True + ).first() + if season_pl: + season_pl.rss_season_number = 1 + season_pl.save(update_fields=['rss_season_number']) + + is_new_playlist = False + if not season_pl: + # Create dynamically for any season (e.g. season 7, 8, etc.) + season_pl = PodcastPlaylist( + title=season_title, + description=channel_desc, + rss_feed_url=self.feed_url, + rss_season_number=s_num, + status=True, + ) + season_pl.slug = generate_smart_slug(season_title, PodcastPlaylist) + season_pl.save() + is_new_playlist = True + + # Download cover if playlist has no thumbnail + if not season_pl.thumbnail and season_cover: + try: + img_res = requests.get(season_cover, timeout=10) + if img_res.status_code == 200 and img_res.content: + ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg" + season_pl.thumbnail.save(f"playlist_{season_pl.id}.{ext}", ContentFile(img_res.content), save=True) + except Exception as img_err: + logger.warning(f"Could not download playlist thumbnail: {img_err}") + + # Collect IDs of podcasts in this season + season_podcast_ids = [existing_guid_map[it["guid"]] for it in season_items if it["guid"] in existing_guid_map] + + # Cross-season cleanup: Remove these podcasts from any OTHER playlists of this feed + other_feed_playlists = PodcastPlaylist.objects.filter( + rss_feed_url=self.feed_url + ).exclude(id=season_pl.id) + if other_feed_playlists.exists(): + PlaylistItem.objects.filter( + playlist__in=other_feed_playlists, + podcast_id__in=season_podcast_ids + ).delete() + + # Now link items to season_pl + existing_links = dict( + PlaylistItem.objects.filter(playlist=season_pl).values_list("podcast_id", "priority") ) - # Step 6: Bulk create PlaylistItem links (1 query!) - all_ids_to_link = existing_podcasts_to_link + newly_created_ids - if all_ids_to_link: - playlist_items_to_create = [] - for pid in all_ids_to_link: - current_priority += 1 - playlist_items_to_create.append( + items_to_create = [] + for idx, it in enumerate(season_items, start=1): + p_id = existing_guid_map.get(it["guid"]) + if not p_id: + continue + priority = it["episode_num"] if (it["episode_num"] and it["episode_num"] > 0) else idx + if p_id not in existing_links: + items_to_create.append( PlaylistItem( - playlist=playlist, - podcast_id=pid, - priority=current_priority, + playlist=season_pl, + podcast_id=p_id, + priority=priority, ) ) - PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200) - # Step 7: Inherit playlist categories to episodes - playlist_categories = list(playlist.categories.all()) + if items_to_create: + PlaylistItem.objects.bulk_create(items_to_create, batch_size=200) + + # Inherit categories from playlist to podcasts if any + playlist_categories = list(season_pl.categories.all()) if playlist_categories: - all_playlist_podcast_ids = list( - PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True) - ) - if all_playlist_podcast_ids: - PodcastCategoryThrough = Podcast.categories.through - through_records = [ - PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id) - for p_id in all_playlist_podcast_ids - for cat in playlist_categories - ] - PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True) - - # Step 8: Update playlist total time & sync date (1 query) - playlist.total_time = playlist.calculate_total_time() - playlist.last_synced_at = timezone.now() - playlist.save(update_fields=['total_time', 'last_synced_at']) + PodcastCategoryThrough = Podcast.categories.through + through_records = [ + PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id) + for p_id in season_podcast_ids + for cat in playlist_categories + ] + PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True) + + # Update total time & sync date + season_pl.total_time = season_pl.calculate_total_time() + season_pl.last_synced_at = timezone.now() + season_pl.save(update_fields=['total_time', 'last_synced_at']) + + synced_playlists_info.append({ + "id": season_pl.id, + "title": season_pl.title, + "season_number": s_num, + "episodes_count": len(season_items), + "is_new": is_new_playlist, + }) # Invalidate caches invalidate_podcast_cache() - skipped_count = total_items_in_feed - created_count + skipped_count = total_items_in_feed - created_podcasts_count + + primary_title = ( + target_playlist.title + if target_playlist + else (f"{len(synced_playlists_info)} Playlists Synced" if len(synced_playlists_info) > 1 else (synced_playlists_info[0]["title"] if synced_playlists_info else "")) + ) result = { "status": "success", - "playlist_id": playlist.id, - "playlist_title": playlist.title, + "playlist_id": target_playlist.id if target_playlist else (synced_playlists_info[0]["id"] if synced_playlists_info else None), + "playlist_title": primary_title, "total_feed_items": total_items_in_feed, - "created": created_count, + "created": created_podcasts_count, "skipped": skipped_count, - "errors": [], - "synced_at": playlist.last_synced_at.isoformat(), + "playlists": synced_playlists_info, + "synced_at": timezone.now().isoformat(), } - logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}") + logger.info(f"RSS multi-season bulk sync completed: {len(synced_playlists_info)} playlists processed, created={created_podcasts_count}, skipped={skipped_count}") return result