Browse Source

feat(podcast): implement dynamic multi-season RSS import and playlist grouping

master
Mohsen Taba 1 week ago
parent
commit
efafb0c83a
  1. 18
      apps/podcast/migrations/0007_podcastplaylist_rss_season_number.py
  2. 1
      apps/podcast/models.py
  3. 2
      apps/podcast/serializers_admin.py
  4. 323
      apps/podcast/services/rss_importer.py

18
apps/podcast/migrations/0007_podcastplaylist_rss_season_number.py

@ -0,0 +1,18 @@
# Generated by Django 4.2.30 on 2026-11-10 09:04
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcast', '0006_podcast_external_audio_url_and_more'),
]
operations = [
migrations.AddField(
model_name='podcastplaylist',
name='rss_season_number',
field=models.PositiveIntegerField(blank=True, db_index=True, null=True, verbose_name='RSS Season Number'),
),
]

1
apps/podcast/models.py

@ -180,6 +180,7 @@ class PodcastPlaylist(LowercaseSlugMixin, models.Model):
# RSS Sync Support # RSS Sync Support
rss_feed_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('RSS Feed URL')) rss_feed_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('RSS Feed URL'))
rss_season_number = models.PositiveIntegerField(null=True, blank=True, db_index=True, verbose_name=_('RSS Season Number'))
last_synced_at = models.DateTimeField(null=True, blank=True, verbose_name=_('Last Synced At')) last_synced_at = models.DateTimeField(null=True, blank=True, verbose_name=_('Last Synced At'))
created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at')) created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at'))

2
apps/podcast/serializers_admin.py

@ -277,6 +277,7 @@ class AdminPodcastPlaylistListSerializer(serializers.ModelSerializer):
"collections_detail", "collections_detail",
"items_count", "items_count",
"rss_feed_url", "rss_feed_url",
"rss_season_number",
"last_synced_at", "last_synced_at",
] ]
read_only_fields = ["id", "view_count", "total_time", "created_at", "updated_at", "items_count"] read_only_fields = ["id", "view_count", "total_time", "created_at", "updated_at", "items_count"]
@ -321,6 +322,7 @@ class AdminPodcastPlaylistDetailSerializer(serializers.ModelSerializer):
"collections_detail", "collections_detail",
"playlist_items_detail", "playlist_items_detail",
"rss_feed_url", "rss_feed_url",
"rss_season_number",
"last_synced_at", "last_synced_at",
"remove_thumbnail", "remove_thumbnail",
] ]

323
apps/podcast/services/rss_importer.py

@ -5,12 +5,13 @@ import html
import logging import logging
import re import re
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from collections import defaultdict, Counter
from typing import Optional, Dict, Any, List from typing import Optional, Dict, Any, List
from urllib.parse import urlparse
import requests import requests
from django.conf import settings from django.conf import settings
from django.db import transaction, close_old_connections, connection
from django.core.files.base import ContentFile
from django.db import transaction
from django.utils import timezone from django.utils import timezone
from django.utils.text import slugify from django.utils.text import slugify
@ -69,10 +70,44 @@ def clean_html_text(raw_text: Optional[str]) -> str:
return text.strip() return text.strip()
def extract_season_title(items: List[Dict[str, Any]], season_num: Optional[int], channel_title: str) -> str:
"""
Dynamically extracts the series/playlist title for a given season.
Works dynamically for any season (e.g. season 7, 8, etc.) without hardcoding.
"""
titles = [it.get("title", "") for it in items if it.get("title")]
candidates = []
for t in titles:
# Strip fraction patterns like ' 36/36' or ' 1/20'
cleaned = re.sub(r'\s+\d+/\d+.*$', '', t)
# Strip episode markers
cleaned = re.split(
r'\s+[—–-]\s*(?:Выпуск|выпуск|Часть|часть|Урок|урок|Серия|серия|Эпизод|эпизод|Беседа|беседа|Episode|episode|Part|part|Лекция|лекция)',
cleaned
)[0]
# Strip trailing dash + number: ' — #1', ' - 12'
cleaned = re.split(r'\s+[—–-]\s*#?\d+', cleaned)[0]
# Strip trailing '#12'
cleaned = re.split(r'\s+#\d+', cleaned)[0]
cleaned = cleaned.strip(' —–-:,')
if cleaned:
candidates.append(cleaned)
if candidates:
counts = Counter(candidates)
most_common, freq = counts.most_common(1)[0]
if freq >= max(1, len(titles) * 0.25):
return most_common
if season_num is not None:
return f"{channel_title} - Сезон {season_num}"
return channel_title
class MaveRSSImporter: class MaveRSSImporter:
""" """
High-performance bulk synchronization service for podcast RSS feeds. High-performance bulk synchronization service for podcast RSS feeds.
Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations.
Dynamically groups episodes into season-based playlists.
""" """
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None): def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None):
@ -93,10 +128,14 @@ class MaveRSSImporter:
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]: def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]:
""" """
Main synchronization method using bulk operations. Main synchronization method using bulk operations.
Dynamically handles multi-season feeds:
- Automatically identifies all seasons in the feed (including any newly added seasons)
- Extracts season titles and season covers dynamically
- Resolves or creates season-specific PodcastPlaylist records
- Correctly assigns episodes to their respective season playlists
""" """
logger.info(f"Starting RSS sync for feed: {self.feed_url}") logger.info(f"Starting RSS sync for feed: {self.feed_url}")
# Step 1: Fetch and parse feed in-memory (0 DB queries)
content = self.fetch_feed_content() content = self.fetch_feed_content()
root = ET.fromstring(content) root = ET.fromstring(content)
@ -116,7 +155,7 @@ class MaveRSSImporter:
raw_items = channel.findall("item") raw_items = channel.findall("item")
total_items_in_feed = len(raw_items) total_items_in_feed = len(raw_items)
# Parse items in-memory (oldest first so priority increments sequentially)
# Parse items in-memory
parsed_items: List[Dict[str, Any]] = [] parsed_items: List[Dict[str, Any]] = []
for item in reversed(raw_items): for item in reversed(raw_items):
guid_el = item.find("guid") guid_el = item.find("guid")
@ -142,6 +181,12 @@ class MaveRSSImporter:
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url
season_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}season")
season_num = int(season_str) if (season_str and season_str.isdigit()) else None
episode_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}episode")
episode_num = int(episode_str) if (episode_str and episode_str.isdigit()) else None
parsed_items.append({ parsed_items.append({
"guid": guid, "guid": guid,
"title": title, "title": title,
@ -149,69 +194,33 @@ class MaveRSSImporter:
"audio_time": audio_time, "audio_time": audio_time,
"audio_url": audio_url, "audio_url": audio_url,
"thumbnail_url": thumbnail_url, "thumbnail_url": thumbnail_url,
"season_num": season_num,
"episode_num": episode_num,
}) })
# Group items by season
seasons_map: Dict[Optional[int], List[Dict[str, Any]]] = defaultdict(list)
for it in parsed_items:
seasons_map[it["season_num"]].append(it)
# Sort items inside each season by episode_num or reverse order
for s_num in seasons_map:
seasons_map[s_num].sort(
key=lambda x: (x["episode_num"] is None, x["episode_num"] or 0)
)
# Step 2: Resolve or create playlist (1-2 queries)
if playlist is None:
playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first()
if not playlist:
playlist = PodcastPlaylist(
title=channel_title,
description=channel_desc,
rss_feed_url=self.feed_url,
status=True,
)
playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist)
playlist.save()
else:
if not playlist.rss_feed_url:
playlist.rss_feed_url = self.feed_url
playlist.save(update_fields=['rss_feed_url'])
# Download playlist cover if missing
if not playlist.thumbnail and channel_img_url:
try:
img_res = requests.get(channel_img_url, timeout=10)
if img_res.status_code == 200 and img_res.content:
from django.core.files.base import ContentFile
ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg"
playlist.thumbnail.save(f"playlist_{playlist.id}.{ext}", ContentFile(img_res.content), save=True)
except Exception as img_err:
logger.warning(f"Could not download playlist thumbnail: {img_err}")
# Step 3: Check which items already exist in DB using a SINGLE IN query!
# Check existing podcasts in DB across all feed items (1 single query)
all_guids = [p["guid"] for p in parsed_items] all_guids = [p["guid"] for p in parsed_items]
existing_guid_map = {
p.rss_guid: p.id
for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid")
}
# Check existing playlist items (1 query)
existing_playlist_podcast_ids = set(
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True)
)
existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first()
current_priority = existing_max_priority.priority if existing_max_priority else 0
existing_podcasts_qs = Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid")
existing_guid_map = {p.rss_guid: p.id for p in existing_podcasts_qs}
# Step 4: Separate new podcasts vs existing ones
new_items_to_create: List[Dict[str, Any]] = []
existing_podcasts_to_link: List[int] = []
# Pre-fetch existing slugs in DB to avoid collisions
# Pre-fetch existing slugs to prevent collisions
existing_slugs = set(Podcast.objects.values_list("slug", flat=True)) existing_slugs = set(Podcast.objects.values_list("slug", flat=True))
new_podcasts_to_create: List[Podcast] = []
for item in parsed_items: for item in parsed_items:
guid = item["guid"]
if guid in existing_guid_map:
podcast_id = existing_guid_map[guid]
if podcast_id not in existing_playlist_podcast_ids:
existing_podcasts_to_link.append(podcast_id)
else:
# Generate unique slug in-memory
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}"
if item["guid"] not in existing_guid_map:
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{item['guid'][:8]}"
slug = base_slug[:200] slug = base_slug[:200]
counter = 1 counter = 1
while slug in existing_slugs: while slug in existing_slugs:
@ -219,17 +228,11 @@ class MaveRSSImporter:
counter += 1 counter += 1
existing_slugs.add(slug) existing_slugs.add(slug)
item["generated_slug"] = slug item["generated_slug"] = slug
new_items_to_create.append(item)
created_count = 0
# Step 5: Bulk create new podcasts (1 query!)
with transaction.atomic():
if new_items_to_create:
podcasts_to_insert = [
new_podcasts_to_create.append(
Podcast( Podcast(
title=item["title"], title=item["title"],
slug=item["generated_slug"],
slug=slug,
description=item["description"], description=item["description"],
audio_time=item["audio_time"], audio_time=item["audio_time"],
external_audio_url=item["audio_url"], external_audio_url=item["audio_url"],
@ -237,68 +240,166 @@ class MaveRSSImporter:
rss_guid=item["guid"], rss_guid=item["guid"],
status=True, status=True,
) )
for item in new_items_to_create
]
Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200)
created_count = len(podcasts_to_insert)
)
created_podcasts_count = len(new_podcasts_to_create)
if new_podcasts_to_create:
Podcast.objects.bulk_create(new_podcasts_to_create, batch_size=200)
# Re-fetch the newly created podcasts to get their DB ids
new_guid_qs = Podcast.objects.filter(
rss_guid__in=[p["guid"] for p in parsed_items if p["guid"] not in existing_guid_map]
).only("id", "rss_guid")
for p in new_guid_qs:
existing_guid_map[p.rss_guid] = p.id
# Now process each season and link to its corresponding PodcastPlaylist
synced_playlists_info = []
target_playlist = playlist
# Sort seasons: 1, 2, 3... (None at end)
sorted_season_keys = sorted(
seasons_map.keys(),
key=lambda x: (x is None, x or 0)
)
# Retrieve newly created podcast IDs (1 query)
new_guids = [item["guid"] for item in new_items_to_create]
newly_created_ids = list(
Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True)
for s_num in sorted_season_keys:
season_items = seasons_map[s_num]
season_title = extract_season_title(season_items, s_num, channel_title)
season_cover = season_items[0].get("thumbnail_url") or channel_img_url
# Find matching playlist for this season
season_pl = None
if target_playlist and (
target_playlist.rss_season_number == s_num
or (target_playlist.rss_season_number is None and s_num == 1)
):
season_pl = target_playlist
if season_pl.rss_season_number is None:
season_pl.rss_season_number = s_num
season_pl.save(update_fields=['rss_season_number'])
else:
if s_num is not None:
season_pl = PodcastPlaylist.objects.filter(
rss_feed_url=self.feed_url,
rss_season_number=s_num
).first()
if not season_pl and s_num == 1:
# Fallback to playlist with null rss_season_number if matching feed
season_pl = PodcastPlaylist.objects.filter(
rss_feed_url=self.feed_url,
rss_season_number__isnull=True
).first()
if season_pl:
season_pl.rss_season_number = 1
season_pl.save(update_fields=['rss_season_number'])
is_new_playlist = False
if not season_pl:
# Create dynamically for any season (e.g. season 7, 8, etc.)
season_pl = PodcastPlaylist(
title=season_title,
description=channel_desc,
rss_feed_url=self.feed_url,
rss_season_number=s_num,
status=True,
)
season_pl.slug = generate_smart_slug(season_title, PodcastPlaylist)
season_pl.save()
is_new_playlist = True
# Download cover if playlist has no thumbnail
if not season_pl.thumbnail and season_cover:
try:
img_res = requests.get(season_cover, timeout=10)
if img_res.status_code == 200 and img_res.content:
ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg"
season_pl.thumbnail.save(f"playlist_{season_pl.id}.{ext}", ContentFile(img_res.content), save=True)
except Exception as img_err:
logger.warning(f"Could not download playlist thumbnail: {img_err}")
# Collect IDs of podcasts in this season
season_podcast_ids = [existing_guid_map[it["guid"]] for it in season_items if it["guid"] in existing_guid_map]
# Cross-season cleanup: Remove these podcasts from any OTHER playlists of this feed
other_feed_playlists = PodcastPlaylist.objects.filter(
rss_feed_url=self.feed_url
).exclude(id=season_pl.id)
if other_feed_playlists.exists():
PlaylistItem.objects.filter(
playlist__in=other_feed_playlists,
podcast_id__in=season_podcast_ids
).delete()
# Now link items to season_pl
existing_links = dict(
PlaylistItem.objects.filter(playlist=season_pl).values_list("podcast_id", "priority")
) )
# Step 6: Bulk create PlaylistItem links (1 query!)
all_ids_to_link = existing_podcasts_to_link + newly_created_ids
if all_ids_to_link:
playlist_items_to_create = []
for pid in all_ids_to_link:
current_priority += 1
playlist_items_to_create.append(
items_to_create = []
for idx, it in enumerate(season_items, start=1):
p_id = existing_guid_map.get(it["guid"])
if not p_id:
continue
priority = it["episode_num"] if (it["episode_num"] and it["episode_num"] > 0) else idx
if p_id not in existing_links:
items_to_create.append(
PlaylistItem( PlaylistItem(
playlist=playlist,
podcast_id=pid,
priority=current_priority,
playlist=season_pl,
podcast_id=p_id,
priority=priority,
) )
) )
PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200)
# Step 7: Inherit playlist categories to episodes
playlist_categories = list(playlist.categories.all())
if items_to_create:
PlaylistItem.objects.bulk_create(items_to_create, batch_size=200)
# Inherit categories from playlist to podcasts if any
playlist_categories = list(season_pl.categories.all())
if playlist_categories: if playlist_categories:
all_playlist_podcast_ids = list(
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True)
)
if all_playlist_podcast_ids:
PodcastCategoryThrough = Podcast.categories.through
through_records = [
PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id)
for p_id in all_playlist_podcast_ids
for cat in playlist_categories
]
PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True)
# Step 8: Update playlist total time & sync date (1 query)
playlist.total_time = playlist.calculate_total_time()
playlist.last_synced_at = timezone.now()
playlist.save(update_fields=['total_time', 'last_synced_at'])
PodcastCategoryThrough = Podcast.categories.through
through_records = [
PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id)
for p_id in season_podcast_ids
for cat in playlist_categories
]
PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True)
# Update total time & sync date
season_pl.total_time = season_pl.calculate_total_time()
season_pl.last_synced_at = timezone.now()
season_pl.save(update_fields=['total_time', 'last_synced_at'])
synced_playlists_info.append({
"id": season_pl.id,
"title": season_pl.title,
"season_number": s_num,
"episodes_count": len(season_items),
"is_new": is_new_playlist,
})
# Invalidate caches # Invalidate caches
invalidate_podcast_cache() invalidate_podcast_cache()
skipped_count = total_items_in_feed - created_count
skipped_count = total_items_in_feed - created_podcasts_count
primary_title = (
target_playlist.title
if target_playlist
else (f"{len(synced_playlists_info)} Playlists Synced" if len(synced_playlists_info) > 1 else (synced_playlists_info[0]["title"] if synced_playlists_info else ""))
)
result = { result = {
"status": "success", "status": "success",
"playlist_id": playlist.id,
"playlist_title": playlist.title,
"playlist_id": target_playlist.id if target_playlist else (synced_playlists_info[0]["id"] if synced_playlists_info else None),
"playlist_title": primary_title,
"total_feed_items": total_items_in_feed, "total_feed_items": total_items_in_feed,
"created": created_count,
"created": created_podcasts_count,
"skipped": skipped_count, "skipped": skipped_count,
"errors": [],
"synced_at": playlist.last_synced_at.isoformat(),
"playlists": synced_playlists_info,
"synced_at": timezone.now().isoformat(),
} }
logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}")
logger.info(f"RSS multi-season bulk sync completed: {len(synced_playlists_info)} playlists processed, created={created_podcasts_count}, skipped={skipped_count}")
return result return result

Loading…
Cancel
Save