Browse Source
feat(podcast): implement Mave RSS sync with direct CDN streaming and admin endpoints
master
feat(podcast): implement Mave RSS sync with direct CDN streaming and admin endpoints
master
11 changed files with 446 additions and 3 deletions
-
38apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py
-
9apps/podcast/models.py
-
20apps/podcast/serializers.py
-
12apps/podcast/serializers_admin.py
-
3apps/podcast/services/__init__.py
-
289apps/podcast/services/rss_importer.py
-
24apps/podcast/tasks.py
-
35apps/podcast/views_admin.py
-
8config/settings/base.py
-
5config/settings/production.py
-
6config/urls.py
@ -0,0 +1,38 @@ |
|||
# Generated by Django 4.2.30 on 2026-11-09 13:26 |
|||
|
|||
from django.db import migrations, models |
|||
|
|||
|
|||
class Migration(migrations.Migration): |
|||
|
|||
dependencies = [ |
|||
('podcast', '0005_alter_podcast_slug'), |
|||
] |
|||
|
|||
operations = [ |
|||
migrations.AddField( |
|||
model_name='podcast', |
|||
name='external_audio_url', |
|||
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Audio URL'), |
|||
), |
|||
migrations.AddField( |
|||
model_name='podcast', |
|||
name='external_thumbnail_url', |
|||
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Thumbnail URL'), |
|||
), |
|||
migrations.AddField( |
|||
model_name='podcast', |
|||
name='rss_guid', |
|||
field=models.CharField(blank=True, db_index=True, max_length=255, null=True, unique=True, verbose_name='RSS GUID'), |
|||
), |
|||
migrations.AddField( |
|||
model_name='podcastplaylist', |
|||
name='last_synced_at', |
|||
field=models.DateTimeField(blank=True, null=True, verbose_name='Last Synced At'), |
|||
), |
|||
migrations.AddField( |
|||
model_name='podcastplaylist', |
|||
name='rss_feed_url', |
|||
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='RSS Feed URL'), |
|||
), |
|||
] |
|||
@ -0,0 +1,3 @@ |
|||
from .rss_importer import MaveRSSImporter, sync_podcast_rss |
|||
|
|||
__all__ = ['MaveRSSImporter', 'sync_podcast_rss'] |
|||
@ -0,0 +1,289 @@ |
|||
from __future__ import annotations |
|||
|
|||
import datetime |
|||
import html |
|||
import logging |
|||
import re |
|||
import xml.etree.ElementTree as ET |
|||
from typing import Optional, Dict, Any, List |
|||
from urllib.parse import urlparse |
|||
|
|||
import requests |
|||
from django.conf import settings |
|||
from django.db import transaction, close_old_connections, connection |
|||
from django.utils import timezone |
|||
from django.utils.text import slugify |
|||
|
|||
from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem |
|||
from apps.podcast.signals import invalidate_podcast_cache |
|||
from utils.slug import generate_smart_slug |
|||
|
|||
logger = logging.getLogger(__name__) |
|||
|
|||
DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205" |
|||
|
|||
|
|||
def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time: |
|||
""" |
|||
Parse itunes:duration into datetime.time. |
|||
Supports formats: |
|||
- Total seconds: '3037' |
|||
- MM:SS: '45:30' |
|||
- HH:MM:SS: '01:15:30' |
|||
""" |
|||
if not duration_str: |
|||
return datetime.time(0, 0, 0) |
|||
|
|||
duration_str = str(duration_str).strip() |
|||
try: |
|||
# Pure seconds |
|||
if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str): |
|||
total_seconds = int(float(duration_str)) |
|||
hours = (total_seconds // 3600) % 24 |
|||
minutes = (total_seconds % 3600) // 60 |
|||
seconds = total_seconds % 60 |
|||
return datetime.time(hours, minutes, seconds) |
|||
|
|||
# HH:MM:SS or MM:SS |
|||
parts = [int(p) for p in duration_str.split(':') if p.isdigit()] |
|||
if len(parts) == 3: |
|||
return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60) |
|||
elif len(parts) == 2: |
|||
return datetime.time(0, parts[0] % 60, parts[1] % 60) |
|||
elif len(parts) == 1: |
|||
return datetime.time(0, 0, parts[0] % 60) |
|||
except Exception as e: |
|||
logger.warning(f"Error parsing duration '{duration_str}': {e}") |
|||
|
|||
return datetime.time(0, 0, 0) |
|||
|
|||
|
|||
def clean_html_text(raw_text: Optional[str]) -> str: |
|||
"""Strips HTML tags and unescapes HTML entities for clean plaintext storage.""" |
|||
if not raw_text: |
|||
return "" |
|||
text = re.sub(r'<[^>]+>', ' ', raw_text) |
|||
text = html.unescape(text) |
|||
text = re.sub(r'[ \t]+', ' ', text) |
|||
text = re.sub(r'\n\s*\n', '\n\n', text) |
|||
return text.strip() |
|||
|
|||
|
|||
class MaveRSSImporter: |
|||
""" |
|||
High-performance bulk synchronization service for podcast RSS feeds. |
|||
Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations. |
|||
""" |
|||
|
|||
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None): |
|||
self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL |
|||
self.user_agent = user_agent or ( |
|||
"Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)" |
|||
) |
|||
|
|||
def fetch_feed_content(self) -> bytes: |
|||
headers = { |
|||
"User-Agent": self.user_agent, |
|||
"Accept": "application/rss+xml, application/xml, text/xml, */*", |
|||
} |
|||
response = requests.get(self.feed_url, headers=headers, timeout=30) |
|||
response.raise_for_status() |
|||
return response.content |
|||
|
|||
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]: |
|||
""" |
|||
Main synchronization method using bulk operations. |
|||
""" |
|||
logger.info(f"Starting RSS sync for feed: {self.feed_url}") |
|||
|
|||
# Step 1: Fetch and parse feed in-memory (0 DB queries) |
|||
content = self.fetch_feed_content() |
|||
root = ET.fromstring(content) |
|||
|
|||
channel = root.find("channel") |
|||
if channel is None: |
|||
raise ValueError("Invalid RSS: <channel> element not found.") |
|||
|
|||
channel_title = (channel.findtext("title") or "پادکست").strip() |
|||
channel_desc = clean_html_text(channel.findtext("description") or "") |
|||
channel_img_el = channel.find("image") |
|||
channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None |
|||
if not channel_img_url: |
|||
itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") |
|||
if itunes_img is not None: |
|||
channel_img_url = itunes_img.get("href") |
|||
|
|||
raw_items = channel.findall("item") |
|||
total_items_in_feed = len(raw_items) |
|||
|
|||
# Parse items in-memory (oldest first so priority increments sequentially) |
|||
parsed_items: List[Dict[str, Any]] = [] |
|||
for item in reversed(raw_items): |
|||
guid_el = item.find("guid") |
|||
guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None |
|||
enclosure = item.find("enclosure") |
|||
audio_url = enclosure.get("url") if enclosure is not None else None |
|||
|
|||
if not audio_url: |
|||
continue |
|||
|
|||
if not guid: |
|||
guid = audio_url |
|||
|
|||
title = (item.findtext("title") or "قسمت پادکست").strip() |
|||
description = clean_html_text( |
|||
item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary") |
|||
or item.findtext("description") |
|||
or "" |
|||
) |
|||
duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration") |
|||
audio_time = parse_duration_to_time(duration_str) |
|||
|
|||
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") |
|||
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url |
|||
|
|||
parsed_items.append({ |
|||
"guid": guid, |
|||
"title": title, |
|||
"description": description, |
|||
"audio_time": audio_time, |
|||
"audio_url": audio_url, |
|||
"thumbnail_url": thumbnail_url, |
|||
}) |
|||
|
|||
|
|||
|
|||
# Step 2: Resolve or create playlist (1-2 queries) |
|||
if playlist is None: |
|||
playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first() |
|||
if not playlist: |
|||
playlist = PodcastPlaylist( |
|||
title=channel_title, |
|||
description=channel_desc, |
|||
rss_feed_url=self.feed_url, |
|||
status=True, |
|||
) |
|||
playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist) |
|||
playlist.save() |
|||
else: |
|||
if not playlist.rss_feed_url: |
|||
playlist.rss_feed_url = self.feed_url |
|||
playlist.save(update_fields=['rss_feed_url']) |
|||
|
|||
# Step 3: Check which items already exist in DB using a SINGLE IN query! |
|||
all_guids = [p["guid"] for p in parsed_items] |
|||
existing_guid_map = { |
|||
p.rss_guid: p.id |
|||
for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid") |
|||
} |
|||
|
|||
# Check existing playlist items (1 query) |
|||
existing_playlist_podcast_ids = set( |
|||
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True) |
|||
) |
|||
|
|||
existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first() |
|||
current_priority = existing_max_priority.priority if existing_max_priority else 0 |
|||
|
|||
# Step 4: Separate new podcasts vs existing ones |
|||
new_items_to_create: List[Dict[str, Any]] = [] |
|||
existing_podcasts_to_link: List[int] = [] |
|||
|
|||
# Pre-fetch existing slugs in DB to avoid collisions |
|||
existing_slugs = set(Podcast.objects.values_list("slug", flat=True)) |
|||
|
|||
for item in parsed_items: |
|||
guid = item["guid"] |
|||
if guid in existing_guid_map: |
|||
podcast_id = existing_guid_map[guid] |
|||
if podcast_id not in existing_playlist_podcast_ids: |
|||
existing_podcasts_to_link.append(podcast_id) |
|||
else: |
|||
# Generate unique slug in-memory |
|||
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}" |
|||
slug = base_slug[:200] |
|||
counter = 1 |
|||
while slug in existing_slugs: |
|||
slug = f"{base_slug[:190]}-{counter}" |
|||
counter += 1 |
|||
existing_slugs.add(slug) |
|||
item["generated_slug"] = slug |
|||
new_items_to_create.append(item) |
|||
|
|||
created_count = 0 |
|||
|
|||
# Step 5: Bulk create new podcasts (1 query!) |
|||
with transaction.atomic(): |
|||
if new_items_to_create: |
|||
podcasts_to_insert = [ |
|||
Podcast( |
|||
title=item["title"], |
|||
slug=item["generated_slug"], |
|||
description=item["description"], |
|||
audio_time=item["audio_time"], |
|||
external_audio_url=item["audio_url"], |
|||
external_thumbnail_url=item["thumbnail_url"], |
|||
rss_guid=item["guid"], |
|||
status=True, |
|||
) |
|||
for item in new_items_to_create |
|||
] |
|||
Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200) |
|||
created_count = len(podcasts_to_insert) |
|||
|
|||
# Retrieve newly created podcast IDs (1 query) |
|||
new_guids = [item["guid"] for item in new_items_to_create] |
|||
newly_created_ids = list( |
|||
Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True) |
|||
) |
|||
|
|||
# Step 6: Bulk create PlaylistItem links (1 query!) |
|||
all_ids_to_link = existing_podcasts_to_link + newly_created_ids |
|||
if all_ids_to_link: |
|||
playlist_items_to_create = [] |
|||
for pid in all_ids_to_link: |
|||
current_priority += 1 |
|||
playlist_items_to_create.append( |
|||
PlaylistItem( |
|||
playlist=playlist, |
|||
podcast_id=pid, |
|||
priority=current_priority, |
|||
) |
|||
) |
|||
PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200) |
|||
|
|||
# Step 7: Update playlist total time & sync date (1 query) |
|||
playlist.total_time = playlist.calculate_total_time() |
|||
playlist.last_synced_at = timezone.now() |
|||
playlist.save(update_fields=['total_time', 'last_synced_at']) |
|||
|
|||
# Invalidate caches |
|||
invalidate_podcast_cache() |
|||
|
|||
skipped_count = total_items_in_feed - created_count |
|||
|
|||
result = { |
|||
"status": "success", |
|||
"playlist_id": playlist.id, |
|||
"playlist_title": playlist.title, |
|||
"total_feed_items": total_items_in_feed, |
|||
"created": created_count, |
|||
"skipped": skipped_count, |
|||
"errors": [], |
|||
"synced_at": playlist.last_synced_at.isoformat(), |
|||
} |
|||
logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}") |
|||
return result |
|||
|
|||
|
|||
def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: |
|||
"""Convenience helper function for manual or automated sync.""" |
|||
playlist = None |
|||
if playlist_id: |
|||
playlist = PodcastPlaylist.objects.get(id=playlist_id) |
|||
if not feed_url: |
|||
feed_url = playlist.rss_feed_url |
|||
|
|||
url = feed_url or DEFAULT_MAVE_FEED_URL |
|||
importer = MaveRSSImporter(feed_url=url) |
|||
return importer.sync(playlist=playlist) |
|||
@ -0,0 +1,24 @@ |
|||
from __future__ import annotations |
|||
|
|||
import logging |
|||
from typing import Optional, Dict, Any |
|||
from celery import shared_task |
|||
from apps.podcast.services.rss_importer import sync_podcast_rss |
|||
|
|||
logger = logging.getLogger(__name__) |
|||
|
|||
|
|||
@shared_task(name='apps.podcast.tasks.sync_podcast_rss_task') |
|||
def sync_podcast_rss_task(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: |
|||
""" |
|||
Celery task to sync podcasts from RSS feed. |
|||
Can be run periodically by Celery Beat or triggered on-demand from the admin panel. |
|||
""" |
|||
logger.info(f"Running sync_podcast_rss_task (playlist_id={playlist_id}, feed_url={feed_url})") |
|||
try: |
|||
result = sync_podcast_rss(playlist_id=playlist_id, feed_url=feed_url) |
|||
logger.info(f"sync_podcast_rss_task finished successfully: {result}") |
|||
return result |
|||
except Exception as e: |
|||
logger.error(f"sync_podcast_rss_task failed: {e}", exc_info=True) |
|||
return {"status": "error", "error": str(e)} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue