Browse Source
feat(podcast): implement Mave RSS sync with direct CDN streaming and admin endpoints
master
feat(podcast): implement Mave RSS sync with direct CDN streaming and admin endpoints
master
11 changed files with 446 additions and 3 deletions
-
38apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py
-
9apps/podcast/models.py
-
20apps/podcast/serializers.py
-
12apps/podcast/serializers_admin.py
-
3apps/podcast/services/__init__.py
-
289apps/podcast/services/rss_importer.py
-
24apps/podcast/tasks.py
-
35apps/podcast/views_admin.py
-
8config/settings/base.py
-
5config/settings/production.py
-
6config/urls.py
@ -0,0 +1,38 @@ |
|||||
|
# Generated by Django 4.2.30 on 2026-11-09 13:26 |
||||
|
|
||||
|
from django.db import migrations, models |
||||
|
|
||||
|
|
||||
|
class Migration(migrations.Migration): |
||||
|
|
||||
|
dependencies = [ |
||||
|
('podcast', '0005_alter_podcast_slug'), |
||||
|
] |
||||
|
|
||||
|
operations = [ |
||||
|
migrations.AddField( |
||||
|
model_name='podcast', |
||||
|
name='external_audio_url', |
||||
|
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Audio URL'), |
||||
|
), |
||||
|
migrations.AddField( |
||||
|
model_name='podcast', |
||||
|
name='external_thumbnail_url', |
||||
|
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Thumbnail URL'), |
||||
|
), |
||||
|
migrations.AddField( |
||||
|
model_name='podcast', |
||||
|
name='rss_guid', |
||||
|
field=models.CharField(blank=True, db_index=True, max_length=255, null=True, unique=True, verbose_name='RSS GUID'), |
||||
|
), |
||||
|
migrations.AddField( |
||||
|
model_name='podcastplaylist', |
||||
|
name='last_synced_at', |
||||
|
field=models.DateTimeField(blank=True, null=True, verbose_name='Last Synced At'), |
||||
|
), |
||||
|
migrations.AddField( |
||||
|
model_name='podcastplaylist', |
||||
|
name='rss_feed_url', |
||||
|
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='RSS Feed URL'), |
||||
|
), |
||||
|
] |
||||
@ -0,0 +1,3 @@ |
|||||
|
from .rss_importer import MaveRSSImporter, sync_podcast_rss |
||||
|
|
||||
|
__all__ = ['MaveRSSImporter', 'sync_podcast_rss'] |
||||
@ -0,0 +1,289 @@ |
|||||
|
from __future__ import annotations |
||||
|
|
||||
|
import datetime |
||||
|
import html |
||||
|
import logging |
||||
|
import re |
||||
|
import xml.etree.ElementTree as ET |
||||
|
from typing import Optional, Dict, Any, List |
||||
|
from urllib.parse import urlparse |
||||
|
|
||||
|
import requests |
||||
|
from django.conf import settings |
||||
|
from django.db import transaction, close_old_connections, connection |
||||
|
from django.utils import timezone |
||||
|
from django.utils.text import slugify |
||||
|
|
||||
|
from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem |
||||
|
from apps.podcast.signals import invalidate_podcast_cache |
||||
|
from utils.slug import generate_smart_slug |
||||
|
|
||||
|
logger = logging.getLogger(__name__) |
||||
|
|
||||
|
DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205" |
||||
|
|
||||
|
|
||||
|
def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time: |
||||
|
""" |
||||
|
Parse itunes:duration into datetime.time. |
||||
|
Supports formats: |
||||
|
- Total seconds: '3037' |
||||
|
- MM:SS: '45:30' |
||||
|
- HH:MM:SS: '01:15:30' |
||||
|
""" |
||||
|
if not duration_str: |
||||
|
return datetime.time(0, 0, 0) |
||||
|
|
||||
|
duration_str = str(duration_str).strip() |
||||
|
try: |
||||
|
# Pure seconds |
||||
|
if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str): |
||||
|
total_seconds = int(float(duration_str)) |
||||
|
hours = (total_seconds // 3600) % 24 |
||||
|
minutes = (total_seconds % 3600) // 60 |
||||
|
seconds = total_seconds % 60 |
||||
|
return datetime.time(hours, minutes, seconds) |
||||
|
|
||||
|
# HH:MM:SS or MM:SS |
||||
|
parts = [int(p) for p in duration_str.split(':') if p.isdigit()] |
||||
|
if len(parts) == 3: |
||||
|
return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60) |
||||
|
elif len(parts) == 2: |
||||
|
return datetime.time(0, parts[0] % 60, parts[1] % 60) |
||||
|
elif len(parts) == 1: |
||||
|
return datetime.time(0, 0, parts[0] % 60) |
||||
|
except Exception as e: |
||||
|
logger.warning(f"Error parsing duration '{duration_str}': {e}") |
||||
|
|
||||
|
return datetime.time(0, 0, 0) |
||||
|
|
||||
|
|
||||
|
def clean_html_text(raw_text: Optional[str]) -> str: |
||||
|
"""Strips HTML tags and unescapes HTML entities for clean plaintext storage.""" |
||||
|
if not raw_text: |
||||
|
return "" |
||||
|
text = re.sub(r'<[^>]+>', ' ', raw_text) |
||||
|
text = html.unescape(text) |
||||
|
text = re.sub(r'[ \t]+', ' ', text) |
||||
|
text = re.sub(r'\n\s*\n', '\n\n', text) |
||||
|
return text.strip() |
||||
|
|
||||
|
|
||||
|
class MaveRSSImporter: |
||||
|
""" |
||||
|
High-performance bulk synchronization service for podcast RSS feeds. |
||||
|
Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations. |
||||
|
""" |
||||
|
|
||||
|
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None): |
||||
|
self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL |
||||
|
self.user_agent = user_agent or ( |
||||
|
"Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)" |
||||
|
) |
||||
|
|
||||
|
def fetch_feed_content(self) -> bytes: |
||||
|
headers = { |
||||
|
"User-Agent": self.user_agent, |
||||
|
"Accept": "application/rss+xml, application/xml, text/xml, */*", |
||||
|
} |
||||
|
response = requests.get(self.feed_url, headers=headers, timeout=30) |
||||
|
response.raise_for_status() |
||||
|
return response.content |
||||
|
|
||||
|
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]: |
||||
|
""" |
||||
|
Main synchronization method using bulk operations. |
||||
|
""" |
||||
|
logger.info(f"Starting RSS sync for feed: {self.feed_url}") |
||||
|
|
||||
|
# Step 1: Fetch and parse feed in-memory (0 DB queries) |
||||
|
content = self.fetch_feed_content() |
||||
|
root = ET.fromstring(content) |
||||
|
|
||||
|
channel = root.find("channel") |
||||
|
if channel is None: |
||||
|
raise ValueError("Invalid RSS: <channel> element not found.") |
||||
|
|
||||
|
channel_title = (channel.findtext("title") or "پادکست").strip() |
||||
|
channel_desc = clean_html_text(channel.findtext("description") or "") |
||||
|
channel_img_el = channel.find("image") |
||||
|
channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None |
||||
|
if not channel_img_url: |
||||
|
itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") |
||||
|
if itunes_img is not None: |
||||
|
channel_img_url = itunes_img.get("href") |
||||
|
|
||||
|
raw_items = channel.findall("item") |
||||
|
total_items_in_feed = len(raw_items) |
||||
|
|
||||
|
# Parse items in-memory (oldest first so priority increments sequentially) |
||||
|
parsed_items: List[Dict[str, Any]] = [] |
||||
|
for item in reversed(raw_items): |
||||
|
guid_el = item.find("guid") |
||||
|
guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None |
||||
|
enclosure = item.find("enclosure") |
||||
|
audio_url = enclosure.get("url") if enclosure is not None else None |
||||
|
|
||||
|
if not audio_url: |
||||
|
continue |
||||
|
|
||||
|
if not guid: |
||||
|
guid = audio_url |
||||
|
|
||||
|
title = (item.findtext("title") or "قسمت پادکست").strip() |
||||
|
description = clean_html_text( |
||||
|
item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary") |
||||
|
or item.findtext("description") |
||||
|
or "" |
||||
|
) |
||||
|
duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration") |
||||
|
audio_time = parse_duration_to_time(duration_str) |
||||
|
|
||||
|
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") |
||||
|
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url |
||||
|
|
||||
|
parsed_items.append({ |
||||
|
"guid": guid, |
||||
|
"title": title, |
||||
|
"description": description, |
||||
|
"audio_time": audio_time, |
||||
|
"audio_url": audio_url, |
||||
|
"thumbnail_url": thumbnail_url, |
||||
|
}) |
||||
|
|
||||
|
|
||||
|
|
||||
|
# Step 2: Resolve or create playlist (1-2 queries) |
||||
|
if playlist is None: |
||||
|
playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first() |
||||
|
if not playlist: |
||||
|
playlist = PodcastPlaylist( |
||||
|
title=channel_title, |
||||
|
description=channel_desc, |
||||
|
rss_feed_url=self.feed_url, |
||||
|
status=True, |
||||
|
) |
||||
|
playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist) |
||||
|
playlist.save() |
||||
|
else: |
||||
|
if not playlist.rss_feed_url: |
||||
|
playlist.rss_feed_url = self.feed_url |
||||
|
playlist.save(update_fields=['rss_feed_url']) |
||||
|
|
||||
|
# Step 3: Check which items already exist in DB using a SINGLE IN query! |
||||
|
all_guids = [p["guid"] for p in parsed_items] |
||||
|
existing_guid_map = { |
||||
|
p.rss_guid: p.id |
||||
|
for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid") |
||||
|
} |
||||
|
|
||||
|
# Check existing playlist items (1 query) |
||||
|
existing_playlist_podcast_ids = set( |
||||
|
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True) |
||||
|
) |
||||
|
|
||||
|
existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first() |
||||
|
current_priority = existing_max_priority.priority if existing_max_priority else 0 |
||||
|
|
||||
|
# Step 4: Separate new podcasts vs existing ones |
||||
|
new_items_to_create: List[Dict[str, Any]] = [] |
||||
|
existing_podcasts_to_link: List[int] = [] |
||||
|
|
||||
|
# Pre-fetch existing slugs in DB to avoid collisions |
||||
|
existing_slugs = set(Podcast.objects.values_list("slug", flat=True)) |
||||
|
|
||||
|
for item in parsed_items: |
||||
|
guid = item["guid"] |
||||
|
if guid in existing_guid_map: |
||||
|
podcast_id = existing_guid_map[guid] |
||||
|
if podcast_id not in existing_playlist_podcast_ids: |
||||
|
existing_podcasts_to_link.append(podcast_id) |
||||
|
else: |
||||
|
# Generate unique slug in-memory |
||||
|
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}" |
||||
|
slug = base_slug[:200] |
||||
|
counter = 1 |
||||
|
while slug in existing_slugs: |
||||
|
slug = f"{base_slug[:190]}-{counter}" |
||||
|
counter += 1 |
||||
|
existing_slugs.add(slug) |
||||
|
item["generated_slug"] = slug |
||||
|
new_items_to_create.append(item) |
||||
|
|
||||
|
created_count = 0 |
||||
|
|
||||
|
# Step 5: Bulk create new podcasts (1 query!) |
||||
|
with transaction.atomic(): |
||||
|
if new_items_to_create: |
||||
|
podcasts_to_insert = [ |
||||
|
Podcast( |
||||
|
title=item["title"], |
||||
|
slug=item["generated_slug"], |
||||
|
description=item["description"], |
||||
|
audio_time=item["audio_time"], |
||||
|
external_audio_url=item["audio_url"], |
||||
|
external_thumbnail_url=item["thumbnail_url"], |
||||
|
rss_guid=item["guid"], |
||||
|
status=True, |
||||
|
) |
||||
|
for item in new_items_to_create |
||||
|
] |
||||
|
Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200) |
||||
|
created_count = len(podcasts_to_insert) |
||||
|
|
||||
|
# Retrieve newly created podcast IDs (1 query) |
||||
|
new_guids = [item["guid"] for item in new_items_to_create] |
||||
|
newly_created_ids = list( |
||||
|
Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True) |
||||
|
) |
||||
|
|
||||
|
# Step 6: Bulk create PlaylistItem links (1 query!) |
||||
|
all_ids_to_link = existing_podcasts_to_link + newly_created_ids |
||||
|
if all_ids_to_link: |
||||
|
playlist_items_to_create = [] |
||||
|
for pid in all_ids_to_link: |
||||
|
current_priority += 1 |
||||
|
playlist_items_to_create.append( |
||||
|
PlaylistItem( |
||||
|
playlist=playlist, |
||||
|
podcast_id=pid, |
||||
|
priority=current_priority, |
||||
|
) |
||||
|
) |
||||
|
PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200) |
||||
|
|
||||
|
# Step 7: Update playlist total time & sync date (1 query) |
||||
|
playlist.total_time = playlist.calculate_total_time() |
||||
|
playlist.last_synced_at = timezone.now() |
||||
|
playlist.save(update_fields=['total_time', 'last_synced_at']) |
||||
|
|
||||
|
# Invalidate caches |
||||
|
invalidate_podcast_cache() |
||||
|
|
||||
|
skipped_count = total_items_in_feed - created_count |
||||
|
|
||||
|
result = { |
||||
|
"status": "success", |
||||
|
"playlist_id": playlist.id, |
||||
|
"playlist_title": playlist.title, |
||||
|
"total_feed_items": total_items_in_feed, |
||||
|
"created": created_count, |
||||
|
"skipped": skipped_count, |
||||
|
"errors": [], |
||||
|
"synced_at": playlist.last_synced_at.isoformat(), |
||||
|
} |
||||
|
logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}") |
||||
|
return result |
||||
|
|
||||
|
|
||||
|
def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: |
||||
|
"""Convenience helper function for manual or automated sync.""" |
||||
|
playlist = None |
||||
|
if playlist_id: |
||||
|
playlist = PodcastPlaylist.objects.get(id=playlist_id) |
||||
|
if not feed_url: |
||||
|
feed_url = playlist.rss_feed_url |
||||
|
|
||||
|
url = feed_url or DEFAULT_MAVE_FEED_URL |
||||
|
importer = MaveRSSImporter(feed_url=url) |
||||
|
return importer.sync(playlist=playlist) |
||||
@ -0,0 +1,24 @@ |
|||||
|
from __future__ import annotations |
||||
|
|
||||
|
import logging |
||||
|
from typing import Optional, Dict, Any |
||||
|
from celery import shared_task |
||||
|
from apps.podcast.services.rss_importer import sync_podcast_rss |
||||
|
|
||||
|
logger = logging.getLogger(__name__) |
||||
|
|
||||
|
|
||||
|
@shared_task(name='apps.podcast.tasks.sync_podcast_rss_task') |
||||
|
def sync_podcast_rss_task(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: |
||||
|
""" |
||||
|
Celery task to sync podcasts from RSS feed. |
||||
|
Can be run periodically by Celery Beat or triggered on-demand from the admin panel. |
||||
|
""" |
||||
|
logger.info(f"Running sync_podcast_rss_task (playlist_id={playlist_id}, feed_url={feed_url})") |
||||
|
try: |
||||
|
result = sync_podcast_rss(playlist_id=playlist_id, feed_url=feed_url) |
||||
|
logger.info(f"sync_podcast_rss_task finished successfully: {result}") |
||||
|
return result |
||||
|
except Exception as e: |
||||
|
logger.error(f"sync_podcast_rss_task failed: {e}", exc_info=True) |
||||
|
return {"status": "error", "error": str(e)} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue