From 3531f12276024fa34e7d7ff097e5cf6bf8370ffe Mon Sep 17 00:00:00 2001 From: mohsentaba Date: Mon, 9 Nov 2026 14:53:08 +0330 Subject: [PATCH] feat(podcast): implement Mave RSS sync with direct CDN streaming and admin endpoints --- ...006_podcast_external_audio_url_and_more.py | 38 +++ apps/podcast/models.py | 9 + apps/podcast/serializers.py | 20 +- apps/podcast/serializers_admin.py | 12 +- apps/podcast/services/__init__.py | 3 + apps/podcast/services/rss_importer.py | 289 ++++++++++++++++++ apps/podcast/tasks.py | 24 ++ apps/podcast/views_admin.py | 35 +++ config/settings/base.py | 8 + config/settings/production.py | 5 + config/urls.py | 6 +- 11 files changed, 446 insertions(+), 3 deletions(-) create mode 100644 apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py create mode 100644 apps/podcast/services/__init__.py create mode 100644 apps/podcast/services/rss_importer.py create mode 100644 apps/podcast/tasks.py diff --git a/apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py b/apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py new file mode 100644 index 0000000..ecccf6f --- /dev/null +++ b/apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.30 on 2026-11-09 13:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('podcast', '0005_alter_podcast_slug'), + ] + + operations = [ + migrations.AddField( + model_name='podcast', + name='external_audio_url', + field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Audio URL'), + ), + migrations.AddField( + model_name='podcast', + name='external_thumbnail_url', + field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Thumbnail URL'), + ), + migrations.AddField( + model_name='podcast', + name='rss_guid', + field=models.CharField(blank=True, db_index=True, max_length=255, null=True, unique=True, verbose_name='RSS GUID'), + ), + migrations.AddField( + model_name='podcastplaylist', + name='last_synced_at', + field=models.DateTimeField(blank=True, null=True, verbose_name='Last Synced At'), + ), + migrations.AddField( + model_name='podcastplaylist', + name='rss_feed_url', + field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='RSS Feed URL'), + ), + ] diff --git a/apps/podcast/models.py b/apps/podcast/models.py index 3a97991..4daf73a 100644 --- a/apps/podcast/models.py +++ b/apps/podcast/models.py @@ -81,6 +81,11 @@ class Podcast(LowercaseSlugMixin, models.Model): collections = models.ManyToManyField(PodcastCollection, related_name='podcasts', verbose_name=_('collections'), blank=True) audio_file = models.FileField(upload_to='podcast/audio/', null=True, blank=True) audio_time = models.TimeField() + + # RSS External Streaming Support + rss_guid = models.CharField(max_length=255, unique=True, null=True, blank=True, db_index=True, verbose_name=_('RSS GUID')) + external_audio_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('External Audio URL')) + external_thumbnail_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('External Thumbnail URL')) view_count = models.PositiveBigIntegerField(default=0, verbose_name=_('view count')) download_count = models.PositiveBigIntegerField(default=0, verbose_name=_('download_count view count')) @@ -172,6 +177,10 @@ class PodcastPlaylist(LowercaseSlugMixin, models.Model): status = models.BooleanField(default=True, verbose_name=_('status')) view_count = models.PositiveBigIntegerField(default=0, verbose_name=_('view count')) total_time = models.DurationField(null=True, blank=True, verbose_name=_('total time')) + + # RSS Sync Support + rss_feed_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('RSS Feed URL')) + last_synced_at = models.DateTimeField(null=True, blank=True, verbose_name=_('Last Synced At')) created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at')) updated_at = models.DateTimeField(auto_now=True, verbose_name=_('updated at')) diff --git a/apps/podcast/serializers.py b/apps/podcast/serializers.py index 9b8f106..4e407c4 100755 --- a/apps/podcast/serializers.py +++ b/apps/podcast/serializers.py @@ -22,7 +22,7 @@ def build_podcast_audio_url(obj, request=None): Bypasses poisoned Cloudflare 301 cache redirects and enforces HTTPS. """ if not obj.audio_file: - return None + return getattr(obj, 'external_audio_url', None) url = obj.audio_file.url if request: full_url = request.build_absolute_uri(url) @@ -52,6 +52,12 @@ class PodcastListSerializer(serializers.ModelSerializer): 'audio_time', 'view_count', 'created_at', 'in_user_playlist', 'share_link'] def get_thumbnail(self, obj): + if not obj.thumbnail and getattr(obj, 'external_thumbnail_url', None): + return { + 'sm': obj.external_thumbnail_url, + 'md': obj.external_thumbnail_url, + 'lg': obj.external_thumbnail_url, + } return get_thumbs(obj.thumbnail, self.context.get('request')) def get_audio_file(self, obj): @@ -99,6 +105,12 @@ class PodcastDetailSerializer(serializers.ModelSerializer): return build_podcast_audio_url(obj, self.context.get('request')) def get_thumbnail(self, obj): + if not obj.thumbnail and getattr(obj, 'external_thumbnail_url', None): + return { + 'sm': obj.external_thumbnail_url, + 'md': obj.external_thumbnail_url, + 'lg': obj.external_thumbnail_url, + } return get_thumbs(obj.thumbnail, self.context.get('request')) def get_bookmark(self, obj): @@ -215,6 +227,12 @@ class PinnedPodcastCollectionSerializer(serializers.ModelSerializer): fields = ['id', 'title', 'slug', 'summary', 'thumbnail', 'order', 'created_at'] def get_thumbnail(self, obj): + if not obj.thumbnail and getattr(obj, 'external_thumbnail_url', None): + return { + 'sm': obj.external_thumbnail_url, + 'md': obj.external_thumbnail_url, + 'lg': obj.external_thumbnail_url, + } return get_thumbs(obj.thumbnail, self.context.get('request')) diff --git a/apps/podcast/serializers_admin.py b/apps/podcast/serializers_admin.py index 5595f60..a7f8787 100644 --- a/apps/podcast/serializers_admin.py +++ b/apps/podcast/serializers_admin.py @@ -92,7 +92,10 @@ class AdminPodcastListSerializer(serializers.ModelSerializer): "updated_at", "categories_detail", "collections_detail", - "audio_file" + "audio_file", + "external_audio_url", + "external_thumbnail_url", + "rss_guid" ] read_only_fields = ["id", "view_count", "download_count", "created_at", "updated_at"] @@ -130,6 +133,9 @@ class AdminPodcastDetailSerializer(serializers.ModelSerializer): "collections", "categories_detail", "collections_detail", + "external_audio_url", + "external_thumbnail_url", + "rss_guid", "remove_thumbnail", "remove_audio_file", ] @@ -237,6 +243,8 @@ class AdminPodcastPlaylistListSerializer(serializers.ModelSerializer): "categories_detail", "collections_detail", "items_count", + "rss_feed_url", + "last_synced_at", ] read_only_fields = ["id", "view_count", "total_time", "created_at", "updated_at", "items_count"] @@ -279,6 +287,8 @@ class AdminPodcastPlaylistDetailSerializer(serializers.ModelSerializer): "categories_detail", "collections_detail", "playlist_items_detail", + "rss_feed_url", + "last_synced_at", "remove_thumbnail", ] read_only_fields = ["id", "view_count", "bookmark_count", "total_time", "created_at", "updated_at", "playlist_items_detail"] diff --git a/apps/podcast/services/__init__.py b/apps/podcast/services/__init__.py new file mode 100644 index 0000000..f68711f --- /dev/null +++ b/apps/podcast/services/__init__.py @@ -0,0 +1,3 @@ +from .rss_importer import MaveRSSImporter, sync_podcast_rss + +__all__ = ['MaveRSSImporter', 'sync_podcast_rss'] diff --git a/apps/podcast/services/rss_importer.py b/apps/podcast/services/rss_importer.py new file mode 100644 index 0000000..efc87af --- /dev/null +++ b/apps/podcast/services/rss_importer.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import datetime +import html +import logging +import re +import xml.etree.ElementTree as ET +from typing import Optional, Dict, Any, List +from urllib.parse import urlparse + +import requests +from django.conf import settings +from django.db import transaction, close_old_connections, connection +from django.utils import timezone +from django.utils.text import slugify + +from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem +from apps.podcast.signals import invalidate_podcast_cache +from utils.slug import generate_smart_slug + +logger = logging.getLogger(__name__) + +DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205" + + +def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time: + """ + Parse itunes:duration into datetime.time. + Supports formats: + - Total seconds: '3037' + - MM:SS: '45:30' + - HH:MM:SS: '01:15:30' + """ + if not duration_str: + return datetime.time(0, 0, 0) + + duration_str = str(duration_str).strip() + try: + # Pure seconds + if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str): + total_seconds = int(float(duration_str)) + hours = (total_seconds // 3600) % 24 + minutes = (total_seconds % 3600) // 60 + seconds = total_seconds % 60 + return datetime.time(hours, minutes, seconds) + + # HH:MM:SS or MM:SS + parts = [int(p) for p in duration_str.split(':') if p.isdigit()] + if len(parts) == 3: + return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60) + elif len(parts) == 2: + return datetime.time(0, parts[0] % 60, parts[1] % 60) + elif len(parts) == 1: + return datetime.time(0, 0, parts[0] % 60) + except Exception as e: + logger.warning(f"Error parsing duration '{duration_str}': {e}") + + return datetime.time(0, 0, 0) + + +def clean_html_text(raw_text: Optional[str]) -> str: + """Strips HTML tags and unescapes HTML entities for clean plaintext storage.""" + if not raw_text: + return "" + text = re.sub(r'<[^>]+>', ' ', raw_text) + text = html.unescape(text) + text = re.sub(r'[ \t]+', ' ', text) + text = re.sub(r'\n\s*\n', '\n\n', text) + return text.strip() + + +class MaveRSSImporter: + """ + High-performance bulk synchronization service for podcast RSS feeds. + Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations. + """ + + def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None): + self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL + self.user_agent = user_agent or ( + "Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)" + ) + + def fetch_feed_content(self) -> bytes: + headers = { + "User-Agent": self.user_agent, + "Accept": "application/rss+xml, application/xml, text/xml, */*", + } + response = requests.get(self.feed_url, headers=headers, timeout=30) + response.raise_for_status() + return response.content + + def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]: + """ + Main synchronization method using bulk operations. + """ + logger.info(f"Starting RSS sync for feed: {self.feed_url}") + + # Step 1: Fetch and parse feed in-memory (0 DB queries) + content = self.fetch_feed_content() + root = ET.fromstring(content) + + channel = root.find("channel") + if channel is None: + raise ValueError("Invalid RSS: element not found.") + + channel_title = (channel.findtext("title") or "پادکست").strip() + channel_desc = clean_html_text(channel.findtext("description") or "") + channel_img_el = channel.find("image") + channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None + if not channel_img_url: + itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") + if itunes_img is not None: + channel_img_url = itunes_img.get("href") + + raw_items = channel.findall("item") + total_items_in_feed = len(raw_items) + + # Parse items in-memory (oldest first so priority increments sequentially) + parsed_items: List[Dict[str, Any]] = [] + for item in reversed(raw_items): + guid_el = item.find("guid") + guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None + enclosure = item.find("enclosure") + audio_url = enclosure.get("url") if enclosure is not None else None + + if not audio_url: + continue + + if not guid: + guid = audio_url + + title = (item.findtext("title") or "قسمت پادکست").strip() + description = clean_html_text( + item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary") + or item.findtext("description") + or "" + ) + duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration") + audio_time = parse_duration_to_time(duration_str) + + thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image") + thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url + + parsed_items.append({ + "guid": guid, + "title": title, + "description": description, + "audio_time": audio_time, + "audio_url": audio_url, + "thumbnail_url": thumbnail_url, + }) + + + + # Step 2: Resolve or create playlist (1-2 queries) + if playlist is None: + playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first() + if not playlist: + playlist = PodcastPlaylist( + title=channel_title, + description=channel_desc, + rss_feed_url=self.feed_url, + status=True, + ) + playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist) + playlist.save() + else: + if not playlist.rss_feed_url: + playlist.rss_feed_url = self.feed_url + playlist.save(update_fields=['rss_feed_url']) + + # Step 3: Check which items already exist in DB using a SINGLE IN query! + all_guids = [p["guid"] for p in parsed_items] + existing_guid_map = { + p.rss_guid: p.id + for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid") + } + + # Check existing playlist items (1 query) + existing_playlist_podcast_ids = set( + PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True) + ) + + existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first() + current_priority = existing_max_priority.priority if existing_max_priority else 0 + + # Step 4: Separate new podcasts vs existing ones + new_items_to_create: List[Dict[str, Any]] = [] + existing_podcasts_to_link: List[int] = [] + + # Pre-fetch existing slugs in DB to avoid collisions + existing_slugs = set(Podcast.objects.values_list("slug", flat=True)) + + for item in parsed_items: + guid = item["guid"] + if guid in existing_guid_map: + podcast_id = existing_guid_map[guid] + if podcast_id not in existing_playlist_podcast_ids: + existing_podcasts_to_link.append(podcast_id) + else: + # Generate unique slug in-memory + base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}" + slug = base_slug[:200] + counter = 1 + while slug in existing_slugs: + slug = f"{base_slug[:190]}-{counter}" + counter += 1 + existing_slugs.add(slug) + item["generated_slug"] = slug + new_items_to_create.append(item) + + created_count = 0 + + # Step 5: Bulk create new podcasts (1 query!) + with transaction.atomic(): + if new_items_to_create: + podcasts_to_insert = [ + Podcast( + title=item["title"], + slug=item["generated_slug"], + description=item["description"], + audio_time=item["audio_time"], + external_audio_url=item["audio_url"], + external_thumbnail_url=item["thumbnail_url"], + rss_guid=item["guid"], + status=True, + ) + for item in new_items_to_create + ] + Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200) + created_count = len(podcasts_to_insert) + + # Retrieve newly created podcast IDs (1 query) + new_guids = [item["guid"] for item in new_items_to_create] + newly_created_ids = list( + Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True) + ) + + # Step 6: Bulk create PlaylistItem links (1 query!) + all_ids_to_link = existing_podcasts_to_link + newly_created_ids + if all_ids_to_link: + playlist_items_to_create = [] + for pid in all_ids_to_link: + current_priority += 1 + playlist_items_to_create.append( + PlaylistItem( + playlist=playlist, + podcast_id=pid, + priority=current_priority, + ) + ) + PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200) + + # Step 7: Update playlist total time & sync date (1 query) + playlist.total_time = playlist.calculate_total_time() + playlist.last_synced_at = timezone.now() + playlist.save(update_fields=['total_time', 'last_synced_at']) + + # Invalidate caches + invalidate_podcast_cache() + + skipped_count = total_items_in_feed - created_count + + result = { + "status": "success", + "playlist_id": playlist.id, + "playlist_title": playlist.title, + "total_feed_items": total_items_in_feed, + "created": created_count, + "skipped": skipped_count, + "errors": [], + "synced_at": playlist.last_synced_at.isoformat(), + } + logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}") + return result + + +def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: + """Convenience helper function for manual or automated sync.""" + playlist = None + if playlist_id: + playlist = PodcastPlaylist.objects.get(id=playlist_id) + if not feed_url: + feed_url = playlist.rss_feed_url + + url = feed_url or DEFAULT_MAVE_FEED_URL + importer = MaveRSSImporter(feed_url=url) + return importer.sync(playlist=playlist) diff --git a/apps/podcast/tasks.py b/apps/podcast/tasks.py new file mode 100644 index 0000000..31b417b --- /dev/null +++ b/apps/podcast/tasks.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import logging +from typing import Optional, Dict, Any +from celery import shared_task +from apps.podcast.services.rss_importer import sync_podcast_rss + +logger = logging.getLogger(__name__) + + +@shared_task(name='apps.podcast.tasks.sync_podcast_rss_task') +def sync_podcast_rss_task(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]: + """ + Celery task to sync podcasts from RSS feed. + Can be run periodically by Celery Beat or triggered on-demand from the admin panel. + """ + logger.info(f"Running sync_podcast_rss_task (playlist_id={playlist_id}, feed_url={feed_url})") + try: + result = sync_podcast_rss(playlist_id=playlist_id, feed_url=feed_url) + logger.info(f"sync_podcast_rss_task finished successfully: {result}") + return result + except Exception as e: + logger.error(f"sync_podcast_rss_task failed: {e}", exc_info=True) + return {"status": "error", "error": str(e)} diff --git a/apps/podcast/views_admin.py b/apps/podcast/views_admin.py index edc95d8..ea16161 100644 --- a/apps/podcast/views_admin.py +++ b/apps/podcast/views_admin.py @@ -13,6 +13,8 @@ from utils.pagination import StandardResultsSetPagination from .models import PlaylistItem, Podcast, PodcastCollection, PodcastCategory, PodcastPlaylist, PodcastPlaylistInCollection from .signals import invalidate_podcast_cache +from apps.podcast.services.rss_importer import sync_podcast_rss +from apps.podcast.tasks import sync_podcast_rss_task from .serializers_admin import ( AdminPodcastCategorySerializer, AdminPodcastCollectionSerializer, @@ -254,3 +256,36 @@ class AdminPodcastPlaylistViewSet(ModelViewSet): ) return Response(serializer.data, status=status.HTTP_200_OK) + + @action(detail=True, methods=["post"], url_path="sync-rss") + @transaction.non_atomic_requests + def sync_rss(self, request, pk=None): + playlist = self.get_object() + feed_url = request.data.get("feed_url") or playlist.rss_feed_url + is_async = request.data.get("async", False) + + if is_async: + sync_podcast_rss_task.delay(playlist_id=playlist.id, feed_url=feed_url) + return Response({"status": "processing", "message": "همگام‌سازی در پس‌زمینه آغاز شد."}) + + try: + result = sync_podcast_rss(playlist_id=playlist.id, feed_url=feed_url) + return Response(result, status=status.HTTP_200_OK) + except Exception as e: + return Response({"status": "error", "detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) + + @action(detail=False, methods=["post"], url_path="sync-global-rss") + @transaction.non_atomic_requests + def sync_global_rss(self, request): + feed_url = request.data.get("feed_url") + is_async = request.data.get("async", False) + + if is_async: + sync_podcast_rss_task.delay(feed_url=feed_url) + return Response({"status": "processing", "message": "همگام‌سازی در پس‌زمینه آغاز شد."}) + + try: + result = sync_podcast_rss(feed_url=feed_url) + return Response(result, status=status.HTTP_200_OK) + except Exception as e: + return Response({"status": "error", "detail": str(e)}, status=status.HTTP_400_BAD_REQUEST) diff --git a/config/settings/base.py b/config/settings/base.py index 2acb750..a4e2096 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -277,6 +277,11 @@ CELERY_BEAT_SCHEDULE = { 'task': 'apps.account.tasks.check_inactive_students_task', 'schedule': crontab(minute=0, hour=9), # Daily at 9:00 AM }, + 'sync_podcast_rss_every_6_hours': { + 'task': 'apps.podcast.tasks.sync_podcast_rss_task', + 'schedule': crontab(minute=0, hour='*/6'), + }, + } # Password validation @@ -831,3 +836,6 @@ PLAUSIBLE_DOMAIN = env("PLAUSIBLE_DOMAIN") INTERNAL_WEBHOOK_SECRET = env('INTERNAL_WEBHOOK_SECRET', default='super-secret-key-12345') FASTAPI_SERVER_URL = env('FASTAPI_SERVER_URL', default='http://88.99.212.243:8020') + +# Mave Podcast RSS Feed URL +MAVE_PODCAST_FEED_URL = env("MAVE_PODCAST_FEED_URL", default="https://cloud.mave.digital/34205") diff --git a/config/settings/production.py b/config/settings/production.py index 24c463a..886d76d 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -55,6 +55,11 @@ CELERY_BEAT_SCHEDULE = { 'task': 'apps.account.tasks.check_inactive_students_task', 'schedule': crontab(minute=0, hour=9), # Daily at 9:00 AM }, + 'sync_podcast_rss_every_6_hours': { + 'task': 'apps.podcast.tasks.sync_podcast_rss_task', + 'schedule': crontab(minute=0, hour='*/6'), + }, + } # CORS_ALLOWED_ORIGINS = [ diff --git a/config/urls.py b/config/urls.py index bf00ea6..cb1741f 100644 --- a/config/urls.py +++ b/config/urls.py @@ -151,7 +151,11 @@ if settings.DEBUG: range_header = request.META.get('HTTP_RANGE', '').strip() if not range_header: - return django_static_serve(request, path, document_root=document_root, **kwargs) + response = django_static_serve(request, path, document_root=document_root, **kwargs) + if path.lower().endswith('.gz') and 'Content-Encoding' in response: + del response['Content-Encoding'] + response['Content-Type'] = 'application/gzip' + return response size = os.path.getsize(fullpath) content_type, encoding = mimetypes.guess_type(fullpath)