Browse Source

feat(podcast): implement Mave RSS sync with direct CDN streaming and admin endpoints

master
Mohsen Taba 1 week ago
parent
commit
3531f12276
  1. 38
      apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py
  2. 9
      apps/podcast/models.py
  3. 20
      apps/podcast/serializers.py
  4. 12
      apps/podcast/serializers_admin.py
  5. 3
      apps/podcast/services/__init__.py
  6. 289
      apps/podcast/services/rss_importer.py
  7. 24
      apps/podcast/tasks.py
  8. 35
      apps/podcast/views_admin.py
  9. 8
      config/settings/base.py
  10. 5
      config/settings/production.py
  11. 6
      config/urls.py

38
apps/podcast/migrations/0006_podcast_external_audio_url_and_more.py

@ -0,0 +1,38 @@
# Generated by Django 4.2.30 on 2026-11-09 13:26
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('podcast', '0005_alter_podcast_slug'),
]
operations = [
migrations.AddField(
model_name='podcast',
name='external_audio_url',
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Audio URL'),
),
migrations.AddField(
model_name='podcast',
name='external_thumbnail_url',
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='External Thumbnail URL'),
),
migrations.AddField(
model_name='podcast',
name='rss_guid',
field=models.CharField(blank=True, db_index=True, max_length=255, null=True, unique=True, verbose_name='RSS GUID'),
),
migrations.AddField(
model_name='podcastplaylist',
name='last_synced_at',
field=models.DateTimeField(blank=True, null=True, verbose_name='Last Synced At'),
),
migrations.AddField(
model_name='podcastplaylist',
name='rss_feed_url',
field=models.URLField(blank=True, max_length=1024, null=True, verbose_name='RSS Feed URL'),
),
]

9
apps/podcast/models.py

@ -81,6 +81,11 @@ class Podcast(LowercaseSlugMixin, models.Model):
collections = models.ManyToManyField(PodcastCollection, related_name='podcasts', verbose_name=_('collections'), blank=True)
audio_file = models.FileField(upload_to='podcast/audio/', null=True, blank=True)
audio_time = models.TimeField()
# RSS External Streaming Support
rss_guid = models.CharField(max_length=255, unique=True, null=True, blank=True, db_index=True, verbose_name=_('RSS GUID'))
external_audio_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('External Audio URL'))
external_thumbnail_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('External Thumbnail URL'))
view_count = models.PositiveBigIntegerField(default=0, verbose_name=_('view count'))
download_count = models.PositiveBigIntegerField(default=0, verbose_name=_('download_count view count'))
@ -172,6 +177,10 @@ class PodcastPlaylist(LowercaseSlugMixin, models.Model):
status = models.BooleanField(default=True, verbose_name=_('status'))
view_count = models.PositiveBigIntegerField(default=0, verbose_name=_('view count'))
total_time = models.DurationField(null=True, blank=True, verbose_name=_('total time'))
# RSS Sync Support
rss_feed_url = models.URLField(max_length=1024, null=True, blank=True, verbose_name=_('RSS Feed URL'))
last_synced_at = models.DateTimeField(null=True, blank=True, verbose_name=_('Last Synced At'))
created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at'))
updated_at = models.DateTimeField(auto_now=True, verbose_name=_('updated at'))

20
apps/podcast/serializers.py

@ -22,7 +22,7 @@ def build_podcast_audio_url(obj, request=None):
Bypasses poisoned Cloudflare 301 cache redirects and enforces HTTPS.
"""
if not obj.audio_file:
return None
return getattr(obj, 'external_audio_url', None)
url = obj.audio_file.url
if request:
full_url = request.build_absolute_uri(url)
@ -52,6 +52,12 @@ class PodcastListSerializer(serializers.ModelSerializer):
'audio_time', 'view_count', 'created_at', 'in_user_playlist', 'share_link']
def get_thumbnail(self, obj):
if not obj.thumbnail and getattr(obj, 'external_thumbnail_url', None):
return {
'sm': obj.external_thumbnail_url,
'md': obj.external_thumbnail_url,
'lg': obj.external_thumbnail_url,
}
return get_thumbs(obj.thumbnail, self.context.get('request'))
def get_audio_file(self, obj):
@ -99,6 +105,12 @@ class PodcastDetailSerializer(serializers.ModelSerializer):
return build_podcast_audio_url(obj, self.context.get('request'))
def get_thumbnail(self, obj):
if not obj.thumbnail and getattr(obj, 'external_thumbnail_url', None):
return {
'sm': obj.external_thumbnail_url,
'md': obj.external_thumbnail_url,
'lg': obj.external_thumbnail_url,
}
return get_thumbs(obj.thumbnail, self.context.get('request'))
def get_bookmark(self, obj):
@ -215,6 +227,12 @@ class PinnedPodcastCollectionSerializer(serializers.ModelSerializer):
fields = ['id', 'title', 'slug', 'summary', 'thumbnail', 'order', 'created_at']
def get_thumbnail(self, obj):
if not obj.thumbnail and getattr(obj, 'external_thumbnail_url', None):
return {
'sm': obj.external_thumbnail_url,
'md': obj.external_thumbnail_url,
'lg': obj.external_thumbnail_url,
}
return get_thumbs(obj.thumbnail, self.context.get('request'))

12
apps/podcast/serializers_admin.py

@ -92,7 +92,10 @@ class AdminPodcastListSerializer(serializers.ModelSerializer):
"updated_at",
"categories_detail",
"collections_detail",
"audio_file"
"audio_file",
"external_audio_url",
"external_thumbnail_url",
"rss_guid"
]
read_only_fields = ["id", "view_count", "download_count", "created_at", "updated_at"]
@ -130,6 +133,9 @@ class AdminPodcastDetailSerializer(serializers.ModelSerializer):
"collections",
"categories_detail",
"collections_detail",
"external_audio_url",
"external_thumbnail_url",
"rss_guid",
"remove_thumbnail",
"remove_audio_file",
]
@ -237,6 +243,8 @@ class AdminPodcastPlaylistListSerializer(serializers.ModelSerializer):
"categories_detail",
"collections_detail",
"items_count",
"rss_feed_url",
"last_synced_at",
]
read_only_fields = ["id", "view_count", "total_time", "created_at", "updated_at", "items_count"]
@ -279,6 +287,8 @@ class AdminPodcastPlaylistDetailSerializer(serializers.ModelSerializer):
"categories_detail",
"collections_detail",
"playlist_items_detail",
"rss_feed_url",
"last_synced_at",
"remove_thumbnail",
]
read_only_fields = ["id", "view_count", "bookmark_count", "total_time", "created_at", "updated_at", "playlist_items_detail"]

3
apps/podcast/services/__init__.py

@ -0,0 +1,3 @@
from .rss_importer import MaveRSSImporter, sync_podcast_rss
__all__ = ['MaveRSSImporter', 'sync_podcast_rss']

289
apps/podcast/services/rss_importer.py

@ -0,0 +1,289 @@
from __future__ import annotations
import datetime
import html
import logging
import re
import xml.etree.ElementTree as ET
from typing import Optional, Dict, Any, List
from urllib.parse import urlparse
import requests
from django.conf import settings
from django.db import transaction, close_old_connections, connection
from django.utils import timezone
from django.utils.text import slugify
from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem
from apps.podcast.signals import invalidate_podcast_cache
from utils.slug import generate_smart_slug
logger = logging.getLogger(__name__)
DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205"
def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time:
"""
Parse itunes:duration into datetime.time.
Supports formats:
- Total seconds: '3037'
- MM:SS: '45:30'
- HH:MM:SS: '01:15:30'
"""
if not duration_str:
return datetime.time(0, 0, 0)
duration_str = str(duration_str).strip()
try:
# Pure seconds
if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str):
total_seconds = int(float(duration_str))
hours = (total_seconds // 3600) % 24
minutes = (total_seconds % 3600) // 60
seconds = total_seconds % 60
return datetime.time(hours, minutes, seconds)
# HH:MM:SS or MM:SS
parts = [int(p) for p in duration_str.split(':') if p.isdigit()]
if len(parts) == 3:
return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60)
elif len(parts) == 2:
return datetime.time(0, parts[0] % 60, parts[1] % 60)
elif len(parts) == 1:
return datetime.time(0, 0, parts[0] % 60)
except Exception as e:
logger.warning(f"Error parsing duration '{duration_str}': {e}")
return datetime.time(0, 0, 0)
def clean_html_text(raw_text: Optional[str]) -> str:
"""Strips HTML tags and unescapes HTML entities for clean plaintext storage."""
if not raw_text:
return ""
text = re.sub(r'<[^>]+>', ' ', raw_text)
text = html.unescape(text)
text = re.sub(r'[ \t]+', ' ', text)
text = re.sub(r'\n\s*\n', '\n\n', text)
return text.strip()
class MaveRSSImporter:
"""
High-performance bulk synchronization service for podcast RSS feeds.
Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations.
"""
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None):
self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL
self.user_agent = user_agent or (
"Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)"
)
def fetch_feed_content(self) -> bytes:
headers = {
"User-Agent": self.user_agent,
"Accept": "application/rss+xml, application/xml, text/xml, */*",
}
response = requests.get(self.feed_url, headers=headers, timeout=30)
response.raise_for_status()
return response.content
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]:
"""
Main synchronization method using bulk operations.
"""
logger.info(f"Starting RSS sync for feed: {self.feed_url}")
# Step 1: Fetch and parse feed in-memory (0 DB queries)
content = self.fetch_feed_content()
root = ET.fromstring(content)
channel = root.find("channel")
if channel is None:
raise ValueError("Invalid RSS: <channel> element not found.")
channel_title = (channel.findtext("title") or "پادکست").strip()
channel_desc = clean_html_text(channel.findtext("description") or "")
channel_img_el = channel.find("image")
channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None
if not channel_img_url:
itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
if itunes_img is not None:
channel_img_url = itunes_img.get("href")
raw_items = channel.findall("item")
total_items_in_feed = len(raw_items)
# Parse items in-memory (oldest first so priority increments sequentially)
parsed_items: List[Dict[str, Any]] = []
for item in reversed(raw_items):
guid_el = item.find("guid")
guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None
enclosure = item.find("enclosure")
audio_url = enclosure.get("url") if enclosure is not None else None
if not audio_url:
continue
if not guid:
guid = audio_url
title = (item.findtext("title") or "قسمت پادکست").strip()
description = clean_html_text(
item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary")
or item.findtext("description")
or ""
)
duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration")
audio_time = parse_duration_to_time(duration_str)
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url
parsed_items.append({
"guid": guid,
"title": title,
"description": description,
"audio_time": audio_time,
"audio_url": audio_url,
"thumbnail_url": thumbnail_url,
})
# Step 2: Resolve or create playlist (1-2 queries)
if playlist is None:
playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first()
if not playlist:
playlist = PodcastPlaylist(
title=channel_title,
description=channel_desc,
rss_feed_url=self.feed_url,
status=True,
)
playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist)
playlist.save()
else:
if not playlist.rss_feed_url:
playlist.rss_feed_url = self.feed_url
playlist.save(update_fields=['rss_feed_url'])
# Step 3: Check which items already exist in DB using a SINGLE IN query!
all_guids = [p["guid"] for p in parsed_items]
existing_guid_map = {
p.rss_guid: p.id
for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid")
}
# Check existing playlist items (1 query)
existing_playlist_podcast_ids = set(
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True)
)
existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first()
current_priority = existing_max_priority.priority if existing_max_priority else 0
# Step 4: Separate new podcasts vs existing ones
new_items_to_create: List[Dict[str, Any]] = []
existing_podcasts_to_link: List[int] = []
# Pre-fetch existing slugs in DB to avoid collisions
existing_slugs = set(Podcast.objects.values_list("slug", flat=True))
for item in parsed_items:
guid = item["guid"]
if guid in existing_guid_map:
podcast_id = existing_guid_map[guid]
if podcast_id not in existing_playlist_podcast_ids:
existing_podcasts_to_link.append(podcast_id)
else:
# Generate unique slug in-memory
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}"
slug = base_slug[:200]
counter = 1
while slug in existing_slugs:
slug = f"{base_slug[:190]}-{counter}"
counter += 1
existing_slugs.add(slug)
item["generated_slug"] = slug
new_items_to_create.append(item)
created_count = 0
# Step 5: Bulk create new podcasts (1 query!)
with transaction.atomic():
if new_items_to_create:
podcasts_to_insert = [
Podcast(
title=item["title"],
slug=item["generated_slug"],
description=item["description"],
audio_time=item["audio_time"],
external_audio_url=item["audio_url"],
external_thumbnail_url=item["thumbnail_url"],
rss_guid=item["guid"],
status=True,
)
for item in new_items_to_create
]
Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200)
created_count = len(podcasts_to_insert)
# Retrieve newly created podcast IDs (1 query)
new_guids = [item["guid"] for item in new_items_to_create]
newly_created_ids = list(
Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True)
)
# Step 6: Bulk create PlaylistItem links (1 query!)
all_ids_to_link = existing_podcasts_to_link + newly_created_ids
if all_ids_to_link:
playlist_items_to_create = []
for pid in all_ids_to_link:
current_priority += 1
playlist_items_to_create.append(
PlaylistItem(
playlist=playlist,
podcast_id=pid,
priority=current_priority,
)
)
PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200)
# Step 7: Update playlist total time & sync date (1 query)
playlist.total_time = playlist.calculate_total_time()
playlist.last_synced_at = timezone.now()
playlist.save(update_fields=['total_time', 'last_synced_at'])
# Invalidate caches
invalidate_podcast_cache()
skipped_count = total_items_in_feed - created_count
result = {
"status": "success",
"playlist_id": playlist.id,
"playlist_title": playlist.title,
"total_feed_items": total_items_in_feed,
"created": created_count,
"skipped": skipped_count,
"errors": [],
"synced_at": playlist.last_synced_at.isoformat(),
}
logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}")
return result
def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]:
"""Convenience helper function for manual or automated sync."""
playlist = None
if playlist_id:
playlist = PodcastPlaylist.objects.get(id=playlist_id)
if not feed_url:
feed_url = playlist.rss_feed_url
url = feed_url or DEFAULT_MAVE_FEED_URL
importer = MaveRSSImporter(feed_url=url)
return importer.sync(playlist=playlist)

24
apps/podcast/tasks.py

@ -0,0 +1,24 @@
from __future__ import annotations
import logging
from typing import Optional, Dict, Any
from celery import shared_task
from apps.podcast.services.rss_importer import sync_podcast_rss
logger = logging.getLogger(__name__)
@shared_task(name='apps.podcast.tasks.sync_podcast_rss_task')
def sync_podcast_rss_task(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]:
"""
Celery task to sync podcasts from RSS feed.
Can be run periodically by Celery Beat or triggered on-demand from the admin panel.
"""
logger.info(f"Running sync_podcast_rss_task (playlist_id={playlist_id}, feed_url={feed_url})")
try:
result = sync_podcast_rss(playlist_id=playlist_id, feed_url=feed_url)
logger.info(f"sync_podcast_rss_task finished successfully: {result}")
return result
except Exception as e:
logger.error(f"sync_podcast_rss_task failed: {e}", exc_info=True)
return {"status": "error", "error": str(e)}

35
apps/podcast/views_admin.py

@ -13,6 +13,8 @@ from utils.pagination import StandardResultsSetPagination
from .models import PlaylistItem, Podcast, PodcastCollection, PodcastCategory, PodcastPlaylist, PodcastPlaylistInCollection
from .signals import invalidate_podcast_cache
from apps.podcast.services.rss_importer import sync_podcast_rss
from apps.podcast.tasks import sync_podcast_rss_task
from .serializers_admin import (
AdminPodcastCategorySerializer,
AdminPodcastCollectionSerializer,
@ -254,3 +256,36 @@ class AdminPodcastPlaylistViewSet(ModelViewSet):
)
return Response(serializer.data, status=status.HTTP_200_OK)
@action(detail=True, methods=["post"], url_path="sync-rss")
@transaction.non_atomic_requests
def sync_rss(self, request, pk=None):
playlist = self.get_object()
feed_url = request.data.get("feed_url") or playlist.rss_feed_url
is_async = request.data.get("async", False)
if is_async:
sync_podcast_rss_task.delay(playlist_id=playlist.id, feed_url=feed_url)
return Response({"status": "processing", "message": "همگام‌سازی در پس‌زمینه آغاز شد."})
try:
result = sync_podcast_rss(playlist_id=playlist.id, feed_url=feed_url)
return Response(result, status=status.HTTP_200_OK)
except Exception as e:
return Response({"status": "error", "detail": str(e)}, status=status.HTTP_400_BAD_REQUEST)
@action(detail=False, methods=["post"], url_path="sync-global-rss")
@transaction.non_atomic_requests
def sync_global_rss(self, request):
feed_url = request.data.get("feed_url")
is_async = request.data.get("async", False)
if is_async:
sync_podcast_rss_task.delay(feed_url=feed_url)
return Response({"status": "processing", "message": "همگام‌سازی در پس‌زمینه آغاز شد."})
try:
result = sync_podcast_rss(feed_url=feed_url)
return Response(result, status=status.HTTP_200_OK)
except Exception as e:
return Response({"status": "error", "detail": str(e)}, status=status.HTTP_400_BAD_REQUEST)

8
config/settings/base.py

@ -277,6 +277,11 @@ CELERY_BEAT_SCHEDULE = {
'task': 'apps.account.tasks.check_inactive_students_task',
'schedule': crontab(minute=0, hour=9), # Daily at 9:00 AM
},
'sync_podcast_rss_every_6_hours': {
'task': 'apps.podcast.tasks.sync_podcast_rss_task',
'schedule': crontab(minute=0, hour='*/6'),
},
}
# Password validation
@ -831,3 +836,6 @@ PLAUSIBLE_DOMAIN = env("PLAUSIBLE_DOMAIN")
INTERNAL_WEBHOOK_SECRET = env('INTERNAL_WEBHOOK_SECRET', default='super-secret-key-12345')
FASTAPI_SERVER_URL = env('FASTAPI_SERVER_URL', default='http://88.99.212.243:8020')
# Mave Podcast RSS Feed URL
MAVE_PODCAST_FEED_URL = env("MAVE_PODCAST_FEED_URL", default="https://cloud.mave.digital/34205")

5
config/settings/production.py

@ -55,6 +55,11 @@ CELERY_BEAT_SCHEDULE = {
'task': 'apps.account.tasks.check_inactive_students_task',
'schedule': crontab(minute=0, hour=9), # Daily at 9:00 AM
},
'sync_podcast_rss_every_6_hours': {
'task': 'apps.podcast.tasks.sync_podcast_rss_task',
'schedule': crontab(minute=0, hour='*/6'),
},
}
# CORS_ALLOWED_ORIGINS = [

6
config/urls.py

@ -151,7 +151,11 @@ if settings.DEBUG:
range_header = request.META.get('HTTP_RANGE', '').strip()
if not range_header:
return django_static_serve(request, path, document_root=document_root, **kwargs)
response = django_static_serve(request, path, document_root=document_root, **kwargs)
if path.lower().endswith('.gz') and 'Content-Encoding' in response:
del response['Content-Encoding']
response['Content-Type'] = 'application/gzip'
return response
size = os.path.getsize(fullpath)
content_type, encoding = mimetypes.guess_type(fullpath)

Loading…
Cancel
Save