You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

315 lines
13 KiB

from __future__ import annotations
import datetime
import html
import logging
import re
import xml.etree.ElementTree as ET
from typing import Optional, Dict, Any, List
from urllib.parse import urlparse
import requests
from django.conf import settings
from django.db import transaction, close_old_connections, connection
from django.utils import timezone
from django.utils.text import slugify
from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem
from apps.podcast.signals import invalidate_podcast_cache
from utils.slug import generate_smart_slug
logger = logging.getLogger(__name__)
DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205"
def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time:
"""
Parse itunes:duration into datetime.time.
Supports formats:
- Total seconds: '3037'
- MM:SS: '45:30'
- HH:MM:SS: '01:15:30'
"""
if not duration_str:
return datetime.time(0, 0, 0)
duration_str = str(duration_str).strip()
try:
# Pure seconds
if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str):
total_seconds = int(float(duration_str))
hours = (total_seconds // 3600) % 24
minutes = (total_seconds % 3600) // 60
seconds = total_seconds % 60
return datetime.time(hours, minutes, seconds)
# HH:MM:SS or MM:SS
parts = [int(p) for p in duration_str.split(':') if p.isdigit()]
if len(parts) == 3:
return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60)
elif len(parts) == 2:
return datetime.time(0, parts[0] % 60, parts[1] % 60)
elif len(parts) == 1:
return datetime.time(0, 0, parts[0] % 60)
except Exception as e:
logger.warning(f"Error parsing duration '{duration_str}': {e}")
return datetime.time(0, 0, 0)
def clean_html_text(raw_text: Optional[str]) -> str:
"""Strips HTML tags and unescapes HTML entities for clean plaintext storage."""
if not raw_text:
return ""
text = re.sub(r'<[^>]+>', ' ', raw_text)
text = html.unescape(text)
text = re.sub(r'[ \t]+', ' ', text)
text = re.sub(r'\n\s*\n', '\n\n', text)
return text.strip()
class MaveRSSImporter:
"""
High-performance bulk synchronization service for podcast RSS feeds.
Minimizes DB queries from hundreds of round-trips down to 3-4 bulk operations.
"""
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None):
self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL
self.user_agent = user_agent or (
"Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)"
)
def fetch_feed_content(self) -> bytes:
headers = {
"User-Agent": self.user_agent,
"Accept": "application/rss+xml, application/xml, text/xml, */*",
}
response = requests.get(self.feed_url, headers=headers, timeout=30)
response.raise_for_status()
return response.content
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]:
"""
Main synchronization method using bulk operations.
"""
logger.info(f"Starting RSS sync for feed: {self.feed_url}")
# Step 1: Fetch and parse feed in-memory (0 DB queries)
content = self.fetch_feed_content()
root = ET.fromstring(content)
channel = root.find("channel")
if channel is None:
raise ValueError("Invalid RSS: <channel> element not found.")
channel_title = (channel.findtext("title") or "پادکست").strip()
channel_desc = clean_html_text(channel.findtext("description") or "")
channel_img_el = channel.find("image")
channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None
if not channel_img_url:
itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
if itunes_img is not None:
channel_img_url = itunes_img.get("href")
raw_items = channel.findall("item")
total_items_in_feed = len(raw_items)
# Parse items in-memory (oldest first so priority increments sequentially)
parsed_items: List[Dict[str, Any]] = []
for item in reversed(raw_items):
guid_el = item.find("guid")
guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None
enclosure = item.find("enclosure")
audio_url = enclosure.get("url") if enclosure is not None else None
if not audio_url:
continue
if not guid:
guid = audio_url
title = (item.findtext("title") or "قسمت پادکست").strip()
description = clean_html_text(
item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary")
or item.findtext("description")
or ""
)
duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration")
audio_time = parse_duration_to_time(duration_str)
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url
parsed_items.append({
"guid": guid,
"title": title,
"description": description,
"audio_time": audio_time,
"audio_url": audio_url,
"thumbnail_url": thumbnail_url,
})
# Step 2: Resolve or create playlist (1-2 queries)
if playlist is None:
playlist = PodcastPlaylist.objects.filter(rss_feed_url=self.feed_url).first()
if not playlist:
playlist = PodcastPlaylist(
title=channel_title,
description=channel_desc,
rss_feed_url=self.feed_url,
status=True,
)
playlist.slug = generate_smart_slug(channel_title, PodcastPlaylist)
playlist.save()
else:
if not playlist.rss_feed_url:
playlist.rss_feed_url = self.feed_url
playlist.save(update_fields=['rss_feed_url'])
# Download playlist cover if missing
if not playlist.thumbnail and channel_img_url:
try:
img_res = requests.get(channel_img_url, timeout=10)
if img_res.status_code == 200 and img_res.content:
from django.core.files.base import ContentFile
ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg"
playlist.thumbnail.save(f"playlist_{playlist.id}.{ext}", ContentFile(img_res.content), save=True)
except Exception as img_err:
logger.warning(f"Could not download playlist thumbnail: {img_err}")
# Step 3: Check which items already exist in DB using a SINGLE IN query!
all_guids = [p["guid"] for p in parsed_items]
existing_guid_map = {
p.rss_guid: p.id
for p in Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid")
}
# Check existing playlist items (1 query)
existing_playlist_podcast_ids = set(
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True)
)
existing_max_priority = PlaylistItem.objects.filter(playlist=playlist).order_by("-priority").first()
current_priority = existing_max_priority.priority if existing_max_priority else 0
# Step 4: Separate new podcasts vs existing ones
new_items_to_create: List[Dict[str, Any]] = []
existing_podcasts_to_link: List[int] = []
# Pre-fetch existing slugs in DB to avoid collisions
existing_slugs = set(Podcast.objects.values_list("slug", flat=True))
for item in parsed_items:
guid = item["guid"]
if guid in existing_guid_map:
podcast_id = existing_guid_map[guid]
if podcast_id not in existing_playlist_podcast_ids:
existing_podcasts_to_link.append(podcast_id)
else:
# Generate unique slug in-memory
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{guid[:8]}"
slug = base_slug[:200]
counter = 1
while slug in existing_slugs:
slug = f"{base_slug[:190]}-{counter}"
counter += 1
existing_slugs.add(slug)
item["generated_slug"] = slug
new_items_to_create.append(item)
created_count = 0
# Step 5: Bulk create new podcasts (1 query!)
with transaction.atomic():
if new_items_to_create:
podcasts_to_insert = [
Podcast(
title=item["title"],
slug=item["generated_slug"],
description=item["description"],
audio_time=item["audio_time"],
external_audio_url=item["audio_url"],
external_thumbnail_url=item["thumbnail_url"],
rss_guid=item["guid"],
status=True,
)
for item in new_items_to_create
]
Podcast.objects.bulk_create(podcasts_to_insert, batch_size=200)
created_count = len(podcasts_to_insert)
# Retrieve newly created podcast IDs (1 query)
new_guids = [item["guid"] for item in new_items_to_create]
newly_created_ids = list(
Podcast.objects.filter(rss_guid__in=new_guids).values_list("id", flat=True)
)
# Step 6: Bulk create PlaylistItem links (1 query!)
all_ids_to_link = existing_podcasts_to_link + newly_created_ids
if all_ids_to_link:
playlist_items_to_create = []
for pid in all_ids_to_link:
current_priority += 1
playlist_items_to_create.append(
PlaylistItem(
playlist=playlist,
podcast_id=pid,
priority=current_priority,
)
)
PlaylistItem.objects.bulk_create(playlist_items_to_create, batch_size=200)
# Step 7: Inherit playlist categories to episodes
playlist_categories = list(playlist.categories.all())
if playlist_categories:
all_playlist_podcast_ids = list(
PlaylistItem.objects.filter(playlist=playlist).values_list("podcast_id", flat=True)
)
if all_playlist_podcast_ids:
PodcastCategoryThrough = Podcast.categories.through
through_records = [
PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id)
for p_id in all_playlist_podcast_ids
for cat in playlist_categories
]
PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True)
# Step 8: Update playlist total time & sync date (1 query)
playlist.total_time = playlist.calculate_total_time()
playlist.last_synced_at = timezone.now()
playlist.save(update_fields=['total_time', 'last_synced_at'])
# Invalidate caches
invalidate_podcast_cache()
skipped_count = total_items_in_feed - created_count
result = {
"status": "success",
"playlist_id": playlist.id,
"playlist_title": playlist.title,
"total_feed_items": total_items_in_feed,
"created": created_count,
"skipped": skipped_count,
"errors": [],
"synced_at": playlist.last_synced_at.isoformat(),
}
logger.info(f"RSS bulk sync completed: playlist_id={playlist.id}, created={created_count}, skipped={skipped_count}")
return result
def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]:
"""Convenience helper function for manual or automated sync."""
playlist = None
if playlist_id:
playlist = PodcastPlaylist.objects.get(id=playlist_id)
if not feed_url:
feed_url = playlist.rss_feed_url
url = feed_url or DEFAULT_MAVE_FEED_URL
importer = MaveRSSImporter(feed_url=url)
return importer.sync(playlist=playlist)