You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

444 lines
18 KiB

from __future__ import annotations
import datetime
import html
import logging
import re
import xml.etree.ElementTree as ET
from collections import defaultdict, Counter
from typing import Optional, Dict, Any, List
import requests
from django.conf import settings
from django.core.files.base import ContentFile
from django.db import transaction
from django.utils import timezone
from django.utils.text import slugify
from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem
from apps.podcast.signals import invalidate_podcast_cache
from utils.slug import generate_smart_slug
logger = logging.getLogger(__name__)
DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205"
def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time:
"""
Parse itunes:duration into datetime.time.
Supports formats:
- Total seconds: '3037'
- MM:SS: '45:30'
- HH:MM:SS: '01:15:30'
"""
if not duration_str:
return datetime.time(0, 0, 0)
duration_str = str(duration_str).strip()
try:
# Pure seconds
if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str):
total_seconds = int(float(duration_str))
hours = (total_seconds // 3600) % 24
minutes = (total_seconds % 3600) // 60
seconds = total_seconds % 60
return datetime.time(hours, minutes, seconds)
# HH:MM:SS or MM:SS
parts = [int(p) for p in duration_str.split(':') if p.isdigit()]
if len(parts) == 3:
return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60)
elif len(parts) == 2:
return datetime.time(0, parts[0] % 60, parts[1] % 60)
elif len(parts) == 1:
return datetime.time(0, 0, parts[0] % 60)
except Exception as e:
logger.warning(f"Error parsing duration '{duration_str}': {e}")
return datetime.time(0, 0, 0)
def clean_html_text(raw_text: Optional[str]) -> str:
"""Strips HTML tags and unescapes HTML entities for clean plaintext storage."""
if not raw_text:
return ""
text = re.sub(r'<[^>]+>', ' ', raw_text)
text = html.unescape(text)
text = re.sub(r'[ \t]+', ' ', text)
text = re.sub(r'\n\s*\n', '\n\n', text)
return text.strip()
def extract_season_title(items: List[Dict[str, Any]], season_num: Optional[int], channel_title: str) -> str:
"""
Dynamically extracts the series/playlist title for a given season.
Works dynamically for any season (e.g. season 7, 8, etc.) without hardcoding.
"""
titles = [it.get("title", "") for it in items if it.get("title")]
candidates = []
for t in titles:
# Strip fraction patterns like ' 36/36' or ' 1/20'
cleaned = re.sub(r'\s+\d+/\d+.*$', '', t)
# Strip episode markers
cleaned = re.split(
r'\s+[—–-]\s*(?:Выпуск|выпуск|Часть|часть|Урок|урок|Серия|серия|Эпизод|эпизод|Беседа|беседа|Episode|episode|Part|part|Лекция|лекция)',
cleaned
)[0]
# Strip trailing dash + number: ' — #1', ' - 12'
cleaned = re.split(r'\s+[—–-]\s*#?\d+', cleaned)[0]
# Strip trailing '#12'
cleaned = re.split(r'\s+#\d+', cleaned)[0]
cleaned = cleaned.strip(' —–-:,')
if cleaned:
candidates.append(cleaned)
if candidates:
counts = Counter(candidates)
most_common, freq = counts.most_common(1)[0]
if freq >= max(1, len(titles) * 0.25):
return most_common
if season_num is not None:
return f"{channel_title} - Сезон {season_num}"
return channel_title
class MaveRSSImporter:
"""
High-performance bulk synchronization service for podcast RSS feeds.
Dynamically groups episodes into season-based playlists.
"""
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None):
self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL
self.user_agent = user_agent or (
"Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)"
)
def fetch_feed_content(self) -> bytes:
headers = {
"User-Agent": self.user_agent,
"Accept": "application/rss+xml, application/xml, text/xml, */*",
}
response = requests.get(self.feed_url, headers=headers, timeout=30)
response.raise_for_status()
return response.content
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]:
"""
Main synchronization method using bulk operations.
Dynamically handles multi-season feeds:
- Automatically identifies all seasons in the feed (including any newly added seasons)
- Extracts season titles and season covers dynamically
- Resolves or creates season-specific PodcastPlaylist records
- Correctly assigns episodes to their respective season playlists
"""
logger.info(f"Starting RSS sync for feed: {self.feed_url}")
content = self.fetch_feed_content()
root = ET.fromstring(content)
channel = root.find("channel")
if channel is None:
raise ValueError("Invalid RSS: <channel> element not found.")
channel_title = (channel.findtext("title") or "پادکست").strip()
channel_desc = clean_html_text(channel.findtext("description") or "")
channel_img_el = channel.find("image")
channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None
if not channel_img_url:
itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
if itunes_img is not None:
channel_img_url = itunes_img.get("href")
raw_items = channel.findall("item")
total_items_in_feed = len(raw_items)
# Parse items in-memory
parsed_items: List[Dict[str, Any]] = []
for item in reversed(raw_items):
guid_el = item.find("guid")
guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None
enclosure = item.find("enclosure")
audio_url = enclosure.get("url") if enclosure is not None else None
if not audio_url:
continue
if not guid:
guid = audio_url
title = (item.findtext("title") or "قسمت پادکست").strip()
description = clean_html_text(
item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary")
or item.findtext("description")
or ""
)
duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration")
audio_time = parse_duration_to_time(duration_str)
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url
season_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}season")
season_num = int(season_str) if (season_str and season_str.isdigit()) else None
episode_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}episode")
episode_num = int(episode_str) if (episode_str and episode_str.isdigit()) else None
parsed_items.append({
"guid": guid,
"title": title,
"description": description,
"audio_time": audio_time,
"audio_url": audio_url,
"thumbnail_url": thumbnail_url,
"season_num": season_num,
"episode_num": episode_num,
})
# Group items by season
seasons_map: Dict[Optional[int], List[Dict[str, Any]]] = defaultdict(list)
for it in parsed_items:
seasons_map[it["season_num"]].append(it)
# Sort items inside each season by episode_num or reverse order
for s_num in seasons_map:
seasons_map[s_num].sort(
key=lambda x: (x["episode_num"] is None, x["episode_num"] or 0)
)
# Check existing podcasts in DB across all feed items (1 single query)
all_guids = [p["guid"] for p in parsed_items]
existing_podcasts_qs = Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid")
existing_guid_map = {p.rss_guid: p.id for p in existing_podcasts_qs}
# Pre-fetch existing slugs to prevent collisions
existing_slugs = set(Podcast.objects.values_list("slug", flat=True))
new_podcasts_to_create: List[Podcast] = []
for item in parsed_items:
if item["guid"] not in existing_guid_map:
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{item['guid'][:8]}"
slug = base_slug[:200]
counter = 1
while slug in existing_slugs:
slug = f"{base_slug[:190]}-{counter}"
counter += 1
existing_slugs.add(slug)
item["generated_slug"] = slug
new_podcasts_to_create.append(
Podcast(
title=item["title"],
slug=slug,
description=item["description"],
audio_time=item["audio_time"],
external_audio_url=item["audio_url"],
external_thumbnail_url=item["thumbnail_url"],
rss_guid=item["guid"],
status=True,
)
)
created_podcasts_count = len(new_podcasts_to_create)
created_guids_set = {p.rss_guid for p in new_podcasts_to_create}
if new_podcasts_to_create:
Podcast.objects.bulk_create(new_podcasts_to_create, batch_size=200)
# Re-fetch the newly created podcasts to get their DB ids
new_guid_qs = Podcast.objects.filter(
rss_guid__in=[p["guid"] for p in parsed_items if p["guid"] not in existing_guid_map]
).only("id", "rss_guid")
for p in new_guid_qs:
existing_guid_map[p.rss_guid] = p.id
# Now process each season and link to its corresponding PodcastPlaylist
synced_playlists_info = []
added_episodes_list: List[Dict[str, Any]] = []
target_playlist = playlist
# Sort seasons: 1, 2, 3... (None at end)
sorted_season_keys = sorted(
seasons_map.keys(),
key=lambda x: (x is None, x or 0)
)
for s_num in sorted_season_keys:
season_items = seasons_map[s_num]
season_title = extract_season_title(season_items, s_num, channel_title)
season_cover = season_items[0].get("thumbnail_url") or channel_img_url
# Find matching playlist for this season
season_pl = None
if target_playlist and (
target_playlist.rss_season_number == s_num
or (target_playlist.rss_season_number is None and s_num == 1)
):
season_pl = target_playlist
if season_pl.rss_season_number is None:
season_pl.rss_season_number = s_num
season_pl.save(update_fields=['rss_season_number'])
else:
if s_num is not None:
season_pl = PodcastPlaylist.objects.filter(
rss_feed_url=self.feed_url,
rss_season_number=s_num
).first()
if not season_pl and s_num == 1:
# Fallback to playlist with null rss_season_number if matching feed
season_pl = PodcastPlaylist.objects.filter(
rss_feed_url=self.feed_url,
rss_season_number__isnull=True
).first()
if season_pl:
season_pl.rss_season_number = 1
season_pl.save(update_fields=['rss_season_number'])
is_new_playlist = False
if not season_pl:
# Create dynamically for any season (e.g. season 7, 8, etc.)
season_pl = PodcastPlaylist(
title=season_title,
description=channel_desc,
rss_feed_url=self.feed_url,
rss_season_number=s_num,
status=True,
)
season_pl.slug = generate_smart_slug(season_title, PodcastPlaylist)
season_pl.save()
is_new_playlist = True
# Download cover if playlist has no thumbnail
if not season_pl.thumbnail and season_cover:
try:
img_res = requests.get(season_cover, timeout=10)
if img_res.status_code == 200 and img_res.content:
ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg"
season_pl.thumbnail.save(f"playlist_{season_pl.id}.{ext}", ContentFile(img_res.content), save=True)
except Exception as img_err:
logger.warning(f"Could not download playlist thumbnail: {img_err}")
# Collect IDs of podcasts in this season
season_podcast_ids = [existing_guid_map[it["guid"]] for it in season_items if it["guid"] in existing_guid_map]
# Cross-season cleanup: Remove these podcasts from any OTHER playlists of this feed
other_feed_playlists = PodcastPlaylist.objects.filter(
rss_feed_url=self.feed_url
).exclude(id=season_pl.id)
if other_feed_playlists.exists():
PlaylistItem.objects.filter(
playlist__in=other_feed_playlists,
podcast_id__in=season_podcast_ids
).delete()
# Now link items to season_pl
existing_links = dict(
PlaylistItem.objects.filter(playlist=season_pl).values_list("podcast_id", "priority")
)
items_to_create = []
for idx, it in enumerate(season_items, start=1):
p_id = existing_guid_map.get(it["guid"])
if not p_id:
continue
priority = it["episode_num"] if (it["episode_num"] and it["episode_num"] > 0) else idx
if p_id not in existing_links:
items_to_create.append(
PlaylistItem(
playlist=season_pl,
podcast_id=p_id,
priority=priority,
)
)
if items_to_create:
PlaylistItem.objects.bulk_create(items_to_create, batch_size=200)
newly_linked_pids = {it.podcast_id for it in items_to_create}
for it in season_items:
p_id = existing_guid_map.get(it["guid"])
if p_id and (it["guid"] in created_guids_set or p_id in newly_linked_pids):
added_episodes_list.append({
"id": p_id,
"title": it["title"],
"season_number": s_num,
"episode_number": it["episode_num"],
"playlist_id": season_pl.id,
"playlist_title": season_pl.title,
"audio_time": str(it["audio_time"]) if it["audio_time"] else None,
})
# Inherit categories from playlist to podcasts if any
playlist_categories = list(season_pl.categories.all())
if playlist_categories:
PodcastCategoryThrough = Podcast.categories.through
through_records = [
PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id)
for p_id in season_podcast_ids
for cat in playlist_categories
]
PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True)
# Update total time & sync date
season_pl.total_time = season_pl.calculate_total_time()
season_pl.last_synced_at = timezone.now()
season_pl.save(update_fields=['total_time', 'last_synced_at'])
synced_playlists_info.append({
"id": season_pl.id,
"title": season_pl.title,
"season_number": s_num,
"episodes_count": len(season_items),
"is_new": is_new_playlist,
})
# Invalidate caches
invalidate_podcast_cache()
skipped_count = total_items_in_feed - created_podcasts_count
primary_title = (
target_playlist.title
if target_playlist
else (f"{len(synced_playlists_info)} Playlists Synced" if len(synced_playlists_info) > 1 else (synced_playlists_info[0]["title"] if synced_playlists_info else ""))
)
created_playlists = [
{
"id": p["id"],
"title": p["title"],
"season_number": p["season_number"],
"episodes_count": p["episodes_count"],
}
for p in synced_playlists_info if p.get("is_new")
]
result = {
"status": "success",
"playlist_id": target_playlist.id if target_playlist else (synced_playlists_info[0]["id"] if synced_playlists_info else None),
"playlist_title": primary_title,
"total_feed_items": total_items_in_feed,
"created": created_podcasts_count,
"skipped": skipped_count,
"created_playlists": created_playlists,
"created_episodes": added_episodes_list,
"playlists": synced_playlists_info,
"synced_at": timezone.now().isoformat(),
}
logger.info(f"RSS multi-season bulk sync completed: {len(synced_playlists_info)} playlists processed, created={created_podcasts_count}, skipped={skipped_count}")
return result
def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]:
"""Convenience helper function for manual or automated sync."""
playlist = None
if playlist_id:
playlist = PodcastPlaylist.objects.get(id=playlist_id)
if not feed_url:
feed_url = playlist.rss_feed_url
url = feed_url or DEFAULT_MAVE_FEED_URL
importer = MaveRSSImporter(feed_url=url)
return importer.sync(playlist=playlist)