You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
416 lines
17 KiB
416 lines
17 KiB
from __future__ import annotations
|
|
|
|
import datetime
|
|
import html
|
|
import logging
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from collections import defaultdict, Counter
|
|
from typing import Optional, Dict, Any, List
|
|
|
|
import requests
|
|
from django.conf import settings
|
|
from django.core.files.base import ContentFile
|
|
from django.db import transaction
|
|
from django.utils import timezone
|
|
from django.utils.text import slugify
|
|
|
|
from apps.podcast.models import Podcast, PodcastPlaylist, PlaylistItem
|
|
from apps.podcast.signals import invalidate_podcast_cache
|
|
from utils.slug import generate_smart_slug
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DEFAULT_MAVE_FEED_URL = getattr(settings, "MAVE_PODCAST_FEED_URL", None) or "https://cloud.mave.digital/34205"
|
|
|
|
|
|
def parse_duration_to_time(duration_str: Optional[str]) -> datetime.time:
|
|
"""
|
|
Parse itunes:duration into datetime.time.
|
|
Supports formats:
|
|
- Total seconds: '3037'
|
|
- MM:SS: '45:30'
|
|
- HH:MM:SS: '01:15:30'
|
|
"""
|
|
if not duration_str:
|
|
return datetime.time(0, 0, 0)
|
|
|
|
duration_str = str(duration_str).strip()
|
|
try:
|
|
# Pure seconds
|
|
if duration_str.isdigit() or re.match(r'^\d+(\.\d+)?$', duration_str):
|
|
total_seconds = int(float(duration_str))
|
|
hours = (total_seconds // 3600) % 24
|
|
minutes = (total_seconds % 3600) // 60
|
|
seconds = total_seconds % 60
|
|
return datetime.time(hours, minutes, seconds)
|
|
|
|
# HH:MM:SS or MM:SS
|
|
parts = [int(p) for p in duration_str.split(':') if p.isdigit()]
|
|
if len(parts) == 3:
|
|
return datetime.time(parts[0] % 24, parts[1] % 60, parts[2] % 60)
|
|
elif len(parts) == 2:
|
|
return datetime.time(0, parts[0] % 60, parts[1] % 60)
|
|
elif len(parts) == 1:
|
|
return datetime.time(0, 0, parts[0] % 60)
|
|
except Exception as e:
|
|
logger.warning(f"Error parsing duration '{duration_str}': {e}")
|
|
|
|
return datetime.time(0, 0, 0)
|
|
|
|
|
|
def clean_html_text(raw_text: Optional[str]) -> str:
|
|
"""Strips HTML tags and unescapes HTML entities for clean plaintext storage."""
|
|
if not raw_text:
|
|
return ""
|
|
text = re.sub(r'<[^>]+>', ' ', raw_text)
|
|
text = html.unescape(text)
|
|
text = re.sub(r'[ \t]+', ' ', text)
|
|
text = re.sub(r'\n\s*\n', '\n\n', text)
|
|
return text.strip()
|
|
|
|
|
|
def extract_season_title(items: List[Dict[str, Any]], season_num: Optional[int], channel_title: str) -> str:
|
|
"""
|
|
Dynamically extracts the series/playlist title for a given season.
|
|
Works dynamically for any season (e.g. season 7, 8, etc.) without hardcoding.
|
|
"""
|
|
titles = [it.get("title", "") for it in items if it.get("title")]
|
|
candidates = []
|
|
for t in titles:
|
|
# Strip fraction patterns like ' 36/36' or ' 1/20'
|
|
cleaned = re.sub(r'\s+\d+/\d+.*$', '', t)
|
|
# Strip episode markers
|
|
cleaned = re.split(
|
|
r'\s+[—–-]\s*(?:Выпуск|выпуск|Часть|часть|Урок|урок|Серия|серия|Эпизод|эпизод|Беседа|беседа|Episode|episode|Part|part|Лекция|лекция)',
|
|
cleaned
|
|
)[0]
|
|
# Strip trailing dash + number: ' — #1', ' - 12'
|
|
cleaned = re.split(r'\s+[—–-]\s*#?\d+', cleaned)[0]
|
|
# Strip trailing '#12'
|
|
cleaned = re.split(r'\s+#\d+', cleaned)[0]
|
|
cleaned = cleaned.strip(' —–-:,')
|
|
if cleaned:
|
|
candidates.append(cleaned)
|
|
|
|
if candidates:
|
|
counts = Counter(candidates)
|
|
most_common, freq = counts.most_common(1)[0]
|
|
if freq >= max(1, len(titles) * 0.25):
|
|
return most_common
|
|
|
|
if season_num is not None:
|
|
return f"{channel_title} - Сезон {season_num}"
|
|
return channel_title
|
|
|
|
|
|
class MaveRSSImporter:
|
|
"""
|
|
High-performance bulk synchronization service for podcast RSS feeds.
|
|
Dynamically groups episodes into season-based playlists.
|
|
"""
|
|
|
|
def __init__(self, feed_url: Optional[str] = None, user_agent: Optional[str] = None):
|
|
self.feed_url = feed_url or DEFAULT_MAVE_FEED_URL
|
|
self.user_agent = user_agent or (
|
|
"Mozilla/5.0 (compatible; DovoodiPodcastBot/1.0; +https://dovoodi.com)"
|
|
)
|
|
|
|
def fetch_feed_content(self) -> bytes:
|
|
headers = {
|
|
"User-Agent": self.user_agent,
|
|
"Accept": "application/rss+xml, application/xml, text/xml, */*",
|
|
}
|
|
response = requests.get(self.feed_url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.content
|
|
|
|
def sync(self, playlist: Optional[PodcastPlaylist] = None) -> Dict[str, Any]:
|
|
"""
|
|
Main synchronization method using bulk operations.
|
|
Dynamically handles multi-season feeds:
|
|
- Automatically identifies all seasons in the feed (including any newly added seasons)
|
|
- Extracts season titles and season covers dynamically
|
|
- Resolves or creates season-specific PodcastPlaylist records
|
|
- Correctly assigns episodes to their respective season playlists
|
|
"""
|
|
logger.info(f"Starting RSS sync for feed: {self.feed_url}")
|
|
|
|
content = self.fetch_feed_content()
|
|
root = ET.fromstring(content)
|
|
|
|
channel = root.find("channel")
|
|
if channel is None:
|
|
raise ValueError("Invalid RSS: <channel> element not found.")
|
|
|
|
channel_title = (channel.findtext("title") or "پادکست").strip()
|
|
channel_desc = clean_html_text(channel.findtext("description") or "")
|
|
channel_img_el = channel.find("image")
|
|
channel_img_url = channel_img_el.findtext("url") if channel_img_el is not None else None
|
|
if not channel_img_url:
|
|
itunes_img = channel.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
|
|
if itunes_img is not None:
|
|
channel_img_url = itunes_img.get("href")
|
|
|
|
raw_items = channel.findall("item")
|
|
total_items_in_feed = len(raw_items)
|
|
|
|
# Parse items in-memory
|
|
parsed_items: List[Dict[str, Any]] = []
|
|
for item in reversed(raw_items):
|
|
guid_el = item.find("guid")
|
|
guid = guid_el.text.strip() if (guid_el is not None and guid_el.text) else None
|
|
enclosure = item.find("enclosure")
|
|
audio_url = enclosure.get("url") if enclosure is not None else None
|
|
|
|
if not audio_url:
|
|
continue
|
|
|
|
if not guid:
|
|
guid = audio_url
|
|
|
|
title = (item.findtext("title") or "قسمت پادکست").strip()
|
|
description = clean_html_text(
|
|
item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}summary")
|
|
or item.findtext("description")
|
|
or ""
|
|
)
|
|
duration_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}duration")
|
|
audio_time = parse_duration_to_time(duration_str)
|
|
|
|
thumb_el = item.find("{http://www.itunes.com/dtds/podcast-1.0.dtd}image")
|
|
thumbnail_url = thumb_el.get("href") if thumb_el is not None else channel_img_url
|
|
|
|
season_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}season")
|
|
season_num = int(season_str) if (season_str and season_str.isdigit()) else None
|
|
|
|
episode_str = item.findtext("{http://www.itunes.com/dtds/podcast-1.0.dtd}episode")
|
|
episode_num = int(episode_str) if (episode_str and episode_str.isdigit()) else None
|
|
|
|
parsed_items.append({
|
|
"guid": guid,
|
|
"title": title,
|
|
"description": description,
|
|
"audio_time": audio_time,
|
|
"audio_url": audio_url,
|
|
"thumbnail_url": thumbnail_url,
|
|
"season_num": season_num,
|
|
"episode_num": episode_num,
|
|
})
|
|
|
|
# Group items by season
|
|
seasons_map: Dict[Optional[int], List[Dict[str, Any]]] = defaultdict(list)
|
|
for it in parsed_items:
|
|
seasons_map[it["season_num"]].append(it)
|
|
|
|
# Sort items inside each season by episode_num or reverse order
|
|
for s_num in seasons_map:
|
|
seasons_map[s_num].sort(
|
|
key=lambda x: (x["episode_num"] is None, x["episode_num"] or 0)
|
|
)
|
|
|
|
# Check existing podcasts in DB across all feed items (1 single query)
|
|
all_guids = [p["guid"] for p in parsed_items]
|
|
existing_podcasts_qs = Podcast.objects.filter(rss_guid__in=all_guids).only("id", "rss_guid")
|
|
existing_guid_map = {p.rss_guid: p.id for p in existing_podcasts_qs}
|
|
|
|
# Pre-fetch existing slugs to prevent collisions
|
|
existing_slugs = set(Podcast.objects.values_list("slug", flat=True))
|
|
|
|
new_podcasts_to_create: List[Podcast] = []
|
|
for item in parsed_items:
|
|
if item["guid"] not in existing_guid_map:
|
|
base_slug = slugify(item["title"], allow_unicode=True) or f"podcast-{item['guid'][:8]}"
|
|
slug = base_slug[:200]
|
|
counter = 1
|
|
while slug in existing_slugs:
|
|
slug = f"{base_slug[:190]}-{counter}"
|
|
counter += 1
|
|
existing_slugs.add(slug)
|
|
item["generated_slug"] = slug
|
|
|
|
new_podcasts_to_create.append(
|
|
Podcast(
|
|
title=item["title"],
|
|
slug=slug,
|
|
description=item["description"],
|
|
audio_time=item["audio_time"],
|
|
external_audio_url=item["audio_url"],
|
|
external_thumbnail_url=item["thumbnail_url"],
|
|
rss_guid=item["guid"],
|
|
status=True,
|
|
)
|
|
)
|
|
|
|
created_podcasts_count = len(new_podcasts_to_create)
|
|
if new_podcasts_to_create:
|
|
Podcast.objects.bulk_create(new_podcasts_to_create, batch_size=200)
|
|
# Re-fetch the newly created podcasts to get their DB ids
|
|
new_guid_qs = Podcast.objects.filter(
|
|
rss_guid__in=[p["guid"] for p in parsed_items if p["guid"] not in existing_guid_map]
|
|
).only("id", "rss_guid")
|
|
for p in new_guid_qs:
|
|
existing_guid_map[p.rss_guid] = p.id
|
|
|
|
# Now process each season and link to its corresponding PodcastPlaylist
|
|
synced_playlists_info = []
|
|
target_playlist = playlist
|
|
|
|
# Sort seasons: 1, 2, 3... (None at end)
|
|
sorted_season_keys = sorted(
|
|
seasons_map.keys(),
|
|
key=lambda x: (x is None, x or 0)
|
|
)
|
|
|
|
for s_num in sorted_season_keys:
|
|
season_items = seasons_map[s_num]
|
|
season_title = extract_season_title(season_items, s_num, channel_title)
|
|
season_cover = season_items[0].get("thumbnail_url") or channel_img_url
|
|
|
|
# Find matching playlist for this season
|
|
season_pl = None
|
|
if target_playlist and (
|
|
target_playlist.rss_season_number == s_num
|
|
or (target_playlist.rss_season_number is None and s_num == 1)
|
|
):
|
|
season_pl = target_playlist
|
|
if season_pl.rss_season_number is None:
|
|
season_pl.rss_season_number = s_num
|
|
season_pl.save(update_fields=['rss_season_number'])
|
|
else:
|
|
if s_num is not None:
|
|
season_pl = PodcastPlaylist.objects.filter(
|
|
rss_feed_url=self.feed_url,
|
|
rss_season_number=s_num
|
|
).first()
|
|
|
|
if not season_pl and s_num == 1:
|
|
# Fallback to playlist with null rss_season_number if matching feed
|
|
season_pl = PodcastPlaylist.objects.filter(
|
|
rss_feed_url=self.feed_url,
|
|
rss_season_number__isnull=True
|
|
).first()
|
|
if season_pl:
|
|
season_pl.rss_season_number = 1
|
|
season_pl.save(update_fields=['rss_season_number'])
|
|
|
|
is_new_playlist = False
|
|
if not season_pl:
|
|
# Create dynamically for any season (e.g. season 7, 8, etc.)
|
|
season_pl = PodcastPlaylist(
|
|
title=season_title,
|
|
description=channel_desc,
|
|
rss_feed_url=self.feed_url,
|
|
rss_season_number=s_num,
|
|
status=True,
|
|
)
|
|
season_pl.slug = generate_smart_slug(season_title, PodcastPlaylist)
|
|
season_pl.save()
|
|
is_new_playlist = True
|
|
|
|
# Download cover if playlist has no thumbnail
|
|
if not season_pl.thumbnail and season_cover:
|
|
try:
|
|
img_res = requests.get(season_cover, timeout=10)
|
|
if img_res.status_code == 200 and img_res.content:
|
|
ext = "png" if "png" in img_res.headers.get("Content-Type", "") else "jpg"
|
|
season_pl.thumbnail.save(f"playlist_{season_pl.id}.{ext}", ContentFile(img_res.content), save=True)
|
|
except Exception as img_err:
|
|
logger.warning(f"Could not download playlist thumbnail: {img_err}")
|
|
|
|
# Collect IDs of podcasts in this season
|
|
season_podcast_ids = [existing_guid_map[it["guid"]] for it in season_items if it["guid"] in existing_guid_map]
|
|
|
|
# Cross-season cleanup: Remove these podcasts from any OTHER playlists of this feed
|
|
other_feed_playlists = PodcastPlaylist.objects.filter(
|
|
rss_feed_url=self.feed_url
|
|
).exclude(id=season_pl.id)
|
|
if other_feed_playlists.exists():
|
|
PlaylistItem.objects.filter(
|
|
playlist__in=other_feed_playlists,
|
|
podcast_id__in=season_podcast_ids
|
|
).delete()
|
|
|
|
# Now link items to season_pl
|
|
existing_links = dict(
|
|
PlaylistItem.objects.filter(playlist=season_pl).values_list("podcast_id", "priority")
|
|
)
|
|
|
|
items_to_create = []
|
|
for idx, it in enumerate(season_items, start=1):
|
|
p_id = existing_guid_map.get(it["guid"])
|
|
if not p_id:
|
|
continue
|
|
priority = it["episode_num"] if (it["episode_num"] and it["episode_num"] > 0) else idx
|
|
if p_id not in existing_links:
|
|
items_to_create.append(
|
|
PlaylistItem(
|
|
playlist=season_pl,
|
|
podcast_id=p_id,
|
|
priority=priority,
|
|
)
|
|
)
|
|
|
|
if items_to_create:
|
|
PlaylistItem.objects.bulk_create(items_to_create, batch_size=200)
|
|
|
|
# Inherit categories from playlist to podcasts if any
|
|
playlist_categories = list(season_pl.categories.all())
|
|
if playlist_categories:
|
|
PodcastCategoryThrough = Podcast.categories.through
|
|
through_records = [
|
|
PodcastCategoryThrough(podcast_id=p_id, podcastcategory_id=cat.id)
|
|
for p_id in season_podcast_ids
|
|
for cat in playlist_categories
|
|
]
|
|
PodcastCategoryThrough.objects.bulk_create(through_records, ignore_conflicts=True)
|
|
|
|
# Update total time & sync date
|
|
season_pl.total_time = season_pl.calculate_total_time()
|
|
season_pl.last_synced_at = timezone.now()
|
|
season_pl.save(update_fields=['total_time', 'last_synced_at'])
|
|
|
|
synced_playlists_info.append({
|
|
"id": season_pl.id,
|
|
"title": season_pl.title,
|
|
"season_number": s_num,
|
|
"episodes_count": len(season_items),
|
|
"is_new": is_new_playlist,
|
|
})
|
|
|
|
# Invalidate caches
|
|
invalidate_podcast_cache()
|
|
|
|
skipped_count = total_items_in_feed - created_podcasts_count
|
|
|
|
primary_title = (
|
|
target_playlist.title
|
|
if target_playlist
|
|
else (f"{len(synced_playlists_info)} Playlists Synced" if len(synced_playlists_info) > 1 else (synced_playlists_info[0]["title"] if synced_playlists_info else ""))
|
|
)
|
|
|
|
result = {
|
|
"status": "success",
|
|
"playlist_id": target_playlist.id if target_playlist else (synced_playlists_info[0]["id"] if synced_playlists_info else None),
|
|
"playlist_title": primary_title,
|
|
"total_feed_items": total_items_in_feed,
|
|
"created": created_podcasts_count,
|
|
"skipped": skipped_count,
|
|
"playlists": synced_playlists_info,
|
|
"synced_at": timezone.now().isoformat(),
|
|
}
|
|
logger.info(f"RSS multi-season bulk sync completed: {len(synced_playlists_info)} playlists processed, created={created_podcasts_count}, skipped={skipped_count}")
|
|
return result
|
|
|
|
|
|
def sync_podcast_rss(playlist_id: Optional[int] = None, feed_url: Optional[str] = None) -> Dict[str, Any]:
|
|
"""Convenience helper function for manual or automated sync."""
|
|
playlist = None
|
|
if playlist_id:
|
|
playlist = PodcastPlaylist.objects.get(id=playlist_id)
|
|
if not feed_url:
|
|
feed_url = playlist.rss_feed_url
|
|
|
|
url = feed_url or DEFAULT_MAVE_FEED_URL
|
|
importer = MaveRSSImporter(feed_url=url)
|
|
return importer.sync(playlist=playlist)
|