You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
358 lines
14 KiB
358 lines
14 KiB
import os
|
|
import json
|
|
import random
|
|
import subprocess
|
|
import tempfile
|
|
from datetime import datetime, time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from django.core.management.base import BaseCommand
|
|
from django.core.files import File
|
|
from django.db import transaction
|
|
from django.utils.text import slugify
|
|
|
|
from apps.video.models import Video
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = 'Download videos from video_link.json and save them to Video model'
|
|
|
|
# Russian titles related to prophets, Quran, and Hadith
|
|
RUSSIAN_TITLES = [
|
|
"Жизнь Пророка Мухаммада (да благословит его Аллах)",
|
|
"История пророка Ибрахима",
|
|
"Коран и его значение в жизни мусульман",
|
|
"Хадисы Пророка о милосердии",
|
|
"Пророк Иса в исламской традиции",
|
|
"Коранические истории о пророках",
|
|
"Хадисы о важности знаний",
|
|
"Жизнь и миссия пророка Мусы",
|
|
"Толкование Корана: суры о вере",
|
|
"Пророк Нух и его призыв к единобожию",
|
|
"Хадисы о праведности и благочестии",
|
|
"Коранические принципы справедливости",
|
|
"История пророка Юсуфа",
|
|
"Хадисы о терпении и благодарности",
|
|
"Пророк Сулейман и его мудрость",
|
|
"Коран о морали и нравственности",
|
|
"Жизнь пророка Закарии",
|
|
"Хадисы о семье и родителях",
|
|
"Пророк Давуд и его псалмы",
|
|
"Коранические учения о добре и зле",
|
|
"Хадисы о щедрости и милосердии",
|
|
"История пророка Салиха",
|
|
"Коран о единобожии и вере",
|
|
"Пророк Худ и его народ",
|
|
"Хадисы о праведных поступках",
|
|
"Коранические истории о терпении",
|
|
"Жизнь пророка Яхьи",
|
|
"Хадисы о молитве и поклонении",
|
|
"Пророк Лут и его призыв",
|
|
"Коран о милости Аллаха",
|
|
"Хадисы о скромности и смирении",
|
|
"История пророка Шуайба",
|
|
"Коранические заповеди о честности",
|
|
"Пророк Идрис и его вознесение",
|
|
"Хадисы о братстве в исламе",
|
|
"Коран о судном дне",
|
|
"Жизнь пророка Исмаила",
|
|
"Хадисы о постоянстве в вере",
|
|
"Пророк Ильяс и его чудеса",
|
|
"Коранические притчи и их мудрость",
|
|
"Хадисы о покаянии и прощении",
|
|
"История пророка Айюба",
|
|
"Коран о защите слабых и угнетенных",
|
|
"Пророк Зу-ль-Кифль и его терпение",
|
|
"Хадисы о правдивости и искренности",
|
|
"Коранические учения о справедливом суде",
|
|
"Жизнь пророка Харуна",
|
|
"Хадисы о стремлении к знаниям",
|
|
"Пророк Юнус и его покаяние",
|
|
"Коран о величии Аллаха",
|
|
"Имам Хусейн и его жертва",
|
|
"Имам Али и его мудрость",
|
|
"Имам Хасан и путь мира",
|
|
"Фатима аз-Захра и ее святость",
|
|
"Имам Махди и ожидание",
|
|
"Ашура и значение Кербелы",
|
|
"Учения Ахль аль-Байт",
|
|
"Двенадцать имамов в шиизме",
|
|
"Имам Риза и его наследие",
|
|
]
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument(
|
|
'--json-file',
|
|
type=str,
|
|
default='video_link.json',
|
|
help='Path to video_link.json file'
|
|
)
|
|
parser.add_argument(
|
|
'--skip-existing',
|
|
action='store_true',
|
|
help='Skip videos that already exist with the same slug'
|
|
)
|
|
parser.add_argument(
|
|
'--dry-run',
|
|
action='store_true',
|
|
help='Show what would be done without actually downloading'
|
|
)
|
|
parser.add_argument(
|
|
'--limit',
|
|
type=int,
|
|
default=None,
|
|
help='Limit number of videos to process (for testing)'
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
json_file = options['json_file']
|
|
skip_existing = options['skip_existing']
|
|
self.dry_run = options.get('dry_run', False)
|
|
limit = options.get('limit')
|
|
|
|
if self.dry_run:
|
|
self.stdout.write(self.style.WARNING('DRY RUN MODE - No actual downloads or saves will be performed'))
|
|
|
|
# Read JSON file
|
|
if not os.path.isabs(json_file):
|
|
# If relative path, make it relative to project root
|
|
from django.conf import settings
|
|
json_file = os.path.join(settings.BASE_DIR, json_file)
|
|
|
|
if not os.path.exists(json_file):
|
|
self.stdout.write(self.style.ERROR(f'File not found: {json_file}'))
|
|
return
|
|
|
|
with open(json_file, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
|
|
videos_list = data.get('videos', [])
|
|
youtube_links = data.get('youtube_links', [])
|
|
|
|
processed_count = 0
|
|
|
|
# Process videos with slugs
|
|
for video_data in videos_list:
|
|
if limit and processed_count >= limit:
|
|
self.stdout.write(self.style.WARNING(f'Reached limit of {limit} videos'))
|
|
break
|
|
|
|
slug = video_data.get('slug')
|
|
video_url = video_data.get('video')
|
|
|
|
if not video_url:
|
|
self.stdout.write(self.style.WARNING(f'Skipping {slug}: No video URL'))
|
|
continue
|
|
|
|
if skip_existing and Video.objects.filter(slug=slug).exists():
|
|
self.stdout.write(self.style.WARNING(f'Skipping {slug}: Already exists'))
|
|
continue
|
|
|
|
self.process_video(video_url, slug)
|
|
processed_count += 1
|
|
|
|
# Process youtube_links (direct links without slugs)
|
|
for idx, video_url in enumerate(youtube_links, start=1):
|
|
if limit and processed_count >= limit:
|
|
self.stdout.write(self.style.WARNING(f'Reached limit of {limit} videos'))
|
|
break
|
|
|
|
# Generate slug from URL filename
|
|
filename = os.path.basename(video_url)
|
|
slug = slugify(os.path.splitext(filename)[0])[:50]
|
|
|
|
if skip_existing and Video.objects.filter(slug=slug).exists():
|
|
self.stdout.write(self.style.WARNING(f'Skipping {slug}: Already exists'))
|
|
continue
|
|
|
|
self.process_video(video_url, slug)
|
|
processed_count += 1
|
|
|
|
self.stdout.write(self.style.SUCCESS(f'Processed {processed_count} videos successfully!'))
|
|
|
|
def process_video(self, video_url, slug):
|
|
"""Process a single video: download, extract thumbnail, save to database"""
|
|
self.stdout.write(f'\nProcessing: {slug}')
|
|
self.stdout.write(f' URL: {video_url}')
|
|
|
|
if self.dry_run:
|
|
title = random.choice(self.RUSSIAN_TITLES)
|
|
self.stdout.write(f' [DRY RUN] Would save as: {title}')
|
|
return
|
|
|
|
temp_dir = None
|
|
try:
|
|
# Create temporary directory
|
|
temp_dir = tempfile.mkdtemp()
|
|
|
|
# Download video
|
|
video_path = self.download_video(video_url, temp_dir, slug)
|
|
if not video_path:
|
|
return
|
|
|
|
# Extract thumbnail
|
|
thumbnail_path = self.extract_thumbnail(video_path, temp_dir, slug)
|
|
|
|
# Get video duration
|
|
duration = self.get_video_duration(video_path)
|
|
|
|
# Generate random title
|
|
title = random.choice(self.RUSSIAN_TITLES)
|
|
|
|
# Ensure unique title by appending counter if needed
|
|
base_title = title
|
|
counter = 1
|
|
while Video.objects.filter(title=title).exists():
|
|
title = f"{base_title} ({counter})"
|
|
counter += 1
|
|
|
|
# Ensure unique slug by appending counter if needed
|
|
final_slug = slug
|
|
slug_counter = 1
|
|
while Video.objects.filter(slug=final_slug).exists():
|
|
final_slug = f"{slug}-{slug_counter}"
|
|
slug_counter += 1
|
|
|
|
if final_slug != slug:
|
|
self.stdout.write(self.style.WARNING(f' ⚠ Slug conflict, using: {final_slug}'))
|
|
|
|
# Create Video object
|
|
with transaction.atomic():
|
|
video = Video(
|
|
title=title,
|
|
slug=final_slug,
|
|
video_type=Video.VedioTypeChoices.VIDEO_FILE,
|
|
video_time=duration,
|
|
status=True,
|
|
)
|
|
|
|
# Save video file
|
|
with open(video_path, 'rb') as video_file:
|
|
video.video_file.save(
|
|
f'{slug}.mp4',
|
|
File(video_file),
|
|
save=False
|
|
)
|
|
|
|
# Save thumbnail if extracted
|
|
if thumbnail_path and os.path.exists(thumbnail_path):
|
|
with open(thumbnail_path, 'rb') as thumb_file:
|
|
video.thumbnail.save(
|
|
f'{slug}_thumb.jpg',
|
|
File(thumb_file),
|
|
save=False
|
|
)
|
|
|
|
video.save()
|
|
self.stdout.write(self.style.SUCCESS(f'✓ Saved: {title} (slug: {final_slug})'))
|
|
|
|
except Exception as e:
|
|
self.stdout.write(self.style.ERROR(f'✗ Error processing {slug}: {str(e)}'))
|
|
|
|
finally:
|
|
# Cleanup temporary files
|
|
if temp_dir and os.path.exists(temp_dir):
|
|
import shutil
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
|
|
def download_video(self, video_url, temp_dir, slug):
|
|
"""Download video to temporary directory"""
|
|
try:
|
|
self.stdout.write(f' Downloading video...')
|
|
video_path = os.path.join(temp_dir, f'{slug}.mp4')
|
|
|
|
response = requests.get(video_url, stream=True, timeout=300)
|
|
response.raise_for_status()
|
|
|
|
total_size = int(response.headers.get('content-length', 0))
|
|
downloaded = 0
|
|
last_percent = 0
|
|
|
|
with open(video_path, 'wb') as f:
|
|
for chunk in response.iter_content(chunk_size=8192):
|
|
if chunk:
|
|
f.write(chunk)
|
|
downloaded += len(chunk)
|
|
if total_size > 0:
|
|
percent = int((downloaded / total_size) * 100)
|
|
# Print progress every 10%
|
|
if percent >= last_percent + 10:
|
|
self.stdout.write(f' Progress: {percent}%')
|
|
last_percent = percent
|
|
|
|
self.stdout.write(f' ✓ Downloaded: {downloaded / (1024*1024):.2f} MB')
|
|
return video_path
|
|
|
|
except Exception as e:
|
|
self.stdout.write(self.style.ERROR(f' ✗ Download failed: {str(e)}'))
|
|
return None
|
|
|
|
def extract_thumbnail(self, video_path, temp_dir, slug):
|
|
"""Extract thumbnail from video using ffmpeg"""
|
|
try:
|
|
self.stdout.write(f' Extracting thumbnail...')
|
|
thumbnail_path = os.path.join(temp_dir, f'{slug}_thumb.jpg')
|
|
|
|
# Extract frame at 1 second
|
|
cmd = [
|
|
'ffmpeg',
|
|
'-i', video_path,
|
|
'-ss', '00:00:01',
|
|
'-vframes', '1',
|
|
'-q:v', '2',
|
|
thumbnail_path,
|
|
'-y'
|
|
]
|
|
|
|
result = subprocess.run(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
timeout=60
|
|
)
|
|
|
|
if result.returncode == 0 and os.path.exists(thumbnail_path):
|
|
self.stdout.write(f' ✓ Thumbnail extracted')
|
|
return thumbnail_path
|
|
else:
|
|
self.stdout.write(self.style.WARNING(f' ⚠ Thumbnail extraction failed'))
|
|
return None
|
|
|
|
except Exception as e:
|
|
self.stdout.write(self.style.WARNING(f' ⚠ Thumbnail error: {str(e)}'))
|
|
return None
|
|
|
|
def get_video_duration(self, video_path):
|
|
"""Get video duration using ffprobe"""
|
|
try:
|
|
cmd = [
|
|
'ffprobe',
|
|
'-v', 'error',
|
|
'-show_entries', 'format=duration',
|
|
'-of', 'default=noprint_wrappers=1:nokey=1',
|
|
video_path
|
|
]
|
|
|
|
result = subprocess.run(
|
|
cmd,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
timeout=30
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
duration_seconds = float(result.stdout.decode().strip())
|
|
hours = int(duration_seconds // 3600)
|
|
minutes = int((duration_seconds % 3600) // 60)
|
|
seconds = int(duration_seconds % 60)
|
|
|
|
self.stdout.write(f' ✓ Duration: {hours:02d}:{minutes:02d}:{seconds:02d}')
|
|
return time(hour=hours, minute=minutes, second=seconds)
|
|
else:
|
|
self.stdout.write(self.style.WARNING(f' ⚠ Could not determine duration, using default'))
|
|
return time(hour=0, minute=0, second=0)
|
|
|
|
except Exception as e:
|
|
self.stdout.write(self.style.WARNING(f' ⚠ Duration error: {str(e)}, using default'))
|
|
return time(hour=0, minute=0, second=0)
|