7 Commits
e0a805a446
...
403e068475
10 changed files with 738 additions and 49 deletions
-
97apps/account/views/user.py
-
362apps/dobodbi_calendar/management/commands/translate_occasions.py
-
35apps/hadis/docs.py
-
7apps/hadis/models/category.py
-
10apps/hadis/views/hadis.py
-
11apps/hadis/views_admin.py
-
6apps/video/serializers_dovodi.py
-
1requirements.txt
-
137utils/__init__.py
-
121utils/excel_exporter.py
@ -0,0 +1,362 @@ |
|||
import json |
|||
import re |
|||
import sys |
|||
import time |
|||
import requests |
|||
import threading |
|||
from concurrent.futures import ThreadPoolExecutor, as_completed |
|||
|
|||
from django.core.management.base import BaseCommand |
|||
from apps.dobodbi_calendar.models import CalendarOccasions |
|||
|
|||
# Target languages from زبانها.md |
|||
TARGET_LANGUAGES = { |
|||
"ar": "Arabic (عربی)", |
|||
"tr": "Turkish (ترکی)", |
|||
"fa": "Persian (فارسی)", |
|||
"ur": "Urdu (اردو)", |
|||
"bn": "Bengali (بنگالی)", |
|||
"id": "Indonesian (اندونزیایی)", |
|||
"fr": "French (فرانسوی)", |
|||
"ru": "Russian (روسی)", |
|||
"uz": "Uzbek (ازبکی)", |
|||
"ky": "Kyrgyz (قرقیزی)", |
|||
"tg": "Tajik (تاجیکی)", |
|||
"az": "Azerbaijani (آذربایجانی)", |
|||
"en": "English (انگلیسی)", |
|||
"de": "German (آلمانی)", |
|||
"zh": "Mandarin Chinese (چینی ماندارین)", |
|||
"ha": "Hausa (هوسا)", |
|||
"sw": "Swahili (سواحیلی)", |
|||
"es": "Spanish (اسپانیایی)", |
|||
} |
|||
|
|||
DEFAULT_API_URL = "http://ai.newhorizonco.uk/v1/chat/completions" |
|||
DEFAULT_API_KEY = "hZwAGW4H8v87Ol8tXvqEEY" |
|||
DEFAULT_MODEL = "gemini-3.6-flash-high" |
|||
|
|||
|
|||
class ThreadSafeRateLimiter: |
|||
""" |
|||
Thread-safe sliding window rate limiter. |
|||
Guarantees max `max_calls` executions per `period` seconds across all threads. |
|||
""" |
|||
def __init__(self, max_calls: int = 5, period: float = 1.0): |
|||
self.max_calls = max_calls |
|||
self.period = period |
|||
self.timestamps = [] |
|||
self.lock = threading.Lock() |
|||
|
|||
def acquire(self): |
|||
with self.lock: |
|||
while True: |
|||
now = time.time() |
|||
# Remove timestamps outside the sliding window |
|||
self.timestamps = [t for t in self.timestamps if now - t < self.period] |
|||
if len(self.timestamps) < self.max_calls: |
|||
self.timestamps.append(now) |
|||
break |
|||
# Wait until the oldest timestamp drops out of the 1-second window |
|||
sleep_time = self.period - (now - self.timestamps[0]) + 0.005 |
|||
if sleep_time > 0: |
|||
time.sleep(sleep_time) |
|||
|
|||
|
|||
class Command(BaseCommand): |
|||
help = "Translates CalendarOccasions titles into target languages concurrently (5 req/sec)." |
|||
|
|||
def add_arguments(self, parser): |
|||
parser.add_argument( |
|||
"--rate-limit", |
|||
type=int, |
|||
default=5, |
|||
help="Maximum API requests per second (default: 5)", |
|||
) |
|||
parser.add_argument( |
|||
"--workers", |
|||
type=int, |
|||
default=5, |
|||
help="Number of concurrent worker threads (default: 5)", |
|||
) |
|||
parser.add_argument( |
|||
"--limit", |
|||
type=int, |
|||
default=0, |
|||
help="Limit number of occasions to translate (0 = all)", |
|||
) |
|||
parser.add_argument( |
|||
"--force", |
|||
action="store_true", |
|||
help="Force re-translation of all target languages even if present", |
|||
) |
|||
parser.add_argument( |
|||
"--api-url", |
|||
type=str, |
|||
default=DEFAULT_API_URL, |
|||
help="API endpoint URL", |
|||
) |
|||
parser.add_argument( |
|||
"--api-key", |
|||
type=str, |
|||
default=DEFAULT_API_KEY, |
|||
help="API bearer token", |
|||
) |
|||
parser.add_argument( |
|||
"--model", |
|||
type=str, |
|||
default=DEFAULT_MODEL, |
|||
help="AI model name", |
|||
) |
|||
|
|||
def safe_write(self, msg: str, style_func=None): |
|||
"""Helper to write to stdout safely, catching Windows charmap encoding errors.""" |
|||
if style_func: |
|||
msg = style_func(msg) |
|||
try: |
|||
self.stdout.write(msg) |
|||
except Exception: |
|||
enc = getattr(sys.stdout, "encoding", "utf-8") or "utf-8" |
|||
safe_msg = msg.encode(enc, errors="replace").decode(enc) |
|||
try: |
|||
self.stdout.write(safe_msg) |
|||
except Exception: |
|||
pass |
|||
|
|||
def handle(self, *args, **options): |
|||
rate_limit = options["rate_limit"] |
|||
max_workers = options["workers"] |
|||
limit = options["limit"] |
|||
force = options["force"] |
|||
api_url = options["api_url"] |
|||
api_key = options["api_key"] |
|||
model = options["model"] |
|||
|
|||
self.safe_write( |
|||
f"Starting Calendar Occasions translation script...\n" |
|||
f" Rate Limit: {rate_limit} req/sec\n" |
|||
f" Workers: {max_workers}\n" |
|||
f" Target Languages: {len(TARGET_LANGUAGES)} languages\n" |
|||
f" Force Re-translate: {force}", |
|||
self.style.WARNING, |
|||
) |
|||
|
|||
rate_limiter = ThreadSafeRateLimiter(max_calls=rate_limit, period=1.0) |
|||
|
|||
queryset = CalendarOccasions.objects.all().order_by("id") |
|||
if limit > 0: |
|||
queryset = queryset[:limit] |
|||
|
|||
occasions = list(queryset) |
|||
total_count = len(occasions) |
|||
|
|||
if total_count == 0: |
|||
self.safe_write("No calendar occasions found in database.", self.style.WARNING) |
|||
return |
|||
|
|||
self.safe_write(f"Found {total_count} occasion objects to process.", self.style.SUCCESS) |
|||
|
|||
success_count = 0 |
|||
skipped_count = 0 |
|||
failed_count = 0 |
|||
lock = threading.Lock() |
|||
|
|||
def process_occasion(occasion): |
|||
nonlocal success_count, skipped_count, failed_count |
|||
|
|||
# Extract current title field structure |
|||
current_titles = occasion.title |
|||
if isinstance(current_titles, str): |
|||
current_titles = [{"text": current_titles, "title": current_titles, "language_code": "fa"}] |
|||
elif not isinstance(current_titles, list): |
|||
current_titles = [] |
|||
|
|||
# Map existing language_codes |
|||
existing_by_lang = {} |
|||
for item in current_titles: |
|||
if isinstance(item, dict) and "language_code" in item: |
|||
existing_by_lang[item["language_code"]] = item |
|||
|
|||
# Determine source Persian text (or any available text) |
|||
source_text = None |
|||
if "fa" in existing_by_lang: |
|||
source_text = existing_by_lang["fa"].get("title") or existing_by_lang["fa"].get("text") |
|||
if not source_text and current_titles: |
|||
first_item = current_titles[0] |
|||
if isinstance(first_item, dict): |
|||
source_text = first_item.get("title") or first_item.get("text") |
|||
elif isinstance(first_item, str): |
|||
source_text = first_item |
|||
|
|||
if not source_text: |
|||
with lock: |
|||
skipped_count += 1 |
|||
self.safe_write(f"[ID {occasion.id}] Skipped - No source text available", self.style.NOTICE) |
|||
return |
|||
|
|||
# Find missing target languages |
|||
if force: |
|||
missing_langs = list(TARGET_LANGUAGES.keys()) |
|||
else: |
|||
missing_langs = [ |
|||
lang for lang in TARGET_LANGUAGES.keys() |
|||
if lang not in existing_by_lang or not (existing_by_lang[lang].get("title") or existing_by_lang[lang].get("text")) |
|||
] |
|||
|
|||
if not missing_langs: |
|||
with lock: |
|||
skipped_count += 1 |
|||
self.safe_write( |
|||
f"[ID {occasion.id}] Skipped - All {len(TARGET_LANGUAGES)} languages already present ({source_text})", |
|||
self.style.SUCCESS, |
|||
) |
|||
return |
|||
|
|||
# Call AI API to translate missing languages |
|||
translations = self.call_translation_api( |
|||
source_text=source_text, |
|||
missing_langs=missing_langs, |
|||
rate_limiter=rate_limiter, |
|||
api_url=api_url, |
|||
api_key=api_key, |
|||
model=model, |
|||
) |
|||
|
|||
if not translations: |
|||
with lock: |
|||
failed_count += 1 |
|||
self.safe_write( |
|||
f"[ID {occasion.id}] Failed - API returned no translations for '{source_text}'", |
|||
self.style.ERROR, |
|||
) |
|||
return |
|||
|
|||
# Merge translations into occasion.title |
|||
for lang_code, translated_val in translations.items(): |
|||
if translated_val and isinstance(translated_val, str): |
|||
clean_val = translated_val.strip() |
|||
existing_by_lang[lang_code] = { |
|||
"text": clean_val, |
|||
"title": clean_val, |
|||
"language_code": lang_code, |
|||
} |
|||
|
|||
# Reconstruct title array keeping TARGET_LANGUAGES order first |
|||
new_title_list = [] |
|||
for code in TARGET_LANGUAGES.keys(): |
|||
if code in existing_by_lang: |
|||
new_title_list.append(existing_by_lang.pop(code)) |
|||
|
|||
# Append any remaining unexpected language objects |
|||
for item in existing_by_lang.values(): |
|||
new_title_list.append(item) |
|||
|
|||
occasion.title = new_title_list |
|||
occasion.save(update_fields=["title", "updated_at"]) |
|||
|
|||
with lock: |
|||
success_count += 1 |
|||
self.safe_write( |
|||
f"[ID {occasion.id}] Translated successfully into {len(translations)} languages ('{source_text}')", |
|||
self.style.SUCCESS, |
|||
) |
|||
|
|||
# Run concurrent workers |
|||
with ThreadPoolExecutor(max_workers=max_workers) as executor: |
|||
futures = [executor.submit(process_occasion, occ) for occ in occasions] |
|||
for future in as_completed(futures): |
|||
try: |
|||
future.result() |
|||
except Exception as exc: |
|||
self.safe_write(f"Unhandled error in thread: {exc}", self.style.ERROR) |
|||
|
|||
self.safe_write( |
|||
f"\nTranslation completed!\n" |
|||
f" Total Processed: {total_count}\n" |
|||
f" Successfully Updated: {success_count}\n" |
|||
f" Skipped (Up to date): {skipped_count}\n" |
|||
f" Failed: {failed_count}", |
|||
self.style.SUCCESS, |
|||
) |
|||
|
|||
def call_translation_api( |
|||
self, |
|||
source_text: str, |
|||
missing_langs: list, |
|||
rate_limiter: ThreadSafeRateLimiter, |
|||
api_url: str, |
|||
api_key: str, |
|||
model: str, |
|||
max_retries: int = 3, |
|||
) -> dict: |
|||
""" |
|||
Calls the AI completions API to translate source_text into missing_langs. |
|||
Enforces rate limiting prior to each request attempt. |
|||
""" |
|||
targets_desc = json.dumps({code: TARGET_LANGUAGES[code] for code in missing_langs}, ensure_ascii=False) |
|||
|
|||
prompt = ( |
|||
f"You are a professional translator for calendar events and occasions.\n" |
|||
f"Translate the following calendar occasion title from Persian into the specified target languages.\n\n" |
|||
f"Original Title: \"{source_text}\"\n\n" |
|||
f"Target Languages (JSON map of code -> Language Name):\n{targets_desc}\n\n" |
|||
f"INSTRUCTIONS:\n" |
|||
f"1. Translate accurately into each requested language code.\n" |
|||
f"2. Output strictly a JSON object mapping language_code to translation string.\n" |
|||
f"3. Do NOT include markdown code fences (```json or ```), explanations, or outside text.\n\n" |
|||
f"Example Output:\n" |
|||
f"{{\"en\": \"New Year\", \"ar\": \"السنة الجديدة\"}}" |
|||
) |
|||
|
|||
headers = { |
|||
"Authorization": f"Bearer {api_key}", |
|||
"Content-Type": "application/json", |
|||
} |
|||
payload = { |
|||
"model": model, |
|||
"stream": False, |
|||
"messages": [ |
|||
{ |
|||
"role": "user", |
|||
"content": prompt, |
|||
} |
|||
], |
|||
} |
|||
|
|||
for attempt in range(1, max_retries + 1): |
|||
rate_limiter.acquire() |
|||
try: |
|||
response = requests.post(api_url, headers=headers, json=payload, timeout=20) |
|||
if response.status_code == 200: |
|||
res_data = response.json() |
|||
raw_content = res_data["choices"][0]["message"]["content"].strip() |
|||
|
|||
# Clean any markdown code blocks if present |
|||
if raw_content.startswith("```"): |
|||
lines = raw_content.splitlines() |
|||
if lines and lines[0].startswith("```"): |
|||
lines = lines[1:] |
|||
if lines and lines[-1].startswith("```"): |
|||
lines = lines[:-1] |
|||
raw_content = "\n".join(lines).strip() |
|||
|
|||
try: |
|||
parsed = json.loads(raw_content) |
|||
if isinstance(parsed, dict): |
|||
# Filter only requested target language keys |
|||
return {k: v for k, v in parsed.items() if k in missing_langs and isinstance(v, str)} |
|||
except json.JSONDecodeError: |
|||
# Regex extract JSON substring if extra text was included |
|||
match = re.search(r"\{.*\}", raw_content, re.DOTALL) |
|||
if match: |
|||
try: |
|||
parsed = json.loads(match.group(0)) |
|||
if isinstance(parsed, dict): |
|||
return {k: v for k, v in parsed.items() if k in missing_langs and isinstance(v, str)} |
|||
except json.JSONDecodeError: |
|||
pass |
|||
except Exception as e: |
|||
if attempt == max_retries: |
|||
self.safe_write(f"API Call exception on attempt {attempt}: {e}", self.style.ERROR) |
|||
time.sleep(1) |
|||
|
|||
return {} |
|||
@ -0,0 +1,121 @@ |
|||
import io |
|||
from datetime import datetime, date |
|||
from typing import List, Any |
|||
from django.http import HttpResponse |
|||
from django.utils import timezone |
|||
import openpyxl |
|||
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side |
|||
from openpyxl.utils import get_column_letter |
|||
|
|||
|
|||
def export_to_excel_response( |
|||
filename: str, |
|||
headers: List[str], |
|||
rows: List[List[Any]], |
|||
sheet_title: str = "Export" |
|||
) -> HttpResponse: |
|||
""" |
|||
Generates a beautifully formatted Excel (.xlsx) file and returns it as a Django HttpResponse. |
|||
|
|||
:param filename: Base name of the file (without extension or with .xlsx) |
|||
:param headers: List of column header names |
|||
:param rows: List of data rows (each row is a list of cell values) |
|||
:param sheet_title: Title for the worksheet |
|||
:return: HttpResponse with .xlsx content type and attachment disposition |
|||
""" |
|||
wb = openpyxl.Workbook() |
|||
ws = wb.active |
|||
ws.title = sheet_title[:31] # Excel sheet title max 31 chars |
|||
|
|||
# Enable gridlines |
|||
ws.views.sheetView[0].showGridLines = True |
|||
|
|||
# 1. Styles Definition |
|||
# Modern dark indigo / slate header style |
|||
header_fill = PatternFill(start_color="1E293B", end_color="1E293B", fill_type="solid") |
|||
header_font = Font(name="Calibri", size=11, bold=True, color="FFFFFF") |
|||
header_alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) |
|||
|
|||
data_font = Font(name="Calibri", size=10, color="0F172A") |
|||
data_alignment_left = Alignment(horizontal="left", vertical="center") |
|||
data_alignment_center = Alignment(horizontal="center", vertical="center") |
|||
|
|||
thin_border_side = Side(border_style="thin", color="E2E8F0") |
|||
border_style = Border( |
|||
left=thin_border_side, |
|||
right=thin_border_side, |
|||
top=thin_border_side, |
|||
bottom=thin_border_side |
|||
) |
|||
|
|||
zebra_fill = PatternFill(start_color="F8FAFC", end_color="F8FAFC", fill_type="solid") |
|||
|
|||
# 2. Write Headers |
|||
ws.append(headers) |
|||
header_row = ws[1] |
|||
ws.row_dimensions[1].height = 28 |
|||
|
|||
for col_idx, cell in enumerate(header_row, 1): |
|||
cell.font = header_font |
|||
cell.fill = header_fill |
|||
cell.alignment = header_alignment |
|||
cell.border = border_style |
|||
|
|||
# 3. Write Data Rows |
|||
for row_idx, row_data in enumerate(rows, start=2): |
|||
formatted_row = [] |
|||
for val in row_data: |
|||
if val is None: |
|||
formatted_row.append("") |
|||
elif isinstance(val, (datetime, date)): |
|||
if isinstance(val, datetime) and timezone.is_aware(val): |
|||
val = timezone.localtime(val) |
|||
formatted_row.append(val.strftime("%Y-%m-%d %H:%M")) |
|||
elif isinstance(val, bool): |
|||
formatted_row.append("Yes" if val else "No") |
|||
else: |
|||
formatted_row.append(str(val)) |
|||
|
|||
ws.append(formatted_row) |
|||
current_row = ws[row_idx] |
|||
ws.row_dimensions[row_idx].height = 22 |
|||
|
|||
is_even = (row_idx % 2 == 0) |
|||
for col_idx, cell in enumerate(current_row, 1): |
|||
cell.font = data_font |
|||
cell.border = border_style |
|||
if is_even: |
|||
cell.fill = zebra_fill |
|||
# Align IDs or numbers center, text left |
|||
val = row_data[col_idx - 1] if col_idx - 1 < len(row_data) else None |
|||
if isinstance(val, (int, float, date, datetime)) or (isinstance(val, str) and val.isdigit()): |
|||
cell.alignment = data_alignment_center |
|||
else: |
|||
cell.alignment = data_alignment_left |
|||
|
|||
# 4. Auto-fit column widths (with min 12 and max 50) |
|||
for col in ws.columns: |
|||
max_len = 0 |
|||
col_letter = get_column_letter(col[0].column) |
|||
for cell in col: |
|||
val_str = str(cell.value or "") |
|||
if len(val_str) > max_len: |
|||
max_len = len(val_str) |
|||
adjusted_width = min(max(max_len + 4, 12), 50) |
|||
ws.column_dimensions[col_letter].width = adjusted_width |
|||
|
|||
# 5. Output to buffer |
|||
output = io.BytesIO() |
|||
wb.save(output) |
|||
output.seek(0) |
|||
|
|||
# Ensure .xlsx extension in filename |
|||
clean_filename = filename if filename.endswith(".xlsx") else f"{filename}.xlsx" |
|||
|
|||
response = HttpResponse( |
|||
output.getvalue(), |
|||
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" |
|||
) |
|||
response["Content-Disposition"] = f'attachment; filename="{clean_filename}"' |
|||
response["Access-Control-Expose-Headers"] = "Content-Disposition" |
|||
return response |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue