7 Commits
e0a805a446
...
403e068475
10 changed files with 738 additions and 49 deletions
-
97apps/account/views/user.py
-
362apps/dobodbi_calendar/management/commands/translate_occasions.py
-
35apps/hadis/docs.py
-
7apps/hadis/models/category.py
-
10apps/hadis/views/hadis.py
-
11apps/hadis/views_admin.py
-
6apps/video/serializers_dovodi.py
-
1requirements.txt
-
137utils/__init__.py
-
121utils/excel_exporter.py
@ -0,0 +1,362 @@ |
|||||
|
import json |
||||
|
import re |
||||
|
import sys |
||||
|
import time |
||||
|
import requests |
||||
|
import threading |
||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed |
||||
|
|
||||
|
from django.core.management.base import BaseCommand |
||||
|
from apps.dobodbi_calendar.models import CalendarOccasions |
||||
|
|
||||
|
# Target languages from زبانها.md |
||||
|
TARGET_LANGUAGES = { |
||||
|
"ar": "Arabic (عربی)", |
||||
|
"tr": "Turkish (ترکی)", |
||||
|
"fa": "Persian (فارسی)", |
||||
|
"ur": "Urdu (اردو)", |
||||
|
"bn": "Bengali (بنگالی)", |
||||
|
"id": "Indonesian (اندونزیایی)", |
||||
|
"fr": "French (فرانسوی)", |
||||
|
"ru": "Russian (روسی)", |
||||
|
"uz": "Uzbek (ازبکی)", |
||||
|
"ky": "Kyrgyz (قرقیزی)", |
||||
|
"tg": "Tajik (تاجیکی)", |
||||
|
"az": "Azerbaijani (آذربایجانی)", |
||||
|
"en": "English (انگلیسی)", |
||||
|
"de": "German (آلمانی)", |
||||
|
"zh": "Mandarin Chinese (چینی ماندارین)", |
||||
|
"ha": "Hausa (هوسا)", |
||||
|
"sw": "Swahili (سواحیلی)", |
||||
|
"es": "Spanish (اسپانیایی)", |
||||
|
} |
||||
|
|
||||
|
DEFAULT_API_URL = "http://ai.newhorizonco.uk/v1/chat/completions" |
||||
|
DEFAULT_API_KEY = "hZwAGW4H8v87Ol8tXvqEEY" |
||||
|
DEFAULT_MODEL = "gemini-3.6-flash-high" |
||||
|
|
||||
|
|
||||
|
class ThreadSafeRateLimiter: |
||||
|
""" |
||||
|
Thread-safe sliding window rate limiter. |
||||
|
Guarantees max `max_calls` executions per `period` seconds across all threads. |
||||
|
""" |
||||
|
def __init__(self, max_calls: int = 5, period: float = 1.0): |
||||
|
self.max_calls = max_calls |
||||
|
self.period = period |
||||
|
self.timestamps = [] |
||||
|
self.lock = threading.Lock() |
||||
|
|
||||
|
def acquire(self): |
||||
|
with self.lock: |
||||
|
while True: |
||||
|
now = time.time() |
||||
|
# Remove timestamps outside the sliding window |
||||
|
self.timestamps = [t for t in self.timestamps if now - t < self.period] |
||||
|
if len(self.timestamps) < self.max_calls: |
||||
|
self.timestamps.append(now) |
||||
|
break |
||||
|
# Wait until the oldest timestamp drops out of the 1-second window |
||||
|
sleep_time = self.period - (now - self.timestamps[0]) + 0.005 |
||||
|
if sleep_time > 0: |
||||
|
time.sleep(sleep_time) |
||||
|
|
||||
|
|
||||
|
class Command(BaseCommand): |
||||
|
help = "Translates CalendarOccasions titles into target languages concurrently (5 req/sec)." |
||||
|
|
||||
|
def add_arguments(self, parser): |
||||
|
parser.add_argument( |
||||
|
"--rate-limit", |
||||
|
type=int, |
||||
|
default=5, |
||||
|
help="Maximum API requests per second (default: 5)", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--workers", |
||||
|
type=int, |
||||
|
default=5, |
||||
|
help="Number of concurrent worker threads (default: 5)", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--limit", |
||||
|
type=int, |
||||
|
default=0, |
||||
|
help="Limit number of occasions to translate (0 = all)", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--force", |
||||
|
action="store_true", |
||||
|
help="Force re-translation of all target languages even if present", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--api-url", |
||||
|
type=str, |
||||
|
default=DEFAULT_API_URL, |
||||
|
help="API endpoint URL", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--api-key", |
||||
|
type=str, |
||||
|
default=DEFAULT_API_KEY, |
||||
|
help="API bearer token", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--model", |
||||
|
type=str, |
||||
|
default=DEFAULT_MODEL, |
||||
|
help="AI model name", |
||||
|
) |
||||
|
|
||||
|
def safe_write(self, msg: str, style_func=None): |
||||
|
"""Helper to write to stdout safely, catching Windows charmap encoding errors.""" |
||||
|
if style_func: |
||||
|
msg = style_func(msg) |
||||
|
try: |
||||
|
self.stdout.write(msg) |
||||
|
except Exception: |
||||
|
enc = getattr(sys.stdout, "encoding", "utf-8") or "utf-8" |
||||
|
safe_msg = msg.encode(enc, errors="replace").decode(enc) |
||||
|
try: |
||||
|
self.stdout.write(safe_msg) |
||||
|
except Exception: |
||||
|
pass |
||||
|
|
||||
|
def handle(self, *args, **options): |
||||
|
rate_limit = options["rate_limit"] |
||||
|
max_workers = options["workers"] |
||||
|
limit = options["limit"] |
||||
|
force = options["force"] |
||||
|
api_url = options["api_url"] |
||||
|
api_key = options["api_key"] |
||||
|
model = options["model"] |
||||
|
|
||||
|
self.safe_write( |
||||
|
f"Starting Calendar Occasions translation script...\n" |
||||
|
f" Rate Limit: {rate_limit} req/sec\n" |
||||
|
f" Workers: {max_workers}\n" |
||||
|
f" Target Languages: {len(TARGET_LANGUAGES)} languages\n" |
||||
|
f" Force Re-translate: {force}", |
||||
|
self.style.WARNING, |
||||
|
) |
||||
|
|
||||
|
rate_limiter = ThreadSafeRateLimiter(max_calls=rate_limit, period=1.0) |
||||
|
|
||||
|
queryset = CalendarOccasions.objects.all().order_by("id") |
||||
|
if limit > 0: |
||||
|
queryset = queryset[:limit] |
||||
|
|
||||
|
occasions = list(queryset) |
||||
|
total_count = len(occasions) |
||||
|
|
||||
|
if total_count == 0: |
||||
|
self.safe_write("No calendar occasions found in database.", self.style.WARNING) |
||||
|
return |
||||
|
|
||||
|
self.safe_write(f"Found {total_count} occasion objects to process.", self.style.SUCCESS) |
||||
|
|
||||
|
success_count = 0 |
||||
|
skipped_count = 0 |
||||
|
failed_count = 0 |
||||
|
lock = threading.Lock() |
||||
|
|
||||
|
def process_occasion(occasion): |
||||
|
nonlocal success_count, skipped_count, failed_count |
||||
|
|
||||
|
# Extract current title field structure |
||||
|
current_titles = occasion.title |
||||
|
if isinstance(current_titles, str): |
||||
|
current_titles = [{"text": current_titles, "title": current_titles, "language_code": "fa"}] |
||||
|
elif not isinstance(current_titles, list): |
||||
|
current_titles = [] |
||||
|
|
||||
|
# Map existing language_codes |
||||
|
existing_by_lang = {} |
||||
|
for item in current_titles: |
||||
|
if isinstance(item, dict) and "language_code" in item: |
||||
|
existing_by_lang[item["language_code"]] = item |
||||
|
|
||||
|
# Determine source Persian text (or any available text) |
||||
|
source_text = None |
||||
|
if "fa" in existing_by_lang: |
||||
|
source_text = existing_by_lang["fa"].get("title") or existing_by_lang["fa"].get("text") |
||||
|
if not source_text and current_titles: |
||||
|
first_item = current_titles[0] |
||||
|
if isinstance(first_item, dict): |
||||
|
source_text = first_item.get("title") or first_item.get("text") |
||||
|
elif isinstance(first_item, str): |
||||
|
source_text = first_item |
||||
|
|
||||
|
if not source_text: |
||||
|
with lock: |
||||
|
skipped_count += 1 |
||||
|
self.safe_write(f"[ID {occasion.id}] Skipped - No source text available", self.style.NOTICE) |
||||
|
return |
||||
|
|
||||
|
# Find missing target languages |
||||
|
if force: |
||||
|
missing_langs = list(TARGET_LANGUAGES.keys()) |
||||
|
else: |
||||
|
missing_langs = [ |
||||
|
lang for lang in TARGET_LANGUAGES.keys() |
||||
|
if lang not in existing_by_lang or not (existing_by_lang[lang].get("title") or existing_by_lang[lang].get("text")) |
||||
|
] |
||||
|
|
||||
|
if not missing_langs: |
||||
|
with lock: |
||||
|
skipped_count += 1 |
||||
|
self.safe_write( |
||||
|
f"[ID {occasion.id}] Skipped - All {len(TARGET_LANGUAGES)} languages already present ({source_text})", |
||||
|
self.style.SUCCESS, |
||||
|
) |
||||
|
return |
||||
|
|
||||
|
# Call AI API to translate missing languages |
||||
|
translations = self.call_translation_api( |
||||
|
source_text=source_text, |
||||
|
missing_langs=missing_langs, |
||||
|
rate_limiter=rate_limiter, |
||||
|
api_url=api_url, |
||||
|
api_key=api_key, |
||||
|
model=model, |
||||
|
) |
||||
|
|
||||
|
if not translations: |
||||
|
with lock: |
||||
|
failed_count += 1 |
||||
|
self.safe_write( |
||||
|
f"[ID {occasion.id}] Failed - API returned no translations for '{source_text}'", |
||||
|
self.style.ERROR, |
||||
|
) |
||||
|
return |
||||
|
|
||||
|
# Merge translations into occasion.title |
||||
|
for lang_code, translated_val in translations.items(): |
||||
|
if translated_val and isinstance(translated_val, str): |
||||
|
clean_val = translated_val.strip() |
||||
|
existing_by_lang[lang_code] = { |
||||
|
"text": clean_val, |
||||
|
"title": clean_val, |
||||
|
"language_code": lang_code, |
||||
|
} |
||||
|
|
||||
|
# Reconstruct title array keeping TARGET_LANGUAGES order first |
||||
|
new_title_list = [] |
||||
|
for code in TARGET_LANGUAGES.keys(): |
||||
|
if code in existing_by_lang: |
||||
|
new_title_list.append(existing_by_lang.pop(code)) |
||||
|
|
||||
|
# Append any remaining unexpected language objects |
||||
|
for item in existing_by_lang.values(): |
||||
|
new_title_list.append(item) |
||||
|
|
||||
|
occasion.title = new_title_list |
||||
|
occasion.save(update_fields=["title", "updated_at"]) |
||||
|
|
||||
|
with lock: |
||||
|
success_count += 1 |
||||
|
self.safe_write( |
||||
|
f"[ID {occasion.id}] Translated successfully into {len(translations)} languages ('{source_text}')", |
||||
|
self.style.SUCCESS, |
||||
|
) |
||||
|
|
||||
|
# Run concurrent workers |
||||
|
with ThreadPoolExecutor(max_workers=max_workers) as executor: |
||||
|
futures = [executor.submit(process_occasion, occ) for occ in occasions] |
||||
|
for future in as_completed(futures): |
||||
|
try: |
||||
|
future.result() |
||||
|
except Exception as exc: |
||||
|
self.safe_write(f"Unhandled error in thread: {exc}", self.style.ERROR) |
||||
|
|
||||
|
self.safe_write( |
||||
|
f"\nTranslation completed!\n" |
||||
|
f" Total Processed: {total_count}\n" |
||||
|
f" Successfully Updated: {success_count}\n" |
||||
|
f" Skipped (Up to date): {skipped_count}\n" |
||||
|
f" Failed: {failed_count}", |
||||
|
self.style.SUCCESS, |
||||
|
) |
||||
|
|
||||
|
def call_translation_api( |
||||
|
self, |
||||
|
source_text: str, |
||||
|
missing_langs: list, |
||||
|
rate_limiter: ThreadSafeRateLimiter, |
||||
|
api_url: str, |
||||
|
api_key: str, |
||||
|
model: str, |
||||
|
max_retries: int = 3, |
||||
|
) -> dict: |
||||
|
""" |
||||
|
Calls the AI completions API to translate source_text into missing_langs. |
||||
|
Enforces rate limiting prior to each request attempt. |
||||
|
""" |
||||
|
targets_desc = json.dumps({code: TARGET_LANGUAGES[code] for code in missing_langs}, ensure_ascii=False) |
||||
|
|
||||
|
prompt = ( |
||||
|
f"You are a professional translator for calendar events and occasions.\n" |
||||
|
f"Translate the following calendar occasion title from Persian into the specified target languages.\n\n" |
||||
|
f"Original Title: \"{source_text}\"\n\n" |
||||
|
f"Target Languages (JSON map of code -> Language Name):\n{targets_desc}\n\n" |
||||
|
f"INSTRUCTIONS:\n" |
||||
|
f"1. Translate accurately into each requested language code.\n" |
||||
|
f"2. Output strictly a JSON object mapping language_code to translation string.\n" |
||||
|
f"3. Do NOT include markdown code fences (```json or ```), explanations, or outside text.\n\n" |
||||
|
f"Example Output:\n" |
||||
|
f"{{\"en\": \"New Year\", \"ar\": \"السنة الجديدة\"}}" |
||||
|
) |
||||
|
|
||||
|
headers = { |
||||
|
"Authorization": f"Bearer {api_key}", |
||||
|
"Content-Type": "application/json", |
||||
|
} |
||||
|
payload = { |
||||
|
"model": model, |
||||
|
"stream": False, |
||||
|
"messages": [ |
||||
|
{ |
||||
|
"role": "user", |
||||
|
"content": prompt, |
||||
|
} |
||||
|
], |
||||
|
} |
||||
|
|
||||
|
for attempt in range(1, max_retries + 1): |
||||
|
rate_limiter.acquire() |
||||
|
try: |
||||
|
response = requests.post(api_url, headers=headers, json=payload, timeout=20) |
||||
|
if response.status_code == 200: |
||||
|
res_data = response.json() |
||||
|
raw_content = res_data["choices"][0]["message"]["content"].strip() |
||||
|
|
||||
|
# Clean any markdown code blocks if present |
||||
|
if raw_content.startswith("```"): |
||||
|
lines = raw_content.splitlines() |
||||
|
if lines and lines[0].startswith("```"): |
||||
|
lines = lines[1:] |
||||
|
if lines and lines[-1].startswith("```"): |
||||
|
lines = lines[:-1] |
||||
|
raw_content = "\n".join(lines).strip() |
||||
|
|
||||
|
try: |
||||
|
parsed = json.loads(raw_content) |
||||
|
if isinstance(parsed, dict): |
||||
|
# Filter only requested target language keys |
||||
|
return {k: v for k, v in parsed.items() if k in missing_langs and isinstance(v, str)} |
||||
|
except json.JSONDecodeError: |
||||
|
# Regex extract JSON substring if extra text was included |
||||
|
match = re.search(r"\{.*\}", raw_content, re.DOTALL) |
||||
|
if match: |
||||
|
try: |
||||
|
parsed = json.loads(match.group(0)) |
||||
|
if isinstance(parsed, dict): |
||||
|
return {k: v for k, v in parsed.items() if k in missing_langs and isinstance(v, str)} |
||||
|
except json.JSONDecodeError: |
||||
|
pass |
||||
|
except Exception as e: |
||||
|
if attempt == max_retries: |
||||
|
self.safe_write(f"API Call exception on attempt {attempt}: {e}", self.style.ERROR) |
||||
|
time.sleep(1) |
||||
|
|
||||
|
return {} |
||||
@ -0,0 +1,121 @@ |
|||||
|
import io |
||||
|
from datetime import datetime, date |
||||
|
from typing import List, Any |
||||
|
from django.http import HttpResponse |
||||
|
from django.utils import timezone |
||||
|
import openpyxl |
||||
|
from openpyxl.styles import Font, PatternFill, Alignment, Border, Side |
||||
|
from openpyxl.utils import get_column_letter |
||||
|
|
||||
|
|
||||
|
def export_to_excel_response( |
||||
|
filename: str, |
||||
|
headers: List[str], |
||||
|
rows: List[List[Any]], |
||||
|
sheet_title: str = "Export" |
||||
|
) -> HttpResponse: |
||||
|
""" |
||||
|
Generates a beautifully formatted Excel (.xlsx) file and returns it as a Django HttpResponse. |
||||
|
|
||||
|
:param filename: Base name of the file (without extension or with .xlsx) |
||||
|
:param headers: List of column header names |
||||
|
:param rows: List of data rows (each row is a list of cell values) |
||||
|
:param sheet_title: Title for the worksheet |
||||
|
:return: HttpResponse with .xlsx content type and attachment disposition |
||||
|
""" |
||||
|
wb = openpyxl.Workbook() |
||||
|
ws = wb.active |
||||
|
ws.title = sheet_title[:31] # Excel sheet title max 31 chars |
||||
|
|
||||
|
# Enable gridlines |
||||
|
ws.views.sheetView[0].showGridLines = True |
||||
|
|
||||
|
# 1. Styles Definition |
||||
|
# Modern dark indigo / slate header style |
||||
|
header_fill = PatternFill(start_color="1E293B", end_color="1E293B", fill_type="solid") |
||||
|
header_font = Font(name="Calibri", size=11, bold=True, color="FFFFFF") |
||||
|
header_alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) |
||||
|
|
||||
|
data_font = Font(name="Calibri", size=10, color="0F172A") |
||||
|
data_alignment_left = Alignment(horizontal="left", vertical="center") |
||||
|
data_alignment_center = Alignment(horizontal="center", vertical="center") |
||||
|
|
||||
|
thin_border_side = Side(border_style="thin", color="E2E8F0") |
||||
|
border_style = Border( |
||||
|
left=thin_border_side, |
||||
|
right=thin_border_side, |
||||
|
top=thin_border_side, |
||||
|
bottom=thin_border_side |
||||
|
) |
||||
|
|
||||
|
zebra_fill = PatternFill(start_color="F8FAFC", end_color="F8FAFC", fill_type="solid") |
||||
|
|
||||
|
# 2. Write Headers |
||||
|
ws.append(headers) |
||||
|
header_row = ws[1] |
||||
|
ws.row_dimensions[1].height = 28 |
||||
|
|
||||
|
for col_idx, cell in enumerate(header_row, 1): |
||||
|
cell.font = header_font |
||||
|
cell.fill = header_fill |
||||
|
cell.alignment = header_alignment |
||||
|
cell.border = border_style |
||||
|
|
||||
|
# 3. Write Data Rows |
||||
|
for row_idx, row_data in enumerate(rows, start=2): |
||||
|
formatted_row = [] |
||||
|
for val in row_data: |
||||
|
if val is None: |
||||
|
formatted_row.append("") |
||||
|
elif isinstance(val, (datetime, date)): |
||||
|
if isinstance(val, datetime) and timezone.is_aware(val): |
||||
|
val = timezone.localtime(val) |
||||
|
formatted_row.append(val.strftime("%Y-%m-%d %H:%M")) |
||||
|
elif isinstance(val, bool): |
||||
|
formatted_row.append("Yes" if val else "No") |
||||
|
else: |
||||
|
formatted_row.append(str(val)) |
||||
|
|
||||
|
ws.append(formatted_row) |
||||
|
current_row = ws[row_idx] |
||||
|
ws.row_dimensions[row_idx].height = 22 |
||||
|
|
||||
|
is_even = (row_idx % 2 == 0) |
||||
|
for col_idx, cell in enumerate(current_row, 1): |
||||
|
cell.font = data_font |
||||
|
cell.border = border_style |
||||
|
if is_even: |
||||
|
cell.fill = zebra_fill |
||||
|
# Align IDs or numbers center, text left |
||||
|
val = row_data[col_idx - 1] if col_idx - 1 < len(row_data) else None |
||||
|
if isinstance(val, (int, float, date, datetime)) or (isinstance(val, str) and val.isdigit()): |
||||
|
cell.alignment = data_alignment_center |
||||
|
else: |
||||
|
cell.alignment = data_alignment_left |
||||
|
|
||||
|
# 4. Auto-fit column widths (with min 12 and max 50) |
||||
|
for col in ws.columns: |
||||
|
max_len = 0 |
||||
|
col_letter = get_column_letter(col[0].column) |
||||
|
for cell in col: |
||||
|
val_str = str(cell.value or "") |
||||
|
if len(val_str) > max_len: |
||||
|
max_len = len(val_str) |
||||
|
adjusted_width = min(max(max_len + 4, 12), 50) |
||||
|
ws.column_dimensions[col_letter].width = adjusted_width |
||||
|
|
||||
|
# 5. Output to buffer |
||||
|
output = io.BytesIO() |
||||
|
wb.save(output) |
||||
|
output.seek(0) |
||||
|
|
||||
|
# Ensure .xlsx extension in filename |
||||
|
clean_filename = filename if filename.endswith(".xlsx") else f"{filename}.xlsx" |
||||
|
|
||||
|
response = HttpResponse( |
||||
|
output.getvalue(), |
||||
|
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" |
||||
|
) |
||||
|
response["Content-Disposition"] = f'attachment; filename="{clean_filename}"' |
||||
|
response["Access-Control-Expose-Headers"] = "Content-Disposition" |
||||
|
return response |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue