2 changed files with 459 additions and 0 deletions
@ -0,0 +1,362 @@ |
|||||
|
import json |
||||
|
import re |
||||
|
import sys |
||||
|
import time |
||||
|
import requests |
||||
|
import threading |
||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed |
||||
|
|
||||
|
from django.core.management.base import BaseCommand |
||||
|
from apps.dobodbi_calendar.models import CalendarOccasions |
||||
|
|
||||
|
# Target languages from زبانها.md |
||||
|
TARGET_LANGUAGES = { |
||||
|
"ar": "Arabic (عربی)", |
||||
|
"tr": "Turkish (ترکی)", |
||||
|
"fa": "Persian (فارسی)", |
||||
|
"ur": "Urdu (اردو)", |
||||
|
"bn": "Bengali (بنگالی)", |
||||
|
"id": "Indonesian (اندونزیایی)", |
||||
|
"fr": "French (فرانسوی)", |
||||
|
"ru": "Russian (روسی)", |
||||
|
"uz": "Uzbek (ازبکی)", |
||||
|
"ky": "Kyrgyz (قرقیزی)", |
||||
|
"tg": "Tajik (تاجیکی)", |
||||
|
"az": "Azerbaijani (آذربایجانی)", |
||||
|
"en": "English (انگلیسی)", |
||||
|
"de": "German (آلمانی)", |
||||
|
"zh": "Mandarin Chinese (چینی ماندارین)", |
||||
|
"ha": "Hausa (هوسا)", |
||||
|
"sw": "Swahili (سواحیلی)", |
||||
|
"es": "Spanish (اسپانیایی)", |
||||
|
} |
||||
|
|
||||
|
DEFAULT_API_URL = "http://ai.newhorizonco.uk/v1/chat/completions" |
||||
|
DEFAULT_API_KEY = "hZwAGW4H8v87Ol8tXvqEEY" |
||||
|
DEFAULT_MODEL = "gemini-3.6-flash-high" |
||||
|
|
||||
|
|
||||
|
class ThreadSafeRateLimiter: |
||||
|
""" |
||||
|
Thread-safe sliding window rate limiter. |
||||
|
Guarantees max `max_calls` executions per `period` seconds across all threads. |
||||
|
""" |
||||
|
def __init__(self, max_calls: int = 5, period: float = 1.0): |
||||
|
self.max_calls = max_calls |
||||
|
self.period = period |
||||
|
self.timestamps = [] |
||||
|
self.lock = threading.Lock() |
||||
|
|
||||
|
def acquire(self): |
||||
|
with self.lock: |
||||
|
while True: |
||||
|
now = time.time() |
||||
|
# Remove timestamps outside the sliding window |
||||
|
self.timestamps = [t for t in self.timestamps if now - t < self.period] |
||||
|
if len(self.timestamps) < self.max_calls: |
||||
|
self.timestamps.append(now) |
||||
|
break |
||||
|
# Wait until the oldest timestamp drops out of the 1-second window |
||||
|
sleep_time = self.period - (now - self.timestamps[0]) + 0.005 |
||||
|
if sleep_time > 0: |
||||
|
time.sleep(sleep_time) |
||||
|
|
||||
|
|
||||
|
class Command(BaseCommand): |
||||
|
help = "Translates CalendarOccasions titles into target languages concurrently (5 req/sec)." |
||||
|
|
||||
|
def add_arguments(self, parser): |
||||
|
parser.add_argument( |
||||
|
"--rate-limit", |
||||
|
type=int, |
||||
|
default=5, |
||||
|
help="Maximum API requests per second (default: 5)", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--workers", |
||||
|
type=int, |
||||
|
default=5, |
||||
|
help="Number of concurrent worker threads (default: 5)", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--limit", |
||||
|
type=int, |
||||
|
default=0, |
||||
|
help="Limit number of occasions to translate (0 = all)", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--force", |
||||
|
action="store_true", |
||||
|
help="Force re-translation of all target languages even if present", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--api-url", |
||||
|
type=str, |
||||
|
default=DEFAULT_API_URL, |
||||
|
help="API endpoint URL", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--api-key", |
||||
|
type=str, |
||||
|
default=DEFAULT_API_KEY, |
||||
|
help="API bearer token", |
||||
|
) |
||||
|
parser.add_argument( |
||||
|
"--model", |
||||
|
type=str, |
||||
|
default=DEFAULT_MODEL, |
||||
|
help="AI model name", |
||||
|
) |
||||
|
|
||||
|
def safe_write(self, msg: str, style_func=None): |
||||
|
"""Helper to write to stdout safely, catching Windows charmap encoding errors.""" |
||||
|
if style_func: |
||||
|
msg = style_func(msg) |
||||
|
try: |
||||
|
self.stdout.write(msg) |
||||
|
except Exception: |
||||
|
enc = getattr(sys.stdout, "encoding", "utf-8") or "utf-8" |
||||
|
safe_msg = msg.encode(enc, errors="replace").decode(enc) |
||||
|
try: |
||||
|
self.stdout.write(safe_msg) |
||||
|
except Exception: |
||||
|
pass |
||||
|
|
||||
|
def handle(self, *args, **options): |
||||
|
rate_limit = options["rate_limit"] |
||||
|
max_workers = options["workers"] |
||||
|
limit = options["limit"] |
||||
|
force = options["force"] |
||||
|
api_url = options["api_url"] |
||||
|
api_key = options["api_key"] |
||||
|
model = options["model"] |
||||
|
|
||||
|
self.safe_write( |
||||
|
f"Starting Calendar Occasions translation script...\n" |
||||
|
f" Rate Limit: {rate_limit} req/sec\n" |
||||
|
f" Workers: {max_workers}\n" |
||||
|
f" Target Languages: {len(TARGET_LANGUAGES)} languages\n" |
||||
|
f" Force Re-translate: {force}", |
||||
|
self.style.WARNING, |
||||
|
) |
||||
|
|
||||
|
rate_limiter = ThreadSafeRateLimiter(max_calls=rate_limit, period=1.0) |
||||
|
|
||||
|
queryset = CalendarOccasions.objects.all().order_by("id") |
||||
|
if limit > 0: |
||||
|
queryset = queryset[:limit] |
||||
|
|
||||
|
occasions = list(queryset) |
||||
|
total_count = len(occasions) |
||||
|
|
||||
|
if total_count == 0: |
||||
|
self.safe_write("No calendar occasions found in database.", self.style.WARNING) |
||||
|
return |
||||
|
|
||||
|
self.safe_write(f"Found {total_count} occasion objects to process.", self.style.SUCCESS) |
||||
|
|
||||
|
success_count = 0 |
||||
|
skipped_count = 0 |
||||
|
failed_count = 0 |
||||
|
lock = threading.Lock() |
||||
|
|
||||
|
def process_occasion(occasion): |
||||
|
nonlocal success_count, skipped_count, failed_count |
||||
|
|
||||
|
# Extract current title field structure |
||||
|
current_titles = occasion.title |
||||
|
if isinstance(current_titles, str): |
||||
|
current_titles = [{"text": current_titles, "title": current_titles, "language_code": "fa"}] |
||||
|
elif not isinstance(current_titles, list): |
||||
|
current_titles = [] |
||||
|
|
||||
|
# Map existing language_codes |
||||
|
existing_by_lang = {} |
||||
|
for item in current_titles: |
||||
|
if isinstance(item, dict) and "language_code" in item: |
||||
|
existing_by_lang[item["language_code"]] = item |
||||
|
|
||||
|
# Determine source Persian text (or any available text) |
||||
|
source_text = None |
||||
|
if "fa" in existing_by_lang: |
||||
|
source_text = existing_by_lang["fa"].get("title") or existing_by_lang["fa"].get("text") |
||||
|
if not source_text and current_titles: |
||||
|
first_item = current_titles[0] |
||||
|
if isinstance(first_item, dict): |
||||
|
source_text = first_item.get("title") or first_item.get("text") |
||||
|
elif isinstance(first_item, str): |
||||
|
source_text = first_item |
||||
|
|
||||
|
if not source_text: |
||||
|
with lock: |
||||
|
skipped_count += 1 |
||||
|
self.safe_write(f"[ID {occasion.id}] Skipped - No source text available", self.style.NOTICE) |
||||
|
return |
||||
|
|
||||
|
# Find missing target languages |
||||
|
if force: |
||||
|
missing_langs = list(TARGET_LANGUAGES.keys()) |
||||
|
else: |
||||
|
missing_langs = [ |
||||
|
lang for lang in TARGET_LANGUAGES.keys() |
||||
|
if lang not in existing_by_lang or not (existing_by_lang[lang].get("title") or existing_by_lang[lang].get("text")) |
||||
|
] |
||||
|
|
||||
|
if not missing_langs: |
||||
|
with lock: |
||||
|
skipped_count += 1 |
||||
|
self.safe_write( |
||||
|
f"[ID {occasion.id}] Skipped - All {len(TARGET_LANGUAGES)} languages already present ({source_text})", |
||||
|
self.style.SUCCESS, |
||||
|
) |
||||
|
return |
||||
|
|
||||
|
# Call AI API to translate missing languages |
||||
|
translations = self.call_translation_api( |
||||
|
source_text=source_text, |
||||
|
missing_langs=missing_langs, |
||||
|
rate_limiter=rate_limiter, |
||||
|
api_url=api_url, |
||||
|
api_key=api_key, |
||||
|
model=model, |
||||
|
) |
||||
|
|
||||
|
if not translations: |
||||
|
with lock: |
||||
|
failed_count += 1 |
||||
|
self.safe_write( |
||||
|
f"[ID {occasion.id}] Failed - API returned no translations for '{source_text}'", |
||||
|
self.style.ERROR, |
||||
|
) |
||||
|
return |
||||
|
|
||||
|
# Merge translations into occasion.title |
||||
|
for lang_code, translated_val in translations.items(): |
||||
|
if translated_val and isinstance(translated_val, str): |
||||
|
clean_val = translated_val.strip() |
||||
|
existing_by_lang[lang_code] = { |
||||
|
"text": clean_val, |
||||
|
"title": clean_val, |
||||
|
"language_code": lang_code, |
||||
|
} |
||||
|
|
||||
|
# Reconstruct title array keeping TARGET_LANGUAGES order first |
||||
|
new_title_list = [] |
||||
|
for code in TARGET_LANGUAGES.keys(): |
||||
|
if code in existing_by_lang: |
||||
|
new_title_list.append(existing_by_lang.pop(code)) |
||||
|
|
||||
|
# Append any remaining unexpected language objects |
||||
|
for item in existing_by_lang.values(): |
||||
|
new_title_list.append(item) |
||||
|
|
||||
|
occasion.title = new_title_list |
||||
|
occasion.save(update_fields=["title", "updated_at"]) |
||||
|
|
||||
|
with lock: |
||||
|
success_count += 1 |
||||
|
self.safe_write( |
||||
|
f"[ID {occasion.id}] Translated successfully into {len(translations)} languages ('{source_text}')", |
||||
|
self.style.SUCCESS, |
||||
|
) |
||||
|
|
||||
|
# Run concurrent workers |
||||
|
with ThreadPoolExecutor(max_workers=max_workers) as executor: |
||||
|
futures = [executor.submit(process_occasion, occ) for occ in occasions] |
||||
|
for future in as_completed(futures): |
||||
|
try: |
||||
|
future.result() |
||||
|
except Exception as exc: |
||||
|
self.safe_write(f"Unhandled error in thread: {exc}", self.style.ERROR) |
||||
|
|
||||
|
self.safe_write( |
||||
|
f"\nTranslation completed!\n" |
||||
|
f" Total Processed: {total_count}\n" |
||||
|
f" Successfully Updated: {success_count}\n" |
||||
|
f" Skipped (Up to date): {skipped_count}\n" |
||||
|
f" Failed: {failed_count}", |
||||
|
self.style.SUCCESS, |
||||
|
) |
||||
|
|
||||
|
def call_translation_api( |
||||
|
self, |
||||
|
source_text: str, |
||||
|
missing_langs: list, |
||||
|
rate_limiter: ThreadSafeRateLimiter, |
||||
|
api_url: str, |
||||
|
api_key: str, |
||||
|
model: str, |
||||
|
max_retries: int = 3, |
||||
|
) -> dict: |
||||
|
""" |
||||
|
Calls the AI completions API to translate source_text into missing_langs. |
||||
|
Enforces rate limiting prior to each request attempt. |
||||
|
""" |
||||
|
targets_desc = json.dumps({code: TARGET_LANGUAGES[code] for code in missing_langs}, ensure_ascii=False) |
||||
|
|
||||
|
prompt = ( |
||||
|
f"You are a professional translator for calendar events and occasions.\n" |
||||
|
f"Translate the following calendar occasion title from Persian into the specified target languages.\n\n" |
||||
|
f"Original Title: \"{source_text}\"\n\n" |
||||
|
f"Target Languages (JSON map of code -> Language Name):\n{targets_desc}\n\n" |
||||
|
f"INSTRUCTIONS:\n" |
||||
|
f"1. Translate accurately into each requested language code.\n" |
||||
|
f"2. Output strictly a JSON object mapping language_code to translation string.\n" |
||||
|
f"3. Do NOT include markdown code fences (```json or ```), explanations, or outside text.\n\n" |
||||
|
f"Example Output:\n" |
||||
|
f"{{\"en\": \"New Year\", \"ar\": \"السنة الجديدة\"}}" |
||||
|
) |
||||
|
|
||||
|
headers = { |
||||
|
"Authorization": f"Bearer {api_key}", |
||||
|
"Content-Type": "application/json", |
||||
|
} |
||||
|
payload = { |
||||
|
"model": model, |
||||
|
"stream": False, |
||||
|
"messages": [ |
||||
|
{ |
||||
|
"role": "user", |
||||
|
"content": prompt, |
||||
|
} |
||||
|
], |
||||
|
} |
||||
|
|
||||
|
for attempt in range(1, max_retries + 1): |
||||
|
rate_limiter.acquire() |
||||
|
try: |
||||
|
response = requests.post(api_url, headers=headers, json=payload, timeout=20) |
||||
|
if response.status_code == 200: |
||||
|
res_data = response.json() |
||||
|
raw_content = res_data["choices"][0]["message"]["content"].strip() |
||||
|
|
||||
|
# Clean any markdown code blocks if present |
||||
|
if raw_content.startswith("```"): |
||||
|
lines = raw_content.splitlines() |
||||
|
if lines and lines[0].startswith("```"): |
||||
|
lines = lines[1:] |
||||
|
if lines and lines[-1].startswith("```"): |
||||
|
lines = lines[:-1] |
||||
|
raw_content = "\n".join(lines).strip() |
||||
|
|
||||
|
try: |
||||
|
parsed = json.loads(raw_content) |
||||
|
if isinstance(parsed, dict): |
||||
|
# Filter only requested target language keys |
||||
|
return {k: v for k, v in parsed.items() if k in missing_langs and isinstance(v, str)} |
||||
|
except json.JSONDecodeError: |
||||
|
# Regex extract JSON substring if extra text was included |
||||
|
match = re.search(r"\{.*\}", raw_content, re.DOTALL) |
||||
|
if match: |
||||
|
try: |
||||
|
parsed = json.loads(match.group(0)) |
||||
|
if isinstance(parsed, dict): |
||||
|
return {k: v for k, v in parsed.items() if k in missing_langs and isinstance(v, str)} |
||||
|
except json.JSONDecodeError: |
||||
|
pass |
||||
|
except Exception as e: |
||||
|
if attempt == max_retries: |
||||
|
self.safe_write(f"API Call exception on attempt {attempt}: {e}", self.style.ERROR) |
||||
|
time.sleep(1) |
||||
|
|
||||
|
return {} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue