|
|
|
@ -1,4 +1,6 @@ |
|
|
|
import os |
|
|
|
import re |
|
|
|
import json |
|
|
|
import html |
|
|
|
import requests |
|
|
|
from openai import OpenAI, AuthenticationError |
|
|
|
@ -23,10 +25,115 @@ class BatchAdEvaluationResult(BaseModel): |
|
|
|
results: list[SingleBatchAdResult] = Field(description="List of evaluation results corresponding to each ad in the batch") |
|
|
|
|
|
|
|
|
|
|
|
def extract_and_parse_json(text: str): |
|
|
|
""" |
|
|
|
Cleans raw LLM text (removing markdown code fences, preambles, thinking process) |
|
|
|
and extracts a valid JSON object or list. |
|
|
|
""" |
|
|
|
if not text: |
|
|
|
raise ValueError("Empty response from LLM") |
|
|
|
|
|
|
|
text = text.strip() |
|
|
|
|
|
|
|
# 1. Check for markdown code fences |
|
|
|
fence_match = re.search(r'```(?:json)?\s*([\s\S]*?)\s*```', text, re.IGNORECASE) |
|
|
|
if fence_match: |
|
|
|
text = fence_match.group(1).strip() |
|
|
|
|
|
|
|
# 2. Try direct json.loads |
|
|
|
try: |
|
|
|
return json.loads(text) |
|
|
|
except json.JSONDecodeError: |
|
|
|
pass |
|
|
|
|
|
|
|
# 3. Find outermost '{' ... '}' or '[' ... ']' |
|
|
|
start_dict = text.find('{') |
|
|
|
end_dict = text.rfind('}') |
|
|
|
start_list = text.find('[') |
|
|
|
end_list = text.rfind(']') |
|
|
|
|
|
|
|
if start_dict != -1 and end_dict != -1 and end_dict > start_dict: |
|
|
|
try: |
|
|
|
return json.loads(text[start_dict:end_dict + 1]) |
|
|
|
except json.JSONDecodeError: |
|
|
|
pass |
|
|
|
|
|
|
|
if start_list != -1 and end_list != -1 and end_list > start_list: |
|
|
|
try: |
|
|
|
return json.loads(text[start_list:end_list + 1]) |
|
|
|
except json.JSONDecodeError: |
|
|
|
pass |
|
|
|
|
|
|
|
raise ValueError(f"Could not extract valid JSON from LLM response: {text[:200]}") |
|
|
|
|
|
|
|
|
|
|
|
def call_llm_with_structured_fallback(client, model_name, system_instruction, user_content, pydantic_cls, max_tokens, timeout): |
|
|
|
""" |
|
|
|
Attempts to call LLM using OpenAI beta parse first, then falls back to standard completions |
|
|
|
with json_object response format and robust markdown-stripping JSON parsing. |
|
|
|
""" |
|
|
|
# 1. First attempt: beta structured parse (ideal for official OpenAI models / compatible endpoints) |
|
|
|
try: |
|
|
|
completion = client.beta.chat.completions.parse( |
|
|
|
model=model_name, |
|
|
|
messages=[ |
|
|
|
{"role": "system", "content": system_instruction}, |
|
|
|
{"role": "user", "content": user_content} |
|
|
|
], |
|
|
|
response_format=pydantic_cls, |
|
|
|
max_tokens=max_tokens, |
|
|
|
timeout=timeout |
|
|
|
) |
|
|
|
if completion.choices and completion.choices[0].message.parsed: |
|
|
|
return completion.choices[0].message.parsed |
|
|
|
except Exception: |
|
|
|
# Fallback to chat completions if beta parse fails (e.g. OpenRouter returning markdown wrappers or non-strict JSON) |
|
|
|
pass |
|
|
|
|
|
|
|
# 2. Second attempt: standard chat completions with json_object response format |
|
|
|
system_instruction_json = ( |
|
|
|
f"{system_instruction}\n" |
|
|
|
"CRITICAL REQUIREMENT: You MUST reply strictly with a valid JSON object. " |
|
|
|
"Do NOT wrap the JSON in markdown code blocks like ```json ... ```. " |
|
|
|
"Do NOT include any introduction, thinking process, or explanatory text before or after the JSON." |
|
|
|
) |
|
|
|
|
|
|
|
try: |
|
|
|
completion = client.chat.completions.create( |
|
|
|
model=model_name, |
|
|
|
messages=[ |
|
|
|
{"role": "system", "content": system_instruction_json}, |
|
|
|
{"role": "user", "content": user_content} |
|
|
|
], |
|
|
|
response_format={"type": "json_object"}, |
|
|
|
max_tokens=max_tokens, |
|
|
|
timeout=timeout |
|
|
|
) |
|
|
|
raw_text = completion.choices[0].message.content or "" |
|
|
|
json_data = extract_and_parse_json(raw_text) |
|
|
|
return pydantic_cls.model_validate(json_data) |
|
|
|
except Exception: |
|
|
|
pass |
|
|
|
|
|
|
|
# 3. Third attempt: standard chat completions without response_format constraint |
|
|
|
completion = client.chat.completions.create( |
|
|
|
model=model_name, |
|
|
|
messages=[ |
|
|
|
{"role": "system", "content": system_instruction_json}, |
|
|
|
{"role": "user", "content": user_content} |
|
|
|
], |
|
|
|
max_tokens=max_tokens, |
|
|
|
timeout=timeout |
|
|
|
) |
|
|
|
raw_text = completion.choices[0].message.content or "" |
|
|
|
json_data = extract_and_parse_json(raw_text) |
|
|
|
return pydantic_cls.model_validate(json_data) |
|
|
|
|
|
|
|
|
|
|
|
@shared_task(bind=True, max_retries=3, default_retry_delay=10) |
|
|
|
def evaluate_ad_with_ai(self, evaluation_id): |
|
|
|
""" |
|
|
|
Asynchronously evaluates a single ad using OpenAI's gpt-4o-mini model |
|
|
|
Asynchronously evaluates a single ad using OpenAI's gpt-4o-mini / OpenRouter model |
|
|
|
against the specific CrawlTask search criteria. |
|
|
|
""" |
|
|
|
try: |
|
|
|
@ -39,7 +146,7 @@ def evaluate_ad_with_ai(self, evaluation_id): |
|
|
|
api_key = os.getenv('OPENAI_API_KEY') |
|
|
|
|
|
|
|
# Fallback if OpenAI API Key is missing |
|
|
|
if not api_key or 'placeholder' in api_key or 'your_openai_api_key' in api_key or 'your-openai-api-key-here' in api_key: |
|
|
|
if not api_key or any(k in api_key for k in ['placeholder', 'your_openai_api_key', 'your-openai-api-key-here']): |
|
|
|
evaluation.is_flagged = False |
|
|
|
evaluation.reason = "AI Evaluation skipped: OpenAI API key is not configured." |
|
|
|
evaluation.confidence = 0.0 |
|
|
|
@ -71,18 +178,16 @@ def evaluate_ad_with_ai(self, evaluation_id): |
|
|
|
default_model = "openrouter/free" if api_key.startswith("sk-or-") else "gpt-4o-mini" |
|
|
|
model_name = os.getenv("OPENAI_MODEL", default_model) |
|
|
|
|
|
|
|
completion = client.beta.chat.completions.parse( |
|
|
|
model=model_name, |
|
|
|
messages=[ |
|
|
|
{"role": "system", "content": system_instruction}, |
|
|
|
{"role": "user", "content": user_content} |
|
|
|
], |
|
|
|
response_format=AdEvaluationResult, |
|
|
|
max_tokens=300, |
|
|
|
timeout=25 |
|
|
|
result = call_llm_with_structured_fallback( |
|
|
|
client=client, |
|
|
|
model_name=model_name, |
|
|
|
system_instruction=system_instruction, |
|
|
|
user_content=user_content, |
|
|
|
pydantic_cls=AdEvaluationResult, |
|
|
|
max_tokens=2500, |
|
|
|
timeout=35 |
|
|
|
) |
|
|
|
|
|
|
|
result = completion.choices[0].message.parsed |
|
|
|
evaluation.is_flagged = result.is_flagged |
|
|
|
evaluation.reason = result.reason |
|
|
|
evaluation.confidence = result.confidence |
|
|
|
@ -116,6 +221,7 @@ def evaluate_ad_with_ai(self, evaluation_id): |
|
|
|
evaluation.confidence = 0.0 |
|
|
|
evaluation.save() |
|
|
|
|
|
|
|
|
|
|
|
@shared_task(bind=True, max_retries=3, default_retry_delay=10) |
|
|
|
def evaluate_ad_batch_with_ai(self, evaluation_ids): |
|
|
|
""" |
|
|
|
@ -159,7 +265,7 @@ def evaluate_ad_batch_with_ai(self, evaluation_ids): |
|
|
|
f"Title: {ev.ad.title}\n" |
|
|
|
f"Price: {ev.ad.price or 'Not specified'}\n" |
|
|
|
f"Category: {ev.ad.category or 'Not specified'}\n" |
|
|
|
f"Description: {ev.ad.description[:600]}\n" |
|
|
|
f"Description:\n{ev.ad.description[:2500]}\n" |
|
|
|
) |
|
|
|
|
|
|
|
user_content = ( |
|
|
|
@ -171,23 +277,31 @@ def evaluate_ad_batch_with_ai(self, evaluation_ids): |
|
|
|
default_model = "openrouter/free" if api_key.startswith("sk-or-") else "gpt-4o-mini" |
|
|
|
model_name = os.getenv("OPENAI_MODEL", default_model) |
|
|
|
|
|
|
|
completion = client.beta.chat.completions.parse( |
|
|
|
model=model_name, |
|
|
|
messages=[ |
|
|
|
{"role": "system", "content": system_instruction}, |
|
|
|
{"role": "user", "content": user_content} |
|
|
|
], |
|
|
|
response_format=BatchAdEvaluationResult, |
|
|
|
max_tokens=1500, |
|
|
|
timeout=45 |
|
|
|
parsed = call_llm_with_structured_fallback( |
|
|
|
client=client, |
|
|
|
model_name=model_name, |
|
|
|
system_instruction=system_instruction, |
|
|
|
user_content=user_content, |
|
|
|
pydantic_cls=BatchAdEvaluationResult, |
|
|
|
max_tokens=8000, |
|
|
|
timeout=90 |
|
|
|
) |
|
|
|
|
|
|
|
parsed = completion.choices[0].message.parsed |
|
|
|
res_map = {str(res.evaluation_id): res for res in parsed.results} |
|
|
|
res_map = {} |
|
|
|
for idx, res in enumerate(parsed.results): |
|
|
|
eval_id_str = str(res.evaluation_id).strip() |
|
|
|
res_map[eval_id_str] = res |
|
|
|
res_map[str(idx)] = res |
|
|
|
res_map[f"Ad #{idx + 1}"] = res |
|
|
|
res_map[f"#{idx + 1}"] = res |
|
|
|
|
|
|
|
flagged_count = 0 |
|
|
|
for ev in evaluations: |
|
|
|
res = res_map.get(str(ev.id)) |
|
|
|
for idx, ev in enumerate(evaluations): |
|
|
|
ev_id_str = str(ev.id).strip() |
|
|
|
res = res_map.get(ev_id_str) |
|
|
|
if not res and idx < len(parsed.results): |
|
|
|
res = parsed.results[idx] |
|
|
|
|
|
|
|
if res: |
|
|
|
ev.is_flagged = res.is_flagged |
|
|
|
ev.reason = res.reason |
|
|
|
|
PouyaKhajavi
1 day ago