diff --git a/backend/ads/tasks.py b/backend/ads/tasks.py index aa16f9e..586907a 100644 --- a/backend/ads/tasks.py +++ b/backend/ads/tasks.py @@ -1,4 +1,6 @@ import os +import re +import json import html import requests from openai import OpenAI, AuthenticationError @@ -23,10 +25,115 @@ class BatchAdEvaluationResult(BaseModel): results: list[SingleBatchAdResult] = Field(description="List of evaluation results corresponding to each ad in the batch") +def extract_and_parse_json(text: str): + """ + Cleans raw LLM text (removing markdown code fences, preambles, thinking process) + and extracts a valid JSON object or list. + """ + if not text: + raise ValueError("Empty response from LLM") + + text = text.strip() + + # 1. Check for markdown code fences + fence_match = re.search(r'```(?:json)?\s*([\s\S]*?)\s*```', text, re.IGNORECASE) + if fence_match: + text = fence_match.group(1).strip() + + # 2. Try direct json.loads + try: + return json.loads(text) + except json.JSONDecodeError: + pass + + # 3. Find outermost '{' ... '}' or '[' ... ']' + start_dict = text.find('{') + end_dict = text.rfind('}') + start_list = text.find('[') + end_list = text.rfind(']') + + if start_dict != -1 and end_dict != -1 and end_dict > start_dict: + try: + return json.loads(text[start_dict:end_dict + 1]) + except json.JSONDecodeError: + pass + + if start_list != -1 and end_list != -1 and end_list > start_list: + try: + return json.loads(text[start_list:end_list + 1]) + except json.JSONDecodeError: + pass + + raise ValueError(f"Could not extract valid JSON from LLM response: {text[:200]}") + + +def call_llm_with_structured_fallback(client, model_name, system_instruction, user_content, pydantic_cls, max_tokens, timeout): + """ + Attempts to call LLM using OpenAI beta parse first, then falls back to standard completions + with json_object response format and robust markdown-stripping JSON parsing. + """ + # 1. First attempt: beta structured parse (ideal for official OpenAI models / compatible endpoints) + try: + completion = client.beta.chat.completions.parse( + model=model_name, + messages=[ + {"role": "system", "content": system_instruction}, + {"role": "user", "content": user_content} + ], + response_format=pydantic_cls, + max_tokens=max_tokens, + timeout=timeout + ) + if completion.choices and completion.choices[0].message.parsed: + return completion.choices[0].message.parsed + except Exception: + # Fallback to chat completions if beta parse fails (e.g. OpenRouter returning markdown wrappers or non-strict JSON) + pass + + # 2. Second attempt: standard chat completions with json_object response format + system_instruction_json = ( + f"{system_instruction}\n" + "CRITICAL REQUIREMENT: You MUST reply strictly with a valid JSON object. " + "Do NOT wrap the JSON in markdown code blocks like ```json ... ```. " + "Do NOT include any introduction, thinking process, or explanatory text before or after the JSON." + ) + + try: + completion = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": system_instruction_json}, + {"role": "user", "content": user_content} + ], + response_format={"type": "json_object"}, + max_tokens=max_tokens, + timeout=timeout + ) + raw_text = completion.choices[0].message.content or "" + json_data = extract_and_parse_json(raw_text) + return pydantic_cls.model_validate(json_data) + except Exception: + pass + + # 3. Third attempt: standard chat completions without response_format constraint + completion = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": system_instruction_json}, + {"role": "user", "content": user_content} + ], + max_tokens=max_tokens, + timeout=timeout + ) + raw_text = completion.choices[0].message.content or "" + json_data = extract_and_parse_json(raw_text) + return pydantic_cls.model_validate(json_data) + + @shared_task(bind=True, max_retries=3, default_retry_delay=10) def evaluate_ad_with_ai(self, evaluation_id): """ - Asynchronously evaluates a single ad using OpenAI's gpt-4o-mini model + Asynchronously evaluates a single ad using OpenAI's gpt-4o-mini / OpenRouter model against the specific CrawlTask search criteria. """ try: @@ -39,7 +146,7 @@ def evaluate_ad_with_ai(self, evaluation_id): api_key = os.getenv('OPENAI_API_KEY') # Fallback if OpenAI API Key is missing - if not api_key or 'placeholder' in api_key or 'your_openai_api_key' in api_key or 'your-openai-api-key-here' in api_key: + if not api_key or any(k in api_key for k in ['placeholder', 'your_openai_api_key', 'your-openai-api-key-here']): evaluation.is_flagged = False evaluation.reason = "AI Evaluation skipped: OpenAI API key is not configured." evaluation.confidence = 0.0 @@ -53,13 +160,13 @@ def evaluate_ad_with_ai(self, evaluation_id): client_kwargs["base_url"] = "https://openrouter.ai/api/v1" client = OpenAI(**client_kwargs) - + system_instruction = ( "You are an expert Iranian market analyst. Your job is to read listing descriptions " "and decide if they match specific target criteria. You MUST reply using the structured JSON response format " "with Persian strings." ) - + user_content = ( f"User Search Criteria: {task.detection_prompt}\n\n" f"Ad Title: {ad.title}\n" @@ -67,22 +174,20 @@ def evaluate_ad_with_ai(self, evaluation_id): f"Ad Category: {ad.category or 'Not specified'}\n" f"Ad Description:\n{ad.description}" ) - + default_model = "openrouter/free" if api_key.startswith("sk-or-") else "gpt-4o-mini" model_name = os.getenv("OPENAI_MODEL", default_model) - - completion = client.beta.chat.completions.parse( - model=model_name, - messages=[ - {"role": "system", "content": system_instruction}, - {"role": "user", "content": user_content} - ], - response_format=AdEvaluationResult, - max_tokens=300, - timeout=25 + + result = call_llm_with_structured_fallback( + client=client, + model_name=model_name, + system_instruction=system_instruction, + user_content=user_content, + pydantic_cls=AdEvaluationResult, + max_tokens=2500, + timeout=35 ) - - result = completion.choices[0].message.parsed + evaluation.is_flagged = result.is_flagged evaluation.reason = result.reason evaluation.confidence = result.confidence @@ -116,6 +221,7 @@ def evaluate_ad_with_ai(self, evaluation_id): evaluation.confidence = 0.0 evaluation.save() + @shared_task(bind=True, max_retries=3, default_retry_delay=10) def evaluate_ad_batch_with_ai(self, evaluation_ids): """ @@ -159,7 +265,7 @@ def evaluate_ad_batch_with_ai(self, evaluation_ids): f"Title: {ev.ad.title}\n" f"Price: {ev.ad.price or 'Not specified'}\n" f"Category: {ev.ad.category or 'Not specified'}\n" - f"Description: {ev.ad.description[:600]}\n" + f"Description:\n{ev.ad.description[:2500]}\n" ) user_content = ( @@ -171,23 +277,31 @@ def evaluate_ad_batch_with_ai(self, evaluation_ids): default_model = "openrouter/free" if api_key.startswith("sk-or-") else "gpt-4o-mini" model_name = os.getenv("OPENAI_MODEL", default_model) - completion = client.beta.chat.completions.parse( - model=model_name, - messages=[ - {"role": "system", "content": system_instruction}, - {"role": "user", "content": user_content} - ], - response_format=BatchAdEvaluationResult, - max_tokens=1500, - timeout=45 + parsed = call_llm_with_structured_fallback( + client=client, + model_name=model_name, + system_instruction=system_instruction, + user_content=user_content, + pydantic_cls=BatchAdEvaluationResult, + max_tokens=8000, + timeout=90 ) - parsed = completion.choices[0].message.parsed - res_map = {str(res.evaluation_id): res for res in parsed.results} + res_map = {} + for idx, res in enumerate(parsed.results): + eval_id_str = str(res.evaluation_id).strip() + res_map[eval_id_str] = res + res_map[str(idx)] = res + res_map[f"Ad #{idx + 1}"] = res + res_map[f"#{idx + 1}"] = res flagged_count = 0 - for ev in evaluations: - res = res_map.get(str(ev.id)) + for idx, ev in enumerate(evaluations): + ev_id_str = str(ev.id).strip() + res = res_map.get(ev_id_str) + if not res and idx < len(parsed.results): + res = parsed.results[idx] + if res: ev.is_flagged = res.is_flagged ev.reason = res.reason