divar-crawler/backend/ads/tests.py

from datetime import time
from rest_framework import status
from rest_framework.test import APITestCase
from crawler.models import CrawlTask
from .models import Ad, AdEvaluation

class AdEvaluationAPITests(APITestCase):
    def setUp(self):
        self.task1 = CrawlTask.objects.create(
            title="Task 1",
            divar_url="https://divar.ir/s/tehran/buy-apartment",
            detection_prompt="Prompt 1",
            interval_minutes=15,
            start_hour=time(8, 0),
            end_hour=time(23, 0)
        )
        self.task2 = CrawlTask.objects.create(
            title="Task 2",
            divar_url="https://divar.ir/s/tehran/rent-apartment",
            detection_prompt="Prompt 2",
            interval_minutes=30,
            start_hour=time(9, 0),
            end_hour=time(22, 0)
        )
        
        self.ad1 = Ad.objects.create(
            divar_token="token1",
            title="Apartment in Tehran",
            description="Beautiful apartment",
            price="10,000,000,000",
            category="buy-apartment",
            images=["http://image1.jpg"],
            url="https://divar.ir/v/token1"
        )
        self.ad2 = Ad.objects.create(
            divar_token="token2",
            title="Rent in Tehran",
            description="Nice place",
            price="500,000,000",
            category="rent-apartment",
            images=[],
            url="https://divar.ir/v/token2"
        )
        
        self.eval1 = AdEvaluation.objects.create(
            crawl_task=self.task1,
            ad=self.ad1,
            is_flagged=True,
            reason="Matches perfectly",
            confidence=0.95,
            extracted_fields={"urgency": "high"}
        )
        self.eval2 = AdEvaluation.objects.create(
            crawl_task=self.task2,
            ad=self.ad2,
            is_flagged=False,
            reason="Does not match",
            confidence=0.1,
            extracted_fields={}
        )
        
        self.list_url = "/api/ads/"

    def test_get_evaluations_list(self):
        response = self.client.get(self.list_url)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 2)
        # Check that ad details are nested
        self.assertEqual(response.data[0]['ad']['divar_token'], self.ad2.divar_token) # Ordered by -evaluated_at
        self.assertEqual(response.data[1]['ad']['divar_token'], self.ad1.divar_token)

    def test_filter_by_crawl_task(self):
        response = self.client.get(self.list_url, {"crawl_task": self.task1.id})
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 1)
        self.assertEqual(response.data[0]['id'], str(self.eval1.id))

    def test_filter_by_is_flagged_true(self):
        response = self.client.get(self.list_url, {"is_flagged": "true"})
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 1)
        self.assertEqual(response.data[0]['id'], str(self.eval1.id))
        self.assertTrue(response.data[0]['is_flagged'])

    def test_filter_by_is_flagged_false(self):
        response = self.client.get(self.list_url, {"is_flagged": "false"})
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 1)
        self.assertEqual(response.data[0]['id'], str(self.eval2.id))
        self.assertFalse(response.data[0]['is_flagged'])

    def test_get_evaluation_detail(self):
        detail_url = f"{self.list_url}{self.eval1.id}/"
        response = self.client.get(detail_url)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['id'], str(self.eval1.id))
        self.assertEqual(response.data['ad']['title'], self.ad1.title)

    def test_delete_ad_evaluation(self):
        detail_url = f"{self.list_url}{self.eval1.id}/"
        response = self.client.delete(detail_url)
        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
        self.assertFalse(AdEvaluation.objects.filter(id=self.eval1.id).exists())


from unittest.mock import MagicMock
from ads.tasks import extract_and_parse_json, call_llm_with_structured_fallback, AdEvaluationResult

class AdTasksHelperTests(APITestCase):
    def test_extract_and_parse_json_raw(self):
        raw = '{"is_flagged": true, "reason": "مناسب است", "confidence": 0.9}'
        res = extract_and_parse_json(raw)
        self.assertTrue(res['is_flagged'])
        self.assertEqual(res['reason'], 'مناسب است')

    def test_extract_and_parse_json_markdown_fence(self):
        raw = '```json\n{\n  "is_flagged": false,\n  "reason": "نامناسب",\n  "confidence": 0.2\n}\n```'
        res = extract_and_parse_json(raw)
        self.assertFalse(res['is_flagged'])

    def test_extract_and_parse_json_with_preamble(self):
        raw = 'Here is the response:\n```json\n{"is_flagged": true, "reason": "خوب", "confidence": 0.8}\n```\nHope this helps!'
        res = extract_and_parse_json(raw)
        self.assertTrue(res['is_flagged'])

    def test_call_llm_fallback_on_markdown(self):
        mock_client = MagicMock()
        # beta parse raises exception (simulating markdown parse failure)
        mock_client.beta.chat.completions.parse.side_effect = Exception("Invalid JSON markdown")
        
        # standard completion returns markdown
        mock_message = MagicMock()
        mock_message.content = '```json\n{"is_flagged": true, "reason": "پرچم گذاری شد", "confidence": 0.95}\n```'
        mock_choice = MagicMock()
        mock_choice.message = mock_message
        mock_completion = MagicMock()
        mock_completion.choices = [mock_choice]
        mock_client.chat.completions.create.return_value = mock_completion

        res = call_llm_with_structured_fallback(
            client=mock_client,
            model_name="openrouter/free",
            system_instruction="sys",
            user_content="user",
            pydantic_cls=AdEvaluationResult,
            max_tokens=2500,
            timeout=35
        )
        self.assertTrue(res.is_flagged)
        self.assertEqual(res.reason, "پرچم گذاری شد")
        self.assertEqual(res.confidence, 0.95)