test(backend): add API and Celery pipeline unit tests

backend/crawler/tests.py

@@ -1,5 +1,6 @@
 from datetime import time
 from django.urls import reverse
+from unittest.mock import patch
 from rest_framework import status
 from rest_framework.test import APITestCase
 from .models import CrawlTask, CrawlRun
@@ -72,11 +73,38 @@ class CrawlTaskAPITests(APITestCase):
         self.assertEqual(response.data['title'], "Updated Task Name")
 
     def test_delete_crawl_task(self):
+        # Create a run and evaluation for the task to verify cascade deletion
+        run = CrawlRun.objects.create(
+            crawl_task=self.task,
+            status='SUCCESS'
+        )
+        from ads.models import Ad, AdEvaluation
+        ad = Ad.objects.create(
+            divar_token="delete_token",
+            title="Delete Test Ad",
+            description="Desc",
+            url="https://divar.ir/v/delete_token"
+        )
+        evaluation = AdEvaluation.objects.create(
+            crawl_task=self.task,
+            ad=ad,
+            is_flagged=True
+        )
+
+        # Ensure they exist initially
+        self.assertTrue(CrawlRun.objects.filter(id=run.id).exists())
+        self.assertTrue(AdEvaluation.objects.filter(id=evaluation.id).exists())
+
         response = self.client.delete(self.detail_url)
         self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
+        
+        # Verify cascade deletion
         self.assertFalse(CrawlTask.objects.filter(id=self.task.id).exists())
+        self.assertFalse(CrawlRun.objects.filter(id=run.id).exists())
+        self.assertFalse(AdEvaluation.objects.filter(id=evaluation.id).exists())
 
-    def test_trigger_crawl_task(self):
+    @patch('crawler.tasks.run_crawl_pipeline.delay')
+    def test_trigger_crawl_task(self, mock_delay):
         runs_before = CrawlRun.objects.filter(crawl_task=self.task).count()
         response = self.client.post(self.trigger_url)
         self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
@@ -84,6 +112,7 @@ class CrawlTaskAPITests(APITestCase):
         self.assertIn('status', response.data)
         runs_after = CrawlRun.objects.filter(crawl_task=self.task).count()
         self.assertEqual(runs_after, runs_before + 1)
+        mock_delay.assert_called_once()
 
     def test_get_crawl_task_runs(self):
         run = CrawlRun.objects.create(
@@ -98,3 +127,36 @@ class CrawlTaskAPITests(APITestCase):
         self.assertEqual(len(response.data), 1)
         self.assertEqual(response.data[0]['id'], str(run.id))
         self.assertEqual(response.data[0]['status'], 'SUCCESS')
+
+class CrawlTaskCeleryTests(APITestCase):
+    def setUp(self):
+        self.task = CrawlTask.objects.create(
+            title="Tehran Apartments",
+            divar_url="https://divar.ir/s/tehran/buy-apartment?query=urgent",
+            detection_prompt="Find urgent sales",
+            interval_minutes=15,
+            start_hour=time(8, 0),
+            end_hour=time(23, 0)
+        )
+        self.run = CrawlRun.objects.create(
+            crawl_task=self.task,
+            status='RUNNING'
+        )
+
+    def test_parse_divar_url(self):
+        from crawler.tasks import parse_divar_url
+        city, category, query_params = parse_divar_url(self.task.divar_url)
+        self.assertEqual(city, 'tehran')
+        self.assertEqual(category, 'buy-apartment')
+        self.assertEqual(query_params, {'query': 'urgent'})
+
+    def test_run_crawl_pipeline_inactive_task(self):
+        self.task.is_active = False
+        self.task.save()
+        
+        from crawler.tasks import run_crawl_pipeline
+        run_crawl_pipeline(self.run.id)
+        
+        self.run.refresh_from_db()
+        self.assertEqual(self.run.status, 'FAILED')
+        self.assertEqual(self.run.error_log, "CrawlTask is inactive.")

backend/crawler/views.py

@@ -1,50 +1,49 @@
-from rest_framework import viewsets, status
-from rest_framework.decorators import action
-from rest_framework.response import Response
-from .models import CrawlTask, CrawlRun
-from .serializers import CrawlTaskSerializer, CrawlRunSerializer
-
-class CrawlTaskViewSet(viewsets.ModelViewSet):
-    """
-    ViewSet for CrawlTask models. Supports CRUD and custom trigger/runs actions.
-    """
-    queryset = CrawlTask.objects.all().order_by('-created_at')
-    serializer_class = CrawlTaskSerializer
-
-    @action(detail=True, methods=['post'], url_path='trigger')
-    def trigger(self, request, pk=None):
-        task = self.get_object()
-        
-        # Create a new CrawlRun with status RUNNING
-        run = CrawlRun.objects.create(
-            crawl_task=task,
-            status='RUNNING'
-        )
-        
-        # Check if tasks can be imported.
-        # This will integrate with crawlers/tasks.py once it is written in a later step.
-        try:
-            from crawler.tasks import run_crawl_pipeline
-            # run asynchronously
-            run_crawl_pipeline.delay(str(run.id))
-            is_queued = True
-        except ImportError:
-            # Fallback if tasks.py doesn't exist yet or run_crawl_pipeline is not defined
-            is_queued = False
-            
-        return Response({
-            "status": "queued" if is_queued else "queued_stub",
-            "run_id": str(run.id)
-        }, status=status.HTTP_202_ACCEPTED)
-
-    @action(detail=True, methods=['get'], url_path='runs')
-    def runs(self, request, pk=None):
-        task = self.get_object()
-        runs = task.runs.all().order_by('-started_at')
-        page = self.paginate_queryset(runs)
-        if page is not None:
-            serializer = CrawlRunSerializer(page, many=True)
-            return self.get_paginated_response(serializer.data)
-            
-        serializer = CrawlRunSerializer(runs, many=True)
-        return Response(serializer.data)
+from rest_framework import viewsets, status
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from .models import CrawlTask, CrawlRun
+from .serializers import CrawlTaskSerializer, CrawlRunSerializer
+
+class CrawlTaskViewSet(viewsets.ModelViewSet):
+    """
+    ViewSet for CrawlTask models. Supports CRUD and custom trigger/runs actions.
+    """
+    queryset = CrawlTask.objects.all().order_by('-created_at')
+    serializer_class = CrawlTaskSerializer
+
+    @action(detail=True, methods=['post'], url_path='trigger')
+    def trigger(self, request, pk=None):
+        task = self.get_object()
+        
+        # Create a new CrawlRun with status RUNNING
+        run = CrawlRun.objects.create(
+            crawl_task=task,
+            status='RUNNING'
+        )
+        
+        # Check if tasks can be imported.
+        try:
+            from crawler.tasks import run_crawl_pipeline
+            # run asynchronously
+            run_crawl_pipeline.delay(str(run.id))
+            is_queued = True
+        except ImportError:
+            # Fallback if tasks.py doesn't exist yet or run_crawl_pipeline is not defined
+            is_queued = False
+            
+        return Response({
+            "status": "queued" if is_queued else "queued_stub",
+            "run_id": str(run.id)
+        }, status=status.HTTP_202_ACCEPTED)
+
+    @action(detail=True, methods=['get'], url_path='runs')
+    def runs(self, request, pk=None):
+        task = self.get_object()
+        runs = task.runs.all().order_by('-started_at')
+        page = self.paginate_queryset(runs)
+        if page is not None:
+            serializer = CrawlRunSerializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+            
+        serializer = CrawlRunSerializer(runs, many=True)
+        return Response(serializer.data)