diff --git a/backend/crawler/tests.py b/backend/crawler/tests.py index 51c8c8a..3792b4c 100644 --- a/backend/crawler/tests.py +++ b/backend/crawler/tests.py @@ -1,5 +1,6 @@ from datetime import time from django.urls import reverse +from unittest.mock import patch from rest_framework import status from rest_framework.test import APITestCase from .models import CrawlTask, CrawlRun @@ -72,11 +73,38 @@ class CrawlTaskAPITests(APITestCase): self.assertEqual(response.data['title'], "Updated Task Name") def test_delete_crawl_task(self): + # Create a run and evaluation for the task to verify cascade deletion + run = CrawlRun.objects.create( + crawl_task=self.task, + status='SUCCESS' + ) + from ads.models import Ad, AdEvaluation + ad = Ad.objects.create( + divar_token="delete_token", + title="Delete Test Ad", + description="Desc", + url="https://divar.ir/v/delete_token" + ) + evaluation = AdEvaluation.objects.create( + crawl_task=self.task, + ad=ad, + is_flagged=True + ) + + # Ensure they exist initially + self.assertTrue(CrawlRun.objects.filter(id=run.id).exists()) + self.assertTrue(AdEvaluation.objects.filter(id=evaluation.id).exists()) + response = self.client.delete(self.detail_url) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) + + # Verify cascade deletion self.assertFalse(CrawlTask.objects.filter(id=self.task.id).exists()) + self.assertFalse(CrawlRun.objects.filter(id=run.id).exists()) + self.assertFalse(AdEvaluation.objects.filter(id=evaluation.id).exists()) - def test_trigger_crawl_task(self): + @patch('crawler.tasks.run_crawl_pipeline.delay') + def test_trigger_crawl_task(self, mock_delay): runs_before = CrawlRun.objects.filter(crawl_task=self.task).count() response = self.client.post(self.trigger_url) self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) @@ -84,6 +112,7 @@ class CrawlTaskAPITests(APITestCase): self.assertIn('status', response.data) runs_after = CrawlRun.objects.filter(crawl_task=self.task).count() self.assertEqual(runs_after, runs_before + 1) + mock_delay.assert_called_once() def test_get_crawl_task_runs(self): run = CrawlRun.objects.create( @@ -98,3 +127,36 @@ class CrawlTaskAPITests(APITestCase): self.assertEqual(len(response.data), 1) self.assertEqual(response.data[0]['id'], str(run.id)) self.assertEqual(response.data[0]['status'], 'SUCCESS') + +class CrawlTaskCeleryTests(APITestCase): + def setUp(self): + self.task = CrawlTask.objects.create( + title="Tehran Apartments", + divar_url="https://divar.ir/s/tehran/buy-apartment?query=urgent", + detection_prompt="Find urgent sales", + interval_minutes=15, + start_hour=time(8, 0), + end_hour=time(23, 0) + ) + self.run = CrawlRun.objects.create( + crawl_task=self.task, + status='RUNNING' + ) + + def test_parse_divar_url(self): + from crawler.tasks import parse_divar_url + city, category, query_params = parse_divar_url(self.task.divar_url) + self.assertEqual(city, 'tehran') + self.assertEqual(category, 'buy-apartment') + self.assertEqual(query_params, {'query': 'urgent'}) + + def test_run_crawl_pipeline_inactive_task(self): + self.task.is_active = False + self.task.save() + + from crawler.tasks import run_crawl_pipeline + run_crawl_pipeline(self.run.id) + + self.run.refresh_from_db() + self.assertEqual(self.run.status, 'FAILED') + self.assertEqual(self.run.error_log, "CrawlTask is inactive.") diff --git a/backend/crawler/views.py b/backend/crawler/views.py index fd72a99..c308596 100644 --- a/backend/crawler/views.py +++ b/backend/crawler/views.py @@ -1,50 +1,49 @@ -from rest_framework import viewsets, status -from rest_framework.decorators import action -from rest_framework.response import Response -from .models import CrawlTask, CrawlRun -from .serializers import CrawlTaskSerializer, CrawlRunSerializer - -class CrawlTaskViewSet(viewsets.ModelViewSet): - """ - ViewSet for CrawlTask models. Supports CRUD and custom trigger/runs actions. - """ - queryset = CrawlTask.objects.all().order_by('-created_at') - serializer_class = CrawlTaskSerializer - - @action(detail=True, methods=['post'], url_path='trigger') - def trigger(self, request, pk=None): - task = self.get_object() - - # Create a new CrawlRun with status RUNNING - run = CrawlRun.objects.create( - crawl_task=task, - status='RUNNING' - ) - - # Check if tasks can be imported. - # This will integrate with crawlers/tasks.py once it is written in a later step. - try: - from crawler.tasks import run_crawl_pipeline - # run asynchronously - run_crawl_pipeline.delay(str(run.id)) - is_queued = True - except ImportError: - # Fallback if tasks.py doesn't exist yet or run_crawl_pipeline is not defined - is_queued = False - - return Response({ - "status": "queued" if is_queued else "queued_stub", - "run_id": str(run.id) - }, status=status.HTTP_202_ACCEPTED) - - @action(detail=True, methods=['get'], url_path='runs') - def runs(self, request, pk=None): - task = self.get_object() - runs = task.runs.all().order_by('-started_at') - page = self.paginate_queryset(runs) - if page is not None: - serializer = CrawlRunSerializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = CrawlRunSerializer(runs, many=True) - return Response(serializer.data) \ No newline at end of file +from rest_framework import viewsets, status +from rest_framework.decorators import action +from rest_framework.response import Response +from .models import CrawlTask, CrawlRun +from .serializers import CrawlTaskSerializer, CrawlRunSerializer + +class CrawlTaskViewSet(viewsets.ModelViewSet): + """ + ViewSet for CrawlTask models. Supports CRUD and custom trigger/runs actions. + """ + queryset = CrawlTask.objects.all().order_by('-created_at') + serializer_class = CrawlTaskSerializer + + @action(detail=True, methods=['post'], url_path='trigger') + def trigger(self, request, pk=None): + task = self.get_object() + + # Create a new CrawlRun with status RUNNING + run = CrawlRun.objects.create( + crawl_task=task, + status='RUNNING' + ) + + # Check if tasks can be imported. + try: + from crawler.tasks import run_crawl_pipeline + # run asynchronously + run_crawl_pipeline.delay(str(run.id)) + is_queued = True + except ImportError: + # Fallback if tasks.py doesn't exist yet or run_crawl_pipeline is not defined + is_queued = False + + return Response({ + "status": "queued" if is_queued else "queued_stub", + "run_id": str(run.id) + }, status=status.HTTP_202_ACCEPTED) + + @action(detail=True, methods=['get'], url_path='runs') + def runs(self, request, pk=None): + task = self.get_object() + runs = task.runs.all().order_by('-started_at') + page = self.paginate_queryset(runs) + if page is not None: + serializer = CrawlRunSerializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = CrawlRunSerializer(runs, many=True) + return Response(serializer.data)