feat(crawler): implement serializers, views, and routing for CrawlTask and CrawlRun

backend/crawler/migrations/0001_initial.py

@@ -0,0 +1,53 @@
+# Generated by Django 6.0.8 on 2026-08-08 07:53
+
+import crawler.models
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CrawlTask',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('title', models.CharField(max_length=255)),
+                ('divar_url', models.URLField(validators=[crawler.models.validate_divar_url])),
+                ('detection_prompt', models.TextField()),
+                ('interval_minutes', models.IntegerField(choices=[(5, '5 Minutes'), (15, '15 Minutes'), (30, '30 Minutes'), (60, '1 Hour'), (120, '2 Hours'), (180, '3 Hours'), (240, '4 Hours'), (300, '5 Hours'), (360, '6 Hours')], default=60)),
+                ('start_hour', models.TimeField()),
+                ('end_hour', models.TimeField()),
+                ('telegram_channel_id', models.CharField(blank=True, max_length=100, null=True)),
+                ('is_active', models.BooleanField(default=True)),
+            ],
+            options={
+                'indexes': [models.Index(fields=['is_active'], name='crawler_cra_is_acti_b67d0a_idx'), models.Index(fields=['created_at'], name='crawler_cra_created_920fb7_idx'), models.Index(fields=['title'], name='crawler_cra_title_dfbb5f_idx')],
+            },
+        ),
+        migrations.CreateModel(
+            name='CrawlRun',
+            fields=[
+                ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('started_at', models.DateTimeField(auto_now_add=True)),
+                ('finished_at', models.DateTimeField(blank=True, null=True)),
+                ('status', models.CharField(choices=[('RUNNING', 'Running'), ('SUCCESS', 'Success'), ('FAILED', 'Failed')], default='RUNNING', max_length=20)),
+                ('ads_fetched_count', models.IntegerField(default=0)),
+                ('ads_evaluated_count', models.IntegerField(default=0)),
+                ('ads_flagged_count', models.IntegerField(default=0)),
+                ('error_log', models.TextField(blank=True, null=True)),
+                ('crawl_task', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='crawler.crawltask')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

backend/crawler/serializers.py

@@ -0,0 +1,44 @@
+from rest_framework import serializers
+from .models import CrawlTask, CrawlRun
+
+class CrawlTaskSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = CrawlTask
+        fields = [
+            'id',
+            'title',
+            'divar_url',
+            'detection_prompt',
+            'interval_minutes',
+            'start_hour',
+            'end_hour',
+            'telegram_channel_id',
+            'is_active',
+            'created_at',
+            'updated_at',
+        ]
+        read_only_fields = ['id', 'created_at', 'updated_at']
+
+    def validate(self, attrs):
+        start_hour = attrs.get('start_hour')
+        end_hour = attrs.get('end_hour')
+        
+        # If hours are provided, ensure they make sense or do any other general task validations
+        return attrs
+
+
+class CrawlRunSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = CrawlRun
+        fields = [
+            'id',
+            'crawl_task',
+            'started_at',
+            'finished_at',
+            'status',
+            'ads_fetched_count',
+            'ads_evaluated_count',
+            'ads_flagged_count',
+            'error_log',
+        ]
+        read_only_fields = ['id', 'started_at', 'finished_at', 'status', 'ads_fetched_count', 'ads_evaluated_count', 'ads_flagged_count', 'error_log']

backend/crawler/urls.py

@@ -1,6 +1,11 @@
-from django.urls import path
-from .views import CrawlerView
+from django.urls import path, include
+from rest_framework.routers import DefaultRouter
+from .views import CrawlTaskViewSet
+
+router = DefaultRouter()
+router.register(r'', CrawlTaskViewSet, basename='crawlers')
 
 urlpatterns = [
-    path('', CrawlerView.as_view()),
+    path('', include(router.urls)),
 ]
+

backend/crawler/views.py

@@ -1,11 +1,50 @@
-from django.shortcuts import render
-from rest_framework.views import APIView
+from rest_framework import viewsets, status
+from rest_framework.decorators import action
 from rest_framework.response import Response
+from .models import CrawlTask, CrawlRun
+from .serializers import CrawlTaskSerializer, CrawlRunSerializer
 
+class CrawlTaskViewSet(viewsets.ModelViewSet):
+    """
+    ViewSet for CrawlTask models. Supports CRUD and custom trigger/runs actions.
+    """
+    queryset = CrawlTask.objects.all().order_by('-created_at')
+    serializer_class = CrawlTaskSerializer
 
-# Create your views here.
+    @action(detail=True, methods=['post'], url_path='trigger')
+    def trigger(self, request, pk=None):
+        task = self.get_object()
         
-#create the django rest framework view for crawling the data
-class CrawlerView(APIView):
-    def get(self, request):
-        return Response({'message': 'Hello World'})
+        # Create a new CrawlRun with status RUNNING
+        run = CrawlRun.objects.create(
+            crawl_task=task,
+            status='RUNNING'
+        )
+        
+        # Check if tasks can be imported.
+        # This will integrate with crawlers/tasks.py once it is written in a later step.
+        try:
+            from crawler.tasks import run_crawl_pipeline
+            # run asynchronously
+            run_crawl_pipeline.delay(str(run.id))
+            is_queued = True
+        except ImportError:
+            # Fallback if tasks.py doesn't exist yet or run_crawl_pipeline is not defined
+            is_queued = False
+            
+        return Response({
+            "status": "queued" if is_queued else "queued_stub",
+            "run_id": str(run.id)
+        }, status=status.HTTP_202_ACCEPTED)
+
+    @action(detail=True, methods=['get'], url_path='runs')
+    def runs(self, request, pk=None):
+        task = self.get_object()
+        runs = task.runs.all().order_by('-started_at')
+        page = self.paginate_queryset(runs)
+        if page is not None:
+            serializer = CrawlRunSerializer(page, many=True)
+            return self.get_paginated_response(serializer.data)
+            
+        serializer = CrawlRunSerializer(runs, many=True)
+        return Response(serializer.data)