diff --git a/backend/crawler/migrations/0001_initial.py b/backend/crawler/migrations/0001_initial.py new file mode 100644 index 0000000..9b393c6 --- /dev/null +++ b/backend/crawler/migrations/0001_initial.py @@ -0,0 +1,53 @@ +# Generated by Django 6.0.8 on 2026-08-08 07:53 + +import crawler.models +import django.db.models.deletion +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='CrawlTask', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('title', models.CharField(max_length=255)), + ('divar_url', models.URLField(validators=[crawler.models.validate_divar_url])), + ('detection_prompt', models.TextField()), + ('interval_minutes', models.IntegerField(choices=[(5, '5 Minutes'), (15, '15 Minutes'), (30, '30 Minutes'), (60, '1 Hour'), (120, '2 Hours'), (180, '3 Hours'), (240, '4 Hours'), (300, '5 Hours'), (360, '6 Hours')], default=60)), + ('start_hour', models.TimeField()), + ('end_hour', models.TimeField()), + ('telegram_channel_id', models.CharField(blank=True, max_length=100, null=True)), + ('is_active', models.BooleanField(default=True)), + ], + options={ + 'indexes': [models.Index(fields=['is_active'], name='crawler_cra_is_acti_b67d0a_idx'), models.Index(fields=['created_at'], name='crawler_cra_created_920fb7_idx'), models.Index(fields=['title'], name='crawler_cra_title_dfbb5f_idx')], + }, + ), + migrations.CreateModel( + name='CrawlRun', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('started_at', models.DateTimeField(auto_now_add=True)), + ('finished_at', models.DateTimeField(blank=True, null=True)), + ('status', models.CharField(choices=[('RUNNING', 'Running'), ('SUCCESS', 'Success'), ('FAILED', 'Failed')], default='RUNNING', max_length=20)), + ('ads_fetched_count', models.IntegerField(default=0)), + ('ads_evaluated_count', models.IntegerField(default=0)), + ('ads_flagged_count', models.IntegerField(default=0)), + ('error_log', models.TextField(blank=True, null=True)), + ('crawl_task', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='runs', to='crawler.crawltask')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/backend/crawler/serializers.py b/backend/crawler/serializers.py index e69de29..9e29733 100644 --- a/backend/crawler/serializers.py +++ b/backend/crawler/serializers.py @@ -0,0 +1,44 @@ +from rest_framework import serializers +from .models import CrawlTask, CrawlRun + +class CrawlTaskSerializer(serializers.ModelSerializer): + class Meta: + model = CrawlTask + fields = [ + 'id', + 'title', + 'divar_url', + 'detection_prompt', + 'interval_minutes', + 'start_hour', + 'end_hour', + 'telegram_channel_id', + 'is_active', + 'created_at', + 'updated_at', + ] + read_only_fields = ['id', 'created_at', 'updated_at'] + + def validate(self, attrs): + start_hour = attrs.get('start_hour') + end_hour = attrs.get('end_hour') + + # If hours are provided, ensure they make sense or do any other general task validations + return attrs + + +class CrawlRunSerializer(serializers.ModelSerializer): + class Meta: + model = CrawlRun + fields = [ + 'id', + 'crawl_task', + 'started_at', + 'finished_at', + 'status', + 'ads_fetched_count', + 'ads_evaluated_count', + 'ads_flagged_count', + 'error_log', + ] + read_only_fields = ['id', 'started_at', 'finished_at', 'status', 'ads_fetched_count', 'ads_evaluated_count', 'ads_flagged_count', 'error_log'] diff --git a/backend/crawler/urls.py b/backend/crawler/urls.py index 69e2f75..50668c6 100644 --- a/backend/crawler/urls.py +++ b/backend/crawler/urls.py @@ -1,6 +1,11 @@ -from django.urls import path -from .views import CrawlerView +from django.urls import path, include +from rest_framework.routers import DefaultRouter +from .views import CrawlTaskViewSet + +router = DefaultRouter() +router.register(r'', CrawlTaskViewSet, basename='crawlers') urlpatterns = [ - path('', CrawlerView.as_view()), + path('', include(router.urls)), ] + diff --git a/backend/crawler/views.py b/backend/crawler/views.py index 83caf61..fd72a99 100644 --- a/backend/crawler/views.py +++ b/backend/crawler/views.py @@ -1,11 +1,50 @@ -from django.shortcuts import render -from rest_framework.views import APIView -from rest_framework.response import Response - - -# Create your views here. - -#create the django rest framework view for crawling the data -class CrawlerView(APIView): - def get(self, request): - return Response({'message': 'Hello World'}) \ No newline at end of file +from rest_framework import viewsets, status +from rest_framework.decorators import action +from rest_framework.response import Response +from .models import CrawlTask, CrawlRun +from .serializers import CrawlTaskSerializer, CrawlRunSerializer + +class CrawlTaskViewSet(viewsets.ModelViewSet): + """ + ViewSet for CrawlTask models. Supports CRUD and custom trigger/runs actions. + """ + queryset = CrawlTask.objects.all().order_by('-created_at') + serializer_class = CrawlTaskSerializer + + @action(detail=True, methods=['post'], url_path='trigger') + def trigger(self, request, pk=None): + task = self.get_object() + + # Create a new CrawlRun with status RUNNING + run = CrawlRun.objects.create( + crawl_task=task, + status='RUNNING' + ) + + # Check if tasks can be imported. + # This will integrate with crawlers/tasks.py once it is written in a later step. + try: + from crawler.tasks import run_crawl_pipeline + # run asynchronously + run_crawl_pipeline.delay(str(run.id)) + is_queued = True + except ImportError: + # Fallback if tasks.py doesn't exist yet or run_crawl_pipeline is not defined + is_queued = False + + return Response({ + "status": "queued" if is_queued else "queued_stub", + "run_id": str(run.id) + }, status=status.HTTP_202_ACCEPTED) + + @action(detail=True, methods=['get'], url_path='runs') + def runs(self, request, pk=None): + task = self.get_object() + runs = task.runs.all().order_by('-started_at') + page = self.paginate_queryset(runs) + if page is not None: + serializer = CrawlRunSerializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = CrawlRunSerializer(runs, many=True) + return Response(serializer.data) \ No newline at end of file