feat(crawler): initialize crawler app with CrawlTask and CrawlRun models and API views

backend/config/settings.py

@@ -48,6 +48,7 @@ INSTALLED_APPS = [
 
     # Local apps
     'core',
+    'crawler',
 ]
 
 MIDDLEWARE = [

backend/config/urls.py

@@ -15,8 +15,9 @@ Including another URLconf
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path
+from django.urls import path,include
 
 urlpatterns = [
     path('admin/', admin.site.urls),
+    path('api/crawlers/', include('crawler.urls')),
 ]

backend/crawler/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

backend/crawler/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CrawlerConfig(AppConfig):
+    name = 'crawler'

backend/crawler/models.py

@@ -0,0 +1,61 @@
+from django.db import models
+from django.core.exceptions import ValidationError
+from core.models import BaseModel, TimeStampedModel
+
+def validate_divar_url(value):
+    """
+    Validator to ensure that the URL is a valid Divar search/category link.
+    """
+    if not value.startswith("https://divar.ir/s/"):
+        raise ValidationError("URL must start with 'https://divar.ir/s/'")
+
+class CrawlTask(TimeStampedModel):
+    INTERVAL_CHOICES = [
+        (5, '5 Minutes'),
+        (15, '15 Minutes'),
+        (30, '30 Minutes'),
+        (60, '1 Hour'),
+        (120, '2 Hours'),
+        (180, '3 Hours'),
+        (240, '4 Hours'),
+        (300, '5 Hours'),
+        (360, '6 Hours'),
+    ]
+
+    title = models.CharField(max_length=255)
+    divar_url = models.URLField(validators=[validate_divar_url])
+    detection_prompt = models.TextField()
+    interval_minutes = models.IntegerField(choices=INTERVAL_CHOICES, default=INTERVAL_CHOICES[3][0])
+    start_hour = models.TimeField()
+    end_hour = models.TimeField()
+    telegram_channel_id = models.CharField(max_length=100, blank=True, null=True)
+    is_active = models.BooleanField(default=True)
+
+    class Meta:
+        indexes = [
+            models.Index(fields=['is_active']),
+            models.Index(fields=['created_at']),
+            models.Index(fields=['title']),
+        ]
+
+    def __str__(self):
+        return f"{self.title} (Active: {self.is_active})"
+
+class CrawlRun(BaseModel):
+    STATUS_CHOICES = [
+        ('RUNNING', 'Running'),
+        ('SUCCESS', 'Success'),
+        ('FAILED', 'Failed'),
+    ]
+
+    crawl_task = models.ForeignKey(CrawlTask, on_delete=models.PROTECT, related_name='runs')
+    started_at = models.DateTimeField(auto_now_add=True)
+    finished_at = models.DateTimeField(null=True, blank=True)
+    status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='RUNNING')
+    ads_fetched_count = models.IntegerField(default=0)
+    ads_evaluated_count = models.IntegerField(default=0)
+    ads_flagged_count = models.IntegerField(default=0)
+    error_log = models.TextField(null=True, blank=True)
+
+    def __str__(self):
+        return f"Run {self.id} for {self.crawl_task.title} ({self.status})"

backend/crawler/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

backend/crawler/urls.py

@@ -0,0 +1,6 @@
+from django.urls import path
+from .views import CrawlerView
+
+urlpatterns = [
+    path('', CrawlerView.as_view()),
+]

backend/crawler/views.py

@@ -0,0 +1,11 @@
+from django.shortcuts import render
+from rest_framework.views import APIView
+from rest_framework.response import Response
+
+
+# Create your views here.
+
+#create the django rest framework view for crawling the data
+class CrawlerView(APIView):
+    def get(self, request):
+        return Response({'message': 'Hello World'})