diff --git a/backend/config/settings.py b/backend/config/settings.py index 014ce18..010f7d8 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -48,6 +48,7 @@ INSTALLED_APPS = [ # Local apps 'core', + 'crawler', ] MIDDLEWARE = [ diff --git a/backend/config/urls.py b/backend/config/urls.py index c74036a..b144811 100644 --- a/backend/config/urls.py +++ b/backend/config/urls.py @@ -15,8 +15,9 @@ Including another URLconf 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path +from django.urls import path,include urlpatterns = [ path('admin/', admin.site.urls), + path('api/crawlers/', include('crawler.urls')), ] diff --git a/backend/crawler/__init__.py b/backend/crawler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/crawler/admin.py b/backend/crawler/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/backend/crawler/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/backend/crawler/apps.py b/backend/crawler/apps.py new file mode 100644 index 0000000..05e02e0 --- /dev/null +++ b/backend/crawler/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class CrawlerConfig(AppConfig): + name = 'crawler' diff --git a/backend/crawler/migrations/__init__.py b/backend/crawler/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/crawler/models.py b/backend/crawler/models.py new file mode 100644 index 0000000..c2fdfda --- /dev/null +++ b/backend/crawler/models.py @@ -0,0 +1,61 @@ +from django.db import models +from django.core.exceptions import ValidationError +from core.models import BaseModel, TimeStampedModel + +def validate_divar_url(value): + """ + Validator to ensure that the URL is a valid Divar search/category link. + """ + if not value.startswith("https://divar.ir/s/"): + raise ValidationError("URL must start with 'https://divar.ir/s/'") + +class CrawlTask(TimeStampedModel): + INTERVAL_CHOICES = [ + (5, '5 Minutes'), + (15, '15 Minutes'), + (30, '30 Minutes'), + (60, '1 Hour'), + (120, '2 Hours'), + (180, '3 Hours'), + (240, '4 Hours'), + (300, '5 Hours'), + (360, '6 Hours'), + ] + + title = models.CharField(max_length=255) + divar_url = models.URLField(validators=[validate_divar_url]) + detection_prompt = models.TextField() + interval_minutes = models.IntegerField(choices=INTERVAL_CHOICES, default=INTERVAL_CHOICES[3][0]) + start_hour = models.TimeField() + end_hour = models.TimeField() + telegram_channel_id = models.CharField(max_length=100, blank=True, null=True) + is_active = models.BooleanField(default=True) + + class Meta: + indexes = [ + models.Index(fields=['is_active']), + models.Index(fields=['created_at']), + models.Index(fields=['title']), + ] + + def __str__(self): + return f"{self.title} (Active: {self.is_active})" + +class CrawlRun(BaseModel): + STATUS_CHOICES = [ + ('RUNNING', 'Running'), + ('SUCCESS', 'Success'), + ('FAILED', 'Failed'), + ] + + crawl_task = models.ForeignKey(CrawlTask, on_delete=models.PROTECT, related_name='runs') + started_at = models.DateTimeField(auto_now_add=True) + finished_at = models.DateTimeField(null=True, blank=True) + status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='RUNNING') + ads_fetched_count = models.IntegerField(default=0) + ads_evaluated_count = models.IntegerField(default=0) + ads_flagged_count = models.IntegerField(default=0) + error_log = models.TextField(null=True, blank=True) + + def __str__(self): + return f"Run {self.id} for {self.crawl_task.title} ({self.status})" diff --git a/backend/crawler/serializers.py b/backend/crawler/serializers.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/crawler/tests.py b/backend/crawler/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/backend/crawler/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/backend/crawler/urls.py b/backend/crawler/urls.py new file mode 100644 index 0000000..69e2f75 --- /dev/null +++ b/backend/crawler/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +from .views import CrawlerView + +urlpatterns = [ + path('', CrawlerView.as_view()), +] diff --git a/backend/crawler/views.py b/backend/crawler/views.py new file mode 100644 index 0000000..83caf61 --- /dev/null +++ b/backend/crawler/views.py @@ -0,0 +1,11 @@ +from django.shortcuts import render +from rest_framework.views import APIView +from rest_framework.response import Response + + +# Create your views here. + +#create the django rest framework view for crawling the data +class CrawlerView(APIView): + def get(self, request): + return Response({'message': 'Hello World'}) \ No newline at end of file