You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
61 lines
2.1 KiB
61 lines
2.1 KiB
from django.db import models
|
|
from django.core.exceptions import ValidationError
|
|
from core.models import BaseModel, TimeStampedModel
|
|
|
|
def validate_divar_url(value):
|
|
"""
|
|
Validator to ensure that the URL is a valid Divar search/category link.
|
|
"""
|
|
if not value.startswith("https://divar.ir/s/"):
|
|
raise ValidationError("URL must start with 'https://divar.ir/s/'")
|
|
|
|
class CrawlTask(TimeStampedModel):
|
|
INTERVAL_CHOICES = [
|
|
(5, '5 Minutes'),
|
|
(15, '15 Minutes'),
|
|
(30, '30 Minutes'),
|
|
(60, '1 Hour'),
|
|
(120, '2 Hours'),
|
|
(180, '3 Hours'),
|
|
(240, '4 Hours'),
|
|
(300, '5 Hours'),
|
|
(360, '6 Hours'),
|
|
]
|
|
|
|
title = models.CharField(max_length=255)
|
|
divar_url = models.URLField(validators=[validate_divar_url])
|
|
detection_prompt = models.TextField()
|
|
interval_minutes = models.IntegerField(choices=INTERVAL_CHOICES, default=INTERVAL_CHOICES[3][0])
|
|
start_hour = models.TimeField()
|
|
end_hour = models.TimeField()
|
|
telegram_channel_id = models.CharField(max_length=100, blank=True, null=True)
|
|
is_active = models.BooleanField(default=True)
|
|
|
|
class Meta:
|
|
indexes = [
|
|
models.Index(fields=['is_active']),
|
|
models.Index(fields=['created_at']),
|
|
models.Index(fields=['title']),
|
|
]
|
|
|
|
def __str__(self):
|
|
return f"{self.title} (Active: {self.is_active})"
|
|
|
|
class CrawlRun(BaseModel):
|
|
STATUS_CHOICES = [
|
|
('RUNNING', 'Running'),
|
|
('SUCCESS', 'Success'),
|
|
('FAILED', 'Failed'),
|
|
]
|
|
|
|
crawl_task = models.ForeignKey(CrawlTask, on_delete=models.PROTECT, related_name='runs')
|
|
started_at = models.DateTimeField(auto_now_add=True)
|
|
finished_at = models.DateTimeField(null=True, blank=True)
|
|
status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='RUNNING')
|
|
ads_fetched_count = models.IntegerField(default=0)
|
|
ads_evaluated_count = models.IntegerField(default=0)
|
|
ads_flagged_count = models.IntegerField(default=0)
|
|
error_log = models.TextField(null=True, blank=True)
|
|
|
|
def __str__(self):
|
|
return f"Run {self.id} for {self.crawl_task.title} ({self.status})"
|