From fa471c37a1430458836b33c1ce4563fb82e1e4d0 Mon Sep 17 00:00:00 2001
From: PouyaKhajavi <pouyakhajavidev@newhorizon.com>
Date: Mon, 10 Aug 2026 12:15:38 +0330
Subject: [PATCH] fix(crawler): implement robust price extraction from Divar
 widget structures and update existing ads

---
 backend/crawler/tasks.py | 56 +++++++++++++++++++++++++++++++++++-----
 backend/crawler/tests.py | 20 +++++++++-----
 2 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/backend/crawler/tasks.py b/backend/crawler/tasks.py
index 445db27..f88f9fd 100644
--- a/backend/crawler/tasks.py
+++ b/backend/crawler/tasks.py
@@ -34,6 +34,52 @@ def parse_divar_url(divar_url):
     query_params = dict(urllib.parse.parse_qsl(parsed.query))
     return city, category, query_params
 
+def extract_price_from_widget(d):
+    """
+    Extracts human-readable price string from various Divar widget structures.
+    """
+    if not isinstance(d, dict):
+        return ''
+        
+    data = d.get('data', {}) if isinstance(d.get('data'), dict) else {}
+    
+    # 1. Direct price string or dict in d or data
+    raw_price = d.get('price') or data.get('price') or data.get('price_text') or data.get('price_str')
+    if isinstance(raw_price, str) and raw_price.strip():
+        return raw_price.strip()
+    elif isinstance(raw_price, dict):
+        text = raw_price.get('text') or raw_price.get('price_text') or raw_price.get('value')
+        if text:
+            return str(text).strip()
+            
+    # 2. Check action_log server_side_params
+    action_log = data.get('action_log', {}) if isinstance(data.get('action_log'), dict) else {}
+    server_params = action_log.get('server_side_params', {}) if isinstance(action_log.get('server_side_params'), dict) else {}
+    if 'price' in server_params:
+        p_val = server_params.get('price')
+        if p_val:
+            return f"{p_val} تومان" if str(p_val).isdigit() else str(p_val)
+
+    # 3. Check middle_description_text, bottom_description_text, top_description_text
+    middle_desc = str(data.get('middle_description_text') or d.get('middle_description_text') or '').strip()
+    bottom_desc = str(data.get('bottom_description_text') or d.get('bottom_description_text') or '').strip()
+    top_desc = str(data.get('top_description_text') or d.get('top_description_text') or '').strip()
+
+    price_keywords = ['تومان', 'ریال', 'توافقی', 'رایگان', 'ودیعه', 'اجاره', 'قیمت']
+    
+    for text in [middle_desc, bottom_desc, top_desc]:
+        if text and any(kw in text for kw in price_keywords):
+            return text
+
+    persian_digits = '۰۱۲۳۴۵۶۷۸۹0123456789'
+    if middle_desc and any(char in middle_desc for char in persian_digits):
+        return middle_desc
+
+    if bottom_desc and any(char in bottom_desc for char in persian_digits):
+        return bottom_desc
+
+    return middle_desc or bottom_desc or top_desc or ''
+
 def extract_widgets_from_dict(d, found_ads):
     """
     Recursively searches a JSON structure for dictionaries matching
@@ -62,12 +108,7 @@ def extract_widgets_from_dict(d, found_ads):
                 ''
             )
             
-            price = (
-                d.get('price') or 
-                d.get('data', {}).get('price_text') or 
-                d.get('data', {}).get('price') or 
-                ''
-            )
+            price = extract_price_from_widget(d)
             
             image_url = ''
             image_data = d.get('image') or d.get('data', {}).get('image') or d.get('data', {}).get('image_url')
@@ -185,6 +226,9 @@ def run_crawl_pipeline(run_id, force=False):
                     'url': f"https://divar.ir/v/{token}"
                 }
             )
+            if not created and ad_info['price'] and (not ad.price or ad.price == 'Not specified'):
+                ad.price = ad_info['price']
+                ad.save(update_fields=['price'])
             
             # Check if an evaluation already exists for this crawler on this ad
             eval_exists = AdEvaluation.objects.filter(ad=ad, crawl_task=task).exists()
diff --git a/backend/crawler/tests.py b/backend/crawler/tests.py
index da9b7a4..bec62c8 100644
--- a/backend/crawler/tests.py
+++ b/backend/crawler/tests.py
@@ -167,12 +167,20 @@ class CrawlTaskCeleryTests(APITestCase):
             status='RUNNING'
         )
 
-    def test_parse_divar_url(self):
-        from crawler.tasks import parse_divar_url
-        city, category, query_params = parse_divar_url(self.task.divar_url)
-        self.assertEqual(city, 'tehran')
-        self.assertEqual(category, 'buy-apartment')
-        self.assertEqual(query_params, {'query': 'urgent'})
+    def test_extract_price_from_widget(self):
+        from crawler.tasks import extract_price_from_widget
+        
+        # Test direct price_text
+        widget1 = {"data": {"title": "Apartment", "price_text": "۴,۵۰۰,۰۰۰ تومان"}}
+        self.assertEqual(extract_price_from_widget(widget1), "۴,۵۰۰,۰۰۰ تومان")
+
+        # Test middle_description_text
+        widget2 = {"data": {"title": "Car", "middle_description_text": "۱۲,۰۰۰,۰۰۰ تومان"}}
+        self.assertEqual(extract_price_from_widget(widget2), "۱۲,۰۰۰,۰۰۰ تومان")
+
+        # Test price dict
+        widget3 = {"data": {"title": "Item", "price": {"text": "توافقی"}}}
+        self.assertEqual(extract_price_from_widget(widget3), "توافقی")
 
     def test_run_crawl_pipeline_inactive_task(self):
         self.task.is_active = False