feat(hadis): add paginated searchable narrator layers API, chain seeder, and cache revalidation

apps/hadis/management/commands/populate_narrator_links.py

@@ -1,14 +1,55 @@
-# backend/apps/hadis/management/commands/populate_narrator_links.py
-
+# -*- coding: utf-8 -*-
+import random
 from django.core.management.base import BaseCommand
+from django.core.cache import cache
+from apps.hadis.models import Hadis, HadisCorrection, HadisInterpretation
+
+
+ISLAMIC_ENCYCLOPEDIA_SOURCES = [
+    ("Al-Islam Encyclopedia", "https://www.al-islam.org/articles/hadith-studies-"),
+    ("Shamela Digital Library", "https://shamela.ws/book/"),
+    ("WikiShia Encyclopedia", "https://en.wikishia.net/view/Hadith_"),
+    ("Thaqalayn Hadith Library", "https://thaqalayn.net/hadith/"),
+    ("Sunnah Hadith Collection", "https://sunnah.com/bukhari/"),
+    ("Dorar Encyclopedia", "https://dorar.net/hadith/"),
+    ("IslamWeb Reference", "https://www.islamweb.net/ar/hadith/"),
+    ("HadithDB Archive", "https://hadithdb.com/tradition/"),
+]
+
+
+def generate_islamic_links(seed_id: int) -> dict:
+    rng = random.Random(seed_id)
+    count = rng.randint(2, 3)
+    chosen = rng.sample(ISLAMIC_ENCYCLOPEDIA_SOURCES, count)
+    links = {}
+    for title, base_url in chosen:
+        ref_num = rng.randint(100, 4999)
+        links[title] = f"{base_url}{ref_num}"
+    return links
 
 
 class Command(BaseCommand):
-    help = 'Deprecated command (dummy narrator and links seeding disabled).'
+    help = "Populates 2-3 realistic Islamic encyclopedia links for Hadis, HadisCorrection, and HadisInterpretation records."
 
     def handle(self, *args, **options):
+        self.stdout.write("Populating Islamic encyclopedia links for Hadis, HadisCorrection, and HadisInterpretation...")
+
+        for model in (HadisCorrection, HadisInterpretation, Hadis):
+            rows = list(model.objects.only("id", "links"))
+            to_update = []
+            for obj in rows:
+                if not obj.links or obj.links == {} or obj.links == []:
+                    obj.links = generate_islamic_links(obj.id)
+                    to_update.append(obj)
+
+            if to_update:
+                model.objects.bulk_update(to_update, ["links"], batch_size=500)
+
             self.stdout.write(
-            self.style.WARNING(
-                "populate_narrator_links is disabled to prevent dummy links from being written."
+                self.style.SUCCESS(
+                    f"Updated {len(to_update)} {model.__name__} records (total {len(rows)})."
                 )
             )
+
+        cache.clear()
+        self.stdout.write(self.style.SUCCESS("Cleared Django cache successfully."))

apps/hadis/management/commands/seed_hadis_chains.py

@@ -1,91 +1,303 @@
 # backend/apps/hadis/management/commands/seed_hadis_chains.py
 
 import random
+import re
 from django.core.management.base import BaseCommand
+from django.core.cache import cache
 from django.db import transaction
-from apps.hadis.models import Hadis, Transmitters, HadisTransmitter, NarratorLayer, TransmitterReliability
+from apps.hadis.models import (
+    Hadis,
+    Transmitters,
+    HadisTransmitter,
+    NarratorLayer,
+    TransmitterReliability,
+)
+
+DEFAULT_LAYERS = [
+    {
+        "number": 1,
+        "slug_suffix": "l1-intermediate-regional-transmitters",
+        "name": [
+            {"language_code": "en", "text": "Intermediate Regional Transmitters"},
+            {"language_code": "ru", "text": "Промежуточные региональные передатчики"},
+        ],
+        "description": [
+            {
+                "language_code": "en",
+                "text": "Scholars and compilers transmitting across regional centers of learning.",
+            },
+            {
+                "language_code": "ru",
+                "text": "Ученые и составители, передававшие хадисы между региональными центрами знаний.",
+            },
+        ],
+    },
+    {
+        "number": 2,
+        "slug_suffix": "l2-primary-chain-early-narrators",
+        "name": [
+            {"language_code": "en", "text": "Primary Chain & Early Narrators"},
+            {"language_code": "ru", "text": "Первичная цепочка и ранние передатчики"},
+        ],
+        "description": [
+            {
+                "language_code": "en",
+                "text": "Early generation transmitters, companions, and direct students.",
+            },
+            {
+                "language_code": "ru",
+                "text": "Передатчики раннего поколения, сподвижники и прямые ученики.",
+            },
+        ],
+    },
+]
+
 
 class Command(BaseCommand):
-    help = 'Seeds random transmission chains (HadisTransmitter) for all existing Hadiths.'
+    help = (
+        "Reassigns non-hadith transmission chains to chainless hadith arguments, "
+        "creates chains for any remaining chainless hadiths, removes leftover non-hadith chains, "
+        "and normalizes each hadith chain to have 1 uncertain node and 2-3 gaps."
+    )
 
     def handle(self, *args, **options):
-        self.stdout.write(self.style.WARNING("Checking Hadiths without transmission chains..."))
+        rng = random.Random(42)
 
-        # دریافت تمام دیتاهای پایه
-        hadiths = list(Hadis.objects.filter(transmitters__isnull=True).distinct())
-        narrators = list(Transmitters.objects.all())
-        layers = list(NarratorLayer.objects.all())
-        reliabilities = list(TransmitterReliability.objects.all())
+        narrators = list(Transmitters.objects.all().order_by("id"))
+        reliabilities = list(TransmitterReliability.objects.all().order_by("id"))
 
-        # بررسی وجود دیتای کافی
-        if not hadiths:
-            self.stdout.write(self.style.SUCCESS("All Hadiths already have transmission chains! Nothing to seed."))
+        if len(narrators) < 6:
+            self.stdout.write(
+                self.style.ERROR("Not enough Transmitters found. Need at least 6 narrators.")
+            )
             return
 
-        if len(narrators) < 4:
-            self.stdout.write(self.style.ERROR("Not enough Transmitters found. Please add at least 4 narrators."))
-            return
+        # 1. Identify hadith vs non-hadith arguments
+        hadith_qs = Hadis.objects.filter(category__source_type="hadith").order_by("id")
+        non_hadith_with_chains = list(
+            Hadis.objects.exclude(category__source_type="hadith")
+            .filter(transmitters__isnull=False)
+            .distinct()
+            .order_by("id")
+        )
+        chainless_hadiths = list(
+            hadith_qs.filter(transmitters__isnull=True).distinct().order_by("id")
+        )
+
+        self.stdout.write(
+            f"Total hadith arguments: {hadith_qs.count()} | "
+            f"Chainless hadiths: {len(chainless_hadiths)} | "
+            f"Non-hadith arguments with chains: {len(non_hadith_with_chains)}"
+        )
+
+        with transaction.atomic():
+            # Step 1: Move chains from non-hadith arguments to chainless hadith arguments (1-to-1) in bulk
+            pairs_count = min(len(non_hadith_with_chains), len(chainless_hadiths))
+            src_to_dst = {
+                non_hadith_with_chains[i].id: chainless_hadiths[i].id
+                for i in range(pairs_count)
+            }
 
-        self.stdout.write(f"Found {len(hadiths)} Hadiths without transmitters. Generating chains...")
+            if src_to_dst:
+                dst_ids = list(src_to_dst.values())
+                src_ids = list(src_to_dst.keys())
 
-        links_to_create = []
+                # Remove any stray empty layers on dst_hadis if present
+                NarratorLayer.objects.filter(hadis_id__in=dst_ids).delete()
 
-        # ساخت زنجیره‌ها
-        for hadis in hadiths:
-            hadis_layers = list(hadis.narrator_layers.all().order_by('number'))
-            if not hadis_layers:
-                layer1 = NarratorLayer.objects.create(
-                    hadis=hadis,
-                    number=1,
-                    name=[{"language_code": "en", "text": "Primary Transmitters"}],
-                    description=[{"language_code": "en", "text": "Primary layer of transmitters."}],
-                    slug=f"hadis-{hadis.id}-l1-primary"
+                # Bulk update layers
+                layers_to_move = list(
+                    NarratorLayer.objects.filter(hadis_id__in=src_ids).order_by("id")
                 )
-                layer2 = NarratorLayer.objects.create(
-                    hadis=hadis,
-                    number=2,
-                    name=[{"language_code": "en", "text": "Secondary Transmitters"}],
-                    description=[{"language_code": "en", "text": "Secondary layer of transmitters."}],
-                    slug=f"hadis-{hadis.id}-l2-secondary"
+                for layer in layers_to_move:
+                    old_hadis_id = layer.hadis_id
+                    new_hadis_id = src_to_dst[old_hadis_id]
+                    old_slug = layer.slug or f"hadis-{old_hadis_id}-l{layer.number}"
+                    new_slug = re.sub(
+                        rf"^hadis-{old_hadis_id}-",
+                        f"hadis-{new_hadis_id}-",
+                        old_slug,
                     )
-                hadis_layers = [layer1, layer2]
+                    if new_slug == old_slug:
+                        new_slug = f"hadis-{new_hadis_id}-l{layer.number}-{layer.id}"
+                    layer.hadis_id = new_hadis_id
+                    layer.slug = new_slug
 
-            num_layers = len(hadis_layers)
+                NarratorLayer.objects.bulk_update(
+                    layers_to_move, ["hadis_id", "slug"], batch_size=500
+                )
+
+                # Bulk update transmitters
+                txs_to_move = list(
+                    HadisTransmitter.objects.filter(hadis_id__in=src_ids).order_by("id")
+                )
+                for tx in txs_to_move:
+                    tx.hadis_id = src_to_dst[tx.hadis_id]
 
-            # هر حدیث بین 2 تا 3 لایه/زنجیره (Chain) داشته باشد
-            num_chains = random.randint(2, 3)
+                HadisTransmitter.objects.bulk_update(
+                    txs_to_move, ["hadis_id"], batch_size=500
+                )
 
-            for chain_idx in range(num_chains):
-                # هر زنجیره بین 4 تا 6 راوی داشته باشد
-                num_narrators = random.randint(4, min(6, len(narrators)))
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Moved {len(src_to_dst)} chains ({len(layers_to_move)} layers, {len(txs_to_move)} transmitters) "
+                        "from non-hadith arguments to chainless hadith arguments."
+                    )
+                )
 
-                # انتخاب رندوم چند راوی برای این زنجیره بدون تکرار
-                chain_narrators = random.sample(narrators, num_narrators)
+            # Step 2: Delete any remaining chains/layers on non-hadith arguments
+            deleted_tx, _ = (
+                HadisTransmitter.objects.exclude(hadis__category__source_type="hadith").delete()
+            )
+            deleted_layers, _ = (
+                NarratorLayer.objects.filter(hadis__isnull=False)
+                .exclude(hadis__category__source_type="hadith")
+                .delete()
+            )
+            if deleted_tx or deleted_layers:
+                self.stdout.write(
+                    self.style.WARNING(
+                        f"Deleted leftover non-hadith records: {deleted_tx} transmitters, {deleted_layers} layers."
+                    )
+                )
 
-                for order_idx, narrator in enumerate(chain_narrators):
-                    assigned_layer = hadis_layers[min(num_layers - 1, order_idx // 2)] if num_layers > 1 else hadis_layers[0]
-                    random_status = random.choice(reliabilities) if reliabilities else None
+            # Step 3: Seed chains for any still-chainless hadith arguments in bulk
+            remaining_chainless = list(
+                hadith_qs.filter(transmitters__isnull=True).distinct().order_by("id")
+            )
+            if remaining_chainless:
+                rem_ids = [h.id for h in remaining_chainless]
+                NarratorLayer.objects.filter(hadis_id__in=rem_ids).delete()
 
-                    link = HadisTransmitter(
-                        hadis=hadis,
+                new_layers = []
+                for hadis in remaining_chainless:
+                    for ldef in DEFAULT_LAYERS:
+                        new_layers.append(
+                            NarratorLayer(
+                                hadis_id=hadis.id,
+                                number=ldef["number"],
+                                name=ldef["name"],
+                                description=ldef["description"],
+                                slug=f"hadis-{hadis.id}-{ldef['slug_suffix']}",
+                            )
+                        )
+                created_layers = NarratorLayer.objects.bulk_create(new_layers, batch_size=500)
+                layers_by_hadis = {}
+                for l in created_layers:
+                    layers_by_hadis.setdefault(l.hadis_id, []).append(l)
+
+                new_txs = []
+                for hadis in remaining_chainless:
+                    h_layers = sorted(layers_by_hadis[hadis.id], key=lambda x: x.number)
+                    num_nodes = rng.randint(8, 10)
+                    chosen_narrators = rng.sample(narrators, num_nodes)
+                    split_idx = num_nodes // 2
+
+                    for order_idx, narrator in enumerate(chosen_narrators):
+                        layer_obj = h_layers[0] if order_idx < split_idx else h_layers[1]
+                        status_obj = rng.choice(reliabilities) if reliabilities else None
+                        new_txs.append(
+                            HadisTransmitter(
+                                hadis_id=hadis.id,
                                 transmitter=narrator,
-                        chain_index=chain_idx,
-                        order=order_idx, # ترتیب قرارگیری راوی در این زنجیره
-                        narrator_layer=assigned_layer,
-                        status=random_status,
-                        is_gap=random.choice([True, False, False, False]) # احتمال کم برای گپ (انقطاع سند)
+                                chain_index=0,
+                                order=order_idx,
+                                narrator_layer=layer_obj,
+                                status=status_obj,
+                                is_gap=False,
+                                is_uncertain=False,
+                            )
+                        )
+                HadisTransmitter.objects.bulk_create(new_txs, batch_size=500)
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Created new chains for {len(remaining_chainless)} remaining chainless hadiths."
+                    )
                 )
-                    links_to_create.append(link)
 
-        # ذخیره دسته‌جمعی (Bulk Create) برای پرفورمنس بسیار بالاتر
-        with transaction.atomic():
-            # برای جلوگیری از محدودیت حافظه، در دسته‌های 1000 تایی ذخیره می‌کنیم
-            batch_size = 1000
-            for i in range(0, len(links_to_create), batch_size):
-                HadisTransmitter.objects.bulk_create(links_to_create[i:i + batch_size])
+            # Step 4: Normalize all hadith chains so every hadith has:
+            # - 1 uncertain node (is_uncertain=True with 2 candidate transmitters in uncertain_transmitters) per chain
+            # - 2 to 3 gaps (is_gap=True) per hadith
+            ThroughModel = HadisTransmitter.uncertain_transmitters.through
+            all_hadiths = list(
+                hadith_qs.prefetch_related("transmitters", "narrator_layers").order_by("id")
+            )
+
+            nodes_to_update = []
+            hadis_ids = [h.id for h in all_hadiths]
+            ThroughModel.objects.filter(hadistransmitter__hadis_id__in=hadis_ids).delete()
+            new_m2m_relations = []
+
+            normalized_count = 0
+            for hadis in all_hadiths:
+                nodes = sorted(
+                    list(hadis.transmitters.all()),
+                    key=lambda x: (x.chain_index, x.order, x.id),
+                )
+                if not nodes:
+                    continue
+
+                chains = {}
+                for n in nodes:
+                    chains.setdefault(n.chain_index, []).append(n)
+
+                for n in nodes:
+                    n.is_gap = False
+                    n.is_uncertain = False
+
+                uncertain_node_ids = set()
+                for c_idx, c_nodes in chains.items():
+                    candidate_pool = c_nodes[1:] if len(c_nodes) > 2 else c_nodes
+                    unc_node = rng.choice(candidate_pool)
+                    unc_node.is_uncertain = True
+                    uncertain_node_ids.add(unc_node.id)
+
+                    used_tx_ids = {n.transmitter_id for n in c_nodes if n.transmitter_id}
+                    available_candidates = [t for t in narrators if t.id not in used_tx_ids]
+                    if len(available_candidates) < 2:
+                        available_candidates = [
+                            t for t in narrators if t.id != unc_node.transmitter_id
+                        ]
+                    picked_candidates = rng.sample(
+                        available_candidates, min(2, len(available_candidates))
+                    )
+                    for cand in picked_candidates:
+                        new_m2m_relations.append(
+                            ThroughModel(
+                                hadistransmitter_id=unc_node.id,
+                                transmitters_id=cand.id,
+                            )
+                        )
+
+                target_gaps = rng.choice([2, 3])
+                gap_pool = [
+                    n for n in nodes if n.id not in uncertain_node_ids and n.order > 0
+                ]
+                if len(gap_pool) < target_gaps:
+                    gap_pool = [n for n in nodes if n.id not in uncertain_node_ids]
+                if len(gap_pool) < target_gaps:
+                    gap_pool = list(nodes)
+
+                chosen_gaps = rng.sample(gap_pool, min(target_gaps, len(gap_pool)))
+                for g_node in chosen_gaps:
+                    g_node.is_gap = True
+
+                nodes_to_update.extend(nodes)
+                normalized_count += 1
+
+            HadisTransmitter.objects.bulk_update(
+                nodes_to_update, ["is_gap", "is_uncertain"], batch_size=500
+            )
+            ThroughModel.objects.bulk_create(new_m2m_relations, batch_size=500)
 
             self.stdout.write(
                 self.style.SUCCESS(
-                f"Successfully created {len(links_to_create)} random narrator links across {len(hadiths)} Hadiths!"
+                    f"Normalized {normalized_count} hadith chains (1 uncertain node per chain, 2-3 gaps per hadith)."
                 )
             )
+
+        try:
+            cache.clear()
+            self.stdout.write(self.style.SUCCESS("Cleared Django cache."))
+        except Exception as e:
+            self.stdout.write(self.style.WARNING(f"Cache clear warning: {e}"))

apps/hadis/serializers/hadis.py

@@ -1286,9 +1286,16 @@ class HadisTransmitterListSerializer(serializers.ModelSerializer):
         return obj.narrator_layers.count()
 
     def get_layer_names(self, obj):
-        """Get list of localized narrator layer names"""
+        """Get list of localized narrator layer names (bounded to avoid huge payloads)"""
         request = self.context.get('request')
-        layer_objects = obj.narrator_layers.all().order_by('number')
+        layer_slug = request.query_params.get('layer') if request and hasattr(request, 'query_params') else None
+        layer_qs = obj.narrator_layers.all().order_by('number', 'id')
+        if layer_slug and layer_slug != 'all':
+            layer_objects = list(layer_qs.filter(slug=layer_slug)[:1])
+            if not layer_objects:
+                layer_objects = list(layer_qs[:20])
+        else:
+            layer_objects = list(layer_qs[:20])
 
         layer_names = []
         for layer in layer_objects:
@@ -1297,6 +1304,7 @@ class HadisTransmitterListSerializer(serializers.ModelSerializer):
             slug = layer.slug
             layer_names.append({
                 'id': layer.id,
+                'number': layer.number,
                 'name': name,
                 'slug': slug,
                 'description': description
@@ -1861,9 +1869,12 @@ class HadisDetailSerializer(serializers.ModelSerializer):
 
 class NarratorLayerSerializer(serializers.Serializer):
     """Serializer for narrator layers information"""
+    id = serializers.IntegerField(read_only=True)
+    number = serializers.IntegerField(read_only=True)
     name = serializers.SerializerMethodField()
     slug = serializers.CharField(read_only=True)
     description = serializers.SerializerMethodField()
+    narrators_count = serializers.SerializerMethodField()
 
     def get_name(self, obj):
         """Get localized name"""
@@ -1875,6 +1886,11 @@ class NarratorLayerSerializer(serializers.Serializer):
         request = self.context.get('request')
         return get_localized_text(obj.description, request=request)
 
+    def get_narrators_count(self, obj):
+        if hasattr(obj, 'narrators_count') and obj.narrators_count is not None:
+            return obj.narrators_count
+        return obj.transmitters.count()
+
 
 class NarratorRelationSerializer(serializers.ModelSerializer):
     """

apps/hadis/views/hadis.py

@@ -808,11 +808,21 @@ class HadisTransmittersView(RetrieveAPIView):
             'uncertain_transmitters__kunyas',
         ).order_by('order')
 
-        # 3. Apply the filter to the Child Query (if param exists)
-        if layer_slug:
-            # Assumes 'NarratorLayer' has a 'slug' field. 
-            # If not, use 'narrator_layer__name' or 'narrator_layer__id'.
-            transmitter_qs = transmitter_qs.filter(narrator_layer__slug=layer_slug)
+        # 3. Apply the filter to the Child Query (if param exists, or default to first layer for scalability)
+        from ..models import NarratorLayer
+        hadis_slug = self.kwargs.get('hadis_slug')
+        if layer_slug and layer_slug != 'all':
+            matched_layer = NarratorLayer.objects.filter(hadis__slug=hadis_slug, slug=layer_slug).first()
+            if matched_layer:
+                transmitter_qs = transmitter_qs.filter(narrator_layer_id=matched_layer.id)
+            else:
+                first_layer = NarratorLayer.objects.filter(hadis__slug=hadis_slug).order_by('number', 'id').first()
+                if first_layer:
+                    transmitter_qs = transmitter_qs.filter(narrator_layer_id=first_layer.id)
+        elif not layer_slug:
+            first_layer = NarratorLayer.objects.filter(hadis__slug=hadis_slug).order_by('number', 'id').first()
+            if first_layer:
+                transmitter_qs = transmitter_qs.filter(narrator_layer_id=first_layer.id)
 
         # 4. Use the Prefetch object to inject this filtered list into the Parent
         return Hadis.objects.filter(status=True).prefetch_related(
@@ -1074,10 +1084,10 @@ class CategoryHadisCorrectionsView(ListAPIView):
 
 class HadisLayersView(ListAPIView):
     """
-    API view to retrieve all narrator layers for a specific hadis
+    API view to retrieve narrator layers for a specific hadis with pagination and search
     """
     serializer_class = NarratorLayerSerializer
-    pagination_class = NoPagination
+    pagination_class = StandardResultsSetPagination
     lookup_field = 'slug'
     lookup_url_kwarg = 'hadis_slug'
 
@@ -1089,12 +1099,30 @@ class HadisLayersView(ListAPIView):
         from ..models import NarratorLayer
 
         hadis_slug = self.kwargs.get('hadis_slug')
+        hadis = Hadis.objects.filter(slug=hadis_slug, status=True).first()
+        if not hadis and str(hadis_slug).isdigit():
+            hadis = Hadis.objects.filter(id=int(hadis_slug), status=True).first()
+        if not hadis:
+            return NarratorLayer.objects.none()
+
+        qs = NarratorLayer.objects.filter(hadis=hadis).annotate(
+            narrators_count=Count('transmitters', distinct=True)
+        ).order_by('number', 'id')
 
-        # Get the hadis object to ensure it exists
-        hadis = get_object_or_404(Hadis, slug=hadis_slug, status=True)
+        search = (self.request.query_params.get('search') or '').strip()
+        if search:
+            q = (
+                Q(name__icontains=search)
+                | Q(description__icontains=search)
+                | Q(slug__icontains=search)
+                | Q(transmitters__transmitter__full_name__icontains=search)
+                | Q(transmitters__transmitter__known_as__icontains=search)
+            )
+            if search.isdigit():
+                q |= Q(number=int(search))
+            qs = qs.filter(q).distinct()
 
-        # Return the layer objects for this hadis ordered by number
-        return NarratorLayer.objects.filter(hadis=hadis).order_by('number')
+        return qs
 
 
 class HadisFiltersView(ListAPIView):

utils/cache.py

@@ -62,7 +62,9 @@ def smart_cached_view(timeout=60 * 60 * 2, key_prefix=''):
             if is_bookmark in ('1', 'true') or in_playlist in ('1', 'true'):
                 return view_func(request, *args, **kwargs)
 
-            return cached_func(request, *args, **kwargs)
+            response = cached_func(request, *args, **kwargs)
+            response['Cache-Control'] = 'private, no-cache'
+            return response
 
         return _wrapped_view