From 8286e24578d02bacaae75ba37803526c5c5c15a2 Mon Sep 17 00:00:00 2001 From: mohsentaba Date: Tue, 27 Oct 2026 19:03:34 +0330 Subject: [PATCH] feat(hadis): add paginated searchable narrator layers API, chain seeder, and cache revalidation --- .../commands/populate_narrator_links.py | 55 ++- .../management/commands/seed_hadis_chains.py | 352 ++++++++++++++---- apps/hadis/serializers/hadis.py | 20 +- apps/hadis/views/hadis.py | 52 ++- utils/cache.py | 4 +- 5 files changed, 391 insertions(+), 92 deletions(-) diff --git a/apps/hadis/management/commands/populate_narrator_links.py b/apps/hadis/management/commands/populate_narrator_links.py index 5d83be3..93d7a87 100644 --- a/apps/hadis/management/commands/populate_narrator_links.py +++ b/apps/hadis/management/commands/populate_narrator_links.py @@ -1,14 +1,55 @@ -# backend/apps/hadis/management/commands/populate_narrator_links.py - +# -*- coding: utf-8 -*- +import random from django.core.management.base import BaseCommand +from django.core.cache import cache +from apps.hadis.models import Hadis, HadisCorrection, HadisInterpretation + + +ISLAMIC_ENCYCLOPEDIA_SOURCES = [ + ("Al-Islam Encyclopedia", "https://www.al-islam.org/articles/hadith-studies-"), + ("Shamela Digital Library", "https://shamela.ws/book/"), + ("WikiShia Encyclopedia", "https://en.wikishia.net/view/Hadith_"), + ("Thaqalayn Hadith Library", "https://thaqalayn.net/hadith/"), + ("Sunnah Hadith Collection", "https://sunnah.com/bukhari/"), + ("Dorar Encyclopedia", "https://dorar.net/hadith/"), + ("IslamWeb Reference", "https://www.islamweb.net/ar/hadith/"), + ("HadithDB Archive", "https://hadithdb.com/tradition/"), +] + + +def generate_islamic_links(seed_id: int) -> dict: + rng = random.Random(seed_id) + count = rng.randint(2, 3) + chosen = rng.sample(ISLAMIC_ENCYCLOPEDIA_SOURCES, count) + links = {} + for title, base_url in chosen: + ref_num = rng.randint(100, 4999) + links[title] = f"{base_url}{ref_num}" + return links class Command(BaseCommand): - help = 'Deprecated command (dummy narrator and links seeding disabled).' + help = "Populates 2-3 realistic Islamic encyclopedia links for Hadis, HadisCorrection, and HadisInterpretation records." def handle(self, *args, **options): - self.stdout.write( - self.style.WARNING( - "populate_narrator_links is disabled to prevent dummy links from being written." + self.stdout.write("Populating Islamic encyclopedia links for Hadis, HadisCorrection, and HadisInterpretation...") + + for model in (HadisCorrection, HadisInterpretation, Hadis): + rows = list(model.objects.only("id", "links")) + to_update = [] + for obj in rows: + if not obj.links or obj.links == {} or obj.links == []: + obj.links = generate_islamic_links(obj.id) + to_update.append(obj) + + if to_update: + model.objects.bulk_update(to_update, ["links"], batch_size=500) + + self.stdout.write( + self.style.SUCCESS( + f"Updated {len(to_update)} {model.__name__} records (total {len(rows)})." + ) ) - ) + + cache.clear() + self.stdout.write(self.style.SUCCESS("Cleared Django cache successfully.")) diff --git a/apps/hadis/management/commands/seed_hadis_chains.py b/apps/hadis/management/commands/seed_hadis_chains.py index df7c2bd..26f8162 100644 --- a/apps/hadis/management/commands/seed_hadis_chains.py +++ b/apps/hadis/management/commands/seed_hadis_chains.py @@ -1,91 +1,303 @@ # backend/apps/hadis/management/commands/seed_hadis_chains.py import random +import re from django.core.management.base import BaseCommand +from django.core.cache import cache from django.db import transaction -from apps.hadis.models import Hadis, Transmitters, HadisTransmitter, NarratorLayer, TransmitterReliability +from apps.hadis.models import ( + Hadis, + Transmitters, + HadisTransmitter, + NarratorLayer, + TransmitterReliability, +) + +DEFAULT_LAYERS = [ + { + "number": 1, + "slug_suffix": "l1-intermediate-regional-transmitters", + "name": [ + {"language_code": "en", "text": "Intermediate Regional Transmitters"}, + {"language_code": "ru", "text": "Промежуточные региональные передатчики"}, + ], + "description": [ + { + "language_code": "en", + "text": "Scholars and compilers transmitting across regional centers of learning.", + }, + { + "language_code": "ru", + "text": "Ученые и составители, передававшие хадисы между региональными центрами знаний.", + }, + ], + }, + { + "number": 2, + "slug_suffix": "l2-primary-chain-early-narrators", + "name": [ + {"language_code": "en", "text": "Primary Chain & Early Narrators"}, + {"language_code": "ru", "text": "Первичная цепочка и ранние передатчики"}, + ], + "description": [ + { + "language_code": "en", + "text": "Early generation transmitters, companions, and direct students.", + }, + { + "language_code": "ru", + "text": "Передатчики раннего поколения, сподвижники и прямые ученики.", + }, + ], + }, +] + class Command(BaseCommand): - help = 'Seeds random transmission chains (HadisTransmitter) for all existing Hadiths.' + help = ( + "Reassigns non-hadith transmission chains to chainless hadith arguments, " + "creates chains for any remaining chainless hadiths, removes leftover non-hadith chains, " + "and normalizes each hadith chain to have 1 uncertain node and 2-3 gaps." + ) def handle(self, *args, **options): - self.stdout.write(self.style.WARNING("Checking Hadiths without transmission chains...")) + rng = random.Random(42) - # دریافت تمام دیتاهای پایه - hadiths = list(Hadis.objects.filter(transmitters__isnull=True).distinct()) - narrators = list(Transmitters.objects.all()) - layers = list(NarratorLayer.objects.all()) - reliabilities = list(TransmitterReliability.objects.all()) + narrators = list(Transmitters.objects.all().order_by("id")) + reliabilities = list(TransmitterReliability.objects.all().order_by("id")) - # بررسی وجود دیتای کافی - if not hadiths: - self.stdout.write(self.style.SUCCESS("All Hadiths already have transmission chains! Nothing to seed.")) - return - - if len(narrators) < 4: - self.stdout.write(self.style.ERROR("Not enough Transmitters found. Please add at least 4 narrators.")) + if len(narrators) < 6: + self.stdout.write( + self.style.ERROR("Not enough Transmitters found. Need at least 6 narrators.") + ) return - self.stdout.write(f"Found {len(hadiths)} Hadiths without transmitters. Generating chains...") + # 1. Identify hadith vs non-hadith arguments + hadith_qs = Hadis.objects.filter(category__source_type="hadith").order_by("id") + non_hadith_with_chains = list( + Hadis.objects.exclude(category__source_type="hadith") + .filter(transmitters__isnull=False) + .distinct() + .order_by("id") + ) + chainless_hadiths = list( + hadith_qs.filter(transmitters__isnull=True).distinct().order_by("id") + ) + + self.stdout.write( + f"Total hadith arguments: {hadith_qs.count()} | " + f"Chainless hadiths: {len(chainless_hadiths)} | " + f"Non-hadith arguments with chains: {len(non_hadith_with_chains)}" + ) + + with transaction.atomic(): + # Step 1: Move chains from non-hadith arguments to chainless hadith arguments (1-to-1) in bulk + pairs_count = min(len(non_hadith_with_chains), len(chainless_hadiths)) + src_to_dst = { + non_hadith_with_chains[i].id: chainless_hadiths[i].id + for i in range(pairs_count) + } + + if src_to_dst: + dst_ids = list(src_to_dst.values()) + src_ids = list(src_to_dst.keys()) + + # Remove any stray empty layers on dst_hadis if present + NarratorLayer.objects.filter(hadis_id__in=dst_ids).delete() + + # Bulk update layers + layers_to_move = list( + NarratorLayer.objects.filter(hadis_id__in=src_ids).order_by("id") + ) + for layer in layers_to_move: + old_hadis_id = layer.hadis_id + new_hadis_id = src_to_dst[old_hadis_id] + old_slug = layer.slug or f"hadis-{old_hadis_id}-l{layer.number}" + new_slug = re.sub( + rf"^hadis-{old_hadis_id}-", + f"hadis-{new_hadis_id}-", + old_slug, + ) + if new_slug == old_slug: + new_slug = f"hadis-{new_hadis_id}-l{layer.number}-{layer.id}" + layer.hadis_id = new_hadis_id + layer.slug = new_slug - links_to_create = [] + NarratorLayer.objects.bulk_update( + layers_to_move, ["hadis_id", "slug"], batch_size=500 + ) - # ساخت زنجیره‌ها - for hadis in hadiths: - hadis_layers = list(hadis.narrator_layers.all().order_by('number')) - if not hadis_layers: - layer1 = NarratorLayer.objects.create( - hadis=hadis, - number=1, - name=[{"language_code": "en", "text": "Primary Transmitters"}], - description=[{"language_code": "en", "text": "Primary layer of transmitters."}], - slug=f"hadis-{hadis.id}-l1-primary" + # Bulk update transmitters + txs_to_move = list( + HadisTransmitter.objects.filter(hadis_id__in=src_ids).order_by("id") ) - layer2 = NarratorLayer.objects.create( - hadis=hadis, - number=2, - name=[{"language_code": "en", "text": "Secondary Transmitters"}], - description=[{"language_code": "en", "text": "Secondary layer of transmitters."}], - slug=f"hadis-{hadis.id}-l2-secondary" + for tx in txs_to_move: + tx.hadis_id = src_to_dst[tx.hadis_id] + + HadisTransmitter.objects.bulk_update( + txs_to_move, ["hadis_id"], batch_size=500 ) - hadis_layers = [layer1, layer2] - - num_layers = len(hadis_layers) - - # هر حدیث بین 2 تا 3 لایه/زنجیره (Chain) داشته باشد - num_chains = random.randint(2, 3) - - for chain_idx in range(num_chains): - # هر زنجیره بین 4 تا 6 راوی داشته باشد - num_narrators = random.randint(4, min(6, len(narrators))) - - # انتخاب رندوم چند راوی برای این زنجیره بدون تکرار - chain_narrators = random.sample(narrators, num_narrators) - - for order_idx, narrator in enumerate(chain_narrators): - assigned_layer = hadis_layers[min(num_layers - 1, order_idx // 2)] if num_layers > 1 else hadis_layers[0] - random_status = random.choice(reliabilities) if reliabilities else None - - link = HadisTransmitter( - hadis=hadis, - transmitter=narrator, - chain_index=chain_idx, - order=order_idx, # ترتیب قرارگیری راوی در این زنجیره - narrator_layer=assigned_layer, - status=random_status, - is_gap=random.choice([True, False, False, False]) # احتمال کم برای گپ (انقطاع سند) + + self.stdout.write( + self.style.SUCCESS( + f"Moved {len(src_to_dst)} chains ({len(layers_to_move)} layers, {len(txs_to_move)} transmitters) " + "from non-hadith arguments to chainless hadith arguments." ) - links_to_create.append(link) + ) - # ذخیره دسته‌جمعی (Bulk Create) برای پرفورمنس بسیار بالاتر - with transaction.atomic(): - # برای جلوگیری از محدودیت حافظه، در دسته‌های 1000 تایی ذخیره می‌کنیم - batch_size = 1000 - for i in range(0, len(links_to_create), batch_size): - HadisTransmitter.objects.bulk_create(links_to_create[i:i + batch_size]) + # Step 2: Delete any remaining chains/layers on non-hadith arguments + deleted_tx, _ = ( + HadisTransmitter.objects.exclude(hadis__category__source_type="hadith").delete() + ) + deleted_layers, _ = ( + NarratorLayer.objects.filter(hadis__isnull=False) + .exclude(hadis__category__source_type="hadith") + .delete() + ) + if deleted_tx or deleted_layers: + self.stdout.write( + self.style.WARNING( + f"Deleted leftover non-hadith records: {deleted_tx} transmitters, {deleted_layers} layers." + ) + ) - self.stdout.write( - self.style.SUCCESS( - f"Successfully created {len(links_to_create)} random narrator links across {len(hadiths)} Hadiths!" + # Step 3: Seed chains for any still-chainless hadith arguments in bulk + remaining_chainless = list( + hadith_qs.filter(transmitters__isnull=True).distinct().order_by("id") ) - ) \ No newline at end of file + if remaining_chainless: + rem_ids = [h.id for h in remaining_chainless] + NarratorLayer.objects.filter(hadis_id__in=rem_ids).delete() + + new_layers = [] + for hadis in remaining_chainless: + for ldef in DEFAULT_LAYERS: + new_layers.append( + NarratorLayer( + hadis_id=hadis.id, + number=ldef["number"], + name=ldef["name"], + description=ldef["description"], + slug=f"hadis-{hadis.id}-{ldef['slug_suffix']}", + ) + ) + created_layers = NarratorLayer.objects.bulk_create(new_layers, batch_size=500) + layers_by_hadis = {} + for l in created_layers: + layers_by_hadis.setdefault(l.hadis_id, []).append(l) + + new_txs = [] + for hadis in remaining_chainless: + h_layers = sorted(layers_by_hadis[hadis.id], key=lambda x: x.number) + num_nodes = rng.randint(8, 10) + chosen_narrators = rng.sample(narrators, num_nodes) + split_idx = num_nodes // 2 + + for order_idx, narrator in enumerate(chosen_narrators): + layer_obj = h_layers[0] if order_idx < split_idx else h_layers[1] + status_obj = rng.choice(reliabilities) if reliabilities else None + new_txs.append( + HadisTransmitter( + hadis_id=hadis.id, + transmitter=narrator, + chain_index=0, + order=order_idx, + narrator_layer=layer_obj, + status=status_obj, + is_gap=False, + is_uncertain=False, + ) + ) + HadisTransmitter.objects.bulk_create(new_txs, batch_size=500) + self.stdout.write( + self.style.SUCCESS( + f"Created new chains for {len(remaining_chainless)} remaining chainless hadiths." + ) + ) + + # Step 4: Normalize all hadith chains so every hadith has: + # - 1 uncertain node (is_uncertain=True with 2 candidate transmitters in uncertain_transmitters) per chain + # - 2 to 3 gaps (is_gap=True) per hadith + ThroughModel = HadisTransmitter.uncertain_transmitters.through + all_hadiths = list( + hadith_qs.prefetch_related("transmitters", "narrator_layers").order_by("id") + ) + + nodes_to_update = [] + hadis_ids = [h.id for h in all_hadiths] + ThroughModel.objects.filter(hadistransmitter__hadis_id__in=hadis_ids).delete() + new_m2m_relations = [] + + normalized_count = 0 + for hadis in all_hadiths: + nodes = sorted( + list(hadis.transmitters.all()), + key=lambda x: (x.chain_index, x.order, x.id), + ) + if not nodes: + continue + + chains = {} + for n in nodes: + chains.setdefault(n.chain_index, []).append(n) + + for n in nodes: + n.is_gap = False + n.is_uncertain = False + + uncertain_node_ids = set() + for c_idx, c_nodes in chains.items(): + candidate_pool = c_nodes[1:] if len(c_nodes) > 2 else c_nodes + unc_node = rng.choice(candidate_pool) + unc_node.is_uncertain = True + uncertain_node_ids.add(unc_node.id) + + used_tx_ids = {n.transmitter_id for n in c_nodes if n.transmitter_id} + available_candidates = [t for t in narrators if t.id not in used_tx_ids] + if len(available_candidates) < 2: + available_candidates = [ + t for t in narrators if t.id != unc_node.transmitter_id + ] + picked_candidates = rng.sample( + available_candidates, min(2, len(available_candidates)) + ) + for cand in picked_candidates: + new_m2m_relations.append( + ThroughModel( + hadistransmitter_id=unc_node.id, + transmitters_id=cand.id, + ) + ) + + target_gaps = rng.choice([2, 3]) + gap_pool = [ + n for n in nodes if n.id not in uncertain_node_ids and n.order > 0 + ] + if len(gap_pool) < target_gaps: + gap_pool = [n for n in nodes if n.id not in uncertain_node_ids] + if len(gap_pool) < target_gaps: + gap_pool = list(nodes) + + chosen_gaps = rng.sample(gap_pool, min(target_gaps, len(gap_pool))) + for g_node in chosen_gaps: + g_node.is_gap = True + + nodes_to_update.extend(nodes) + normalized_count += 1 + + HadisTransmitter.objects.bulk_update( + nodes_to_update, ["is_gap", "is_uncertain"], batch_size=500 + ) + ThroughModel.objects.bulk_create(new_m2m_relations, batch_size=500) + + self.stdout.write( + self.style.SUCCESS( + f"Normalized {normalized_count} hadith chains (1 uncertain node per chain, 2-3 gaps per hadith)." + ) + ) + + try: + cache.clear() + self.stdout.write(self.style.SUCCESS("Cleared Django cache.")) + except Exception as e: + self.stdout.write(self.style.WARNING(f"Cache clear warning: {e}")) diff --git a/apps/hadis/serializers/hadis.py b/apps/hadis/serializers/hadis.py index e49b88d..5c661ec 100644 --- a/apps/hadis/serializers/hadis.py +++ b/apps/hadis/serializers/hadis.py @@ -1286,9 +1286,16 @@ class HadisTransmitterListSerializer(serializers.ModelSerializer): return obj.narrator_layers.count() def get_layer_names(self, obj): - """Get list of localized narrator layer names""" + """Get list of localized narrator layer names (bounded to avoid huge payloads)""" request = self.context.get('request') - layer_objects = obj.narrator_layers.all().order_by('number') + layer_slug = request.query_params.get('layer') if request and hasattr(request, 'query_params') else None + layer_qs = obj.narrator_layers.all().order_by('number', 'id') + if layer_slug and layer_slug != 'all': + layer_objects = list(layer_qs.filter(slug=layer_slug)[:1]) + if not layer_objects: + layer_objects = list(layer_qs[:20]) + else: + layer_objects = list(layer_qs[:20]) layer_names = [] for layer in layer_objects: @@ -1297,6 +1304,7 @@ class HadisTransmitterListSerializer(serializers.ModelSerializer): slug = layer.slug layer_names.append({ 'id': layer.id, + 'number': layer.number, 'name': name, 'slug': slug, 'description': description @@ -1861,9 +1869,12 @@ class HadisDetailSerializer(serializers.ModelSerializer): class NarratorLayerSerializer(serializers.Serializer): """Serializer for narrator layers information""" + id = serializers.IntegerField(read_only=True) + number = serializers.IntegerField(read_only=True) name = serializers.SerializerMethodField() slug = serializers.CharField(read_only=True) description = serializers.SerializerMethodField() + narrators_count = serializers.SerializerMethodField() def get_name(self, obj): """Get localized name""" @@ -1875,6 +1886,11 @@ class NarratorLayerSerializer(serializers.Serializer): request = self.context.get('request') return get_localized_text(obj.description, request=request) + def get_narrators_count(self, obj): + if hasattr(obj, 'narrators_count') and obj.narrators_count is not None: + return obj.narrators_count + return obj.transmitters.count() + class NarratorRelationSerializer(serializers.ModelSerializer): """ diff --git a/apps/hadis/views/hadis.py b/apps/hadis/views/hadis.py index 1b2e984..bf7cba9 100644 --- a/apps/hadis/views/hadis.py +++ b/apps/hadis/views/hadis.py @@ -808,11 +808,21 @@ class HadisTransmittersView(RetrieveAPIView): 'uncertain_transmitters__kunyas', ).order_by('order') - # 3. Apply the filter to the Child Query (if param exists) - if layer_slug: - # Assumes 'NarratorLayer' has a 'slug' field. - # If not, use 'narrator_layer__name' or 'narrator_layer__id'. - transmitter_qs = transmitter_qs.filter(narrator_layer__slug=layer_slug) + # 3. Apply the filter to the Child Query (if param exists, or default to first layer for scalability) + from ..models import NarratorLayer + hadis_slug = self.kwargs.get('hadis_slug') + if layer_slug and layer_slug != 'all': + matched_layer = NarratorLayer.objects.filter(hadis__slug=hadis_slug, slug=layer_slug).first() + if matched_layer: + transmitter_qs = transmitter_qs.filter(narrator_layer_id=matched_layer.id) + else: + first_layer = NarratorLayer.objects.filter(hadis__slug=hadis_slug).order_by('number', 'id').first() + if first_layer: + transmitter_qs = transmitter_qs.filter(narrator_layer_id=first_layer.id) + elif not layer_slug: + first_layer = NarratorLayer.objects.filter(hadis__slug=hadis_slug).order_by('number', 'id').first() + if first_layer: + transmitter_qs = transmitter_qs.filter(narrator_layer_id=first_layer.id) # 4. Use the Prefetch object to inject this filtered list into the Parent return Hadis.objects.filter(status=True).prefetch_related( @@ -1074,10 +1084,10 @@ class CategoryHadisCorrectionsView(ListAPIView): class HadisLayersView(ListAPIView): """ - API view to retrieve all narrator layers for a specific hadis + API view to retrieve narrator layers for a specific hadis with pagination and search """ serializer_class = NarratorLayerSerializer - pagination_class = NoPagination + pagination_class = StandardResultsSetPagination lookup_field = 'slug' lookup_url_kwarg = 'hadis_slug' @@ -1089,12 +1099,30 @@ class HadisLayersView(ListAPIView): from ..models import NarratorLayer hadis_slug = self.kwargs.get('hadis_slug') + hadis = Hadis.objects.filter(slug=hadis_slug, status=True).first() + if not hadis and str(hadis_slug).isdigit(): + hadis = Hadis.objects.filter(id=int(hadis_slug), status=True).first() + if not hadis: + return NarratorLayer.objects.none() + + qs = NarratorLayer.objects.filter(hadis=hadis).annotate( + narrators_count=Count('transmitters', distinct=True) + ).order_by('number', 'id') + + search = (self.request.query_params.get('search') or '').strip() + if search: + q = ( + Q(name__icontains=search) + | Q(description__icontains=search) + | Q(slug__icontains=search) + | Q(transmitters__transmitter__full_name__icontains=search) + | Q(transmitters__transmitter__known_as__icontains=search) + ) + if search.isdigit(): + q |= Q(number=int(search)) + qs = qs.filter(q).distinct() - # Get the hadis object to ensure it exists - hadis = get_object_or_404(Hadis, slug=hadis_slug, status=True) - - # Return the layer objects for this hadis ordered by number - return NarratorLayer.objects.filter(hadis=hadis).order_by('number') + return qs class HadisFiltersView(ListAPIView): diff --git a/utils/cache.py b/utils/cache.py index 8017629..8d5ad1a 100644 --- a/utils/cache.py +++ b/utils/cache.py @@ -62,7 +62,9 @@ def smart_cached_view(timeout=60 * 60 * 2, key_prefix=''): if is_bookmark in ('1', 'true') or in_playlist in ('1', 'true'): return view_func(request, *args, **kwargs) - return cached_func(request, *args, **kwargs) + response = cached_func(request, *args, **kwargs) + response['Cache-Control'] = 'private, no-cache' + return response return _wrapped_view