Browse Source
feat(hadis): add paginated searchable narrator layers API, chain seeder, and cache revalidation
master
feat(hadis): add paginated searchable narrator layers API, chain seeder, and cache revalidation
master
5 changed files with 391 additions and 92 deletions
-
51apps/hadis/management/commands/populate_narrator_links.py
-
330apps/hadis/management/commands/seed_hadis_chains.py
-
20apps/hadis/serializers/hadis.py
-
50apps/hadis/views/hadis.py
-
4utils/cache.py
@ -1,14 +1,55 @@ |
|||
# backend/apps/hadis/management/commands/populate_narrator_links.py |
|||
|
|||
# -*- coding: utf-8 -*- |
|||
import random |
|||
from django.core.management.base import BaseCommand |
|||
from django.core.cache import cache |
|||
from apps.hadis.models import Hadis, HadisCorrection, HadisInterpretation |
|||
|
|||
|
|||
ISLAMIC_ENCYCLOPEDIA_SOURCES = [ |
|||
("Al-Islam Encyclopedia", "https://www.al-islam.org/articles/hadith-studies-"), |
|||
("Shamela Digital Library", "https://shamela.ws/book/"), |
|||
("WikiShia Encyclopedia", "https://en.wikishia.net/view/Hadith_"), |
|||
("Thaqalayn Hadith Library", "https://thaqalayn.net/hadith/"), |
|||
("Sunnah Hadith Collection", "https://sunnah.com/bukhari/"), |
|||
("Dorar Encyclopedia", "https://dorar.net/hadith/"), |
|||
("IslamWeb Reference", "https://www.islamweb.net/ar/hadith/"), |
|||
("HadithDB Archive", "https://hadithdb.com/tradition/"), |
|||
] |
|||
|
|||
|
|||
def generate_islamic_links(seed_id: int) -> dict: |
|||
rng = random.Random(seed_id) |
|||
count = rng.randint(2, 3) |
|||
chosen = rng.sample(ISLAMIC_ENCYCLOPEDIA_SOURCES, count) |
|||
links = {} |
|||
for title, base_url in chosen: |
|||
ref_num = rng.randint(100, 4999) |
|||
links[title] = f"{base_url}{ref_num}" |
|||
return links |
|||
|
|||
|
|||
class Command(BaseCommand): |
|||
help = 'Deprecated command (dummy narrator and links seeding disabled).' |
|||
help = "Populates 2-3 realistic Islamic encyclopedia links for Hadis, HadisCorrection, and HadisInterpretation records." |
|||
|
|||
def handle(self, *args, **options): |
|||
self.stdout.write("Populating Islamic encyclopedia links for Hadis, HadisCorrection, and HadisInterpretation...") |
|||
|
|||
for model in (HadisCorrection, HadisInterpretation, Hadis): |
|||
rows = list(model.objects.only("id", "links")) |
|||
to_update = [] |
|||
for obj in rows: |
|||
if not obj.links or obj.links == {} or obj.links == []: |
|||
obj.links = generate_islamic_links(obj.id) |
|||
to_update.append(obj) |
|||
|
|||
if to_update: |
|||
model.objects.bulk_update(to_update, ["links"], batch_size=500) |
|||
|
|||
self.stdout.write( |
|||
self.style.WARNING( |
|||
"populate_narrator_links is disabled to prevent dummy links from being written." |
|||
self.style.SUCCESS( |
|||
f"Updated {len(to_update)} {model.__name__} records (total {len(rows)})." |
|||
) |
|||
) |
|||
|
|||
cache.clear() |
|||
self.stdout.write(self.style.SUCCESS("Cleared Django cache successfully.")) |
|||
@ -1,91 +1,303 @@ |
|||
# backend/apps/hadis/management/commands/seed_hadis_chains.py |
|||
|
|||
import random |
|||
import re |
|||
from django.core.management.base import BaseCommand |
|||
from django.core.cache import cache |
|||
from django.db import transaction |
|||
from apps.hadis.models import Hadis, Transmitters, HadisTransmitter, NarratorLayer, TransmitterReliability |
|||
from apps.hadis.models import ( |
|||
Hadis, |
|||
Transmitters, |
|||
HadisTransmitter, |
|||
NarratorLayer, |
|||
TransmitterReliability, |
|||
) |
|||
|
|||
DEFAULT_LAYERS = [ |
|||
{ |
|||
"number": 1, |
|||
"slug_suffix": "l1-intermediate-regional-transmitters", |
|||
"name": [ |
|||
{"language_code": "en", "text": "Intermediate Regional Transmitters"}, |
|||
{"language_code": "ru", "text": "Промежуточные региональные передатчики"}, |
|||
], |
|||
"description": [ |
|||
{ |
|||
"language_code": "en", |
|||
"text": "Scholars and compilers transmitting across regional centers of learning.", |
|||
}, |
|||
{ |
|||
"language_code": "ru", |
|||
"text": "Ученые и составители, передававшие хадисы между региональными центрами знаний.", |
|||
}, |
|||
], |
|||
}, |
|||
{ |
|||
"number": 2, |
|||
"slug_suffix": "l2-primary-chain-early-narrators", |
|||
"name": [ |
|||
{"language_code": "en", "text": "Primary Chain & Early Narrators"}, |
|||
{"language_code": "ru", "text": "Первичная цепочка и ранние передатчики"}, |
|||
], |
|||
"description": [ |
|||
{ |
|||
"language_code": "en", |
|||
"text": "Early generation transmitters, companions, and direct students.", |
|||
}, |
|||
{ |
|||
"language_code": "ru", |
|||
"text": "Передатчики раннего поколения, сподвижники и прямые ученики.", |
|||
}, |
|||
], |
|||
}, |
|||
] |
|||
|
|||
|
|||
class Command(BaseCommand): |
|||
help = 'Seeds random transmission chains (HadisTransmitter) for all existing Hadiths.' |
|||
help = ( |
|||
"Reassigns non-hadith transmission chains to chainless hadith arguments, " |
|||
"creates chains for any remaining chainless hadiths, removes leftover non-hadith chains, " |
|||
"and normalizes each hadith chain to have 1 uncertain node and 2-3 gaps." |
|||
) |
|||
|
|||
def handle(self, *args, **options): |
|||
self.stdout.write(self.style.WARNING("Checking Hadiths without transmission chains...")) |
|||
rng = random.Random(42) |
|||
|
|||
# دریافت تمام دیتاهای پایه |
|||
hadiths = list(Hadis.objects.filter(transmitters__isnull=True).distinct()) |
|||
narrators = list(Transmitters.objects.all()) |
|||
layers = list(NarratorLayer.objects.all()) |
|||
reliabilities = list(TransmitterReliability.objects.all()) |
|||
narrators = list(Transmitters.objects.all().order_by("id")) |
|||
reliabilities = list(TransmitterReliability.objects.all().order_by("id")) |
|||
|
|||
# بررسی وجود دیتای کافی |
|||
if not hadiths: |
|||
self.stdout.write(self.style.SUCCESS("All Hadiths already have transmission chains! Nothing to seed.")) |
|||
if len(narrators) < 6: |
|||
self.stdout.write( |
|||
self.style.ERROR("Not enough Transmitters found. Need at least 6 narrators.") |
|||
) |
|||
return |
|||
|
|||
if len(narrators) < 4: |
|||
self.stdout.write(self.style.ERROR("Not enough Transmitters found. Please add at least 4 narrators.")) |
|||
return |
|||
# 1. Identify hadith vs non-hadith arguments |
|||
hadith_qs = Hadis.objects.filter(category__source_type="hadith").order_by("id") |
|||
non_hadith_with_chains = list( |
|||
Hadis.objects.exclude(category__source_type="hadith") |
|||
.filter(transmitters__isnull=False) |
|||
.distinct() |
|||
.order_by("id") |
|||
) |
|||
chainless_hadiths = list( |
|||
hadith_qs.filter(transmitters__isnull=True).distinct().order_by("id") |
|||
) |
|||
|
|||
self.stdout.write(f"Found {len(hadiths)} Hadiths without transmitters. Generating chains...") |
|||
self.stdout.write( |
|||
f"Total hadith arguments: {hadith_qs.count()} | " |
|||
f"Chainless hadiths: {len(chainless_hadiths)} | " |
|||
f"Non-hadith arguments with chains: {len(non_hadith_with_chains)}" |
|||
) |
|||
|
|||
links_to_create = [] |
|||
with transaction.atomic(): |
|||
# Step 1: Move chains from non-hadith arguments to chainless hadith arguments (1-to-1) in bulk |
|||
pairs_count = min(len(non_hadith_with_chains), len(chainless_hadiths)) |
|||
src_to_dst = { |
|||
non_hadith_with_chains[i].id: chainless_hadiths[i].id |
|||
for i in range(pairs_count) |
|||
} |
|||
|
|||
# ساخت زنجیرهها |
|||
for hadis in hadiths: |
|||
hadis_layers = list(hadis.narrator_layers.all().order_by('number')) |
|||
if not hadis_layers: |
|||
layer1 = NarratorLayer.objects.create( |
|||
hadis=hadis, |
|||
number=1, |
|||
name=[{"language_code": "en", "text": "Primary Transmitters"}], |
|||
description=[{"language_code": "en", "text": "Primary layer of transmitters."}], |
|||
slug=f"hadis-{hadis.id}-l1-primary" |
|||
if src_to_dst: |
|||
dst_ids = list(src_to_dst.values()) |
|||
src_ids = list(src_to_dst.keys()) |
|||
|
|||
# Remove any stray empty layers on dst_hadis if present |
|||
NarratorLayer.objects.filter(hadis_id__in=dst_ids).delete() |
|||
|
|||
# Bulk update layers |
|||
layers_to_move = list( |
|||
NarratorLayer.objects.filter(hadis_id__in=src_ids).order_by("id") |
|||
) |
|||
layer2 = NarratorLayer.objects.create( |
|||
hadis=hadis, |
|||
number=2, |
|||
name=[{"language_code": "en", "text": "Secondary Transmitters"}], |
|||
description=[{"language_code": "en", "text": "Secondary layer of transmitters."}], |
|||
slug=f"hadis-{hadis.id}-l2-secondary" |
|||
for layer in layers_to_move: |
|||
old_hadis_id = layer.hadis_id |
|||
new_hadis_id = src_to_dst[old_hadis_id] |
|||
old_slug = layer.slug or f"hadis-{old_hadis_id}-l{layer.number}" |
|||
new_slug = re.sub( |
|||
rf"^hadis-{old_hadis_id}-", |
|||
f"hadis-{new_hadis_id}-", |
|||
old_slug, |
|||
) |
|||
hadis_layers = [layer1, layer2] |
|||
if new_slug == old_slug: |
|||
new_slug = f"hadis-{new_hadis_id}-l{layer.number}-{layer.id}" |
|||
layer.hadis_id = new_hadis_id |
|||
layer.slug = new_slug |
|||
|
|||
num_layers = len(hadis_layers) |
|||
NarratorLayer.objects.bulk_update( |
|||
layers_to_move, ["hadis_id", "slug"], batch_size=500 |
|||
) |
|||
|
|||
# هر حدیث بین 2 تا 3 لایه/زنجیره (Chain) داشته باشد |
|||
num_chains = random.randint(2, 3) |
|||
# Bulk update transmitters |
|||
txs_to_move = list( |
|||
HadisTransmitter.objects.filter(hadis_id__in=src_ids).order_by("id") |
|||
) |
|||
for tx in txs_to_move: |
|||
tx.hadis_id = src_to_dst[tx.hadis_id] |
|||
|
|||
HadisTransmitter.objects.bulk_update( |
|||
txs_to_move, ["hadis_id"], batch_size=500 |
|||
) |
|||
|
|||
self.stdout.write( |
|||
self.style.SUCCESS( |
|||
f"Moved {len(src_to_dst)} chains ({len(layers_to_move)} layers, {len(txs_to_move)} transmitters) " |
|||
"from non-hadith arguments to chainless hadith arguments." |
|||
) |
|||
) |
|||
|
|||
# Step 2: Delete any remaining chains/layers on non-hadith arguments |
|||
deleted_tx, _ = ( |
|||
HadisTransmitter.objects.exclude(hadis__category__source_type="hadith").delete() |
|||
) |
|||
deleted_layers, _ = ( |
|||
NarratorLayer.objects.filter(hadis__isnull=False) |
|||
.exclude(hadis__category__source_type="hadith") |
|||
.delete() |
|||
) |
|||
if deleted_tx or deleted_layers: |
|||
self.stdout.write( |
|||
self.style.WARNING( |
|||
f"Deleted leftover non-hadith records: {deleted_tx} transmitters, {deleted_layers} layers." |
|||
) |
|||
) |
|||
|
|||
for chain_idx in range(num_chains): |
|||
# هر زنجیره بین 4 تا 6 راوی داشته باشد |
|||
num_narrators = random.randint(4, min(6, len(narrators))) |
|||
# Step 3: Seed chains for any still-chainless hadith arguments in bulk |
|||
remaining_chainless = list( |
|||
hadith_qs.filter(transmitters__isnull=True).distinct().order_by("id") |
|||
) |
|||
if remaining_chainless: |
|||
rem_ids = [h.id for h in remaining_chainless] |
|||
NarratorLayer.objects.filter(hadis_id__in=rem_ids).delete() |
|||
|
|||
# انتخاب رندوم چند راوی برای این زنجیره بدون تکرار |
|||
chain_narrators = random.sample(narrators, num_narrators) |
|||
new_layers = [] |
|||
for hadis in remaining_chainless: |
|||
for ldef in DEFAULT_LAYERS: |
|||
new_layers.append( |
|||
NarratorLayer( |
|||
hadis_id=hadis.id, |
|||
number=ldef["number"], |
|||
name=ldef["name"], |
|||
description=ldef["description"], |
|||
slug=f"hadis-{hadis.id}-{ldef['slug_suffix']}", |
|||
) |
|||
) |
|||
created_layers = NarratorLayer.objects.bulk_create(new_layers, batch_size=500) |
|||
layers_by_hadis = {} |
|||
for l in created_layers: |
|||
layers_by_hadis.setdefault(l.hadis_id, []).append(l) |
|||
|
|||
for order_idx, narrator in enumerate(chain_narrators): |
|||
assigned_layer = hadis_layers[min(num_layers - 1, order_idx // 2)] if num_layers > 1 else hadis_layers[0] |
|||
random_status = random.choice(reliabilities) if reliabilities else None |
|||
new_txs = [] |
|||
for hadis in remaining_chainless: |
|||
h_layers = sorted(layers_by_hadis[hadis.id], key=lambda x: x.number) |
|||
num_nodes = rng.randint(8, 10) |
|||
chosen_narrators = rng.sample(narrators, num_nodes) |
|||
split_idx = num_nodes // 2 |
|||
|
|||
link = HadisTransmitter( |
|||
hadis=hadis, |
|||
for order_idx, narrator in enumerate(chosen_narrators): |
|||
layer_obj = h_layers[0] if order_idx < split_idx else h_layers[1] |
|||
status_obj = rng.choice(reliabilities) if reliabilities else None |
|||
new_txs.append( |
|||
HadisTransmitter( |
|||
hadis_id=hadis.id, |
|||
transmitter=narrator, |
|||
chain_index=chain_idx, |
|||
order=order_idx, # ترتیب قرارگیری راوی در این زنجیره |
|||
narrator_layer=assigned_layer, |
|||
status=random_status, |
|||
is_gap=random.choice([True, False, False, False]) # احتمال کم برای گپ (انقطاع سند) |
|||
chain_index=0, |
|||
order=order_idx, |
|||
narrator_layer=layer_obj, |
|||
status=status_obj, |
|||
is_gap=False, |
|||
is_uncertain=False, |
|||
) |
|||
) |
|||
HadisTransmitter.objects.bulk_create(new_txs, batch_size=500) |
|||
self.stdout.write( |
|||
self.style.SUCCESS( |
|||
f"Created new chains for {len(remaining_chainless)} remaining chainless hadiths." |
|||
) |
|||
) |
|||
links_to_create.append(link) |
|||
|
|||
# ذخیره دستهجمعی (Bulk Create) برای پرفورمنس بسیار بالاتر |
|||
with transaction.atomic(): |
|||
# برای جلوگیری از محدودیت حافظه، در دستههای 1000 تایی ذخیره میکنیم |
|||
batch_size = 1000 |
|||
for i in range(0, len(links_to_create), batch_size): |
|||
HadisTransmitter.objects.bulk_create(links_to_create[i:i + batch_size]) |
|||
# Step 4: Normalize all hadith chains so every hadith has: |
|||
# - 1 uncertain node (is_uncertain=True with 2 candidate transmitters in uncertain_transmitters) per chain |
|||
# - 2 to 3 gaps (is_gap=True) per hadith |
|||
ThroughModel = HadisTransmitter.uncertain_transmitters.through |
|||
all_hadiths = list( |
|||
hadith_qs.prefetch_related("transmitters", "narrator_layers").order_by("id") |
|||
) |
|||
|
|||
nodes_to_update = [] |
|||
hadis_ids = [h.id for h in all_hadiths] |
|||
ThroughModel.objects.filter(hadistransmitter__hadis_id__in=hadis_ids).delete() |
|||
new_m2m_relations = [] |
|||
|
|||
normalized_count = 0 |
|||
for hadis in all_hadiths: |
|||
nodes = sorted( |
|||
list(hadis.transmitters.all()), |
|||
key=lambda x: (x.chain_index, x.order, x.id), |
|||
) |
|||
if not nodes: |
|||
continue |
|||
|
|||
chains = {} |
|||
for n in nodes: |
|||
chains.setdefault(n.chain_index, []).append(n) |
|||
|
|||
for n in nodes: |
|||
n.is_gap = False |
|||
n.is_uncertain = False |
|||
|
|||
uncertain_node_ids = set() |
|||
for c_idx, c_nodes in chains.items(): |
|||
candidate_pool = c_nodes[1:] if len(c_nodes) > 2 else c_nodes |
|||
unc_node = rng.choice(candidate_pool) |
|||
unc_node.is_uncertain = True |
|||
uncertain_node_ids.add(unc_node.id) |
|||
|
|||
used_tx_ids = {n.transmitter_id for n in c_nodes if n.transmitter_id} |
|||
available_candidates = [t for t in narrators if t.id not in used_tx_ids] |
|||
if len(available_candidates) < 2: |
|||
available_candidates = [ |
|||
t for t in narrators if t.id != unc_node.transmitter_id |
|||
] |
|||
picked_candidates = rng.sample( |
|||
available_candidates, min(2, len(available_candidates)) |
|||
) |
|||
for cand in picked_candidates: |
|||
new_m2m_relations.append( |
|||
ThroughModel( |
|||
hadistransmitter_id=unc_node.id, |
|||
transmitters_id=cand.id, |
|||
) |
|||
) |
|||
|
|||
target_gaps = rng.choice([2, 3]) |
|||
gap_pool = [ |
|||
n for n in nodes if n.id not in uncertain_node_ids and n.order > 0 |
|||
] |
|||
if len(gap_pool) < target_gaps: |
|||
gap_pool = [n for n in nodes if n.id not in uncertain_node_ids] |
|||
if len(gap_pool) < target_gaps: |
|||
gap_pool = list(nodes) |
|||
|
|||
chosen_gaps = rng.sample(gap_pool, min(target_gaps, len(gap_pool))) |
|||
for g_node in chosen_gaps: |
|||
g_node.is_gap = True |
|||
|
|||
nodes_to_update.extend(nodes) |
|||
normalized_count += 1 |
|||
|
|||
HadisTransmitter.objects.bulk_update( |
|||
nodes_to_update, ["is_gap", "is_uncertain"], batch_size=500 |
|||
) |
|||
ThroughModel.objects.bulk_create(new_m2m_relations, batch_size=500) |
|||
|
|||
self.stdout.write( |
|||
self.style.SUCCESS( |
|||
f"Successfully created {len(links_to_create)} random narrator links across {len(hadiths)} Hadiths!" |
|||
f"Normalized {normalized_count} hadith chains (1 uncertain node per chain, 2-3 gaps per hadith)." |
|||
) |
|||
) |
|||
|
|||
try: |
|||
cache.clear() |
|||
self.stdout.write(self.style.SUCCESS("Cleared Django cache.")) |
|||
except Exception as e: |
|||
self.stdout.write(self.style.WARNING(f"Cache clear warning: {e}")) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue