Browse Source
feat(hadis): add narrator filtering and cleanup non-hadith narrator records
feat(hadis): add narrator filtering and cleanup non-hadith narrator records
- Add narrator and transmitter filtering support to HadisMainListView and HadisListView by slug or ID - Include available narrators list with localized names in HadisFiltersView endpoint - Document narrator query parameter in Swagger schema - Add cleanup_non_hadith_narrators.py script to remove orphan/erroneous narrator chains and layers from non-hadith argumentsmaster
3 changed files with 279 additions and 5 deletions
@ -0,0 +1,222 @@ |
|||
#!/usr/bin/env python3 |
|||
""" |
|||
Script to clean up Narrator and Narrative Layer data for non-hadith arguments: |
|||
1. Preserves all narrator records (HadisTransmitter, NarratorLayer, title_narrator) |
|||
ONLY for arguments of type 'hadith' (source_type == 'hadith'). |
|||
2. For all other arguments (quran, history, fatwa, quote, or uncategorized): |
|||
- Deletes all associated HadisTransmitter records. |
|||
- Deletes all associated NarratorLayer records. |
|||
- Clears title_narrator and updates normalized_text. |
|||
3. Supports --dry-run mode to preview changes before applying them. |
|||
""" |
|||
|
|||
import os |
|||
import sys |
|||
import argparse |
|||
from pathlib import Path |
|||
|
|||
# Setup Django environment |
|||
BASE_DIR = Path(__file__).resolve().parent.parent |
|||
sys.path.append(str(BASE_DIR)) |
|||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.base') |
|||
|
|||
import django |
|||
django.setup() |
|||
|
|||
# Configure UTF-8 output on Windows |
|||
if sys.platform == "win32": |
|||
try: |
|||
sys.stdout.reconfigure(encoding="utf-8") |
|||
sys.stderr.reconfigure(encoding="utf-8") |
|||
except Exception: |
|||
pass |
|||
|
|||
from django.db import transaction |
|||
from django.db.models import Count, Q |
|||
from apps.hadis.models import Hadis, HadisCategory, HadisTransmitter, NarratorLayer |
|||
from utils.text_normalizer import build_search_blob |
|||
|
|||
|
|||
def print_separator(char="=", length=70): |
|||
print(char * length) |
|||
|
|||
|
|||
def audit_narrator_data(): |
|||
"""Returns a dictionary summary of narrator records grouped by source_type.""" |
|||
summary = {} |
|||
for st in HadisCategory.SourceType.values: |
|||
hadis_count = Hadis.objects.filter(category__source_type=st).count() |
|||
transmitter_count = HadisTransmitter.objects.filter(hadis__category__source_type=st).count() |
|||
layer_count = NarratorLayer.objects.filter(hadis__category__source_type=st).count() |
|||
has_title_narrator = Hadis.objects.filter(category__source_type=st).exclude( |
|||
Q(title_narrator=[]) | Q(title_narrator__isnull=True) |
|||
).count() |
|||
summary[st] = { |
|||
"hadis_count": hadis_count, |
|||
"transmitters": transmitter_count, |
|||
"layers": layer_count, |
|||
"has_title_narrator": has_title_narrator, |
|||
} |
|||
|
|||
# Arguments without a category or with null source_type |
|||
null_hadis = Hadis.objects.filter(Q(category__isnull=True) | Q(category__source_type__isnull=True)).count() |
|||
null_transmitters = HadisTransmitter.objects.filter( |
|||
Q(hadis__category__isnull=True) | Q(hadis__category__source_type__isnull=True) |
|||
).count() |
|||
null_layers = NarratorLayer.objects.filter( |
|||
Q(hadis__category__isnull=True) | Q(hadis__category__source_type__isnull=True) |
|||
).count() |
|||
null_title_narrator = Hadis.objects.filter( |
|||
Q(category__isnull=True) | Q(category__source_type__isnull=True) |
|||
).exclude(Q(title_narrator=[]) | Q(title_narrator__isnull=True)).count() |
|||
|
|||
summary["_none_"] = { |
|||
"hadis_count": null_hadis, |
|||
"transmitters": null_transmitters, |
|||
"layers": null_layers, |
|||
"has_title_narrator": null_title_narrator, |
|||
} |
|||
|
|||
return summary |
|||
|
|||
|
|||
def display_summary(summary, title="وضعیت رکوردها"): |
|||
print_separator() |
|||
print(f"📊 {title}:") |
|||
print_separator("-") |
|||
print(f"{'نوع منبع (Source Type)':<25} | {'تعداد استدلال':<12} | {'راویان (Transmitters)':<20} | {'لایهها (Layers)':<16} | {'عنوان راوی':<12}") |
|||
print_separator("-") |
|||
|
|||
for st, stats in summary.items(): |
|||
label = st if st != "_none_" else "بدون دستهبندی (None)" |
|||
marker = " ✅ (حفظ میشود)" if st == HadisCategory.SourceType.HADITH else " ❌ (حذف میشود)" |
|||
print( |
|||
f"{label + marker:<25} | " |
|||
f"{stats['hadis_count']:<12} | " |
|||
f"{stats['transmitters']:<20} | " |
|||
f"{stats['layers']:<16} | " |
|||
f"{stats['has_title_narrator']:<12}" |
|||
) |
|||
print_separator() |
|||
|
|||
|
|||
def cleanup_non_hadith_narrators(dry_run=False): |
|||
""" |
|||
Performs cleanup of narrator layers and transmitters for all non-hadith arguments. |
|||
""" |
|||
print_separator() |
|||
mode_text = "[حالت آزمایشی - DRY RUN (بدون اعمال تغییر)]" if dry_run else "[حالت اجرایی واقعی - LIVE EXECUTION]" |
|||
print(f"🚀 شروع پاکسازی راویان و لایههای روایت برای استدلالهای غیر حدیث {mode_text}") |
|||
print_separator() |
|||
|
|||
# 1. Audit before |
|||
before_summary = audit_narrator_data() |
|||
display_summary(before_summary, "وضعیت قبل از پاکسازی") |
|||
|
|||
# 2. Identify non-hadith arguments |
|||
non_hadith_hadis_qs = Hadis.objects.exclude( |
|||
category__source_type=HadisCategory.SourceType.HADITH |
|||
) |
|||
non_hadith_ids = list(non_hadith_hadis_qs.values_list("id", flat=True)) |
|||
|
|||
transmitters_to_delete_qs = HadisTransmitter.objects.filter(hadis_id__in=non_hadith_ids) |
|||
layers_to_delete_qs = NarratorLayer.objects.filter(hadis_id__in=non_hadith_ids) |
|||
hadis_with_title_narrator_qs = non_hadith_hadis_qs.exclude( |
|||
Q(title_narrator=[]) | Q(title_narrator__isnull=True) |
|||
) |
|||
|
|||
total_transmitters_to_delete = transmitters_to_delete_qs.count() |
|||
total_layers_to_delete = layers_to_delete_qs.count() |
|||
total_title_narrator_to_clear = hadis_with_title_narrator_qs.count() |
|||
|
|||
print(f"🎯 موارد شناسایی شده برای پاکسازی:") |
|||
print(f" • تعداد استدلالهای غیر حدیث: {len(non_hadith_ids)}") |
|||
print(f" • رکوردهای HadisTransmitter غیر حدیث جهت حذف: {total_transmitters_to_delete}") |
|||
print(f" • رکوردهای NarratorLayer غیر حدیث جهت حذف: {total_layers_to_delete}") |
|||
print(f" • فیلدهای title_narrator غیر حدیث جهت خالیسازی: {total_title_narrator_to_clear}") |
|||
print_separator() |
|||
|
|||
if dry_run: |
|||
print("⚠️ حالت آزمایشی (dry-run) فعال است. هیچ دادهای از دیتابیس پاک نشد.") |
|||
print("برای اعمال قطعی تغییرات، اسکریپت را بدون فلگ --dry-run اجرا نمایید.") |
|||
return |
|||
|
|||
# 3. Execution inside atomic transaction |
|||
with transaction.atomic(): |
|||
print("⏳ در حال حذف رکوردهای HadisTransmitter غیر حدیث...") |
|||
deleted_transmitters_count, _ = transmitters_to_delete_qs.delete() |
|||
print(f" ✔ {deleted_transmitters_count} رکورد HadisTransmitter با موفقیت حذف شد.") |
|||
|
|||
print("⏳ در حال حذف رکوردهای NarratorLayer غیر حدیث...") |
|||
deleted_layers_count, _ = layers_to_delete_qs.delete() |
|||
print(f" ✔ {deleted_layers_count} رکورد NarratorLayer با موفقیت حذف شد.") |
|||
|
|||
print("⏳ در حال خالیسازی title_narrator و بهروزرسانی normalized_text استدلالهای غیر حدیث...") |
|||
hadis_to_update = [] |
|||
for hadis in hadis_with_title_narrator_qs.only("id", "text", "title", "translation", "title_narrator"): |
|||
hadis.title_narrator = [] |
|||
hadis.normalized_text = build_search_blob( |
|||
hadis.text, |
|||
hadis.title, |
|||
[], |
|||
hadis.translation |
|||
) |
|||
hadis_to_update.append(hadis) |
|||
|
|||
if hadis_to_update: |
|||
Hadis.objects.bulk_update(hadis_to_update, ["title_narrator", "normalized_text"], batch_size=500) |
|||
print(f" ✔ {len(hadis_to_update)} استدلال با موفقیت اصلاح و نرمالسازی شد.") |
|||
|
|||
# Invalidate cache once at the end |
|||
try: |
|||
from apps.hadis.signals import invalidate_hadis_cache |
|||
invalidate_hadis_cache() |
|||
print(" ✔ کش ردیس با موفقیت بازنشانی شد.") |
|||
except Exception: |
|||
pass |
|||
|
|||
print_separator() |
|||
print("✨ عملیات پاکسازی با موفقیت تکمیل شد! بررسی وضعیت نهایی:") |
|||
after_summary = audit_narrator_data() |
|||
display_summary(after_summary, "وضعیت بعد از پاکسازی") |
|||
|
|||
# Sanity checks |
|||
non_hadith_transmitters_remaining = sum( |
|||
stats["transmitters"] for st, stats in after_summary.items() if st != HadisCategory.SourceType.HADITH |
|||
) |
|||
non_hadith_layers_remaining = sum( |
|||
stats["layers"] for st, stats in after_summary.items() if st != HadisCategory.SourceType.HADITH |
|||
) |
|||
non_hadith_title_narrator_remaining = sum( |
|||
stats["has_title_narrator"] for st, stats in after_summary.items() if st != HadisCategory.SourceType.HADITH |
|||
) |
|||
|
|||
hadith_transmitters_preserved = after_summary[HadisCategory.SourceType.HADITH]["transmitters"] |
|||
hadith_layers_preserved = after_summary[HadisCategory.SourceType.HADITH]["layers"] |
|||
|
|||
print_separator("-") |
|||
print(f"🔒 نتایج تضمین سلامت دادهها:") |
|||
print(f" • رکوردهای راوی باقیمانده در موارد غیر حدیث: {non_hadith_transmitters_remaining} (باید 0 باشد)") |
|||
print(f" • رکوردهای لایه روایت باقیمانده در موارد غیر حدیث: {non_hadith_layers_remaining} (باید 0 باشد)") |
|||
print(f" • عنوان راوی باقیمانده در موارد غیر حدیث: {non_hadith_title_narrator_remaining} (باید 0 باشد)") |
|||
print(f" • رکوردهای راوی حفظشده احادیث (Hadith): {hadith_transmitters_preserved} (کاملاً دستنخورده)") |
|||
print(f" • رکوردهای لایه روایت حفظشده احادیث (Hadith): {hadith_layers_preserved} (کاملاً دستنخورده)") |
|||
print_separator() |
|||
|
|||
|
|||
def main(): |
|||
parser = argparse.ArgumentParser( |
|||
description="Clean up narrator chains and layers for all non-hadith arguments." |
|||
) |
|||
parser.add_argument( |
|||
"--dry-run", |
|||
action="store_true", |
|||
help="Simulate the cleanup and display statistics without deleting anything.", |
|||
) |
|||
args = parser.parse_args() |
|||
|
|||
cleanup_non_hadith_narrators(dry_run=args.dry_run) |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
main() |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue