From f8d0ba8666337f9a0eef80de799703890bda829e Mon Sep 17 00:00:00 2001 From: mohsentaba Date: Sun, 11 Oct 2026 17:51:41 +0330 Subject: [PATCH] feat(hadis): add narrator filtering and cleanup non-hadith narrator records - Add narrator and transmitter filtering support to HadisMainListView and HadisListView by slug or ID - Include available narrators list with localized names in HadisFiltersView endpoint - Document narrator query parameter in Swagger schema - Add cleanup_non_hadith_narrators.py script to remove orphan/erroneous narrator chains and layers from non-hadith arguments --- apps/hadis/docs.py | 8 + apps/hadis/views/hadis.py | 54 +++++- scripts/cleanup_non_hadith_narrators.py | 222 ++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 scripts/cleanup_non_hadith_narrators.py diff --git a/apps/hadis/docs.py b/apps/hadis/docs.py index 65817d7..5df70dd 100644 --- a/apps/hadis/docs.py +++ b/apps/hadis/docs.py @@ -2672,6 +2672,14 @@ hadis_main_list_swagger = swagger_auto_schema( required=False, example="prayer" ), + openapi.Parameter( + 'narrator', + openapi.IN_QUERY, + description="Filter hadis by narrator/transmitter slug or ID (comma-separated for multiple).", + type=openapi.TYPE_STRING, + required=False, + example="ali-ibn-ibrahim" + ), openapi.Parameter( 'Accept-Language', openapi.IN_HEADER, diff --git a/apps/hadis/views/hadis.py b/apps/hadis/views/hadis.py index 1023775..48c65b0 100644 --- a/apps/hadis/views/hadis.py +++ b/apps/hadis/views/hadis.py @@ -377,11 +377,24 @@ class HadisListView(ListAPIView): status_slugs = [s.strip() for s in status_filter.split(',')] queryset = queryset.filter(hadis_status__slug__in=status_slugs) - # 👇 3. Apply Transmitter Filter (Supports multiple comma-separated slugs) - transmitter_filter = self.request.query_params.get('transmitter', None) + # 👇 3. Apply Transmitter / Narrator Filter (Supports multiple comma-separated slugs or IDs) + transmitter_filter = ( + self.request.query_params.get('transmitter', None) or + self.request.query_params.get('transmitters', None) or + self.request.query_params.get('narrator', None) or + self.request.query_params.get('narrators', None) + ) if transmitter_filter: - transmitter_slugs = [t.strip() for t in transmitter_filter.split(',')] - queryset = queryset.filter(transmitters__transmitter__slug__in=transmitter_slugs) + t_items = [t.strip() for t in transmitter_filter.split(',') if t.strip()] + numeric_ids = [int(t) for t in t_items if t.isdigit()] + slugs = [t for t in t_items if not t.isdigit()] + t_q = Q() + if numeric_ids: + t_q |= Q(transmitters__transmitter_id__in=numeric_ids) + if slugs: + t_q |= Q(transmitters__transmitter__slug__in=slugs) + if t_q: + queryset = queryset.filter(t_q) # 👇 4. Apply Source Filter (Supports multiple comma-separated slugs) source_filter = self.request.query_params.get('source', None) @@ -493,6 +506,12 @@ class HadisMainListView(ListAPIView): source_filter = self.request.query_params.get('source', None) category_filter = self.request.query_params.get('category', None) or self.request.query_params.get('category_slug', None) tag_filter = self.request.query_params.get('tag', None) or self.request.query_params.get('tags', None) + transmitter_filter = ( + self.request.query_params.get('transmitter', None) or + self.request.query_params.get('transmitters', None) or + self.request.query_params.get('narrator', None) or + self.request.query_params.get('narrators', None) + ) # Apply search filter if search_query: @@ -552,6 +571,19 @@ class HadisMainListView(ListAPIView): if tag_q: queryset = queryset.filter(tag_q) + # Apply transmitter / narrator filter + if transmitter_filter: + t_items = [t.strip() for t in transmitter_filter.split(',') if t.strip()] + numeric_ids = [int(t) for t in t_items if t.isdigit()] + slugs = [t for t in t_items if not t.isdigit()] + t_q = Q() + if numeric_ids: + t_q |= Q(transmitters__transmitter_id__in=numeric_ids) + if slugs: + t_q |= Q(transmitters__transmitter__slug__in=slugs) + if t_q: + queryset = queryset.filter(t_q) + # Filter by bookmarks if provided is_bookmark = self.request.query_params.get('is_bookmark', '').lower() if is_bookmark == 'true' and self.request.user.is_authenticated: @@ -1078,9 +1110,21 @@ class HadisFiltersView(ListAPIView): 'slug': category.slug }) + # Get narrators from Transmitters model + narrators = [] + for narrator in Transmitters.objects.all().order_by('id'): + title_text = get_localized_text(narrator.full_name, request) + if title_text and narrator.slug: + narrators.append({ + 'id': narrator.id, + 'text': title_text, + 'slug': narrator.slug + }) + response_data = { 'statuses': statuses, - 'categories': categories + 'categories': categories, + 'narrators': narrators } return Response(response_data) diff --git a/scripts/cleanup_non_hadith_narrators.py b/scripts/cleanup_non_hadith_narrators.py new file mode 100644 index 0000000..eba4ccb --- /dev/null +++ b/scripts/cleanup_non_hadith_narrators.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +""" +Script to clean up Narrator and Narrative Layer data for non-hadith arguments: +1. Preserves all narrator records (HadisTransmitter, NarratorLayer, title_narrator) + ONLY for arguments of type 'hadith' (source_type == 'hadith'). +2. For all other arguments (quran, history, fatwa, quote, or uncategorized): + - Deletes all associated HadisTransmitter records. + - Deletes all associated NarratorLayer records. + - Clears title_narrator and updates normalized_text. +3. Supports --dry-run mode to preview changes before applying them. +""" + +import os +import sys +import argparse +from pathlib import Path + +# Setup Django environment +BASE_DIR = Path(__file__).resolve().parent.parent +sys.path.append(str(BASE_DIR)) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.base') + +import django +django.setup() + +# Configure UTF-8 output on Windows +if sys.platform == "win32": + try: + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") + except Exception: + pass + +from django.db import transaction +from django.db.models import Count, Q +from apps.hadis.models import Hadis, HadisCategory, HadisTransmitter, NarratorLayer +from utils.text_normalizer import build_search_blob + + +def print_separator(char="=", length=70): + print(char * length) + + +def audit_narrator_data(): + """Returns a dictionary summary of narrator records grouped by source_type.""" + summary = {} + for st in HadisCategory.SourceType.values: + hadis_count = Hadis.objects.filter(category__source_type=st).count() + transmitter_count = HadisTransmitter.objects.filter(hadis__category__source_type=st).count() + layer_count = NarratorLayer.objects.filter(hadis__category__source_type=st).count() + has_title_narrator = Hadis.objects.filter(category__source_type=st).exclude( + Q(title_narrator=[]) | Q(title_narrator__isnull=True) + ).count() + summary[st] = { + "hadis_count": hadis_count, + "transmitters": transmitter_count, + "layers": layer_count, + "has_title_narrator": has_title_narrator, + } + + # Arguments without a category or with null source_type + null_hadis = Hadis.objects.filter(Q(category__isnull=True) | Q(category__source_type__isnull=True)).count() + null_transmitters = HadisTransmitter.objects.filter( + Q(hadis__category__isnull=True) | Q(hadis__category__source_type__isnull=True) + ).count() + null_layers = NarratorLayer.objects.filter( + Q(hadis__category__isnull=True) | Q(hadis__category__source_type__isnull=True) + ).count() + null_title_narrator = Hadis.objects.filter( + Q(category__isnull=True) | Q(category__source_type__isnull=True) + ).exclude(Q(title_narrator=[]) | Q(title_narrator__isnull=True)).count() + + summary["_none_"] = { + "hadis_count": null_hadis, + "transmitters": null_transmitters, + "layers": null_layers, + "has_title_narrator": null_title_narrator, + } + + return summary + + +def display_summary(summary, title="وضعیت رکوردها"): + print_separator() + print(f"📊 {title}:") + print_separator("-") + print(f"{'نوع منبع (Source Type)':<25} | {'تعداد استدلال':<12} | {'راویان (Transmitters)':<20} | {'لایه‌ها (Layers)':<16} | {'عنوان راوی':<12}") + print_separator("-") + + for st, stats in summary.items(): + label = st if st != "_none_" else "بدون دسته‌بندی (None)" + marker = " ✅ (حفظ می‌شود)" if st == HadisCategory.SourceType.HADITH else " ❌ (حذف می‌شود)" + print( + f"{label + marker:<25} | " + f"{stats['hadis_count']:<12} | " + f"{stats['transmitters']:<20} | " + f"{stats['layers']:<16} | " + f"{stats['has_title_narrator']:<12}" + ) + print_separator() + + +def cleanup_non_hadith_narrators(dry_run=False): + """ + Performs cleanup of narrator layers and transmitters for all non-hadith arguments. + """ + print_separator() + mode_text = "[حالت آزمایشی - DRY RUN (بدون اعمال تغییر)]" if dry_run else "[حالت اجرایی واقعی - LIVE EXECUTION]" + print(f"🚀 شروع پاکسازی راویان و لایه‌های روایت برای استدلال‌های غیر حدیث {mode_text}") + print_separator() + + # 1. Audit before + before_summary = audit_narrator_data() + display_summary(before_summary, "وضعیت قبل از پاکسازی") + + # 2. Identify non-hadith arguments + non_hadith_hadis_qs = Hadis.objects.exclude( + category__source_type=HadisCategory.SourceType.HADITH + ) + non_hadith_ids = list(non_hadith_hadis_qs.values_list("id", flat=True)) + + transmitters_to_delete_qs = HadisTransmitter.objects.filter(hadis_id__in=non_hadith_ids) + layers_to_delete_qs = NarratorLayer.objects.filter(hadis_id__in=non_hadith_ids) + hadis_with_title_narrator_qs = non_hadith_hadis_qs.exclude( + Q(title_narrator=[]) | Q(title_narrator__isnull=True) + ) + + total_transmitters_to_delete = transmitters_to_delete_qs.count() + total_layers_to_delete = layers_to_delete_qs.count() + total_title_narrator_to_clear = hadis_with_title_narrator_qs.count() + + print(f"🎯 موارد شناسایی شده برای پاکسازی:") + print(f" • تعداد استدلال‌های غیر حدیث: {len(non_hadith_ids)}") + print(f" • رکوردهای HadisTransmitter غیر حدیث جهت حذف: {total_transmitters_to_delete}") + print(f" • رکوردهای NarratorLayer غیر حدیث جهت حذف: {total_layers_to_delete}") + print(f" • فیلدهای title_narrator غیر حدیث جهت خالی‌سازی: {total_title_narrator_to_clear}") + print_separator() + + if dry_run: + print("⚠️ حالت آزمایشی (dry-run) فعال است. هیچ داده‌ای از دیتابیس پاک نشد.") + print("برای اعمال قطعی تغییرات، اسکریپت را بدون فلگ --dry-run اجرا نمایید.") + return + + # 3. Execution inside atomic transaction + with transaction.atomic(): + print("⏳ در حال حذف رکوردهای HadisTransmitter غیر حدیث...") + deleted_transmitters_count, _ = transmitters_to_delete_qs.delete() + print(f" ✔ {deleted_transmitters_count} رکورد HadisTransmitter با موفقیت حذف شد.") + + print("⏳ در حال حذف رکوردهای NarratorLayer غیر حدیث...") + deleted_layers_count, _ = layers_to_delete_qs.delete() + print(f" ✔ {deleted_layers_count} رکورد NarratorLayer با موفقیت حذف شد.") + + print("⏳ در حال خالی‌سازی title_narrator و به‌روزرسانی normalized_text استدلال‌های غیر حدیث...") + hadis_to_update = [] + for hadis in hadis_with_title_narrator_qs.only("id", "text", "title", "translation", "title_narrator"): + hadis.title_narrator = [] + hadis.normalized_text = build_search_blob( + hadis.text, + hadis.title, + [], + hadis.translation + ) + hadis_to_update.append(hadis) + + if hadis_to_update: + Hadis.objects.bulk_update(hadis_to_update, ["title_narrator", "normalized_text"], batch_size=500) + print(f" ✔ {len(hadis_to_update)} استدلال با موفقیت اصلاح و نرمال‌سازی شد.") + + # Invalidate cache once at the end + try: + from apps.hadis.signals import invalidate_hadis_cache + invalidate_hadis_cache() + print(" ✔ کش ردیس با موفقیت بازنشانی شد.") + except Exception: + pass + + print_separator() + print("✨ عملیات پاکسازی با موفقیت تکمیل شد! بررسی وضعیت نهایی:") + after_summary = audit_narrator_data() + display_summary(after_summary, "وضعیت بعد از پاکسازی") + + # Sanity checks + non_hadith_transmitters_remaining = sum( + stats["transmitters"] for st, stats in after_summary.items() if st != HadisCategory.SourceType.HADITH + ) + non_hadith_layers_remaining = sum( + stats["layers"] for st, stats in after_summary.items() if st != HadisCategory.SourceType.HADITH + ) + non_hadith_title_narrator_remaining = sum( + stats["has_title_narrator"] for st, stats in after_summary.items() if st != HadisCategory.SourceType.HADITH + ) + + hadith_transmitters_preserved = after_summary[HadisCategory.SourceType.HADITH]["transmitters"] + hadith_layers_preserved = after_summary[HadisCategory.SourceType.HADITH]["layers"] + + print_separator("-") + print(f"🔒 نتایج تضمین سلامت داده‌ها:") + print(f" • رکوردهای راوی باقیمانده در موارد غیر حدیث: {non_hadith_transmitters_remaining} (باید 0 باشد)") + print(f" • رکوردهای لایه روایت باقیمانده در موارد غیر حدیث: {non_hadith_layers_remaining} (باید 0 باشد)") + print(f" • عنوان راوی باقیمانده در موارد غیر حدیث: {non_hadith_title_narrator_remaining} (باید 0 باشد)") + print(f" • رکوردهای راوی حفظ‌شده احادیث (Hadith): {hadith_transmitters_preserved} (کاملاً دست‌نخورده)") + print(f" • رکوردهای لایه روایت حفظ‌شده احادیث (Hadith): {hadith_layers_preserved} (کاملاً دست‌نخورده)") + print_separator() + + +def main(): + parser = argparse.ArgumentParser( + description="Clean up narrator chains and layers for all non-hadith arguments." + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Simulate the cleanup and display statistics without deleting anything.", + ) + args = parser.parse_args() + + cleanup_non_hadith_narrators(dry_run=args.dry_run) + + +if __name__ == "__main__": + main()