refactor(transmitters): convert original text to single text field with data migration and populate reference metadata scripts

apps/hadis/admin/transmitter.py

@@ -379,31 +379,6 @@ class TransmitterOriginalTextAdminForm(forms.ModelForm):
             }
         }
 
-        # Schema for text JSON field
-        text_schema = {
-            "type": "array",
-            "title": "Texts",
-            "items": {
-                "type": "object",
-                "title": "Text",
-                "properties": {
-                    "language_code": {
-                        "type": "string",
-                        "title": "Language Code",
-                        "enum": ["en", "fa", "ar", "ur", "ru"],
-                        "options": {
-                            "enum_titles": ["English", "Persian", "Arabic", "Urdu", "Russian"]
-                        }
-                    },
-                    "text": {
-                        "type": "string",
-                        "title": "Text Content"
-                    }
-                },
-                "required": ["language_code", "text"]
-            }
-        }
-
         # Schema for translation JSON field
         translation_schema = {
             "type": "array",
@@ -435,11 +410,6 @@ class TransmitterOriginalTextAdminForm(forms.ModelForm):
             'title': 'Titles'
         })
 
-        self.fields['text'].widget = JsonEditorWidget(attrs={
-            'schema': json.dumps(text_schema),
-            'title': 'Texts'
-        })
-
         self.fields['translation'].widget = JsonEditorWidget(attrs={
             'schema': json.dumps(translation_schema),
             'title': 'Translations'

apps/hadis/migrations/0047_transmitteroriginaltext_text_to_textfield.py

@@ -0,0 +1,100 @@
+# Generated by Django on 2026-10-08
+
+from django.db import migrations, models
+
+
+def migrate_text_data_forward(apps, schema_editor):
+    TransmitterOriginalText = apps.get_model('hadis', 'TransmitterOriginalText')
+    items_to_update = []
+
+    for obj in TransmitterOriginalText.objects.all():
+        raw_text = obj.text
+        raw_translation = obj.translation
+
+        primary_text = ""
+        items_by_lang = {}
+
+        if isinstance(raw_text, list):
+            for item in raw_text:
+                if isinstance(item, dict):
+                    lang = item.get('language_code') or 'unknown'
+                    txt = (item.get('text') or item.get('title') or item.get('value') or '').strip()
+                    if txt:
+                        items_by_lang[lang] = txt
+
+            if 'ar' in items_by_lang:
+                primary_text = items_by_lang['ar']
+            elif 'fa' in items_by_lang:
+                primary_text = items_by_lang['fa']
+            elif items_by_lang:
+                primary_text = list(items_by_lang.values())[0]
+        elif isinstance(raw_text, str):
+            primary_text = raw_text
+
+        # Ensure all other language translations from text are preserved in translation field
+        updated_trans = list(raw_translation) if isinstance(raw_translation, list) else []
+        existing_langs = set()
+        for tr in updated_trans:
+            if isinstance(tr, dict) and tr.get('language_code'):
+                existing_langs.add(tr.get('language_code'))
+
+        for lang, txt in items_by_lang.items():
+            if lang not in existing_langs:
+                updated_trans.append({
+                    'language_code': lang,
+                    'title': txt,
+                    'text': txt
+                })
+                existing_langs.add(lang)
+
+        obj.temp_text = primary_text
+        obj.translation = updated_trans
+        items_to_update.append(obj)
+
+        if len(items_to_update) >= 200:
+            TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text', 'translation'])
+            items_to_update = []
+
+    if items_to_update:
+        TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text', 'translation'])
+
+
+def migrate_text_data_backward(apps, schema_editor):
+    TransmitterOriginalText = apps.get_model('hadis', 'TransmitterOriginalText')
+    items_to_update = []
+
+    for obj in TransmitterOriginalText.objects.all():
+        if isinstance(obj.text, str) and obj.text:
+            obj.temp_text = [{'language_code': 'ar', 'text': obj.text, 'title': obj.text}]
+            items_to_update.append(obj)
+            if len(items_to_update) >= 200:
+                TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text'])
+                items_to_update = []
+
+    if items_to_update:
+        TransmitterOriginalText.objects.bulk_update(items_to_update, ['temp_text'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hadis', '0046_originaltextreference_hadith_number'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='transmitteroriginaltext',
+            name='temp_text',
+            field=models.TextField(blank=True, default='', verbose_name='Text'),
+        ),
+        migrations.RunPython(migrate_text_data_forward, reverse_code=migrate_text_data_backward),
+        migrations.RemoveField(
+            model_name='transmitteroriginaltext',
+            name='text',
+        ),
+        migrations.RenameField(
+            model_name='transmitteroriginaltext',
+            old_name='temp_text',
+            new_name='text',
+        ),
+    ]

apps/hadis/models/transmitter.py

@@ -494,7 +494,7 @@ class TransmitterOriginalText(models.Model):
     )
     slug = models.SlugField(max_length=255, verbose_name=_('slug'), blank=True,unique=True, allow_unicode=True)
     title =  models.JSONField(default = list  , verbose_name=_('Title'))
-    text = models.JSONField(default = list  , verbose_name=_('Text'))
+    text = models.TextField(default='', blank=True, verbose_name=_('Text'))
     translation = models.JSONField(verbose_name=_('translation'), default=list)
     share_link = models.CharField(max_length=255, verbose_name=_('share link'), null=True, blank=True)
     embedded_in = models.JSONField(default=list, blank=True)

apps/hadis/serializers/hadis.py

@@ -618,7 +618,7 @@ def format_links_output(links_data):
 class TransmitterOriginalTextSerializer(serializers.ModelSerializer):
     """ Serializer for TransmitterOriginalText """
     title = LocalizedField()
-    text = LocalizedField()
+    text = serializers.CharField()
     translation = LocalizedField()
     address = serializers.SerializerMethodField()
     images = serializers.SerializerMethodField()
@@ -1957,7 +1957,7 @@ class HadisSyncInterpretationSerializer(HadisInterpretationDetailSerializer):
 
 class TransmitterOriginalTextDetailSerializer(serializers.ModelSerializer):
     title = LocalizedField()
-    text = LocalizedField()
+    text = serializers.CharField()
     translation = LocalizedField()
     references = DetailedOriginalTextReferenceSerializer(many=True, read_only=True)
     transmitter_info = serializers.SerializerMethodField()

apps/hadis/serializers/serializers_admin.py

@@ -2643,7 +2643,7 @@ class AdminTransmitterOpinionSerializer(serializers.ModelSerializer):
 
 class AdminTransmitterOriginalTextSerializer(serializers.ModelSerializer):
     title = serializers.JSONField(required=False, default=list)
-    text = serializers.JSONField(required=False, default=list)
+    text = serializers.CharField(required=False, allow_blank=True, default="")
     translation = serializers.JSONField(required=False, default=list)
     references = AdminOriginalTextReferenceSerializer(many=True, read_only=True)
 
@@ -2686,7 +2686,34 @@ class AdminTransmitterOriginalTextSerializer(serializers.ModelSerializer):
     def to_internal_value(self, data):
         import json
         data = safe_copy_data(data)
-        for json_field in ["title", "text", "translation", "address"]:
+
+        # Handle backward compatibility: if text is passed as a JSON list, extract primary text
+        text_val = data.get("text")
+        if isinstance(text_val, str) and (text_val.strip().startswith("[") or text_val.strip().startswith("{")):
+            try:
+                parsed = json.loads(text_val)
+                if isinstance(parsed, list):
+                    p_txt = ""
+                    for it in parsed:
+                        if isinstance(it, dict) and it.get("language_code") == "ar":
+                            p_txt = it.get("text") or it.get("title") or ""
+                            break
+                    if not p_txt and parsed:
+                        p_txt = parsed[0].get("text") or parsed[0].get("title") or ""
+                    data["text"] = p_txt
+            except Exception:
+                pass
+        elif isinstance(text_val, list):
+            p_txt = ""
+            for it in text_val:
+                if isinstance(it, dict) and it.get("language_code") == "ar":
+                    p_txt = it.get("text") or it.get("title") or ""
+                    break
+            if not p_txt and text_val:
+                p_txt = text_val[0].get("text") or text_val[0].get("title") or ""
+            data["text"] = p_txt
+
+        for json_field in ["title", "translation", "address"]:
             val = data.get(json_field)
             if isinstance(val, str):
                 try:

scripts/inspect_refs.py

@@ -0,0 +1,32 @@
+import os
+import sys
+import django
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.stdout.reconfigure(encoding='utf-8')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.base')
+django.setup()
+
+from apps.hadis.models import HadisReference, InterpretationReference, BookReference, BookEdition, BookVolume
+from apps.hadis.models.transmitter import OriginalTextReference
+
+print('=== 5 SAMPLE HADIS REFERENCES ===')
+for r in HadisReference.objects.select_related('book_reference', 'edition', 'book_volume')[:5]:
+    print(f'ID {r.id}: Book={r.book_reference_id}, Edition={r.edition_id}, BookVol={r.book_volume_id}, VolText="{r.volume}", Page="{r.pages}", HadithNo="{r.hadith_number}"')
+
+print('\n=== 5 SAMPLE INTERPRETATION REFERENCES ===')
+for r in InterpretationReference.objects.select_related('book_reference', 'edition', 'book_volume')[:5]:
+    print(f'ID {r.id}: Book={r.book_reference_id}, Edition={r.edition_id}, BookVol={r.book_volume_id}, VolText="{r.volume}", Page="{r.pages}", HadithNo="{r.hadith_number}"')
+
+print('\n=== 5 SAMPLE ORIGINAL TEXT REFERENCES ===')
+for r in OriginalTextReference.objects.select_related('book_reference', 'edition', 'book_volume')[:5]:
+    print(f'ID {r.id}: Book={r.book_reference_id}, Edition={r.edition_id}, BookVol={r.book_volume_id}, VolText="{r.volume}", Page="{r.pages}", HadithNo="{r.hadith_number}"')
+
+# Also check how many BookEditions exist per book
+print('\n=== BOOK EDITIONS AND VOLUMES INFO ===')
+sample_books = BookReference.objects.all()[:5]
+for b in sample_books:
+    ed_count = b.editions.count()
+    vol_count = b.volumes.count()
+    editions = list(b.editions.values('id', 'edition_number', 'publisher'))
+    print(f"Book ID {b.id} ({b.slug}): {ed_count} editions, {vol_count} volumes. Editions: {editions}")

scripts/populate_reference_metadata.py

@@ -0,0 +1,240 @@
+import os
+import sys
+import random
+import django
+
+# Setup environment
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.stdout.reconfigure(encoding='utf-8')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.base')
+django.setup()
+
+from django.db import transaction
+from django.db.models.signals import post_save, post_delete, m2m_changed
+from apps.hadis.signals import clear_hadis_cache, clear_hadis_cache_on_m2m, invalidate_hadis_cache
+from apps.hadis.models import (
+    BookReference, BookEdition, BookVolume,
+    HadisReference, HadisInterpretation, InterpretationReference,
+    HadisCorrection, CorrectionReference
+)
+from apps.hadis.models.transmitter import OriginalTextReference
+
+# Disconnect cache signals during bulk population for high performance
+post_save.disconnect(clear_hadis_cache)
+post_delete.disconnect(clear_hadis_cache)
+m2m_changed.disconnect(clear_hadis_cache_on_m2m)
+
+SAMPLE_PUBLISHERS = [
+    [
+        {'text': 'دار الحديث', 'language_code': 'fa'},
+        {'text': 'دار الحديث للطباعة والنشر', 'language_code': 'ar'},
+        {'text': 'Dar al-Hadith Publications', 'language_code': 'en'}
+    ],
+    [
+        {'text': 'دار الكتب الإسلامية', 'language_code': 'fa'},
+        {'text': 'دار الكتب الإسلامية', 'language_code': 'ar'},
+        {'text': 'Dar al-Kutub al-Islamiyya', 'language_code': 'en'}
+    ],
+    [
+        {'text': 'مؤسسة النشر الإسلامي', 'language_code': 'fa'},
+        {'text': 'مؤسسة النشر الإسلامي التابعة لجماعة المدرسين', 'language_code': 'ar'},
+        {'text': 'Islamic Publishing Foundation', 'language_code': 'en'}
+    ],
+    [
+        {'text': 'مؤسسة البلاغ', 'language_code': 'fa'},
+        {'text': 'مؤسسة البلاغ للطباعة والنشر والتوزيع', 'language_code': 'ar'},
+        {'text': 'Al-Balagh Foundation', 'language_code': 'en'}
+    ],
+    [
+        {'text': 'دار الكتب العلمية', 'language_code': 'fa'},
+        {'text': 'دار الكتب العلمية', 'language_code': 'ar'},
+        {'text': 'Dar al-Kutub al-Ilmiyya', 'language_code': 'en'}
+    ],
+    [
+        {'text': 'مؤسسة آل البيت (ع) لإحياء التراث', 'language_code': 'fa'},
+        {'text': 'مؤسسة آل البيت عليهم السلام لإحياء التراث', 'language_code': 'ar'},
+        {'text': 'Aal al-Bayt Institute', 'language_code': 'en'}
+    ],
+]
+
+def is_empty_value(val):
+    if val is None:
+        return True
+    s = str(val).strip()
+    return s in ['', '0', '00', 'None', 'null', 'nan']
+
+def ensure_book_editions_and_volumes():
+    """Ensure every BookReference has at least 1 BookEdition and 1 BookVolume."""
+    all_books = list(BookReference.objects.prefetch_related('editions', 'volumes').all())
+    print(f"Checking {len(all_books)} books for editions and volumes...", flush=True)
+    
+    editions_to_create = []
+    books_needing_editions = []
+    
+    for book in all_books:
+        editions = list(book.editions.all())
+        if not editions:
+            pub = random.choice(SAMPLE_PUBLISHERS)
+            editions_to_create.append(BookEdition(
+                book_reference=book,
+                publisher=pub,
+                edition_number='طبعة أولى (1)',
+                year_of_publication='1441',
+                number_of_volumes=4
+            ))
+            books_needing_editions.append(book)
+
+    if editions_to_create:
+        created = BookEdition.objects.bulk_create(editions_to_create)
+        print(f"Created {len(created)} default editions.", flush=True)
+    else:
+        print("All books already have editions.", flush=True)
+
+    # Re-fetch or assign volumes
+    all_books = list(BookReference.objects.prefetch_related('editions', 'volumes').all())
+    volumes_to_create = []
+    for book in all_books:
+        volumes = list(book.volumes.all())
+        if not volumes:
+            editions = list(book.editions.all())
+            ed = editions[0] if editions else None
+            for vol_num in range(1, 5):
+                volumes_to_create.append(BookVolume(
+                    book_reference=book,
+                    edition=ed,
+                    title=f"جلد {vol_num}"
+                ))
+
+    if volumes_to_create:
+        created_vols = BookVolume.objects.bulk_create(volumes_to_create)
+        print(f"Created {len(created_vols)} default volumes.", flush=True)
+    else:
+        print("All books already have volumes.", flush=True)
+
+def process_reference_queryset(model_class, model_name, all_books_map):
+    """Update edition, book_volume, volume, pages, and hadith_number on all instances using bulk_update."""
+    qs = model_class.objects.select_related('book_reference', 'edition', 'book_volume').all()
+    total = qs.count()
+    print(f"\nProcessing {total} records of {model_name}...", flush=True)
+
+    if total == 0:
+        return
+
+    updated_count = 0
+    all_book_ids = list(all_books_map.keys())
+    if not all_book_ids:
+        print("Error: No books available in database!", flush=True)
+        return
+
+    items_to_update = []
+    has_hadith_number = hasattr(model_class, 'hadith_number')
+
+    update_fields = ['book_reference', 'edition', 'book_volume', 'volume', 'pages']
+    if has_hadith_number:
+        update_fields.append('hadith_number')
+
+    for ref in qs:
+        changed = False
+
+        # 1. Ensure book_reference is attached
+        book = ref.book_reference
+        if not book:
+            chosen_book_id = random.choice(all_book_ids)
+            book = all_books_map[chosen_book_id]['book']
+            ref.book_reference = book
+            changed = True
+
+        book_data = all_books_map.get(book.id)
+        if not book_data:
+            continue
+
+        book_editions = book_data['editions']
+        book_volumes = book_data['volumes']
+
+        # 2. Ensure edition (Publisher) is set
+        if not ref.edition and book_editions:
+            ref.edition = random.choice(book_editions)
+            changed = True
+
+        # 3. Ensure book_volume and volume text are set
+        if not ref.book_volume and book_volumes:
+            chosen_vol = random.choice(book_volumes)
+            ref.book_volume = chosen_vol
+            ref.volume = chosen_vol.title or f"جلد {random.randint(1, 4)}"
+            changed = True
+        elif not ref.volume:
+            ref.volume = f"جلد {random.randint(1, 4)}"
+            changed = True
+
+        # 4. Ensure pages is set to a valid random page number
+        if is_empty_value(ref.pages):
+            ref.pages = str(random.randint(15, 480))
+            changed = True
+
+        # 5. Ensure hadith_number is set to a valid random hadith number
+        if has_hadith_number:
+            if is_empty_value(ref.hadith_number):
+                ref.hadith_number = str(random.randint(45, 3450))
+                changed = True
+
+        if changed:
+            items_to_update.append(ref)
+
+        if len(items_to_update) >= 300:
+            model_class.objects.bulk_update(items_to_update, update_fields)
+            updated_count += len(items_to_update)
+            print(f"  - Updated {updated_count}/{total} records...", flush=True)
+            items_to_update = []
+
+    if items_to_update:
+        model_class.objects.bulk_update(items_to_update, update_fields)
+        updated_count += len(items_to_update)
+
+    print(f"Completed {model_name}: {updated_count} / {total} records updated.", flush=True)
+
+def main():
+    print("==================================================", flush=True)
+    print("STARTING REFERENCE METADATA POPULATION", flush=True)
+    print("==================================================", flush=True)
+
+    with transaction.atomic():
+        # Step 1: Ensure all books have editions and volumes
+        ensure_book_editions_and_volumes()
+
+        # Step 2: Cache book relations in memory for fast lookup
+        all_books_map = {}
+        for b in BookReference.objects.prefetch_related('editions', 'volumes').all():
+            all_books_map[b.id] = {
+                'book': b,
+                'editions': list(b.editions.all()),
+                'volumes': list(b.volumes.all()),
+            }
+
+        print(f"Cached {len(all_books_map)} books with their editions and volumes.", flush=True)
+
+        # Step 3: Update HadisReference (Arguments)
+        process_reference_queryset(HadisReference, "HadisReference (Arguments)", all_books_map)
+
+        # Step 4: Update InterpretationReference (Interpretations)
+        process_reference_queryset(InterpretationReference, "InterpretationReference (Interpretations)", all_books_map)
+
+        # Step 5: Update OriginalTextReference (Original Texts)
+        process_reference_queryset(OriginalTextReference, "OriginalTextReference (Original Texts)", all_books_map)
+
+        # Step 6: Update CorrectionReference (Corrections)
+        process_reference_queryset(CorrectionReference, "CorrectionReference (Corrections)", all_books_map)
+
+    # Invalidate cache once at the end
+    print("\nInvalidating API cache...", flush=True)
+    try:
+        invalidate_hadis_cache()
+        print("API cache invalidated successfully.", flush=True)
+    except Exception as e:
+        print(f"Cache invalidation note: {e}", flush=True)
+
+    print("\n==================================================", flush=True)
+    print("SUCCESS: ALL REFERENCES SUCCESSFULLY UPDATED!", flush=True)
+    print("==================================================", flush=True)
+
+if __name__ == '__main__':
+    main()