import os import gzip import json import sqlite3 import hashlib from pathlib import Path from django.core.management.base import BaseCommand from django.conf import settings from django.db import transaction from apps.hadis.models import ( Hadis, HadisCategory, HadisInterpretation, HadisCorrection, HadisDatabaseVersion, HadisStatus, HadisTag, BookReference, BookAuthor, Transmitters, ) from apps.hadis.serializers.hadis import ( DetailedCorrectionReferenceSerializer, DetailedInterpretationReferenceSerializer, get_cached_edition_metadata, ) from apps.hadis.serializers.category import get_arabic_localized_text from apps.hadis.utils.links import format_links_output class Command(BaseCommand): help = "Export Hadis and Categories into an optimized, compressed SQLite database for offline mobile sync." def add_arguments(self, parser): parser.add_argument( '--notes', type=str, default='', help='Release notes for this SQLite database version' ) parser.add_argument( '--force-version', type=int, default=None, help='Force a specific version number' ) def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS("Starting Hadis SQLite database generation (Single-File Architecture)...")) # 1. Determine destination and version number latest_ver = HadisDatabaseVersion.objects.order_by('-version').first() if options['force_version'] is not None: new_version = options['force_version'] else: new_version = (latest_ver.version + 1) if latest_ver else 1 output_dir = Path(settings.MEDIA_ROOT) / 'hadis_db' output_dir.mkdir(parents=True, exist_ok=True) temp_db_path = output_dir / "temp_hadis_build.sqlite" final_gz_name = "hadis_offline.sqlite.gz" final_gz_path = output_dir / final_gz_name if temp_db_path.exists(): temp_db_path.unlink() # 2. Initialize SQLite database conn = sqlite3.connect(str(temp_db_path)) cursor = conn.cursor() # Optimize SQLite for bulk writes cursor.execute("PRAGMA synchronous = OFF;") cursor.execute("PRAGMA journal_mode = MEMORY;") cursor.execute("PRAGMA encoding = 'UTF-8';") self.stdout.write("Creating SQLite tables and indexes...") self._create_tables(cursor) # 3. Export data categories_count, cat_info_map = self._export_categories(cursor) sources_count, book_image_map = self._export_sources(cursor) authors_count = self._export_authors(cursor) narrators_count = self._export_narrators(cursor) statuses_count = self._export_statuses(cursor) tags_count = self._export_tags(cursor) interp_count = self._export_category_interpretations(cursor, book_image_map) hadis_count, corr_count = self._export_hadiths_and_corrections(cursor, cat_info_map, book_image_map) conn.commit() # Create indexes after insertion for maximum performance self.stdout.write("Building indexes...") self._create_indexes(cursor) conn.commit() conn.close() # 4. Gzip compress directly to the single target file self.stdout.write(f"Compressing database to single file: {final_gz_name}...") hasher = hashlib.md5() temp_gz_path = output_dir / "temp_hadis_build.sqlite.gz" with open(temp_db_path, 'rb') as f_in: with gzip.open(temp_gz_path, 'wb', compresslevel=9) as f_out: while chunk := f_in.read(1024 * 1024): f_out.write(chunk) hasher.update(chunk) checksum = hasher.hexdigest() temp_size = temp_db_path.stat().st_size # Atomic replacement: replace final_gz_path with temp_gz_path if final_gz_path.exists(): final_gz_path.unlink() temp_gz_path.rename(final_gz_path) file_size = final_gz_path.stat().st_size # Clean up any remaining temp files in output_dir temp_db_path.unlink(missing_ok=True) for extra_file in output_dir.iterdir(): if extra_file.name != final_gz_name: try: extra_file.unlink(missing_ok=True) except OSError: pass # 5. Record version in database (Single Active Record) with transaction.atomic(): # Delete old records to keep DB clean HadisDatabaseVersion.objects.all().delete() db_version = HadisDatabaseVersion.objects.create( version=new_version, file=f"hadis_db/{final_gz_name}", file_size=file_size, checksum_md5=checksum, hadis_count=hadis_count, categories_count=categories_count, is_active=True, notes=options['notes'], ) self.stdout.write(self.style.SUCCESS( f"Successfully updated single Hadis SQLite DB to v{new_version}!\n" f" - Uncompressed size: {temp_size / (1024*1024):.2f} MB\n" f" - Gzipped size: {file_size / (1024*1024):.2f} MB\n" f" - Hadiths: {hadis_count}\n" f" - Categories: {categories_count}\n" f" - Sources: {sources_count}\n" f" - Authors: {authors_count}\n" f" - Narrators: {narrators_count}\n" f" - Statuses: {statuses_count}\n" f" - Tags: {tags_count}\n" f" - Interpretations: {interp_count}\n" f" - Corrections: {corr_count}\n" f" - MD5 Checksum: {checksum}\n" f" - File path: {final_gz_path}" )) def _create_tables(self, cursor): cursor.executescript(""" CREATE TABLE IF NOT EXISTS categories ( id INTEGER PRIMARY KEY, title TEXT, slug TEXT UNIQUE, source_type TEXT, sect_type TEXT, is_topic INTEGER DEFAULT 0 ); CREATE TABLE IF NOT EXISTS hadith_categories ( hadis_id INTEGER, category_id INTEGER, category_slug TEXT, source_type TEXT, sect_type TEXT, PRIMARY KEY (hadis_id, category_id), FOREIGN KEY (hadis_id) REFERENCES hadiths (id), FOREIGN KEY (category_id) REFERENCES categories (id) ); CREATE TABLE IF NOT EXISTS sources ( id INTEGER PRIMARY KEY, slug TEXT UNIQUE, title TEXT, image TEXT, number_of_volumes INTEGER, authors_json TEXT ); CREATE TABLE IF NOT EXISTS authors ( id INTEGER PRIMARY KEY, slug TEXT UNIQUE, name TEXT, death_year_hijri TEXT ); CREATE TABLE IF NOT EXISTS narrators ( id INTEGER PRIMARY KEY, slug TEXT UNIQUE, name TEXT, title TEXT, arabic_full_name TEXT, known_as TEXT, arabic_known_as TEXT, nickname TEXT, arabic_nickname TEXT, kunya TEXT, arabic_kunya TEXT, gender TEXT, birth_year_hijri TEXT, death_year_hijri TEXT, reliability_json TEXT ); CREATE TABLE IF NOT EXISTS statuses ( id INTEGER PRIMARY KEY, title TEXT, color TEXT, main_color_code TEXT, description TEXT ); CREATE TABLE IF NOT EXISTS tags ( id INTEGER PRIMARY KEY, title TEXT ); CREATE TABLE IF NOT EXISTS category_interpretations ( id INTEGER PRIMARY KEY, category_id INTEGER, title TEXT, slug TEXT, narrator TEXT, text TEXT, translation TEXT, references_json TEXT, links_json TEXT, FOREIGN KEY (category_id) REFERENCES categories (id) ); CREATE TABLE IF NOT EXISTS hadiths ( id INTEGER PRIMARY KEY, slug TEXT UNIQUE, category_id INTEGER, category_slug TEXT, source_type TEXT, sect_type TEXT, categories_json TEXT, title TEXT, title_narrator TEXT, text TEXT, translation_json TEXT, address TEXT, share_link TEXT, hadis_status_id INTEGER, status_title TEXT, status_color TEXT, status_main_color_code TEXT, status_text TEXT, tags_json TEXT, references_json TEXT, reference_images_json TEXT, narrators_json TEXT, explanations_json TEXT, links_json TEXT, FOREIGN KEY (category_id) REFERENCES categories (id) ); CREATE TABLE IF NOT EXISTS hadith_corrections ( id INTEGER PRIMARY KEY, hadis_id INTEGER, title TEXT, slug TEXT, narrator TEXT, description TEXT, translation TEXT, share_link TEXT, address TEXT, images_json TEXT, references_json TEXT, links_json TEXT, FOREIGN KEY (hadis_id) REFERENCES hadiths (id) ); """) def _create_indexes(self, cursor): cursor.executescript(""" CREATE INDEX IF NOT EXISTS idx_cat_slug ON categories(slug); CREATE INDEX IF NOT EXISTS idx_cat_source_type ON categories(source_type); CREATE INDEX IF NOT EXISTS idx_cat_sect_type ON categories(sect_type); CREATE INDEX IF NOT EXISTS idx_interp_cat ON category_interpretations(category_id); CREATE INDEX IF NOT EXISTS idx_hadith_cat ON hadiths(category_id); CREATE INDEX IF NOT EXISTS idx_hadith_slug ON hadiths(slug); CREATE INDEX IF NOT EXISTS idx_hadith_source_type ON hadiths(source_type); CREATE INDEX IF NOT EXISTS idx_hadith_sect_type ON hadiths(sect_type); CREATE INDEX IF NOT EXISTS idx_hadith_status_id ON hadiths(hadis_status_id); CREATE INDEX IF NOT EXISTS idx_corr_hadis ON hadith_corrections(hadis_id); CREATE INDEX IF NOT EXISTS idx_hcat_cat_id ON hadith_categories(category_id); CREATE INDEX IF NOT EXISTS idx_hcat_cat_slug ON hadith_categories(category_slug); CREATE INDEX IF NOT EXISTS idx_hcat_hadis_id ON hadith_categories(hadis_id); CREATE INDEX IF NOT EXISTS idx_sources_slug ON sources(slug); CREATE INDEX IF NOT EXISTS idx_authors_slug ON authors(slug); CREATE INDEX IF NOT EXISTS idx_narrators_slug ON narrators(slug); """) def _is_empty_localized_data(self, val): if not val: return True if isinstance(val, (list, tuple)): if len(val) == 0: return True for item in val: if isinstance(item, dict): txt = item.get('text') or item.get('title') or item.get('description') or '' if str(txt).strip(): return False elif str(item).strip(): return False return True if isinstance(val, dict): if not val: return True for k, v in val.items(): if str(v).strip(): return False return True if isinstance(val, str) and not val.strip(): return True return False def _export_categories(self, cursor): cats = list(HadisCategory.objects.select_related('sect').all()) by_id = {c.id: c for c in cats} def resolve_source_type(cat_obj): cur = cat_obj visited = set() while cur and cur.id not in visited: visited.add(cur.id) st = getattr(cur, 'source_type', None) if st: return st cur = by_id.get(getattr(cur, 'parent_id', None)) return 'hadith' def resolve_sect_type(cat_obj): cur = cat_obj visited = set() while cur and cur.id not in visited: visited.add(cur.id) if getattr(cur, 'sect_id', None) and getattr(cur, 'sect', None) and getattr(cur.sect, 'sect_type', None): return cur.sect.sect_type st = getattr(cur, 'sect_type', None) if st: return st cur = by_id.get(getattr(cur, 'parent_id', None)) return 'shia' rows = [] cat_info_map = {} for c in cats: source_type = resolve_source_type(c) sect_type = resolve_sect_type(c) cat_info_map[c.id] = { 'id': c.id, 'slug': c.slug, 'title': c.title, 'source_type': source_type, 'sect_type': sect_type, } rows.append(( c.id, json.dumps(c.title, ensure_ascii=False) if isinstance(c.title, (dict, list)) else (c.title or ''), c.slug, source_type, sect_type, 1 if getattr(c, 'is_topic', False) else 0, )) cursor.executemany( "INSERT INTO categories (id, title, slug, source_type, sect_type, is_topic) VALUES (?, ?, ?, ?, ?, ?)", rows ) return len(rows), cat_info_map def _export_sources(self, cursor): edition_meta = get_cached_edition_metadata() first_by_book_id = edition_meta.get('first_by_book_id', {}) by_book_id = edition_meta.get('by_book_id', {}) book_volumes_count = edition_meta.get('book_volumes_count', {}) books = list( BookReference.objects .select_related('author') .prefetch_related('images', 'volumes') .all() ) rows = [] book_image_map = {} for b in books: img_url = None b_imgs = list(b.images.all()) if b_imgs and getattr(b_imgs[0], 'image', None): img_url = self._make_full_url(b_imgs[0].image.url) if not img_url: for v in b.volumes.all(): if getattr(v, 'image', None): img_url = self._make_full_url(v.image.url) break book_image_map[b.id] = img_url num_vols = getattr(b, 'number_of_volumes', None) if num_vols is None: ed = first_by_book_id.get(b.id) or (by_book_id.get(b.id, [None])[0]) if ed: vol_cnt = getattr(ed, '_cached_vol_count', 0) num_vols = vol_cnt if vol_cnt > 0 else getattr(ed, 'number_of_volumes', None) if num_vols is None: num_vols = book_volumes_count.get(b.id) or len(b.volumes.all()) or None authors_list = [] if b.author: authors_list.append({ 'id': b.author.id, 'slug': b.author.slug, 'name': b.author.name, 'death_year_hijri': b.author.death_year_hijri or '', }) rows.append(( b.id, b.slug, json.dumps(b.title, ensure_ascii=False) if isinstance(b.title, (dict, list)) else (b.title or ''), img_url or '', num_vols, json.dumps(authors_list, ensure_ascii=False), )) cursor.executemany( "INSERT INTO sources (id, slug, title, image, number_of_volumes, authors_json) VALUES (?, ?, ?, ?, ?, ?)", rows ) return len(rows), book_image_map def _export_authors(self, cursor): authors = BookAuthor.objects.all() rows = [ ( a.id, a.slug, json.dumps(a.name, ensure_ascii=False) if isinstance(a.name, (dict, list)) else (a.name or ''), a.death_year_hijri or '', ) for a in authors ] cursor.executemany( "INSERT INTO authors (id, slug, name, death_year_hijri) VALUES (?, ?, ?, ?)", rows ) return len(rows) def _build_kunya_values(self, t): if not t: return [], '' kunya_items = list(t.kunyas.all()) if hasattr(t, 'kunyas') else [] if kunya_items: by_lang = {} for item in kunya_items: raw_title = getattr(item, 'title', None) if isinstance(raw_title, list): for entry in raw_title: if isinstance(entry, dict): lc = entry.get('language_code') or 'en' txt = (entry.get('text') or entry.get('title') or '').strip() if txt: by_lang.setdefault(lc, []).append(txt) if by_lang: multilingual = [ { 'language_code': lc, 'text': ('، ' if lc in ('ar', 'fa', 'ur') else ', ').join(vals), } for lc, vals in by_lang.items() ] ar_kunya = ('، '.join(by_lang.get('ar', []))) or get_arabic_localized_text(multilingual) or '' return multilingual, ar_kunya raw_kunya = getattr(t, 'kunya', None) ar_kunya = get_arabic_localized_text(raw_kunya) if raw_kunya else '' return (raw_kunya or []), (ar_kunya or '') def _serialize_narrator_person(self, t): if not t: return None rel = getattr(t, 'reliability', None) rel_dict = { 'id': rel.id, 'title': rel.title, 'slug': rel.slug, 'color': rel.color, 'main_color_code': rel.main_color_code, 'description': getattr(rel, 'description', None), } if rel else None kunya_val, arabic_kunya_val = self._build_kunya_values(t) arabic_full_name = get_arabic_localized_text(t.full_name) if t.full_name else '' arabic_known_as = get_arabic_localized_text(t.known_as) if getattr(t, 'known_as', None) else '' arabic_nickname = get_arabic_localized_text(t.nickname) if getattr(t, 'nickname', None) else '' return { 'id': t.id, 'slug': t.slug, 'name': t.full_name, 'title': t.full_name, 'full_name': t.full_name, 'arabic_full_name': arabic_full_name or '', 'known_as': getattr(t, 'known_as', None) or [], 'arabic_known_as': arabic_known_as or '', 'nickname': getattr(t, 'nickname', None) or [], 'arabic_nickname': arabic_nickname or '', 'kunya': kunya_val, 'arabic_kunya': arabic_kunya_val or '', 'gender': t.gender or '', 'birth_year_hijri': t.birth_year_hijri or '', 'death_year_hijri': t.death_year_hijri or '', 'birth_year_raw': t.birth_year_hijri or '', 'death_year_raw': t.death_year_hijri or '', 'reliability': rel_dict, } def _export_narrators(self, cursor): narrators = Transmitters.objects.select_related('reliability').prefetch_related('kunyas').all() rows = [] for t in narrators: p = self._serialize_narrator_person(t) name_str = json.dumps(p['full_name'], ensure_ascii=False) if isinstance(p['full_name'], (dict, list)) else (p['full_name'] or '') known_as_str = json.dumps(p['known_as'], ensure_ascii=False) if isinstance(p['known_as'], (dict, list)) else (p['known_as'] or '') nickname_str = json.dumps(p['nickname'], ensure_ascii=False) if isinstance(p['nickname'], (dict, list)) else (p['nickname'] or '') kunya_str = json.dumps(p['kunya'], ensure_ascii=False) if isinstance(p['kunya'], (dict, list)) else (p['kunya'] or '') rows.append(( t.id, t.slug, name_str, name_str, p['arabic_full_name'], known_as_str, p['arabic_known_as'], nickname_str, p['arabic_nickname'], kunya_str, p['arabic_kunya'], p['gender'], p['birth_year_hijri'], p['death_year_hijri'], json.dumps(p['reliability'], ensure_ascii=False), )) cursor.executemany( """INSERT INTO narrators (id, slug, name, title, arabic_full_name, known_as, arabic_known_as, nickname, arabic_nickname, kunya, arabic_kunya, gender, birth_year_hijri, death_year_hijri, reliability_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", rows ) return len(rows) def _export_statuses(self, cursor): statuses = HadisStatus.objects.all() rows = [ ( s.id, json.dumps(s.title, ensure_ascii=False) if isinstance(s.title, (dict, list)) else (s.title or ''), s.color or '', s.main_color_code or '', json.dumps(s.description, ensure_ascii=False) if isinstance(s.description, (dict, list)) else (s.description or ''), ) for s in statuses ] cursor.executemany( "INSERT INTO statuses (id, title, color, main_color_code, description) VALUES (?, ?, ?, ?, ?)", rows ) return len(rows) def _export_tags(self, cursor): tags = HadisTag.objects.all() rows = [ ( t.id, json.dumps(t.title, ensure_ascii=False) if isinstance(t.title, (dict, list)) else (t.title or ''), ) for t in tags ] cursor.executemany( "INSERT INTO tags (id, title) VALUES (?, ?)", rows ) return len(rows) def _make_full_url(self, url): if not url: return '' if url.startswith('http://') or url.startswith('https://'): return url if not url.startswith('/'): url = '/' + url return f'https://dovodi.newhorizonco.uk{url}' def _resolve_edition_for_ref(self, ref, edition_meta): by_id = edition_meta.get('by_id', {}) by_book_id = edition_meta.get('by_book_id', {}) first_by_book_id = edition_meta.get('first_by_book_id', {}) ed_id = getattr(ref, 'edition_id', None) if ed_id and ed_id in by_id: return by_id[ed_id] if getattr(ref, 'edition', None): return by_id.get(ref.edition.id, ref.edition) vol_obj = getattr(ref, 'book_volume', None) if vol_obj: bv_ed_id = getattr(vol_obj, 'edition_id', None) if bv_ed_id and bv_ed_id in by_id: return by_id[bv_ed_id] if getattr(vol_obj, 'edition', None): return by_id.get(vol_obj.edition.id, vol_obj.edition) book_id = getattr(ref, 'book_reference_id', None) if book_id: if book_id in first_by_book_id: return first_by_book_id[book_id] book_eds = by_book_id.get(book_id, []) if book_eds: return book_eds[0] return None def _serialize_contributor_obj(self, c): author_obj = getattr(c, 'author', None) author_data = { 'id': author_obj.id, 'slug': author_obj.slug, 'name': author_obj.name, } if author_obj else None role_obj = getattr(c, 'role', None) role_data = { 'id': role_obj.id, 'slug': role_obj.slug, 'title': role_obj.title, 'order': getattr(role_obj, 'order', 0), } if role_obj else None ret = { 'id': c.id, 'name': getattr(c, 'name', None) or (author_obj.name if author_obj else ''), 'slug': author_obj.slug if (author_obj and author_obj.slug) else getattr(c, 'slug', None), 'author_id': getattr(c, 'author_id', None), 'author': author_data, } if role_data is not None: ret['role_id'] = role_obj.id ret['role'] = role_data if hasattr(c, 'volumes'): vols = list(c.volumes.all()) ret['volume_ids'] = [v.id for v in vols] ret['volumes'] = [{'id': v.id, 'title': v.title} for v in vols] return ret def _serialize_edition(self, ed): if not ed: return None pub = ( ed.publisher_item.title if (getattr(ed, 'publisher_item_id', None) and getattr(ed, 'publisher_item', None) and ed.publisher_item.title) else getattr(ed, 'publisher', None) ) city = ( ed.city_of_publication_city.title if (getattr(ed, 'city_of_publication_city_id', None) and getattr(ed, 'city_of_publication_city', None) and ed.city_of_publication_city.title) else getattr(ed, 'city_of_publication', None) ) country = ( ed.country_of_publication_country.title if (getattr(ed, 'country_of_publication_country_id', None) and getattr(ed, 'country_of_publication_country', None) and ed.country_of_publication_country.title) else getattr(ed, 'country_of_publication', None) ) if hasattr(ed, '_cached_vol_count'): vol_count = ed._cached_vol_count elif hasattr(ed, '_prefetched_objects_cache') and 'volumes' in ed._prefetched_objects_cache: vol_count = len(ed.volumes.all()) elif hasattr(ed, 'volumes'): vol_count = ed.volumes.count() else: vol_count = 0 num_vols = vol_count if vol_count > 0 else getattr(ed, 'number_of_volumes', None) raw_researchers = ed._cached_researchers if hasattr(ed, '_cached_researchers') else (ed.researchers.all() if hasattr(ed, 'researchers') else []) raw_editors = ed._cached_editors if hasattr(ed, '_cached_editors') else (ed.editors.all() if hasattr(ed, 'editors') else []) raw_contributors = list(ed._cached_contributors if hasattr(ed, '_cached_contributors') else (ed.contributors.all() if hasattr(ed, 'contributors') else [])) raw_contributors.sort( key=lambda c: ( getattr(c.role, 'order', 999999) if getattr(c, 'role_id', None) and getattr(c, 'role', None) else 999999, c.id or 0, ) ) return { 'id': ed.id, 'title': getattr(ed, 'title', None) or '', 'language': getattr(ed, 'language', 'ar') or 'ar', 'edition_number': ed.edition_number, 'publisher': pub or '', 'publication_place': { 'city': city or '', 'country': country or '', }, 'year_of_publication': getattr(ed, 'year_of_publication', None) or '', 'number_of_volumes': num_vols, 'source_url': getattr(ed, 'source_url', None) or '', 'researchers': [self._serialize_contributor_obj(r) for r in raw_researchers], 'editors': [self._serialize_contributor_obj(e) for e in raw_editors], 'contributors': [self._serialize_contributor_obj(c) for c in raw_contributors], } def _serialize_reference(self, ref, edition_meta=None, book_image_map=None): if edition_meta is None: edition_meta = get_cached_edition_metadata() book = getattr(ref, 'book_reference', None) author = book.author if book and getattr(book, 'author', None) else None ed = self._resolve_reference_edition(ref, edition_meta) if hasattr(self, '_resolve_reference_edition') else self._resolve_edition_for_ref(ref, edition_meta) vol_obj = getattr(ref, 'book_volume', None) images_list = [] if hasattr(ref, 'images'): for img in sorted(ref.images.all(), key=lambda x: getattr(x, 'priority', 0)): img_file = getattr(img, 'image', None) or getattr(img, 'thumbnail', None) if img_file: img_url = self._make_full_url(img_file.url) thumb_file = getattr(img, 'thumbnail', None) or img_file thumb_url = self._make_full_url(thumb_file.url) images_list.append({ 'id': img.id, 'image': img_url, 'thumbnail': thumb_url, 'priority': getattr(img, 'priority', 0), }) edition_dict = self._serialize_edition(ed) book_dict = None if book: book_pub = getattr(book, 'publisher', None) if not book_pub and edition_dict: book_pub = edition_dict.get('publisher') book_year = getattr(book, 'year_of_publication', None) if not book_year and edition_dict: book_year = edition_dict.get('year_of_publication') book_num_vols = getattr(book, 'number_of_volumes', None) if book_num_vols is None and edition_dict: book_num_vols = edition_dict.get('number_of_volumes') if book_num_vols is None: book_num_vols = edition_meta.get('book_volumes_count', {}).get(book.id) book_img = book_image_map.get(book.id) if book_image_map else None if not book_img and vol_obj and getattr(vol_obj, 'image', None): book_img = self._make_full_url(vol_obj.image.url) book_dict = { 'id': book.id, 'slug': book.slug, 'title': book.title, 'image': book_img or '', 'publisher': book_pub or '', 'year_of_publication': book_year or '', 'number_of_volumes': book_num_vols, } author_dict = { 'id': author.id, 'slug': author.slug, 'name': author.name, 'death_year_hijri': getattr(author, 'death_year_hijri', None) or '', } if author else None book_authors = [{ 'id': author.id, 'slug': author.slug, 'name': author.name, 'death_year_hijri': getattr(author, 'death_year_hijri', None) or '', }] if author else [] volume_info = { 'id': vol_obj.id, 'title': vol_obj.title, } if vol_obj else None open_book_url = None if vol_obj and getattr(vol_obj, 'file', None): open_book_url = self._make_full_url(vol_obj.file.url) elif getattr(ref, 'url', None): open_book_url = getattr(ref, 'url', None) elif ed and getattr(ed, 'source_url', None): open_book_url = ed.source_url raw_vol = getattr(ref, 'volume', None) vol_val = ( raw_vol if (raw_vol and str(raw_vol).strip().lower() != 'none') else (vol_obj.title if (vol_obj and vol_obj.title) else None) ) raw_pg = getattr(ref, 'pages', None) pg_val = raw_pg if (raw_pg and str(raw_pg).strip().lower() != 'none') else None raw_num = getattr(ref, 'hadith_number', None) num_val = raw_num if (raw_num and str(raw_num).strip().lower() != 'none') else None parts = [] if vol_val: v_str = str(vol_val).strip() parts.append(v_str if v_str.startswith("جلد") else f"جلد {v_str}") if pg_val: p_str = str(pg_val).strip() parts.append(p_str if (p_str.startswith("ص") or p_str.startswith("صفحه")) else f"صفحه {p_str}") if num_val: n_str = str(num_val).strip() parts.append(n_str if (n_str.startswith("ح") or n_str.startswith("حدیث") or n_str.startswith("شماره")) else f"شماره حدیث {n_str}") citation_summary = " • ".join(parts) if parts else "" return { 'id': ref.id, 'title': getattr(book, 'title', None) if book else None, 'authors': book_authors, 'share_link': getattr(book, 'share_link', None) if book else None, 'book': book_dict, 'author': author_dict, 'edition': edition_dict, 'edition_info': edition_dict, 'volume_info': volume_info, 'hadith_number': num_val, 'volume': vol_val, 'pages': pg_val, 'address': getattr(ref, 'address', None), 'url': getattr(ref, 'url', None), 'open_book_url': open_book_url, 'citation_summary': citation_summary, 'images': images_list, 'images_count': len(images_list), 'book_title': getattr(book, 'title', None) if book else None, 'book_authors': book_authors, } def _export_category_interpretations(self, cursor, book_image_map=None): edition_meta = get_cached_edition_metadata() interps = HadisInterpretation.objects.select_related('category').prefetch_related( 'references__book_reference__author', 'references__images', 'references__edition', 'references__book_volume', ).all() rows = [] for inp in interps: refs_list = [self._serialize_reference(ref, edition_meta, book_image_map) for ref in inp.references.all()] rows.append(( inp.id, inp.category_id, json.dumps(inp.title, ensure_ascii=False) if isinstance(inp.title, (dict, list)) else (inp.title or ''), inp.slug, inp.narrator or '', inp.text or '', json.dumps(inp.translation, ensure_ascii=False) if isinstance(inp.translation, (dict, list)) else (inp.translation or ''), json.dumps(refs_list, ensure_ascii=False), json.dumps(format_links_output(inp.links), ensure_ascii=False), )) cursor.executemany( """INSERT INTO category_interpretations (id, category_id, title, slug, narrator, text, translation, references_json, links_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", rows ) return len(rows) def _export_hadiths_and_corrections(self, cursor, cat_info_map=None, book_image_map=None): edition_meta = get_cached_edition_metadata() cat_info_map = cat_info_map or {} hadiths = ( Hadis.objects .filter(status=True) .select_related('category', 'category__sect', 'hadis_status') .prefetch_related( 'categories__sect', 'tags', 'narrator_layers', 'references__book_reference__author', 'references__edition', 'references__book_volume', 'references__images', 'transmitters__status', 'transmitters__transmitter__reliability', 'transmitters__transmitter__kunyas', 'transmitters__uncertain_transmitters__reliability', 'transmitters__uncertain_transmitters__kunyas', 'transmitters__narrator_layer', 'hadiscorrection_set__references__book_reference__author', 'hadiscorrection_set__references__edition', 'hadiscorrection_set__references__book_volume', 'hadiscorrection_set__references__images', ) .order_by('id') ) hadis_rows = [] correction_rows = [] hadith_category_rows = [] for h in hadiths: # 1. Status Block status_id = h.hadis_status.id if h.hadis_status else None status_title = h.hadis_status.title if h.hadis_status else None status_color = h.hadis_status.color if h.hadis_status else None status_code = h.hadis_status.main_color_code if h.hadis_status else None # 2. Tags tags_list = [{'id': t.id, 'title': t.title} for t in h.tags.all()] # 3. References & Reference Images refs_list = [] ref_imgs_list = [] for ref in h.references.all(): ref_data = self._serialize_reference(ref, edition_meta, book_image_map) refs_list.append(ref_data) if ref_data.get('images'): ref_imgs_list.extend(ref_data['images']) # 4. Narrator Layers & Narrators sorted_layers = sorted( h.narrator_layers.all(), key=lambda l: (l.number if l.number is not None else 0, l.id or 0), ) layer_names = [ { 'id': layer.id, 'number': layer.number, 'slug': layer.slug, 'name': layer.name or [], 'description': layer.description or [], } for layer in sorted_layers ] sorted_transmitters = sorted( h.transmitters.all(), key=lambda item: ( getattr(item, 'chain_index', 0) or 0, item.order if item.order is not None else 0, item.id or 0, ), ) transmitters_data = [] for tr in sorted_transmitters: layer_obj = tr.narrator_layer uncertain_list = [] if tr.is_uncertain: for ut in tr.uncertain_transmitters.all(): uncertain_list.append(self._serialize_narrator_person(ut)) has_uncertain = bool(tr.is_uncertain and len(uncertain_list) > 0) t = None if has_uncertain else tr.transmitter person_dict = self._serialize_narrator_person(t) if t else None tr_status = getattr(tr, 'status', None) tr_status_dict = { 'id': tr_status.id, 'title': tr_status.title, 'slug': tr_status.slug, 'color': tr_status.color, 'main_color_code': tr_status.main_color_code, 'description': getattr(tr_status, 'description', None), } if tr_status else None transmitters_data.append({ 'id': person_dict['id'] if person_dict else None, 'hadis_transmitter_id': tr.id, 'order': tr.order, 'chain_index': getattr(tr, 'chain_index', 0), 'is_gap': bool(tr.is_gap), 'is_uncertain': bool(tr.is_uncertain), 'layer': layer_obj.slug if layer_obj else None, 'layer_slug': layer_obj.slug if layer_obj else None, 'layer_id': tr.narrator_layer_id, 'narrator_layer_id': tr.narrator_layer_id, 'layer_level': layer_obj.number if layer_obj else None, 'layer_name': layer_obj.name if layer_obj else None, 'name': person_dict['name'] if person_dict else None, 'title': person_dict['title'] if person_dict else None, 'full_name': person_dict['full_name'] if person_dict else None, 'arabic_full_name': person_dict['arabic_full_name'] if person_dict else '', 'slug': person_dict['slug'] if person_dict else None, 'known_as': person_dict['known_as'] if person_dict else [], 'arabic_known_as': person_dict['arabic_known_as'] if person_dict else '', 'nickname': person_dict['nickname'] if person_dict else [], 'arabic_nickname': person_dict['arabic_nickname'] if person_dict else '', 'kunya': person_dict['kunya'] if person_dict else [], 'arabic_kunya': person_dict['arabic_kunya'] if person_dict else '', 'gender': person_dict['gender'] if person_dict else '', 'birth_year_hijri': person_dict['birth_year_hijri'] if person_dict else '', 'death_year_hijri': person_dict['death_year_hijri'] if person_dict else '', 'birth_year_raw': person_dict['birth_year_raw'] if person_dict else '', 'death_year_raw': person_dict['death_year_raw'] if person_dict else '', 'reliability': person_dict['reliability'] if person_dict else None, 'uncertain_transmitters': uncertain_list, 'status': tr_status_dict, }) # 5. Explanations explanations_data = [] if hasattr(h, 'explanations') and h.explanations: for exp in h.explanations.all(): explanations_data.append({ 'title': exp.title, 'detail': getattr(exp, 'detail', '') }) # 5.5 Multi-category M2M relations (plus primary category) & source_type / sect_type seen_cat_ids = set() hadith_cats_list = [] def add_cat_entry(cat_obj): if not cat_obj or cat_obj.id in seen_cat_ids: return seen_cat_ids.add(cat_obj.id) c_info = cat_info_map.get(cat_obj.id) if not c_info: c_source = getattr(cat_obj, 'source_type', None) or 'hadith' c_sect = ( cat_obj.sect.sect_type if (getattr(cat_obj, 'sect_id', None) and getattr(cat_obj, 'sect', None)) else 'shia' ) c_info = { 'id': cat_obj.id, 'slug': cat_obj.slug, 'title': cat_obj.title, 'source_type': c_source, 'sect_type': c_sect, } hadith_cats_list.append(c_info) hadith_category_rows.append(( h.id, cat_obj.id, cat_obj.slug, c_info['source_type'], c_info['sect_type'], )) if h.category: add_cat_entry(h.category) for cat in h.categories.all(): add_cat_entry(cat) primary_cat_info = hadith_cats_list[0] if hadith_cats_list else None hadis_source_type = primary_cat_info['source_type'] if primary_cat_info else 'hadith' hadis_sect_type = primary_cat_info['sect_type'] if primary_cat_info else 'shia' primary_cat_id = h.category_id or (primary_cat_info['id'] if primary_cat_info else None) primary_cat_slug = (h.category.slug if h.category else None) or (primary_cat_info['slug'] if primary_cat_info else None) # Status description fallback: use hadis_status_text if present and non-empty, otherwise fallback to hadis_status.description status_desc_raw = h.hadis_status_text if self._is_empty_localized_data(status_desc_raw) and h.hadis_status: status_desc_raw = h.hadis_status.description status_text_val = json.dumps(status_desc_raw, ensure_ascii=False) if isinstance(status_desc_raw, (dict, list)) else (status_desc_raw or '') hadis_rows.append(( h.id, h.slug, primary_cat_id, primary_cat_slug, hadis_source_type, hadis_sect_type, json.dumps(hadith_cats_list, ensure_ascii=False), json.dumps(h.title, ensure_ascii=False) if isinstance(h.title, (dict, list)) else (h.title or ''), json.dumps(h.title_narrator, ensure_ascii=False) if isinstance(h.title_narrator, (dict, list)) else (h.title_narrator or ''), h.text or '', json.dumps(h.translation or [], ensure_ascii=False), json.dumps(h.address, ensure_ascii=False) if isinstance(h.address, (dict, list)) else (h.address or ''), h.share_link or '', status_id, json.dumps(status_title, ensure_ascii=False) if isinstance(status_title, (dict, list)) else (status_title or ''), status_color or '', status_code or '', status_text_val, json.dumps(tags_list, ensure_ascii=False), json.dumps(refs_list, ensure_ascii=False), json.dumps(ref_imgs_list, ensure_ascii=False), json.dumps( { 'layer_count': len(layer_names), 'layer_names': layer_names, 'transmitters': transmitters_data, }, ensure_ascii=False, ), json.dumps(explanations_data, ensure_ascii=False), json.dumps(format_links_output(h.links), ensure_ascii=False), )) # 6. Corrections for corr in h.hadiscorrection_set.all(): corr_imgs = [] corr_refs = [] for cref in corr.references.all(): ref_data = self._serialize_reference(cref, edition_meta, book_image_map) corr_refs.append(ref_data) if ref_data.get('images'): corr_imgs.extend(ref_data['images']) corr_address = corr_refs[0].get('address') or '' if corr_refs else '' correction_rows.append(( corr.id, h.id, json.dumps(corr.title, ensure_ascii=False) if isinstance(corr.title, (dict, list)) else (corr.title or ''), corr.slug, corr.narrator or '', corr.text or '', json.dumps(corr.translation, ensure_ascii=False) if isinstance(corr.translation, (dict, list)) else (corr.translation or ''), corr.share_link or '', corr_address, json.dumps(corr_imgs, ensure_ascii=False), json.dumps(corr_refs, ensure_ascii=False), json.dumps(format_links_output(corr.links), ensure_ascii=False), )) cursor.executemany( """INSERT INTO hadiths ( id, slug, category_id, category_slug, source_type, sect_type, categories_json, title, title_narrator, text, translation_json, address, share_link, hadis_status_id, status_title, status_color, status_main_color_code, status_text, tags_json, references_json, reference_images_json, narrators_json, explanations_json, links_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", hadis_rows ) cursor.executemany( """INSERT OR IGNORE INTO hadith_categories ( hadis_id, category_id, category_slug, source_type, sect_type ) VALUES (?, ?, ?, ?, ?)""", hadith_category_rows ) cursor.executemany( """INSERT INTO hadith_corrections ( id, hadis_id, title, slug, narrator, description, translation, share_link, address, images_json, references_json, links_json ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", correction_rows ) return len(hadis_rows), len(correction_rows)