You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

1040 lines
42 KiB

import os
import gzip
import json
import sqlite3
import hashlib
from pathlib import Path
from django.core.management.base import BaseCommand
from django.conf import settings
from django.db import transaction
from apps.hadis.models import (
Hadis,
HadisCategory,
HadisInterpretation,
HadisCorrection,
HadisDatabaseVersion,
HadisStatus,
HadisTag,
BookReference,
BookAuthor,
Transmitters,
)
from apps.hadis.serializers.hadis import (
DetailedCorrectionReferenceSerializer,
DetailedInterpretationReferenceSerializer,
get_cached_edition_metadata,
)
from apps.hadis.utils.links import format_links_output
class Command(BaseCommand):
help = "Export Hadis and Categories into an optimized, compressed SQLite database for offline mobile sync."
def add_arguments(self, parser):
parser.add_argument(
'--notes',
type=str,
default='',
help='Release notes for this SQLite database version'
)
parser.add_argument(
'--force-version',
type=int,
default=None,
help='Force a specific version number'
)
def handle(self, *args, **options):
self.stdout.write(self.style.SUCCESS("Starting Hadis SQLite database generation (Single-File Architecture)..."))
# 1. Determine destination and version number
latest_ver = HadisDatabaseVersion.objects.order_by('-version').first()
if options['force_version'] is not None:
new_version = options['force_version']
else:
new_version = (latest_ver.version + 1) if latest_ver else 1
output_dir = Path(settings.MEDIA_ROOT) / 'hadis_db'
output_dir.mkdir(parents=True, exist_ok=True)
temp_db_path = output_dir / "temp_hadis_build.sqlite"
final_gz_name = "hadis_offline.sqlite.gz"
final_gz_path = output_dir / final_gz_name
if temp_db_path.exists():
temp_db_path.unlink()
# 2. Initialize SQLite database
conn = sqlite3.connect(str(temp_db_path))
cursor = conn.cursor()
# Optimize SQLite for bulk writes
cursor.execute("PRAGMA synchronous = OFF;")
cursor.execute("PRAGMA journal_mode = MEMORY;")
cursor.execute("PRAGMA encoding = 'UTF-8';")
self.stdout.write("Creating SQLite tables and indexes...")
self._create_tables(cursor)
# 3. Export data
categories_count, cat_info_map = self._export_categories(cursor)
sources_count, book_image_map = self._export_sources(cursor)
authors_count = self._export_authors(cursor)
narrators_count = self._export_narrators(cursor)
statuses_count = self._export_statuses(cursor)
tags_count = self._export_tags(cursor)
interp_count = self._export_category_interpretations(cursor, book_image_map)
hadis_count, corr_count = self._export_hadiths_and_corrections(cursor, cat_info_map, book_image_map)
conn.commit()
# Create indexes after insertion for maximum performance
self.stdout.write("Building indexes...")
self._create_indexes(cursor)
conn.commit()
conn.close()
# 4. Gzip compress directly to the single target file
self.stdout.write(f"Compressing database to single file: {final_gz_name}...")
hasher = hashlib.md5()
temp_gz_path = output_dir / "temp_hadis_build.sqlite.gz"
with open(temp_db_path, 'rb') as f_in:
with gzip.open(temp_gz_path, 'wb', compresslevel=9) as f_out:
while chunk := f_in.read(1024 * 1024):
f_out.write(chunk)
hasher.update(chunk)
checksum = hasher.hexdigest()
temp_size = temp_db_path.stat().st_size
# Atomic replacement: replace final_gz_path with temp_gz_path
if final_gz_path.exists():
final_gz_path.unlink()
temp_gz_path.rename(final_gz_path)
file_size = final_gz_path.stat().st_size
# Clean up any remaining temp files in output_dir
temp_db_path.unlink(missing_ok=True)
for extra_file in output_dir.iterdir():
if extra_file.name != final_gz_name:
try:
extra_file.unlink(missing_ok=True)
except OSError:
pass
# 5. Record version in database (Single Active Record)
with transaction.atomic():
# Delete old records to keep DB clean
HadisDatabaseVersion.objects.all().delete()
db_version = HadisDatabaseVersion.objects.create(
version=new_version,
file=f"hadis_db/{final_gz_name}",
file_size=file_size,
checksum_md5=checksum,
hadis_count=hadis_count,
categories_count=categories_count,
is_active=True,
notes=options['notes'],
)
self.stdout.write(self.style.SUCCESS(
f"Successfully updated single Hadis SQLite DB to v{new_version}!\n"
f" - Uncompressed size: {temp_size / (1024*1024):.2f} MB\n"
f" - Gzipped size: {file_size / (1024*1024):.2f} MB\n"
f" - Hadiths: {hadis_count}\n"
f" - Categories: {categories_count}\n"
f" - Sources: {sources_count}\n"
f" - Authors: {authors_count}\n"
f" - Narrators: {narrators_count}\n"
f" - Statuses: {statuses_count}\n"
f" - Tags: {tags_count}\n"
f" - Interpretations: {interp_count}\n"
f" - Corrections: {corr_count}\n"
f" - MD5 Checksum: {checksum}\n"
f" - File path: {final_gz_path}"
))
def _create_tables(self, cursor):
cursor.executescript("""
CREATE TABLE IF NOT EXISTS categories (
id INTEGER PRIMARY KEY,
title TEXT,
slug TEXT UNIQUE,
source_type TEXT,
sect_type TEXT,
is_topic INTEGER DEFAULT 0
);
CREATE TABLE IF NOT EXISTS hadith_categories (
hadis_id INTEGER,
category_id INTEGER,
category_slug TEXT,
source_type TEXT,
sect_type TEXT,
PRIMARY KEY (hadis_id, category_id),
FOREIGN KEY (hadis_id) REFERENCES hadiths (id),
FOREIGN KEY (category_id) REFERENCES categories (id)
);
CREATE TABLE IF NOT EXISTS sources (
id INTEGER PRIMARY KEY,
slug TEXT UNIQUE,
title TEXT,
image TEXT,
number_of_volumes INTEGER,
authors_json TEXT
);
CREATE TABLE IF NOT EXISTS authors (
id INTEGER PRIMARY KEY,
slug TEXT UNIQUE,
name TEXT,
death_year_hijri TEXT
);
CREATE TABLE IF NOT EXISTS narrators (
id INTEGER PRIMARY KEY,
slug TEXT UNIQUE,
name TEXT,
title TEXT,
gender TEXT,
birth_year_hijri TEXT,
death_year_hijri TEXT,
reliability_json TEXT
);
CREATE TABLE IF NOT EXISTS statuses (
id INTEGER PRIMARY KEY,
title TEXT,
color TEXT,
main_color_code TEXT,
description TEXT
);
CREATE TABLE IF NOT EXISTS tags (
id INTEGER PRIMARY KEY,
title TEXT
);
CREATE TABLE IF NOT EXISTS category_interpretations (
id INTEGER PRIMARY KEY,
category_id INTEGER,
title TEXT,
slug TEXT,
narrator TEXT,
text TEXT,
translation TEXT,
references_json TEXT,
links_json TEXT,
FOREIGN KEY (category_id) REFERENCES categories (id)
);
CREATE TABLE IF NOT EXISTS hadiths (
id INTEGER PRIMARY KEY,
slug TEXT UNIQUE,
category_id INTEGER,
category_slug TEXT,
source_type TEXT,
sect_type TEXT,
categories_json TEXT,
title TEXT,
title_narrator TEXT,
text TEXT,
translation_json TEXT,
address TEXT,
share_link TEXT,
hadis_status_id INTEGER,
status_title TEXT,
status_color TEXT,
status_main_color_code TEXT,
status_text TEXT,
tags_json TEXT,
references_json TEXT,
reference_images_json TEXT,
narrators_json TEXT,
explanations_json TEXT,
links_json TEXT,
FOREIGN KEY (category_id) REFERENCES categories (id)
);
CREATE TABLE IF NOT EXISTS hadith_corrections (
id INTEGER PRIMARY KEY,
hadis_id INTEGER,
title TEXT,
slug TEXT,
narrator TEXT,
description TEXT,
translation TEXT,
share_link TEXT,
address TEXT,
images_json TEXT,
references_json TEXT,
links_json TEXT,
FOREIGN KEY (hadis_id) REFERENCES hadiths (id)
);
""")
def _create_indexes(self, cursor):
cursor.executescript("""
CREATE INDEX IF NOT EXISTS idx_cat_slug ON categories(slug);
CREATE INDEX IF NOT EXISTS idx_cat_source_type ON categories(source_type);
CREATE INDEX IF NOT EXISTS idx_cat_sect_type ON categories(sect_type);
CREATE INDEX IF NOT EXISTS idx_interp_cat ON category_interpretations(category_id);
CREATE INDEX IF NOT EXISTS idx_hadith_cat ON hadiths(category_id);
CREATE INDEX IF NOT EXISTS idx_hadith_slug ON hadiths(slug);
CREATE INDEX IF NOT EXISTS idx_hadith_source_type ON hadiths(source_type);
CREATE INDEX IF NOT EXISTS idx_hadith_sect_type ON hadiths(sect_type);
CREATE INDEX IF NOT EXISTS idx_hadith_status_id ON hadiths(hadis_status_id);
CREATE INDEX IF NOT EXISTS idx_corr_hadis ON hadith_corrections(hadis_id);
CREATE INDEX IF NOT EXISTS idx_hcat_cat_id ON hadith_categories(category_id);
CREATE INDEX IF NOT EXISTS idx_hcat_cat_slug ON hadith_categories(category_slug);
CREATE INDEX IF NOT EXISTS idx_hcat_hadis_id ON hadith_categories(hadis_id);
CREATE INDEX IF NOT EXISTS idx_sources_slug ON sources(slug);
CREATE INDEX IF NOT EXISTS idx_authors_slug ON authors(slug);
CREATE INDEX IF NOT EXISTS idx_narrators_slug ON narrators(slug);
""")
def _is_empty_localized_data(self, val):
if not val:
return True
if isinstance(val, (list, tuple)):
if len(val) == 0:
return True
for item in val:
if isinstance(item, dict):
txt = item.get('text') or item.get('title') or item.get('description') or ''
if str(txt).strip():
return False
elif str(item).strip():
return False
return True
if isinstance(val, dict):
if not val:
return True
for k, v in val.items():
if str(v).strip():
return False
return True
if isinstance(val, str) and not val.strip():
return True
return False
def _export_categories(self, cursor):
cats = list(HadisCategory.objects.select_related('sect').all())
by_id = {c.id: c for c in cats}
def resolve_source_type(cat_obj):
cur = cat_obj
visited = set()
while cur and cur.id not in visited:
visited.add(cur.id)
st = getattr(cur, 'source_type', None)
if st:
return st
cur = by_id.get(getattr(cur, 'parent_id', None))
return 'hadith'
def resolve_sect_type(cat_obj):
cur = cat_obj
visited = set()
while cur and cur.id not in visited:
visited.add(cur.id)
if getattr(cur, 'sect_id', None) and getattr(cur, 'sect', None) and getattr(cur.sect, 'sect_type', None):
return cur.sect.sect_type
st = getattr(cur, 'sect_type', None)
if st:
return st
cur = by_id.get(getattr(cur, 'parent_id', None))
return 'shia'
rows = []
cat_info_map = {}
for c in cats:
source_type = resolve_source_type(c)
sect_type = resolve_sect_type(c)
cat_info_map[c.id] = {
'id': c.id,
'slug': c.slug,
'title': c.title,
'source_type': source_type,
'sect_type': sect_type,
}
rows.append((
c.id,
json.dumps(c.title, ensure_ascii=False) if isinstance(c.title, (dict, list)) else (c.title or ''),
c.slug,
source_type,
sect_type,
1 if getattr(c, 'is_topic', False) else 0,
))
cursor.executemany(
"INSERT INTO categories (id, title, slug, source_type, sect_type, is_topic) VALUES (?, ?, ?, ?, ?, ?)",
rows
)
return len(rows), cat_info_map
def _export_sources(self, cursor):
edition_meta = get_cached_edition_metadata()
first_by_book_id = edition_meta.get('first_by_book_id', {})
by_book_id = edition_meta.get('by_book_id', {})
book_volumes_count = edition_meta.get('book_volumes_count', {})
books = list(
BookReference.objects
.select_related('author')
.prefetch_related('images', 'volumes')
.all()
)
rows = []
book_image_map = {}
for b in books:
img_url = None
b_imgs = list(b.images.all())
if b_imgs and getattr(b_imgs[0], 'image', None):
img_url = self._make_full_url(b_imgs[0].image.url)
if not img_url:
for v in b.volumes.all():
if getattr(v, 'image', None):
img_url = self._make_full_url(v.image.url)
break
book_image_map[b.id] = img_url
num_vols = getattr(b, 'number_of_volumes', None)
if num_vols is None:
ed = first_by_book_id.get(b.id) or (by_book_id.get(b.id, [None])[0])
if ed:
vol_cnt = getattr(ed, '_cached_vol_count', 0)
num_vols = vol_cnt if vol_cnt > 0 else getattr(ed, 'number_of_volumes', None)
if num_vols is None:
num_vols = book_volumes_count.get(b.id) or len(b.volumes.all()) or None
authors_list = []
if b.author:
authors_list.append({
'id': b.author.id,
'slug': b.author.slug,
'name': b.author.name,
'death_year_hijri': b.author.death_year_hijri or '',
})
rows.append((
b.id,
b.slug,
json.dumps(b.title, ensure_ascii=False) if isinstance(b.title, (dict, list)) else (b.title or ''),
img_url or '',
num_vols,
json.dumps(authors_list, ensure_ascii=False),
))
cursor.executemany(
"INSERT INTO sources (id, slug, title, image, number_of_volumes, authors_json) VALUES (?, ?, ?, ?, ?, ?)",
rows
)
return len(rows), book_image_map
def _export_authors(self, cursor):
authors = BookAuthor.objects.all()
rows = [
(
a.id,
a.slug,
json.dumps(a.name, ensure_ascii=False) if isinstance(a.name, (dict, list)) else (a.name or ''),
a.death_year_hijri or '',
)
for a in authors
]
cursor.executemany(
"INSERT INTO authors (id, slug, name, death_year_hijri) VALUES (?, ?, ?, ?)",
rows
)
return len(rows)
def _export_narrators(self, cursor):
narrators = Transmitters.objects.select_related('reliability').all()
rows = []
for t in narrators:
rel = t.reliability
rel_dict = {
'id': rel.id,
'title': rel.title,
'slug': rel.slug,
'color': rel.color,
'main_color_code': rel.main_color_code,
'description': getattr(rel, 'description', None),
} if rel else None
name_str = json.dumps(t.full_name, ensure_ascii=False) if isinstance(t.full_name, (dict, list)) else (t.full_name or '')
rows.append((
t.id,
t.slug,
name_str,
name_str,
t.gender or '',
t.birth_year_hijri or '',
t.death_year_hijri or '',
json.dumps(rel_dict, ensure_ascii=False),
))
cursor.executemany(
"""INSERT INTO narrators
(id, slug, name, title, gender, birth_year_hijri, death_year_hijri, reliability_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
rows
)
return len(rows)
def _export_statuses(self, cursor):
statuses = HadisStatus.objects.all()
rows = [
(
s.id,
json.dumps(s.title, ensure_ascii=False) if isinstance(s.title, (dict, list)) else (s.title or ''),
s.color or '',
s.main_color_code or '',
json.dumps(s.description, ensure_ascii=False) if isinstance(s.description, (dict, list)) else (s.description or ''),
)
for s in statuses
]
cursor.executemany(
"INSERT INTO statuses (id, title, color, main_color_code, description) VALUES (?, ?, ?, ?, ?)",
rows
)
return len(rows)
def _export_tags(self, cursor):
tags = HadisTag.objects.all()
rows = [
(
t.id,
json.dumps(t.title, ensure_ascii=False) if isinstance(t.title, (dict, list)) else (t.title or ''),
)
for t in tags
]
cursor.executemany(
"INSERT INTO tags (id, title) VALUES (?, ?)",
rows
)
return len(rows)
def _make_full_url(self, url):
if not url:
return ''
if url.startswith('http://') or url.startswith('https://'):
return url
if not url.startswith('/'):
url = '/' + url
return f'https://dovodi.newhorizonco.uk{url}'
def _resolve_edition_for_ref(self, ref, edition_meta):
by_id = edition_meta.get('by_id', {})
by_book_id = edition_meta.get('by_book_id', {})
first_by_book_id = edition_meta.get('first_by_book_id', {})
ed_id = getattr(ref, 'edition_id', None)
if ed_id and ed_id in by_id:
return by_id[ed_id]
if getattr(ref, 'edition', None):
return by_id.get(ref.edition.id, ref.edition)
vol_obj = getattr(ref, 'book_volume', None)
if vol_obj:
bv_ed_id = getattr(vol_obj, 'edition_id', None)
if bv_ed_id and bv_ed_id in by_id:
return by_id[bv_ed_id]
if getattr(vol_obj, 'edition', None):
return by_id.get(vol_obj.edition.id, vol_obj.edition)
book_id = getattr(ref, 'book_reference_id', None)
if book_id:
if book_id in first_by_book_id:
return first_by_book_id[book_id]
book_eds = by_book_id.get(book_id, [])
if book_eds:
return book_eds[0]
return None
def _serialize_contributor_obj(self, c):
author_obj = getattr(c, 'author', None)
author_data = {
'id': author_obj.id,
'slug': author_obj.slug,
'name': author_obj.name,
} if author_obj else None
role_obj = getattr(c, 'role', None)
role_data = {
'id': role_obj.id,
'slug': role_obj.slug,
'title': role_obj.title,
'order': getattr(role_obj, 'order', 0),
} if role_obj else None
ret = {
'id': c.id,
'name': getattr(c, 'name', None) or (author_obj.name if author_obj else ''),
'slug': author_obj.slug if (author_obj and author_obj.slug) else getattr(c, 'slug', None),
'author_id': getattr(c, 'author_id', None),
'author': author_data,
}
if role_data is not None:
ret['role_id'] = role_obj.id
ret['role'] = role_data
if hasattr(c, 'volumes'):
vols = list(c.volumes.all())
ret['volume_ids'] = [v.id for v in vols]
ret['volumes'] = [{'id': v.id, 'title': v.title} for v in vols]
return ret
def _serialize_edition(self, ed):
if not ed:
return None
pub = (
ed.publisher_item.title
if (getattr(ed, 'publisher_item_id', None) and getattr(ed, 'publisher_item', None) and ed.publisher_item.title)
else getattr(ed, 'publisher', None)
)
city = (
ed.city_of_publication_city.title
if (getattr(ed, 'city_of_publication_city_id', None) and getattr(ed, 'city_of_publication_city', None) and ed.city_of_publication_city.title)
else getattr(ed, 'city_of_publication', None)
)
country = (
ed.country_of_publication_country.title
if (getattr(ed, 'country_of_publication_country_id', None) and getattr(ed, 'country_of_publication_country', None) and ed.country_of_publication_country.title)
else getattr(ed, 'country_of_publication', None)
)
if hasattr(ed, '_cached_vol_count'):
vol_count = ed._cached_vol_count
elif hasattr(ed, '_prefetched_objects_cache') and 'volumes' in ed._prefetched_objects_cache:
vol_count = len(ed.volumes.all())
elif hasattr(ed, 'volumes'):
vol_count = ed.volumes.count()
else:
vol_count = 0
num_vols = vol_count if vol_count > 0 else getattr(ed, 'number_of_volumes', None)
raw_researchers = ed._cached_researchers if hasattr(ed, '_cached_researchers') else (ed.researchers.all() if hasattr(ed, 'researchers') else [])
raw_editors = ed._cached_editors if hasattr(ed, '_cached_editors') else (ed.editors.all() if hasattr(ed, 'editors') else [])
raw_contributors = list(ed._cached_contributors if hasattr(ed, '_cached_contributors') else (ed.contributors.all() if hasattr(ed, 'contributors') else []))
raw_contributors.sort(
key=lambda c: (
getattr(c.role, 'order', 999999) if getattr(c, 'role_id', None) and getattr(c, 'role', None) else 999999,
c.id or 0,
)
)
return {
'id': ed.id,
'title': getattr(ed, 'title', None) or '',
'language': getattr(ed, 'language', 'ar') or 'ar',
'edition_number': ed.edition_number,
'publisher': pub or '',
'publication_place': {
'city': city or '',
'country': country or '',
},
'year_of_publication': getattr(ed, 'year_of_publication', None) or '',
'number_of_volumes': num_vols,
'source_url': getattr(ed, 'source_url', None) or '',
'researchers': [self._serialize_contributor_obj(r) for r in raw_researchers],
'editors': [self._serialize_contributor_obj(e) for e in raw_editors],
'contributors': [self._serialize_contributor_obj(c) for c in raw_contributors],
}
def _serialize_reference(self, ref, edition_meta=None, book_image_map=None):
if edition_meta is None:
edition_meta = get_cached_edition_metadata()
book = getattr(ref, 'book_reference', None)
author = book.author if book and getattr(book, 'author', None) else None
ed = self._resolve_reference_edition(ref, edition_meta) if hasattr(self, '_resolve_reference_edition') else self._resolve_edition_for_ref(ref, edition_meta)
vol_obj = getattr(ref, 'book_volume', None)
images_list = []
if hasattr(ref, 'images'):
for img in sorted(ref.images.all(), key=lambda x: getattr(x, 'priority', 0)):
img_file = getattr(img, 'image', None) or getattr(img, 'thumbnail', None)
if img_file:
img_url = self._make_full_url(img_file.url)
thumb_file = getattr(img, 'thumbnail', None) or img_file
thumb_url = self._make_full_url(thumb_file.url)
images_list.append({
'id': img.id,
'image': img_url,
'thumbnail': thumb_url,
'priority': getattr(img, 'priority', 0),
})
edition_dict = self._serialize_edition(ed)
book_dict = None
if book:
book_pub = getattr(book, 'publisher', None)
if not book_pub and edition_dict:
book_pub = edition_dict.get('publisher')
book_year = getattr(book, 'year_of_publication', None)
if not book_year and edition_dict:
book_year = edition_dict.get('year_of_publication')
book_num_vols = getattr(book, 'number_of_volumes', None)
if book_num_vols is None and edition_dict:
book_num_vols = edition_dict.get('number_of_volumes')
if book_num_vols is None:
book_num_vols = edition_meta.get('book_volumes_count', {}).get(book.id)
book_img = book_image_map.get(book.id) if book_image_map else None
if not book_img and vol_obj and getattr(vol_obj, 'image', None):
book_img = self._make_full_url(vol_obj.image.url)
book_dict = {
'id': book.id,
'slug': book.slug,
'title': book.title,
'image': book_img or '',
'publisher': book_pub or '',
'year_of_publication': book_year or '',
'number_of_volumes': book_num_vols,
}
author_dict = {
'id': author.id,
'slug': author.slug,
'name': author.name,
'death_year_hijri': getattr(author, 'death_year_hijri', None) or '',
} if author else None
book_authors = [{
'id': author.id,
'slug': author.slug,
'name': author.name,
'death_year_hijri': getattr(author, 'death_year_hijri', None) or '',
}] if author else []
volume_info = {
'id': vol_obj.id,
'title': vol_obj.title,
} if vol_obj else None
open_book_url = None
if vol_obj and getattr(vol_obj, 'file', None):
open_book_url = self._make_full_url(vol_obj.file.url)
elif getattr(ref, 'url', None):
open_book_url = getattr(ref, 'url', None)
elif ed and getattr(ed, 'source_url', None):
open_book_url = ed.source_url
raw_vol = getattr(ref, 'volume', None)
vol_val = (
raw_vol
if (raw_vol and str(raw_vol).strip().lower() != 'none')
else (vol_obj.title if (vol_obj and vol_obj.title) else None)
)
raw_pg = getattr(ref, 'pages', None)
pg_val = raw_pg if (raw_pg and str(raw_pg).strip().lower() != 'none') else None
raw_num = getattr(ref, 'hadith_number', None)
num_val = raw_num if (raw_num and str(raw_num).strip().lower() != 'none') else None
parts = []
if vol_val:
v_str = str(vol_val).strip()
parts.append(v_str if v_str.startswith("جلد") else f"جلد {v_str}")
if pg_val:
p_str = str(pg_val).strip()
parts.append(p_str if (p_str.startswith("ص") or p_str.startswith("صفحه")) else f"صفحه {p_str}")
if num_val:
n_str = str(num_val).strip()
parts.append(n_str if (n_str.startswith("ح") or n_str.startswith("حدیث") or n_str.startswith("شماره")) else f"شماره حدیث {n_str}")
citation_summary = " • ".join(parts) if parts else ""
return {
'id': ref.id,
'title': getattr(book, 'title', None) if book else None,
'authors': book_authors,
'share_link': getattr(book, 'share_link', None) if book else None,
'book': book_dict,
'author': author_dict,
'edition': edition_dict,
'edition_info': edition_dict,
'volume_info': volume_info,
'hadith_number': num_val,
'volume': vol_val,
'pages': pg_val,
'address': getattr(ref, 'address', None),
'url': getattr(ref, 'url', None),
'open_book_url': open_book_url,
'citation_summary': citation_summary,
'images': images_list,
'images_count': len(images_list),
'book_title': getattr(book, 'title', None) if book else None,
'book_authors': book_authors,
}
def _export_category_interpretations(self, cursor, book_image_map=None):
edition_meta = get_cached_edition_metadata()
interps = HadisInterpretation.objects.select_related('category').prefetch_related(
'references__book_reference__author',
'references__images',
'references__edition',
'references__book_volume',
).all()
rows = []
for inp in interps:
refs_list = [self._serialize_reference(ref, edition_meta, book_image_map) for ref in inp.references.all()]
rows.append((
inp.id,
inp.category_id,
json.dumps(inp.title, ensure_ascii=False) if isinstance(inp.title, (dict, list)) else (inp.title or ''),
inp.slug,
inp.narrator or '',
inp.text or '',
json.dumps(inp.translation, ensure_ascii=False) if isinstance(inp.translation, (dict, list)) else (inp.translation or ''),
json.dumps(refs_list, ensure_ascii=False),
json.dumps(format_links_output(inp.links), ensure_ascii=False),
))
cursor.executemany(
"""INSERT INTO category_interpretations
(id, category_id, title, slug, narrator, text, translation, references_json, links_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
rows
)
return len(rows)
def _export_hadiths_and_corrections(self, cursor, cat_info_map=None, book_image_map=None):
edition_meta = get_cached_edition_metadata()
cat_info_map = cat_info_map or {}
hadiths = (
Hadis.objects
.filter(status=True)
.select_related('category', 'category__sect', 'hadis_status')
.prefetch_related(
'categories__sect',
'tags',
'references__book_reference__author',
'references__edition',
'references__book_volume',
'references__images',
'transmitters__transmitter__reliability',
'transmitters__uncertain_transmitters__reliability',
'transmitters__narrator_layer',
'hadiscorrection_set__references__book_reference__author',
'hadiscorrection_set__references__edition',
'hadiscorrection_set__references__book_volume',
'hadiscorrection_set__references__images',
)
.order_by('id')
)
hadis_rows = []
correction_rows = []
hadith_category_rows = []
for h in hadiths:
# 1. Status Block
status_id = h.hadis_status.id if h.hadis_status else None
status_title = h.hadis_status.title if h.hadis_status else None
status_color = h.hadis_status.color if h.hadis_status else None
status_code = h.hadis_status.main_color_code if h.hadis_status else None
# 2. Tags
tags_list = [{'id': t.id, 'title': t.title} for t in h.tags.all()]
# 3. References & Reference Images
refs_list = []
ref_imgs_list = []
for ref in h.references.all():
ref_data = self._serialize_reference(ref, edition_meta, book_image_map)
refs_list.append(ref_data)
if ref_data.get('images'):
ref_imgs_list.extend(ref_data['images'])
# 4. Narrators
transmitters_data = []
for tr in h.transmitters.all():
t = tr.transmitter
rel = t.reliability if t else None
uncertain_list = []
if tr.is_uncertain:
for ut in tr.uncertain_transmitters.all():
ut_rel = ut.reliability
uncertain_list.append({
'id': ut.id,
'name': ut.full_name,
'title': ut.full_name,
'slug': ut.slug,
'gender': ut.gender or '',
'birth_year_hijri': ut.birth_year_hijri or '',
'death_year_hijri': ut.death_year_hijri or '',
'reliability': {
'id': ut_rel.id,
'title': ut_rel.title,
'slug': ut_rel.slug,
'color': ut_rel.color,
'main_color_code': ut_rel.main_color_code,
} if ut_rel else None,
})
transmitters_data.append({
'id': t.id if t else None,
'name': t.full_name if t else None,
'title': t.full_name if t else None,
'slug': t.slug if t else None,
'gender': t.gender if t else '',
'birth_year_hijri': t.birth_year_hijri if t else '',
'death_year_hijri': t.death_year_hijri if t else '',
'layer_id': tr.narrator_layer_id,
'is_uncertain': tr.is_uncertain,
'uncertain_transmitters': uncertain_list,
'reliability': {
'id': rel.id,
'title': rel.title,
'slug': rel.slug,
'color': rel.color,
'main_color_code': rel.main_color_code,
} if rel else None,
})
# 5. Explanations
explanations_data = []
if hasattr(h, 'explanations') and h.explanations:
for exp in h.explanations.all():
explanations_data.append({
'title': exp.title,
'detail': getattr(exp, 'detail', '')
})
# 5.5 Multi-category M2M relations (plus primary category) & source_type / sect_type
seen_cat_ids = set()
hadith_cats_list = []
def add_cat_entry(cat_obj):
if not cat_obj or cat_obj.id in seen_cat_ids:
return
seen_cat_ids.add(cat_obj.id)
c_info = cat_info_map.get(cat_obj.id)
if not c_info:
c_source = getattr(cat_obj, 'source_type', None) or 'hadith'
c_sect = (
cat_obj.sect.sect_type
if (getattr(cat_obj, 'sect_id', None) and getattr(cat_obj, 'sect', None))
else 'shia'
)
c_info = {
'id': cat_obj.id,
'slug': cat_obj.slug,
'title': cat_obj.title,
'source_type': c_source,
'sect_type': c_sect,
}
hadith_cats_list.append(c_info)
hadith_category_rows.append((
h.id,
cat_obj.id,
cat_obj.slug,
c_info['source_type'],
c_info['sect_type'],
))
if h.category:
add_cat_entry(h.category)
for cat in h.categories.all():
add_cat_entry(cat)
primary_cat_info = hadith_cats_list[0] if hadith_cats_list else None
hadis_source_type = primary_cat_info['source_type'] if primary_cat_info else 'hadith'
hadis_sect_type = primary_cat_info['sect_type'] if primary_cat_info else 'shia'
primary_cat_id = h.category_id or (primary_cat_info['id'] if primary_cat_info else None)
primary_cat_slug = (h.category.slug if h.category else None) or (primary_cat_info['slug'] if primary_cat_info else None)
# Status description fallback: use hadis_status_text if present and non-empty, otherwise fallback to hadis_status.description
status_desc_raw = h.hadis_status_text
if self._is_empty_localized_data(status_desc_raw) and h.hadis_status:
status_desc_raw = h.hadis_status.description
status_text_val = json.dumps(status_desc_raw, ensure_ascii=False) if isinstance(status_desc_raw, (dict, list)) else (status_desc_raw or '')
hadis_rows.append((
h.id,
h.slug,
primary_cat_id,
primary_cat_slug,
hadis_source_type,
hadis_sect_type,
json.dumps(hadith_cats_list, ensure_ascii=False),
json.dumps(h.title, ensure_ascii=False) if isinstance(h.title, (dict, list)) else (h.title or ''),
json.dumps(h.title_narrator, ensure_ascii=False) if isinstance(h.title_narrator, (dict, list)) else (h.title_narrator or ''),
h.text or '',
json.dumps(h.translation or [], ensure_ascii=False),
json.dumps(h.address, ensure_ascii=False) if isinstance(h.address, (dict, list)) else (h.address or ''),
h.share_link or '',
status_id,
json.dumps(status_title, ensure_ascii=False) if isinstance(status_title, (dict, list)) else (status_title or ''),
status_color or '',
status_code or '',
status_text_val,
json.dumps(tags_list, ensure_ascii=False),
json.dumps(refs_list, ensure_ascii=False),
json.dumps(ref_imgs_list, ensure_ascii=False),
json.dumps({'transmitters': transmitters_data}, ensure_ascii=False),
json.dumps(explanations_data, ensure_ascii=False),
json.dumps(format_links_output(h.links), ensure_ascii=False),
))
# 6. Corrections
for corr in h.hadiscorrection_set.all():
corr_imgs = []
corr_refs = []
for cref in corr.references.all():
ref_data = self._serialize_reference(cref, edition_meta, book_image_map)
corr_refs.append(ref_data)
if ref_data.get('images'):
corr_imgs.extend(ref_data['images'])
corr_address = corr_refs[0].get('address') or '' if corr_refs else ''
correction_rows.append((
corr.id,
h.id,
json.dumps(corr.title, ensure_ascii=False) if isinstance(corr.title, (dict, list)) else (corr.title or ''),
corr.slug,
corr.narrator or '',
corr.text or '',
json.dumps(corr.translation, ensure_ascii=False) if isinstance(corr.translation, (dict, list)) else (corr.translation or ''),
corr.share_link or '',
corr_address,
json.dumps(corr_imgs, ensure_ascii=False),
json.dumps(corr_refs, ensure_ascii=False),
json.dumps(format_links_output(corr.links), ensure_ascii=False),
))
cursor.executemany(
"""INSERT INTO hadiths (
id, slug, category_id, category_slug, source_type, sect_type, categories_json,
title, title_narrator, text,
translation_json, address, share_link, hadis_status_id, status_title,
status_color, status_main_color_code, status_text, tags_json,
references_json, reference_images_json, narrators_json, explanations_json,
links_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
hadis_rows
)
cursor.executemany(
"""INSERT OR IGNORE INTO hadith_categories (
hadis_id, category_id, category_slug, source_type, sect_type
) VALUES (?, ?, ?, ?, ?)""",
hadith_category_rows
)
cursor.executemany(
"""INSERT INTO hadith_corrections (
id, hadis_id, title, slug, narrator, description, translation,
share_link, address, images_json, references_json, links_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
correction_rows
)
return len(hadis_rows), len(correction_rows)