From 27c291c68205e515b3d8b44639c118b57db15487 Mon Sep 17 00:00:00 2001 From: mohsentaba Date: Sat, 12 Sep 2026 13:18:56 +0330 Subject: [PATCH] refactor(open_book_url): strictly return volume file url or null without fallbacks --- .../commands/export_hadis_sqlite.py | 8 +- .../commands/export_hadis_sqlite.py.bak | 415 ++++++++++++++++++ apps/hadis/serializers/hadis.py | 8 +- 3 files changed, 417 insertions(+), 14 deletions(-) create mode 100644 apps/hadis/management/commands/export_hadis_sqlite.py.bak diff --git a/apps/hadis/management/commands/export_hadis_sqlite.py b/apps/hadis/management/commands/export_hadis_sqlite.py index c0037e0..5da82e1 100644 --- a/apps/hadis/management/commands/export_hadis_sqlite.py +++ b/apps/hadis/management/commands/export_hadis_sqlite.py @@ -311,14 +311,8 @@ class Command(BaseCommand): } if vol_obj else None open_book_url = None - if getattr(ref, 'url', None): - open_book_url = ref.url - elif vol_obj and getattr(vol_obj, 'file', None): + if vol_obj and getattr(vol_obj, 'file', None): open_book_url = self._make_full_url(vol_obj.file.url) - elif ed and getattr(ed, 'source_url', None): - open_book_url = ed.source_url - elif book and getattr(book, 'slug', None): - open_book_url = f"/arguments/sources/{book.slug}" parts = [] vol_val = getattr(ref, 'volume', None) or (vol_obj.title if vol_obj else None) diff --git a/apps/hadis/management/commands/export_hadis_sqlite.py.bak b/apps/hadis/management/commands/export_hadis_sqlite.py.bak new file mode 100644 index 0000000..67b79cd --- /dev/null +++ b/apps/hadis/management/commands/export_hadis_sqlite.py.bak @@ -0,0 +1,415 @@ +import os +import gzip +import json +import sqlite3 +import hashlib +from pathlib import Path +from django.core.management.base import BaseCommand +from django.conf import settings +from django.db import transaction +from apps.hadis.models import ( + Hadis, + HadisCategory, + HadisInterpretation, + HadisCorrection, + HadisDatabaseVersion +) + + +class Command(BaseCommand): + help = "Export Hadis and Categories into an optimized, compressed SQLite database for offline mobile sync." + + def add_arguments(self, parser): + parser.add_argument( + '--notes', + type=str, + default='', + help='Release notes for this SQLite database version' + ) + parser.add_argument( + '--force-version', + type=int, + default=None, + help='Force a specific version number' + ) + + def handle(self, *args, **options): + self.stdout.write(self.style.SUCCESS("Starting Hadis SQLite database generation (Single-File Architecture)...")) + + # 1. Determine destination and version number + latest_ver = HadisDatabaseVersion.objects.order_by('-version').first() + if options['force_version'] is not None: + new_version = options['force_version'] + else: + new_version = (latest_ver.version + 1) if latest_ver else 1 + + output_dir = Path(settings.MEDIA_ROOT) / 'hadis_db' + output_dir.mkdir(parents=True, exist_ok=True) + + temp_db_path = output_dir / "temp_hadis_build.sqlite" + final_gz_name = "hadis_offline.sqlite.gz" + final_gz_path = output_dir / final_gz_name + + if temp_db_path.exists(): + temp_db_path.unlink() + + # 2. Initialize SQLite database + conn = sqlite3.connect(str(temp_db_path)) + cursor = conn.cursor() + + # Optimize SQLite for bulk writes + cursor.execute("PRAGMA synchronous = OFF;") + cursor.execute("PRAGMA journal_mode = MEMORY;") + cursor.execute("PRAGMA encoding = 'UTF-8';") + + self.stdout.write("Creating SQLite tables and indexes...") + self._create_tables(cursor) + + # 3. Export data + categories_count = self._export_categories(cursor) + interp_count = self._export_category_interpretations(cursor) + hadis_count, corr_count = self._export_hadiths_and_corrections(cursor) + + conn.commit() + + # Create indexes after insertion for maximum performance + self.stdout.write("Building indexes...") + self._create_indexes(cursor) + conn.commit() + conn.close() + + # 4. Gzip compress directly to the single target file + self.stdout.write(f"Compressing database to single file: {final_gz_name}...") + hasher = hashlib.md5() + temp_gz_path = output_dir / "temp_hadis_build.sqlite.gz" + + with open(temp_db_path, 'rb') as f_in: + with gzip.open(temp_gz_path, 'wb', compresslevel=9) as f_out: + while chunk := f_in.read(1024 * 1024): + f_out.write(chunk) + hasher.update(chunk) + + checksum = hasher.hexdigest() + temp_size = temp_db_path.stat().st_size + + # Atomic replacement: replace final_gz_path with temp_gz_path + if final_gz_path.exists(): + final_gz_path.unlink() + temp_gz_path.rename(final_gz_path) + file_size = final_gz_path.stat().st_size + + # Clean up any remaining temp files in output_dir + temp_db_path.unlink(missing_ok=True) + for extra_file in output_dir.iterdir(): + if extra_file.name != final_gz_name: + try: + extra_file.unlink(missing_ok=True) + except OSError: + pass + + # 5. Record version in database (Single Active Record) + with transaction.atomic(): + # Delete old records to keep DB clean + HadisDatabaseVersion.objects.all().delete() + db_version = HadisDatabaseVersion.objects.create( + version=new_version, + file=f"hadis_db/{final_gz_name}", + file_size=file_size, + checksum_md5=checksum, + hadis_count=hadis_count, + categories_count=categories_count, + is_active=True, + notes=options['notes'], + ) + + self.stdout.write(self.style.SUCCESS( + f"Successfully updated single Hadis SQLite DB to v{new_version}!\n" + f" - Uncompressed size: {temp_size / (1024*1024):.2f} MB\n" + f" - Gzipped size: {file_size / (1024*1024):.2f} MB\n" + f" - Hadiths: {hadis_count}\n" + f" - Categories: {categories_count}\n" + f" - Interpretations: {interp_count}\n" + f" - Corrections: {corr_count}\n" + f" - MD5 Checksum: {checksum}\n" + f" - File path: {final_gz_path}" + )) + + def _create_tables(self, cursor): + cursor.executescript(""" + CREATE TABLE IF NOT EXISTS categories ( + id INTEGER PRIMARY KEY, + title TEXT, + slug TEXT UNIQUE, + source_type TEXT, + sect_type TEXT + ); + + CREATE TABLE IF NOT EXISTS category_interpretations ( + id INTEGER PRIMARY KEY, + category_id INTEGER, + title TEXT, + slug TEXT, + narrator TEXT, + text TEXT, + translation TEXT, + references_json TEXT, + links_json TEXT, + FOREIGN KEY (category_id) REFERENCES categories (id) + ); + + CREATE TABLE IF NOT EXISTS hadiths ( + id INTEGER PRIMARY KEY, + slug TEXT UNIQUE, + category_id INTEGER, + category_slug TEXT, + title TEXT, + title_narrator TEXT, + text TEXT, + translation_json TEXT, + address TEXT, + share_link TEXT, + hadis_status_id INTEGER, + status_title TEXT, + status_color TEXT, + status_main_color_code TEXT, + status_text TEXT, + tags_json TEXT, + references_json TEXT, + reference_images_json TEXT, + narrators_json TEXT, + explanations_json TEXT, + FOREIGN KEY (category_id) REFERENCES categories (id) + ); + + CREATE TABLE IF NOT EXISTS hadith_corrections ( + id INTEGER PRIMARY KEY, + hadis_id INTEGER, + title TEXT, + slug TEXT, + narrator TEXT, + description TEXT, + translation TEXT, + share_link TEXT, + address TEXT, + images_json TEXT, + references_json TEXT, + links_json TEXT, + FOREIGN KEY (hadis_id) REFERENCES hadiths (id) + ); + """) + + def _create_indexes(self, cursor): + cursor.executescript(""" + CREATE INDEX IF NOT EXISTS idx_cat_slug ON categories(slug); + CREATE INDEX IF NOT EXISTS idx_interp_cat ON category_interpretations(category_id); + CREATE INDEX IF NOT EXISTS idx_hadith_cat ON hadiths(category_id); + CREATE INDEX IF NOT EXISTS idx_hadith_slug ON hadiths(slug); + CREATE INDEX IF NOT EXISTS idx_corr_hadis ON hadith_corrections(hadis_id); + """) + + def _export_categories(self, cursor): + cats = HadisCategory.objects.all() + rows = [ + ( + c.id, + json.dumps(c.title, ensure_ascii=False) if isinstance(c.title, (dict, list)) else (c.title or ''), + c.slug, + getattr(c, 'source_type', None), + getattr(c, 'sect_type', None), + ) + for c in cats + ] + cursor.executemany( + "INSERT INTO categories (id, title, slug, source_type, sect_type) VALUES (?, ?, ?, ?, ?)", + rows + ) + return len(rows) + + def _export_category_interpretations(self, cursor): + interps = HadisInterpretation.objects.select_related('category').prefetch_related( + 'references__book_reference', 'references__images' + ).all() + + rows = [] + for inp in interps: + refs_list = [] + for ref in inp.references.all(): + book = ref.book_reference + refs_list.append({ + 'id': ref.id, + 'book_title': getattr(book, 'title', None), + 'address': getattr(ref, 'address', None), + 'edition': getattr(ref, 'edition_id', None), + 'volume': getattr(ref, 'volume_id', None), + 'pages': getattr(ref, 'pages', None), + 'hadith_number': getattr(ref, 'hadith_number', None), + }) + + rows.append(( + inp.id, + inp.category_id, + json.dumps(inp.title, ensure_ascii=False) if isinstance(inp.title, (dict, list)) else (inp.title or ''), + inp.slug, + inp.narrator or '', + inp.text or '', + json.dumps(inp.translation, ensure_ascii=False) if isinstance(inp.translation, (dict, list)) else (inp.translation or ''), + json.dumps(refs_list, ensure_ascii=False), + json.dumps(inp.links or [], ensure_ascii=False), + )) + + cursor.executemany( + """INSERT INTO category_interpretations + (id, category_id, title, slug, narrator, text, translation, references_json, links_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + rows + ) + return len(rows) + + def _export_hadiths_and_corrections(self, cursor): + hadiths = ( + Hadis.objects + .filter(status=True) + .select_related('category', 'hadis_status') + .prefetch_related( + 'tags', + 'references__book_reference__author', + 'references__images', + 'transmitters__transmitter__reliability', + 'transmitters__uncertain_transmitters__reliability', + 'transmitters__narrator_layer', + 'hadiscorrection_set__references__images', + ) + .order_by('id') + ) + + hadis_rows = [] + correction_rows = [] + + for h in hadiths: + # 1. Status Block + status_id = h.hadis_status.id if h.hadis_status else None + status_title = h.hadis_status.title if h.hadis_status else None + status_color = h.hadis_status.color if h.hadis_status else None + status_code = h.hadis_status.main_color_code if h.hadis_status else None + + # 2. Tags + tags_list = [{'id': t.id, 'title': t.title} for t in h.tags.all()] + + # 3. References & Reference Images + refs_list = [] + ref_imgs_list = [] + for ref in h.references.all(): + book = ref.book_reference + authors = [{'id': book.author.id, 'name': book.author.name}] if book and book.author else [] + refs_list.append({ + 'id': ref.id, + 'title': book.title if book else None, + 'authors': authors, + 'share_link': getattr(book, 'share_link', None) if book else None, + }) + for img in ref.images.all(): + ref_imgs_list.append({ + 'id': img.id, + 'thumbnail': img.thumbnail.url if img.thumbnail else None, + 'priority': img.priority, + }) + + # 4. Narrators + transmitters_data = [] + for tr in h.transmitters.all(): + t = tr.transmitter + rel = t.reliability if t else None + transmitters_data.append({ + 'id': t.id if t else None, + 'name': t.full_name if t else None, + 'slug': t.slug if t else None, + 'layer_id': tr.narrator_layer_id, + 'is_uncertain': tr.is_uncertain, + 'reliability': { + 'id': rel.id, + 'title': rel.title, + 'slug': rel.slug, + 'color': rel.color, + 'main_color_code': rel.main_color_code, + } if rel else None, + }) + + # 5. Explanations + explanations_data = [] + if hasattr(h, 'explanations') and h.explanations: + for exp in h.explanations.all(): + explanations_data.append({ + 'title': exp.title, + 'detail': getattr(exp, 'detail', '') + }) + + hadis_rows.append(( + h.id, + h.slug, + h.category_id, + h.category.slug if h.category else None, + json.dumps(h.title, ensure_ascii=False) if isinstance(h.title, (dict, list)) else (h.title or ''), + json.dumps(h.title_narrator, ensure_ascii=False) if isinstance(h.title_narrator, (dict, list)) else (h.title_narrator or ''), + h.text or '', + json.dumps(h.translation or [], ensure_ascii=False), + json.dumps(h.address, ensure_ascii=False) if isinstance(h.address, (dict, list)) else (h.address or ''), + h.share_link or '', + status_id, + json.dumps(status_title, ensure_ascii=False) if isinstance(status_title, (dict, list)) else (status_title or ''), + status_color or '', + status_code or '', + json.dumps(h.hadis_status_text, ensure_ascii=False) if isinstance(h.hadis_status_text, (dict, list)) else (h.hadis_status_text or ''), + json.dumps(tags_list, ensure_ascii=False), + json.dumps(refs_list, ensure_ascii=False), + json.dumps(ref_imgs_list, ensure_ascii=False), + json.dumps({'transmitters': transmitters_data}, ensure_ascii=False), + json.dumps(explanations_data, ensure_ascii=False), + )) + + # 6. Corrections + for corr in h.hadiscorrection_set.all(): + corr_imgs = [] + corr_refs = [] + for cref in corr.references.all(): + for cimg in cref.images.all(): + if cimg.image: + corr_imgs.append({'id': cimg.id, 'image': cimg.image.url}) + corr_refs.append({ + 'id': cref.id, + 'address': getattr(cref, 'address', None), + }) + + correction_rows.append(( + corr.id, + h.id, + json.dumps(corr.title, ensure_ascii=False) if isinstance(corr.title, (dict, list)) else (corr.title or ''), + corr.slug, + corr.narrator or '', + corr.text or '', + json.dumps(corr.translation, ensure_ascii=False) if isinstance(corr.translation, (dict, list)) else (corr.translation or ''), + corr.share_link or '', + corr.address if hasattr(corr, 'address') else '', + json.dumps(corr_imgs, ensure_ascii=False), + json.dumps(corr_refs, ensure_ascii=False), + json.dumps(corr.links or [], ensure_ascii=False), + )) + + cursor.executemany( + """INSERT INTO hadiths ( + id, slug, category_id, category_slug, title, title_narrator, text, + translation_json, address, share_link, hadis_status_id, status_title, + status_color, status_main_color_code, status_text, tags_json, + references_json, reference_images_json, narrators_json, explanations_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + hadis_rows + ) + + cursor.executemany( + """INSERT INTO hadith_corrections ( + id, hadis_id, title, slug, narrator, description, translation, + share_link, address, images_json, references_json, links_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + correction_rows + ) + + return len(hadis_rows), len(correction_rows) diff --git a/apps/hadis/serializers/hadis.py b/apps/hadis/serializers/hadis.py index db69b99..b9c4321 100644 --- a/apps/hadis/serializers/hadis.py +++ b/apps/hadis/serializers/hadis.py @@ -1576,14 +1576,8 @@ class DetailedCorrectionReferenceSerializer(BaseDetailedReferenceSerializer): def get_open_book_url(self, obj): request = self.context.get('request') - if obj.url: - return obj.url if obj.book_volume and obj.book_volume.file: - return absolute_https_url(obj.book_volume.file.url, request) if request else obj.book_volume.file.url - if obj.edition and obj.edition.source_url: - return obj.edition.source_url - if obj.book_reference and obj.book_reference.slug: - return f"/arguments/sources/{obj.book_reference.slug}" + return absolute_https_url(obj.book_volume.file.url, request) if request else absolute_https_url(obj.book_volume.file.url) return None volume = serializers.SerializerMethodField()