Browse Source
refactor(open_book_url): strictly return volume file url or null without fallbacks
master
refactor(open_book_url): strictly return volume file url or null without fallbacks
master
3 changed files with 417 additions and 14 deletions
-
8apps/hadis/management/commands/export_hadis_sqlite.py
-
415apps/hadis/management/commands/export_hadis_sqlite.py.bak
-
8apps/hadis/serializers/hadis.py
@ -0,0 +1,415 @@ |
|||
import os |
|||
import gzip |
|||
import json |
|||
import sqlite3 |
|||
import hashlib |
|||
from pathlib import Path |
|||
from django.core.management.base import BaseCommand |
|||
from django.conf import settings |
|||
from django.db import transaction |
|||
from apps.hadis.models import ( |
|||
Hadis, |
|||
HadisCategory, |
|||
HadisInterpretation, |
|||
HadisCorrection, |
|||
HadisDatabaseVersion |
|||
) |
|||
|
|||
|
|||
class Command(BaseCommand): |
|||
help = "Export Hadis and Categories into an optimized, compressed SQLite database for offline mobile sync." |
|||
|
|||
def add_arguments(self, parser): |
|||
parser.add_argument( |
|||
'--notes', |
|||
type=str, |
|||
default='', |
|||
help='Release notes for this SQLite database version' |
|||
) |
|||
parser.add_argument( |
|||
'--force-version', |
|||
type=int, |
|||
default=None, |
|||
help='Force a specific version number' |
|||
) |
|||
|
|||
def handle(self, *args, **options): |
|||
self.stdout.write(self.style.SUCCESS("Starting Hadis SQLite database generation (Single-File Architecture)...")) |
|||
|
|||
# 1. Determine destination and version number |
|||
latest_ver = HadisDatabaseVersion.objects.order_by('-version').first() |
|||
if options['force_version'] is not None: |
|||
new_version = options['force_version'] |
|||
else: |
|||
new_version = (latest_ver.version + 1) if latest_ver else 1 |
|||
|
|||
output_dir = Path(settings.MEDIA_ROOT) / 'hadis_db' |
|||
output_dir.mkdir(parents=True, exist_ok=True) |
|||
|
|||
temp_db_path = output_dir / "temp_hadis_build.sqlite" |
|||
final_gz_name = "hadis_offline.sqlite.gz" |
|||
final_gz_path = output_dir / final_gz_name |
|||
|
|||
if temp_db_path.exists(): |
|||
temp_db_path.unlink() |
|||
|
|||
# 2. Initialize SQLite database |
|||
conn = sqlite3.connect(str(temp_db_path)) |
|||
cursor = conn.cursor() |
|||
|
|||
# Optimize SQLite for bulk writes |
|||
cursor.execute("PRAGMA synchronous = OFF;") |
|||
cursor.execute("PRAGMA journal_mode = MEMORY;") |
|||
cursor.execute("PRAGMA encoding = 'UTF-8';") |
|||
|
|||
self.stdout.write("Creating SQLite tables and indexes...") |
|||
self._create_tables(cursor) |
|||
|
|||
# 3. Export data |
|||
categories_count = self._export_categories(cursor) |
|||
interp_count = self._export_category_interpretations(cursor) |
|||
hadis_count, corr_count = self._export_hadiths_and_corrections(cursor) |
|||
|
|||
conn.commit() |
|||
|
|||
# Create indexes after insertion for maximum performance |
|||
self.stdout.write("Building indexes...") |
|||
self._create_indexes(cursor) |
|||
conn.commit() |
|||
conn.close() |
|||
|
|||
# 4. Gzip compress directly to the single target file |
|||
self.stdout.write(f"Compressing database to single file: {final_gz_name}...") |
|||
hasher = hashlib.md5() |
|||
temp_gz_path = output_dir / "temp_hadis_build.sqlite.gz" |
|||
|
|||
with open(temp_db_path, 'rb') as f_in: |
|||
with gzip.open(temp_gz_path, 'wb', compresslevel=9) as f_out: |
|||
while chunk := f_in.read(1024 * 1024): |
|||
f_out.write(chunk) |
|||
hasher.update(chunk) |
|||
|
|||
checksum = hasher.hexdigest() |
|||
temp_size = temp_db_path.stat().st_size |
|||
|
|||
# Atomic replacement: replace final_gz_path with temp_gz_path |
|||
if final_gz_path.exists(): |
|||
final_gz_path.unlink() |
|||
temp_gz_path.rename(final_gz_path) |
|||
file_size = final_gz_path.stat().st_size |
|||
|
|||
# Clean up any remaining temp files in output_dir |
|||
temp_db_path.unlink(missing_ok=True) |
|||
for extra_file in output_dir.iterdir(): |
|||
if extra_file.name != final_gz_name: |
|||
try: |
|||
extra_file.unlink(missing_ok=True) |
|||
except OSError: |
|||
pass |
|||
|
|||
# 5. Record version in database (Single Active Record) |
|||
with transaction.atomic(): |
|||
# Delete old records to keep DB clean |
|||
HadisDatabaseVersion.objects.all().delete() |
|||
db_version = HadisDatabaseVersion.objects.create( |
|||
version=new_version, |
|||
file=f"hadis_db/{final_gz_name}", |
|||
file_size=file_size, |
|||
checksum_md5=checksum, |
|||
hadis_count=hadis_count, |
|||
categories_count=categories_count, |
|||
is_active=True, |
|||
notes=options['notes'], |
|||
) |
|||
|
|||
self.stdout.write(self.style.SUCCESS( |
|||
f"Successfully updated single Hadis SQLite DB to v{new_version}!\n" |
|||
f" - Uncompressed size: {temp_size / (1024*1024):.2f} MB\n" |
|||
f" - Gzipped size: {file_size / (1024*1024):.2f} MB\n" |
|||
f" - Hadiths: {hadis_count}\n" |
|||
f" - Categories: {categories_count}\n" |
|||
f" - Interpretations: {interp_count}\n" |
|||
f" - Corrections: {corr_count}\n" |
|||
f" - MD5 Checksum: {checksum}\n" |
|||
f" - File path: {final_gz_path}" |
|||
)) |
|||
|
|||
def _create_tables(self, cursor): |
|||
cursor.executescript(""" |
|||
CREATE TABLE IF NOT EXISTS categories ( |
|||
id INTEGER PRIMARY KEY, |
|||
title TEXT, |
|||
slug TEXT UNIQUE, |
|||
source_type TEXT, |
|||
sect_type TEXT |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS category_interpretations ( |
|||
id INTEGER PRIMARY KEY, |
|||
category_id INTEGER, |
|||
title TEXT, |
|||
slug TEXT, |
|||
narrator TEXT, |
|||
text TEXT, |
|||
translation TEXT, |
|||
references_json TEXT, |
|||
links_json TEXT, |
|||
FOREIGN KEY (category_id) REFERENCES categories (id) |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS hadiths ( |
|||
id INTEGER PRIMARY KEY, |
|||
slug TEXT UNIQUE, |
|||
category_id INTEGER, |
|||
category_slug TEXT, |
|||
title TEXT, |
|||
title_narrator TEXT, |
|||
text TEXT, |
|||
translation_json TEXT, |
|||
address TEXT, |
|||
share_link TEXT, |
|||
hadis_status_id INTEGER, |
|||
status_title TEXT, |
|||
status_color TEXT, |
|||
status_main_color_code TEXT, |
|||
status_text TEXT, |
|||
tags_json TEXT, |
|||
references_json TEXT, |
|||
reference_images_json TEXT, |
|||
narrators_json TEXT, |
|||
explanations_json TEXT, |
|||
FOREIGN KEY (category_id) REFERENCES categories (id) |
|||
); |
|||
|
|||
CREATE TABLE IF NOT EXISTS hadith_corrections ( |
|||
id INTEGER PRIMARY KEY, |
|||
hadis_id INTEGER, |
|||
title TEXT, |
|||
slug TEXT, |
|||
narrator TEXT, |
|||
description TEXT, |
|||
translation TEXT, |
|||
share_link TEXT, |
|||
address TEXT, |
|||
images_json TEXT, |
|||
references_json TEXT, |
|||
links_json TEXT, |
|||
FOREIGN KEY (hadis_id) REFERENCES hadiths (id) |
|||
); |
|||
""") |
|||
|
|||
def _create_indexes(self, cursor): |
|||
cursor.executescript(""" |
|||
CREATE INDEX IF NOT EXISTS idx_cat_slug ON categories(slug); |
|||
CREATE INDEX IF NOT EXISTS idx_interp_cat ON category_interpretations(category_id); |
|||
CREATE INDEX IF NOT EXISTS idx_hadith_cat ON hadiths(category_id); |
|||
CREATE INDEX IF NOT EXISTS idx_hadith_slug ON hadiths(slug); |
|||
CREATE INDEX IF NOT EXISTS idx_corr_hadis ON hadith_corrections(hadis_id); |
|||
""") |
|||
|
|||
def _export_categories(self, cursor): |
|||
cats = HadisCategory.objects.all() |
|||
rows = [ |
|||
( |
|||
c.id, |
|||
json.dumps(c.title, ensure_ascii=False) if isinstance(c.title, (dict, list)) else (c.title or ''), |
|||
c.slug, |
|||
getattr(c, 'source_type', None), |
|||
getattr(c, 'sect_type', None), |
|||
) |
|||
for c in cats |
|||
] |
|||
cursor.executemany( |
|||
"INSERT INTO categories (id, title, slug, source_type, sect_type) VALUES (?, ?, ?, ?, ?)", |
|||
rows |
|||
) |
|||
return len(rows) |
|||
|
|||
def _export_category_interpretations(self, cursor): |
|||
interps = HadisInterpretation.objects.select_related('category').prefetch_related( |
|||
'references__book_reference', 'references__images' |
|||
).all() |
|||
|
|||
rows = [] |
|||
for inp in interps: |
|||
refs_list = [] |
|||
for ref in inp.references.all(): |
|||
book = ref.book_reference |
|||
refs_list.append({ |
|||
'id': ref.id, |
|||
'book_title': getattr(book, 'title', None), |
|||
'address': getattr(ref, 'address', None), |
|||
'edition': getattr(ref, 'edition_id', None), |
|||
'volume': getattr(ref, 'volume_id', None), |
|||
'pages': getattr(ref, 'pages', None), |
|||
'hadith_number': getattr(ref, 'hadith_number', None), |
|||
}) |
|||
|
|||
rows.append(( |
|||
inp.id, |
|||
inp.category_id, |
|||
json.dumps(inp.title, ensure_ascii=False) if isinstance(inp.title, (dict, list)) else (inp.title or ''), |
|||
inp.slug, |
|||
inp.narrator or '', |
|||
inp.text or '', |
|||
json.dumps(inp.translation, ensure_ascii=False) if isinstance(inp.translation, (dict, list)) else (inp.translation or ''), |
|||
json.dumps(refs_list, ensure_ascii=False), |
|||
json.dumps(inp.links or [], ensure_ascii=False), |
|||
)) |
|||
|
|||
cursor.executemany( |
|||
"""INSERT INTO category_interpretations |
|||
(id, category_id, title, slug, narrator, text, translation, references_json, links_json) |
|||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", |
|||
rows |
|||
) |
|||
return len(rows) |
|||
|
|||
def _export_hadiths_and_corrections(self, cursor): |
|||
hadiths = ( |
|||
Hadis.objects |
|||
.filter(status=True) |
|||
.select_related('category', 'hadis_status') |
|||
.prefetch_related( |
|||
'tags', |
|||
'references__book_reference__author', |
|||
'references__images', |
|||
'transmitters__transmitter__reliability', |
|||
'transmitters__uncertain_transmitters__reliability', |
|||
'transmitters__narrator_layer', |
|||
'hadiscorrection_set__references__images', |
|||
) |
|||
.order_by('id') |
|||
) |
|||
|
|||
hadis_rows = [] |
|||
correction_rows = [] |
|||
|
|||
for h in hadiths: |
|||
# 1. Status Block |
|||
status_id = h.hadis_status.id if h.hadis_status else None |
|||
status_title = h.hadis_status.title if h.hadis_status else None |
|||
status_color = h.hadis_status.color if h.hadis_status else None |
|||
status_code = h.hadis_status.main_color_code if h.hadis_status else None |
|||
|
|||
# 2. Tags |
|||
tags_list = [{'id': t.id, 'title': t.title} for t in h.tags.all()] |
|||
|
|||
# 3. References & Reference Images |
|||
refs_list = [] |
|||
ref_imgs_list = [] |
|||
for ref in h.references.all(): |
|||
book = ref.book_reference |
|||
authors = [{'id': book.author.id, 'name': book.author.name}] if book and book.author else [] |
|||
refs_list.append({ |
|||
'id': ref.id, |
|||
'title': book.title if book else None, |
|||
'authors': authors, |
|||
'share_link': getattr(book, 'share_link', None) if book else None, |
|||
}) |
|||
for img in ref.images.all(): |
|||
ref_imgs_list.append({ |
|||
'id': img.id, |
|||
'thumbnail': img.thumbnail.url if img.thumbnail else None, |
|||
'priority': img.priority, |
|||
}) |
|||
|
|||
# 4. Narrators |
|||
transmitters_data = [] |
|||
for tr in h.transmitters.all(): |
|||
t = tr.transmitter |
|||
rel = t.reliability if t else None |
|||
transmitters_data.append({ |
|||
'id': t.id if t else None, |
|||
'name': t.full_name if t else None, |
|||
'slug': t.slug if t else None, |
|||
'layer_id': tr.narrator_layer_id, |
|||
'is_uncertain': tr.is_uncertain, |
|||
'reliability': { |
|||
'id': rel.id, |
|||
'title': rel.title, |
|||
'slug': rel.slug, |
|||
'color': rel.color, |
|||
'main_color_code': rel.main_color_code, |
|||
} if rel else None, |
|||
}) |
|||
|
|||
# 5. Explanations |
|||
explanations_data = [] |
|||
if hasattr(h, 'explanations') and h.explanations: |
|||
for exp in h.explanations.all(): |
|||
explanations_data.append({ |
|||
'title': exp.title, |
|||
'detail': getattr(exp, 'detail', '') |
|||
}) |
|||
|
|||
hadis_rows.append(( |
|||
h.id, |
|||
h.slug, |
|||
h.category_id, |
|||
h.category.slug if h.category else None, |
|||
json.dumps(h.title, ensure_ascii=False) if isinstance(h.title, (dict, list)) else (h.title or ''), |
|||
json.dumps(h.title_narrator, ensure_ascii=False) if isinstance(h.title_narrator, (dict, list)) else (h.title_narrator or ''), |
|||
h.text or '', |
|||
json.dumps(h.translation or [], ensure_ascii=False), |
|||
json.dumps(h.address, ensure_ascii=False) if isinstance(h.address, (dict, list)) else (h.address or ''), |
|||
h.share_link or '', |
|||
status_id, |
|||
json.dumps(status_title, ensure_ascii=False) if isinstance(status_title, (dict, list)) else (status_title or ''), |
|||
status_color or '', |
|||
status_code or '', |
|||
json.dumps(h.hadis_status_text, ensure_ascii=False) if isinstance(h.hadis_status_text, (dict, list)) else (h.hadis_status_text or ''), |
|||
json.dumps(tags_list, ensure_ascii=False), |
|||
json.dumps(refs_list, ensure_ascii=False), |
|||
json.dumps(ref_imgs_list, ensure_ascii=False), |
|||
json.dumps({'transmitters': transmitters_data}, ensure_ascii=False), |
|||
json.dumps(explanations_data, ensure_ascii=False), |
|||
)) |
|||
|
|||
# 6. Corrections |
|||
for corr in h.hadiscorrection_set.all(): |
|||
corr_imgs = [] |
|||
corr_refs = [] |
|||
for cref in corr.references.all(): |
|||
for cimg in cref.images.all(): |
|||
if cimg.image: |
|||
corr_imgs.append({'id': cimg.id, 'image': cimg.image.url}) |
|||
corr_refs.append({ |
|||
'id': cref.id, |
|||
'address': getattr(cref, 'address', None), |
|||
}) |
|||
|
|||
correction_rows.append(( |
|||
corr.id, |
|||
h.id, |
|||
json.dumps(corr.title, ensure_ascii=False) if isinstance(corr.title, (dict, list)) else (corr.title or ''), |
|||
corr.slug, |
|||
corr.narrator or '', |
|||
corr.text or '', |
|||
json.dumps(corr.translation, ensure_ascii=False) if isinstance(corr.translation, (dict, list)) else (corr.translation or ''), |
|||
corr.share_link or '', |
|||
corr.address if hasattr(corr, 'address') else '', |
|||
json.dumps(corr_imgs, ensure_ascii=False), |
|||
json.dumps(corr_refs, ensure_ascii=False), |
|||
json.dumps(corr.links or [], ensure_ascii=False), |
|||
)) |
|||
|
|||
cursor.executemany( |
|||
"""INSERT INTO hadiths ( |
|||
id, slug, category_id, category_slug, title, title_narrator, text, |
|||
translation_json, address, share_link, hadis_status_id, status_title, |
|||
status_color, status_main_color_code, status_text, tags_json, |
|||
references_json, reference_images_json, narrators_json, explanations_json |
|||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", |
|||
hadis_rows |
|||
) |
|||
|
|||
cursor.executemany( |
|||
"""INSERT INTO hadith_corrections ( |
|||
id, hadis_id, title, slug, narrator, description, translation, |
|||
share_link, address, images_json, references_json, links_json |
|||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", |
|||
correction_rows |
|||
) |
|||
|
|||
return len(hadis_rows), len(correction_rows) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue