You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
415 lines
16 KiB
415 lines
16 KiB
import os
|
|
import gzip
|
|
import json
|
|
import sqlite3
|
|
import hashlib
|
|
from pathlib import Path
|
|
from django.core.management.base import BaseCommand
|
|
from django.conf import settings
|
|
from django.db import transaction
|
|
from apps.hadis.models import (
|
|
Hadis,
|
|
HadisCategory,
|
|
HadisInterpretation,
|
|
HadisCorrection,
|
|
HadisDatabaseVersion
|
|
)
|
|
|
|
|
|
class Command(BaseCommand):
|
|
help = "Export Hadis and Categories into an optimized, compressed SQLite database for offline mobile sync."
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument(
|
|
'--notes',
|
|
type=str,
|
|
default='',
|
|
help='Release notes for this SQLite database version'
|
|
)
|
|
parser.add_argument(
|
|
'--force-version',
|
|
type=int,
|
|
default=None,
|
|
help='Force a specific version number'
|
|
)
|
|
|
|
def handle(self, *args, **options):
|
|
self.stdout.write(self.style.SUCCESS("Starting Hadis SQLite database generation (Single-File Architecture)..."))
|
|
|
|
# 1. Determine destination and version number
|
|
latest_ver = HadisDatabaseVersion.objects.order_by('-version').first()
|
|
if options['force_version'] is not None:
|
|
new_version = options['force_version']
|
|
else:
|
|
new_version = (latest_ver.version + 1) if latest_ver else 1
|
|
|
|
output_dir = Path(settings.MEDIA_ROOT) / 'hadis_db'
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
temp_db_path = output_dir / "temp_hadis_build.sqlite"
|
|
final_gz_name = "hadis_offline.sqlite.gz"
|
|
final_gz_path = output_dir / final_gz_name
|
|
|
|
if temp_db_path.exists():
|
|
temp_db_path.unlink()
|
|
|
|
# 2. Initialize SQLite database
|
|
conn = sqlite3.connect(str(temp_db_path))
|
|
cursor = conn.cursor()
|
|
|
|
# Optimize SQLite for bulk writes
|
|
cursor.execute("PRAGMA synchronous = OFF;")
|
|
cursor.execute("PRAGMA journal_mode = MEMORY;")
|
|
cursor.execute("PRAGMA encoding = 'UTF-8';")
|
|
|
|
self.stdout.write("Creating SQLite tables and indexes...")
|
|
self._create_tables(cursor)
|
|
|
|
# 3. Export data
|
|
categories_count = self._export_categories(cursor)
|
|
interp_count = self._export_category_interpretations(cursor)
|
|
hadis_count, corr_count = self._export_hadiths_and_corrections(cursor)
|
|
|
|
conn.commit()
|
|
|
|
# Create indexes after insertion for maximum performance
|
|
self.stdout.write("Building indexes...")
|
|
self._create_indexes(cursor)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
# 4. Gzip compress directly to the single target file
|
|
self.stdout.write(f"Compressing database to single file: {final_gz_name}...")
|
|
hasher = hashlib.md5()
|
|
temp_gz_path = output_dir / "temp_hadis_build.sqlite.gz"
|
|
|
|
with open(temp_db_path, 'rb') as f_in:
|
|
with gzip.open(temp_gz_path, 'wb', compresslevel=9) as f_out:
|
|
while chunk := f_in.read(1024 * 1024):
|
|
f_out.write(chunk)
|
|
hasher.update(chunk)
|
|
|
|
checksum = hasher.hexdigest()
|
|
temp_size = temp_db_path.stat().st_size
|
|
|
|
# Atomic replacement: replace final_gz_path with temp_gz_path
|
|
if final_gz_path.exists():
|
|
final_gz_path.unlink()
|
|
temp_gz_path.rename(final_gz_path)
|
|
file_size = final_gz_path.stat().st_size
|
|
|
|
# Clean up any remaining temp files in output_dir
|
|
temp_db_path.unlink(missing_ok=True)
|
|
for extra_file in output_dir.iterdir():
|
|
if extra_file.name != final_gz_name:
|
|
try:
|
|
extra_file.unlink(missing_ok=True)
|
|
except OSError:
|
|
pass
|
|
|
|
# 5. Record version in database (Single Active Record)
|
|
with transaction.atomic():
|
|
# Delete old records to keep DB clean
|
|
HadisDatabaseVersion.objects.all().delete()
|
|
db_version = HadisDatabaseVersion.objects.create(
|
|
version=new_version,
|
|
file=f"hadis_db/{final_gz_name}",
|
|
file_size=file_size,
|
|
checksum_md5=checksum,
|
|
hadis_count=hadis_count,
|
|
categories_count=categories_count,
|
|
is_active=True,
|
|
notes=options['notes'],
|
|
)
|
|
|
|
self.stdout.write(self.style.SUCCESS(
|
|
f"Successfully updated single Hadis SQLite DB to v{new_version}!\n"
|
|
f" - Uncompressed size: {temp_size / (1024*1024):.2f} MB\n"
|
|
f" - Gzipped size: {file_size / (1024*1024):.2f} MB\n"
|
|
f" - Hadiths: {hadis_count}\n"
|
|
f" - Categories: {categories_count}\n"
|
|
f" - Interpretations: {interp_count}\n"
|
|
f" - Corrections: {corr_count}\n"
|
|
f" - MD5 Checksum: {checksum}\n"
|
|
f" - File path: {final_gz_path}"
|
|
))
|
|
|
|
def _create_tables(self, cursor):
|
|
cursor.executescript("""
|
|
CREATE TABLE IF NOT EXISTS categories (
|
|
id INTEGER PRIMARY KEY,
|
|
title TEXT,
|
|
slug TEXT UNIQUE,
|
|
source_type TEXT,
|
|
sect_type TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS category_interpretations (
|
|
id INTEGER PRIMARY KEY,
|
|
category_id INTEGER,
|
|
title TEXT,
|
|
slug TEXT,
|
|
narrator TEXT,
|
|
text TEXT,
|
|
translation TEXT,
|
|
references_json TEXT,
|
|
links_json TEXT,
|
|
FOREIGN KEY (category_id) REFERENCES categories (id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS hadiths (
|
|
id INTEGER PRIMARY KEY,
|
|
slug TEXT UNIQUE,
|
|
category_id INTEGER,
|
|
category_slug TEXT,
|
|
title TEXT,
|
|
title_narrator TEXT,
|
|
text TEXT,
|
|
translation_json TEXT,
|
|
address TEXT,
|
|
share_link TEXT,
|
|
hadis_status_id INTEGER,
|
|
status_title TEXT,
|
|
status_color TEXT,
|
|
status_main_color_code TEXT,
|
|
status_text TEXT,
|
|
tags_json TEXT,
|
|
references_json TEXT,
|
|
reference_images_json TEXT,
|
|
narrators_json TEXT,
|
|
explanations_json TEXT,
|
|
FOREIGN KEY (category_id) REFERENCES categories (id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS hadith_corrections (
|
|
id INTEGER PRIMARY KEY,
|
|
hadis_id INTEGER,
|
|
title TEXT,
|
|
slug TEXT,
|
|
narrator TEXT,
|
|
description TEXT,
|
|
translation TEXT,
|
|
share_link TEXT,
|
|
address TEXT,
|
|
images_json TEXT,
|
|
references_json TEXT,
|
|
links_json TEXT,
|
|
FOREIGN KEY (hadis_id) REFERENCES hadiths (id)
|
|
);
|
|
""")
|
|
|
|
def _create_indexes(self, cursor):
|
|
cursor.executescript("""
|
|
CREATE INDEX IF NOT EXISTS idx_cat_slug ON categories(slug);
|
|
CREATE INDEX IF NOT EXISTS idx_interp_cat ON category_interpretations(category_id);
|
|
CREATE INDEX IF NOT EXISTS idx_hadith_cat ON hadiths(category_id);
|
|
CREATE INDEX IF NOT EXISTS idx_hadith_slug ON hadiths(slug);
|
|
CREATE INDEX IF NOT EXISTS idx_corr_hadis ON hadith_corrections(hadis_id);
|
|
""")
|
|
|
|
def _export_categories(self, cursor):
|
|
cats = HadisCategory.objects.all()
|
|
rows = [
|
|
(
|
|
c.id,
|
|
json.dumps(c.title, ensure_ascii=False) if isinstance(c.title, (dict, list)) else (c.title or ''),
|
|
c.slug,
|
|
getattr(c, 'source_type', None),
|
|
getattr(c, 'sect_type', None),
|
|
)
|
|
for c in cats
|
|
]
|
|
cursor.executemany(
|
|
"INSERT INTO categories (id, title, slug, source_type, sect_type) VALUES (?, ?, ?, ?, ?)",
|
|
rows
|
|
)
|
|
return len(rows)
|
|
|
|
def _export_category_interpretations(self, cursor):
|
|
interps = HadisInterpretation.objects.select_related('category').prefetch_related(
|
|
'references__book_reference', 'references__images'
|
|
).all()
|
|
|
|
rows = []
|
|
for inp in interps:
|
|
refs_list = []
|
|
for ref in inp.references.all():
|
|
book = ref.book_reference
|
|
refs_list.append({
|
|
'id': ref.id,
|
|
'book_title': getattr(book, 'title', None),
|
|
'address': getattr(ref, 'address', None),
|
|
'edition': getattr(ref, 'edition_id', None),
|
|
'volume': getattr(ref, 'volume_id', None),
|
|
'pages': getattr(ref, 'pages', None),
|
|
'hadith_number': getattr(ref, 'hadith_number', None),
|
|
})
|
|
|
|
rows.append((
|
|
inp.id,
|
|
inp.category_id,
|
|
json.dumps(inp.title, ensure_ascii=False) if isinstance(inp.title, (dict, list)) else (inp.title or ''),
|
|
inp.slug,
|
|
inp.narrator or '',
|
|
inp.text or '',
|
|
json.dumps(inp.translation, ensure_ascii=False) if isinstance(inp.translation, (dict, list)) else (inp.translation or ''),
|
|
json.dumps(refs_list, ensure_ascii=False),
|
|
json.dumps(inp.links or [], ensure_ascii=False),
|
|
))
|
|
|
|
cursor.executemany(
|
|
"""INSERT INTO category_interpretations
|
|
(id, category_id, title, slug, narrator, text, translation, references_json, links_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
rows
|
|
)
|
|
return len(rows)
|
|
|
|
def _export_hadiths_and_corrections(self, cursor):
|
|
hadiths = (
|
|
Hadis.objects
|
|
.filter(status=True)
|
|
.select_related('category', 'hadis_status')
|
|
.prefetch_related(
|
|
'tags',
|
|
'references__book_reference__author',
|
|
'references__images',
|
|
'transmitters__transmitter__reliability',
|
|
'transmitters__uncertain_transmitters__reliability',
|
|
'transmitters__narrator_layer',
|
|
'hadiscorrection_set__references__images',
|
|
)
|
|
.order_by('id')
|
|
)
|
|
|
|
hadis_rows = []
|
|
correction_rows = []
|
|
|
|
for h in hadiths:
|
|
# 1. Status Block
|
|
status_id = h.hadis_status.id if h.hadis_status else None
|
|
status_title = h.hadis_status.title if h.hadis_status else None
|
|
status_color = h.hadis_status.color if h.hadis_status else None
|
|
status_code = h.hadis_status.main_color_code if h.hadis_status else None
|
|
|
|
# 2. Tags
|
|
tags_list = [{'id': t.id, 'title': t.title} for t in h.tags.all()]
|
|
|
|
# 3. References & Reference Images
|
|
refs_list = []
|
|
ref_imgs_list = []
|
|
for ref in h.references.all():
|
|
book = ref.book_reference
|
|
authors = [{'id': book.author.id, 'name': book.author.name}] if book and book.author else []
|
|
refs_list.append({
|
|
'id': ref.id,
|
|
'title': book.title if book else None,
|
|
'authors': authors,
|
|
'share_link': getattr(book, 'share_link', None) if book else None,
|
|
})
|
|
for img in ref.images.all():
|
|
ref_imgs_list.append({
|
|
'id': img.id,
|
|
'thumbnail': img.thumbnail.url if img.thumbnail else None,
|
|
'priority': img.priority,
|
|
})
|
|
|
|
# 4. Narrators
|
|
transmitters_data = []
|
|
for tr in h.transmitters.all():
|
|
t = tr.transmitter
|
|
rel = t.reliability if t else None
|
|
transmitters_data.append({
|
|
'id': t.id if t else None,
|
|
'name': t.full_name if t else None,
|
|
'slug': t.slug if t else None,
|
|
'layer_id': tr.narrator_layer_id,
|
|
'is_uncertain': tr.is_uncertain,
|
|
'reliability': {
|
|
'id': rel.id,
|
|
'title': rel.title,
|
|
'slug': rel.slug,
|
|
'color': rel.color,
|
|
'main_color_code': rel.main_color_code,
|
|
} if rel else None,
|
|
})
|
|
|
|
# 5. Explanations
|
|
explanations_data = []
|
|
if hasattr(h, 'explanations') and h.explanations:
|
|
for exp in h.explanations.all():
|
|
explanations_data.append({
|
|
'title': exp.title,
|
|
'detail': getattr(exp, 'detail', '')
|
|
})
|
|
|
|
hadis_rows.append((
|
|
h.id,
|
|
h.slug,
|
|
h.category_id,
|
|
h.category.slug if h.category else None,
|
|
json.dumps(h.title, ensure_ascii=False) if isinstance(h.title, (dict, list)) else (h.title or ''),
|
|
json.dumps(h.title_narrator, ensure_ascii=False) if isinstance(h.title_narrator, (dict, list)) else (h.title_narrator or ''),
|
|
h.text or '',
|
|
json.dumps(h.translation or [], ensure_ascii=False),
|
|
json.dumps(h.address, ensure_ascii=False) if isinstance(h.address, (dict, list)) else (h.address or ''),
|
|
h.share_link or '',
|
|
status_id,
|
|
json.dumps(status_title, ensure_ascii=False) if isinstance(status_title, (dict, list)) else (status_title or ''),
|
|
status_color or '',
|
|
status_code or '',
|
|
json.dumps(h.hadis_status_text, ensure_ascii=False) if isinstance(h.hadis_status_text, (dict, list)) else (h.hadis_status_text or ''),
|
|
json.dumps(tags_list, ensure_ascii=False),
|
|
json.dumps(refs_list, ensure_ascii=False),
|
|
json.dumps(ref_imgs_list, ensure_ascii=False),
|
|
json.dumps({'transmitters': transmitters_data}, ensure_ascii=False),
|
|
json.dumps(explanations_data, ensure_ascii=False),
|
|
))
|
|
|
|
# 6. Corrections
|
|
for corr in h.hadiscorrection_set.all():
|
|
corr_imgs = []
|
|
corr_refs = []
|
|
for cref in corr.references.all():
|
|
for cimg in cref.images.all():
|
|
if cimg.image:
|
|
corr_imgs.append({'id': cimg.id, 'image': cimg.image.url})
|
|
corr_refs.append({
|
|
'id': cref.id,
|
|
'address': getattr(cref, 'address', None),
|
|
})
|
|
|
|
correction_rows.append((
|
|
corr.id,
|
|
h.id,
|
|
json.dumps(corr.title, ensure_ascii=False) if isinstance(corr.title, (dict, list)) else (corr.title or ''),
|
|
corr.slug,
|
|
corr.narrator or '',
|
|
corr.text or '',
|
|
json.dumps(corr.translation, ensure_ascii=False) if isinstance(corr.translation, (dict, list)) else (corr.translation or ''),
|
|
corr.share_link or '',
|
|
corr.address if hasattr(corr, 'address') else '',
|
|
json.dumps(corr_imgs, ensure_ascii=False),
|
|
json.dumps(corr_refs, ensure_ascii=False),
|
|
json.dumps(corr.links or [], ensure_ascii=False),
|
|
))
|
|
|
|
cursor.executemany(
|
|
"""INSERT INTO hadiths (
|
|
id, slug, category_id, category_slug, title, title_narrator, text,
|
|
translation_json, address, share_link, hadis_status_id, status_title,
|
|
status_color, status_main_color_code, status_text, tags_json,
|
|
references_json, reference_images_json, narrators_json, explanations_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
hadis_rows
|
|
)
|
|
|
|
cursor.executemany(
|
|
"""INSERT INTO hadith_corrections (
|
|
id, hadis_id, title, slug, narrator, description, translation,
|
|
share_link, address, images_json, references_json, links_json
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
correction_rows
|
|
)
|
|
|
|
return len(hadis_rows), len(correction_rows)
|