diff --git a/apps/hadis/management/commands/export_hadis_sqlite.py b/apps/hadis/management/commands/export_hadis_sqlite.py index 7f2abea..064a051 100644 --- a/apps/hadis/management/commands/export_hadis_sqlite.py +++ b/apps/hadis/management/commands/export_hadis_sqlite.py @@ -12,7 +12,12 @@ from apps.hadis.models import ( HadisCategory, HadisInterpretation, HadisCorrection, - HadisDatabaseVersion + HadisDatabaseVersion, + HadisStatus, + HadisTag, + BookReference, + BookAuthor, + Transmitters, ) from apps.hadis.serializers.hadis import ( DetailedCorrectionReferenceSerializer, @@ -72,9 +77,14 @@ class Command(BaseCommand): self._create_tables(cursor) # 3. Export data - categories_count = self._export_categories(cursor) - interp_count = self._export_category_interpretations(cursor) - hadis_count, corr_count = self._export_hadiths_and_corrections(cursor) + categories_count, cat_info_map = self._export_categories(cursor) + sources_count, book_image_map = self._export_sources(cursor) + authors_count = self._export_authors(cursor) + narrators_count = self._export_narrators(cursor) + statuses_count = self._export_statuses(cursor) + tags_count = self._export_tags(cursor) + interp_count = self._export_category_interpretations(cursor, book_image_map) + hadis_count, corr_count = self._export_hadiths_and_corrections(cursor, cat_info_map, book_image_map) conn.commit() @@ -134,6 +144,11 @@ class Command(BaseCommand): f" - Gzipped size: {file_size / (1024*1024):.2f} MB\n" f" - Hadiths: {hadis_count}\n" f" - Categories: {categories_count}\n" + f" - Sources: {sources_count}\n" + f" - Authors: {authors_count}\n" + f" - Narrators: {narrators_count}\n" + f" - Statuses: {statuses_count}\n" + f" - Tags: {tags_count}\n" f" - Interpretations: {interp_count}\n" f" - Corrections: {corr_count}\n" f" - MD5 Checksum: {checksum}\n" @@ -155,11 +170,53 @@ class Command(BaseCommand): hadis_id INTEGER, category_id INTEGER, category_slug TEXT, + source_type TEXT, + sect_type TEXT, PRIMARY KEY (hadis_id, category_id), FOREIGN KEY (hadis_id) REFERENCES hadiths (id), FOREIGN KEY (category_id) REFERENCES categories (id) ); + CREATE TABLE IF NOT EXISTS sources ( + id INTEGER PRIMARY KEY, + slug TEXT UNIQUE, + title TEXT, + image TEXT, + number_of_volumes INTEGER, + authors_json TEXT + ); + + CREATE TABLE IF NOT EXISTS authors ( + id INTEGER PRIMARY KEY, + slug TEXT UNIQUE, + name TEXT, + death_year_hijri TEXT + ); + + CREATE TABLE IF NOT EXISTS narrators ( + id INTEGER PRIMARY KEY, + slug TEXT UNIQUE, + name TEXT, + title TEXT, + gender TEXT, + birth_year_hijri TEXT, + death_year_hijri TEXT, + reliability_json TEXT + ); + + CREATE TABLE IF NOT EXISTS statuses ( + id INTEGER PRIMARY KEY, + title TEXT, + color TEXT, + main_color_code TEXT, + description TEXT + ); + + CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY, + title TEXT + ); + CREATE TABLE IF NOT EXISTS category_interpretations ( id INTEGER PRIMARY KEY, category_id INTEGER, @@ -178,6 +235,9 @@ class Command(BaseCommand): slug TEXT UNIQUE, category_id INTEGER, category_slug TEXT, + source_type TEXT, + sect_type TEXT, + categories_json TEXT, title TEXT, title_narrator TEXT, text TEXT, @@ -218,13 +278,21 @@ class Command(BaseCommand): def _create_indexes(self, cursor): cursor.executescript(""" CREATE INDEX IF NOT EXISTS idx_cat_slug ON categories(slug); + CREATE INDEX IF NOT EXISTS idx_cat_source_type ON categories(source_type); + CREATE INDEX IF NOT EXISTS idx_cat_sect_type ON categories(sect_type); CREATE INDEX IF NOT EXISTS idx_interp_cat ON category_interpretations(category_id); CREATE INDEX IF NOT EXISTS idx_hadith_cat ON hadiths(category_id); CREATE INDEX IF NOT EXISTS idx_hadith_slug ON hadiths(slug); + CREATE INDEX IF NOT EXISTS idx_hadith_source_type ON hadiths(source_type); + CREATE INDEX IF NOT EXISTS idx_hadith_sect_type ON hadiths(sect_type); + CREATE INDEX IF NOT EXISTS idx_hadith_status_id ON hadiths(hadis_status_id); CREATE INDEX IF NOT EXISTS idx_corr_hadis ON hadith_corrections(hadis_id); CREATE INDEX IF NOT EXISTS idx_hcat_cat_id ON hadith_categories(category_id); CREATE INDEX IF NOT EXISTS idx_hcat_cat_slug ON hadith_categories(category_slug); CREATE INDEX IF NOT EXISTS idx_hcat_hadis_id ON hadith_categories(hadis_id); + CREATE INDEX IF NOT EXISTS idx_sources_slug ON sources(slug); + CREATE INDEX IF NOT EXISTS idx_authors_slug ON authors(slug); + CREATE INDEX IF NOT EXISTS idx_narrators_slug ON narrators(slug); """) def _is_empty_localized_data(self, val): @@ -253,20 +321,198 @@ class Command(BaseCommand): return False def _export_categories(self, cursor): - cats = HadisCategory.objects.all() - rows = [ - ( + cats = list(HadisCategory.objects.select_related('sect').all()) + by_id = {c.id: c for c in cats} + + def resolve_source_type(cat_obj): + cur = cat_obj + visited = set() + while cur and cur.id not in visited: + visited.add(cur.id) + st = getattr(cur, 'source_type', None) + if st: + return st + cur = by_id.get(getattr(cur, 'parent_id', None)) + return 'hadith' + + def resolve_sect_type(cat_obj): + cur = cat_obj + visited = set() + while cur and cur.id not in visited: + visited.add(cur.id) + if getattr(cur, 'sect_id', None) and getattr(cur, 'sect', None) and getattr(cur.sect, 'sect_type', None): + return cur.sect.sect_type + st = getattr(cur, 'sect_type', None) + if st: + return st + cur = by_id.get(getattr(cur, 'parent_id', None)) + return 'shia' + + rows = [] + cat_info_map = {} + for c in cats: + source_type = resolve_source_type(c) + sect_type = resolve_sect_type(c) + cat_info_map[c.id] = { + 'id': c.id, + 'slug': c.slug, + 'title': c.title, + 'source_type': source_type, + 'sect_type': sect_type, + } + rows.append(( c.id, json.dumps(c.title, ensure_ascii=False) if isinstance(c.title, (dict, list)) else (c.title or ''), c.slug, - getattr(c, 'source_type', None), - getattr(c, 'sect_type', None), + source_type, + sect_type, 1 if getattr(c, 'is_topic', False) else 0, + )) + + cursor.executemany( + "INSERT INTO categories (id, title, slug, source_type, sect_type, is_topic) VALUES (?, ?, ?, ?, ?, ?)", + rows + ) + return len(rows), cat_info_map + + def _export_sources(self, cursor): + edition_meta = get_cached_edition_metadata() + first_by_book_id = edition_meta.get('first_by_book_id', {}) + by_book_id = edition_meta.get('by_book_id', {}) + book_volumes_count = edition_meta.get('book_volumes_count', {}) + + books = list( + BookReference.objects + .select_related('author') + .prefetch_related('images', 'volumes') + .all() + ) + + rows = [] + book_image_map = {} + for b in books: + img_url = None + b_imgs = list(b.images.all()) + if b_imgs and getattr(b_imgs[0], 'image', None): + img_url = self._make_full_url(b_imgs[0].image.url) + if not img_url: + for v in b.volumes.all(): + if getattr(v, 'image', None): + img_url = self._make_full_url(v.image.url) + break + book_image_map[b.id] = img_url + + num_vols = getattr(b, 'number_of_volumes', None) + if num_vols is None: + ed = first_by_book_id.get(b.id) or (by_book_id.get(b.id, [None])[0]) + if ed: + vol_cnt = getattr(ed, '_cached_vol_count', 0) + num_vols = vol_cnt if vol_cnt > 0 else getattr(ed, 'number_of_volumes', None) + if num_vols is None: + num_vols = book_volumes_count.get(b.id) or len(b.volumes.all()) or None + + authors_list = [] + if b.author: + authors_list.append({ + 'id': b.author.id, + 'slug': b.author.slug, + 'name': b.author.name, + 'death_year_hijri': b.author.death_year_hijri or '', + }) + + rows.append(( + b.id, + b.slug, + json.dumps(b.title, ensure_ascii=False) if isinstance(b.title, (dict, list)) else (b.title or ''), + img_url or '', + num_vols, + json.dumps(authors_list, ensure_ascii=False), + )) + + cursor.executemany( + "INSERT INTO sources (id, slug, title, image, number_of_volumes, authors_json) VALUES (?, ?, ?, ?, ?, ?)", + rows + ) + return len(rows), book_image_map + + def _export_authors(self, cursor): + authors = BookAuthor.objects.all() + rows = [ + ( + a.id, + a.slug, + json.dumps(a.name, ensure_ascii=False) if isinstance(a.name, (dict, list)) else (a.name or ''), + a.death_year_hijri or '', ) - for c in cats + for a in authors ] cursor.executemany( - "INSERT INTO categories (id, title, slug, source_type, sect_type, is_topic) VALUES (?, ?, ?, ?, ?, ?)", + "INSERT INTO authors (id, slug, name, death_year_hijri) VALUES (?, ?, ?, ?)", + rows + ) + return len(rows) + + def _export_narrators(self, cursor): + narrators = Transmitters.objects.select_related('reliability').all() + rows = [] + for t in narrators: + rel = t.reliability + rel_dict = { + 'id': rel.id, + 'title': rel.title, + 'slug': rel.slug, + 'color': rel.color, + 'main_color_code': rel.main_color_code, + 'description': getattr(rel, 'description', None), + } if rel else None + name_str = json.dumps(t.full_name, ensure_ascii=False) if isinstance(t.full_name, (dict, list)) else (t.full_name or '') + rows.append(( + t.id, + t.slug, + name_str, + name_str, + t.gender or '', + t.birth_year_hijri or '', + t.death_year_hijri or '', + json.dumps(rel_dict, ensure_ascii=False), + )) + cursor.executemany( + """INSERT INTO narrators + (id, slug, name, title, gender, birth_year_hijri, death_year_hijri, reliability_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + rows + ) + return len(rows) + + def _export_statuses(self, cursor): + statuses = HadisStatus.objects.all() + rows = [ + ( + s.id, + json.dumps(s.title, ensure_ascii=False) if isinstance(s.title, (dict, list)) else (s.title or ''), + s.color or '', + s.main_color_code or '', + json.dumps(s.description, ensure_ascii=False) if isinstance(s.description, (dict, list)) else (s.description or ''), + ) + for s in statuses + ] + cursor.executemany( + "INSERT INTO statuses (id, title, color, main_color_code, description) VALUES (?, ?, ?, ?, ?)", + rows + ) + return len(rows) + + def _export_tags(self, cursor): + tags = HadisTag.objects.all() + rows = [ + ( + t.id, + json.dumps(t.title, ensure_ascii=False) if isinstance(t.title, (dict, list)) else (t.title or ''), + ) + for t in tags + ] + cursor.executemany( + "INSERT INTO tags (id, title) VALUES (?, ?)", rows ) return len(rows) @@ -398,7 +644,7 @@ class Command(BaseCommand): 'contributors': [self._serialize_contributor_obj(c) for c in raw_contributors], } - def _serialize_reference(self, ref, edition_meta=None): + def _serialize_reference(self, ref, edition_meta=None, book_image_map=None): if edition_meta is None: edition_meta = get_cached_edition_metadata() @@ -438,10 +684,15 @@ class Command(BaseCommand): if book_num_vols is None: book_num_vols = edition_meta.get('book_volumes_count', {}).get(book.id) + book_img = book_image_map.get(book.id) if book_image_map else None + if not book_img and vol_obj and getattr(vol_obj, 'image', None): + book_img = self._make_full_url(vol_obj.image.url) + book_dict = { 'id': book.id, 'slug': book.slug, 'title': book.title, + 'image': book_img or '', 'publisher': book_pub or '', 'year_of_publication': book_year or '', 'number_of_volumes': book_num_vols, @@ -451,9 +702,15 @@ class Command(BaseCommand): 'id': author.id, 'slug': author.slug, 'name': author.name, + 'death_year_hijri': getattr(author, 'death_year_hijri', None) or '', } if author else None - book_authors = [{'id': author.id, 'slug': author.slug, 'name': author.name}] if author else [] + book_authors = [{ + 'id': author.id, + 'slug': author.slug, + 'name': author.name, + 'death_year_hijri': getattr(author, 'death_year_hijri', None) or '', + }] if author else [] volume_info = { 'id': vol_obj.id, @@ -514,7 +771,7 @@ class Command(BaseCommand): 'book_authors': book_authors, } - def _export_category_interpretations(self, cursor): + def _export_category_interpretations(self, cursor, book_image_map=None): edition_meta = get_cached_edition_metadata() interps = HadisInterpretation.objects.select_related('category').prefetch_related( 'references__book_reference__author', @@ -525,7 +782,7 @@ class Command(BaseCommand): rows = [] for inp in interps: - refs_list = [self._serialize_reference(ref, edition_meta) for ref in inp.references.all()] + refs_list = [self._serialize_reference(ref, edition_meta, book_image_map) for ref in inp.references.all()] rows.append(( inp.id, @@ -547,14 +804,15 @@ class Command(BaseCommand): ) return len(rows) - def _export_hadiths_and_corrections(self, cursor): + def _export_hadiths_and_corrections(self, cursor, cat_info_map=None, book_image_map=None): edition_meta = get_cached_edition_metadata() + cat_info_map = cat_info_map or {} hadiths = ( Hadis.objects .filter(status=True) - .select_related('category', 'hadis_status') + .select_related('category', 'category__sect', 'hadis_status') .prefetch_related( - 'categories', + 'categories__sect', 'tags', 'references__book_reference__author', 'references__edition', @@ -589,7 +847,7 @@ class Command(BaseCommand): refs_list = [] ref_imgs_list = [] for ref in h.references.all(): - ref_data = self._serialize_reference(ref, edition_meta) + ref_data = self._serialize_reference(ref, edition_meta, book_image_map) refs_list.append(ref_data) if ref_data.get('images'): ref_imgs_list.extend(ref_data['images']) @@ -599,12 +857,37 @@ class Command(BaseCommand): for tr in h.transmitters.all(): t = tr.transmitter rel = t.reliability if t else None + uncertain_list = [] + if tr.is_uncertain: + for ut in tr.uncertain_transmitters.all(): + ut_rel = ut.reliability + uncertain_list.append({ + 'id': ut.id, + 'name': ut.full_name, + 'title': ut.full_name, + 'slug': ut.slug, + 'gender': ut.gender or '', + 'birth_year_hijri': ut.birth_year_hijri or '', + 'death_year_hijri': ut.death_year_hijri or '', + 'reliability': { + 'id': ut_rel.id, + 'title': ut_rel.title, + 'slug': ut_rel.slug, + 'color': ut_rel.color, + 'main_color_code': ut_rel.main_color_code, + } if ut_rel else None, + }) transmitters_data.append({ 'id': t.id if t else None, 'name': t.full_name if t else None, + 'title': t.full_name if t else None, 'slug': t.slug if t else None, + 'gender': t.gender if t else '', + 'birth_year_hijri': t.birth_year_hijri if t else '', + 'death_year_hijri': t.death_year_hijri if t else '', 'layer_id': tr.narrator_layer_id, 'is_uncertain': tr.is_uncertain, + 'uncertain_transmitters': uncertain_list, 'reliability': { 'id': rel.id, 'title': rel.title, @@ -623,6 +906,49 @@ class Command(BaseCommand): 'detail': getattr(exp, 'detail', '') }) + # 5.5 Multi-category M2M relations (plus primary category) & source_type / sect_type + seen_cat_ids = set() + hadith_cats_list = [] + + def add_cat_entry(cat_obj): + if not cat_obj or cat_obj.id in seen_cat_ids: + return + seen_cat_ids.add(cat_obj.id) + c_info = cat_info_map.get(cat_obj.id) + if not c_info: + c_source = getattr(cat_obj, 'source_type', None) or 'hadith' + c_sect = ( + cat_obj.sect.sect_type + if (getattr(cat_obj, 'sect_id', None) and getattr(cat_obj, 'sect', None)) + else 'shia' + ) + c_info = { + 'id': cat_obj.id, + 'slug': cat_obj.slug, + 'title': cat_obj.title, + 'source_type': c_source, + 'sect_type': c_sect, + } + hadith_cats_list.append(c_info) + hadith_category_rows.append(( + h.id, + cat_obj.id, + cat_obj.slug, + c_info['source_type'], + c_info['sect_type'], + )) + + if h.category: + add_cat_entry(h.category) + for cat in h.categories.all(): + add_cat_entry(cat) + + primary_cat_info = hadith_cats_list[0] if hadith_cats_list else None + hadis_source_type = primary_cat_info['source_type'] if primary_cat_info else 'hadith' + hadis_sect_type = primary_cat_info['sect_type'] if primary_cat_info else 'shia' + primary_cat_id = h.category_id or (primary_cat_info['id'] if primary_cat_info else None) + primary_cat_slug = (h.category.slug if h.category else None) or (primary_cat_info['slug'] if primary_cat_info else None) + # Status description fallback: use hadis_status_text if present and non-empty, otherwise fallback to hadis_status.description status_desc_raw = h.hadis_status_text if self._is_empty_localized_data(status_desc_raw) and h.hadis_status: @@ -633,8 +959,11 @@ class Command(BaseCommand): hadis_rows.append(( h.id, h.slug, - h.category_id, - h.category.slug if h.category else None, + primary_cat_id, + primary_cat_slug, + hadis_source_type, + hadis_sect_type, + json.dumps(hadith_cats_list, ensure_ascii=False), json.dumps(h.title, ensure_ascii=False) if isinstance(h.title, (dict, list)) else (h.title or ''), json.dumps(h.title_narrator, ensure_ascii=False) if isinstance(h.title_narrator, (dict, list)) else (h.title_narrator or ''), h.text or '', @@ -654,30 +983,12 @@ class Command(BaseCommand): json.dumps(format_links_output(h.links), ensure_ascii=False), )) - # 5.5 Multi-category M2M relations (plus primary category) - seen_cat_ids = set() - if h.category_id: - seen_cat_ids.add(h.category_id) - hadith_category_rows.append(( - h.id, - h.category_id, - h.category.slug if h.category else None, - )) - for cat in h.categories.all(): - if cat.id not in seen_cat_ids: - seen_cat_ids.add(cat.id) - hadith_category_rows.append(( - h.id, - cat.id, - cat.slug, - )) - # 6. Corrections for corr in h.hadiscorrection_set.all(): corr_imgs = [] corr_refs = [] for cref in corr.references.all(): - ref_data = self._serialize_reference(cref, edition_meta) + ref_data = self._serialize_reference(cref, edition_meta, book_image_map) corr_refs.append(ref_data) if ref_data.get('images'): corr_imgs.extend(ref_data['images']) @@ -701,19 +1012,20 @@ class Command(BaseCommand): cursor.executemany( """INSERT INTO hadiths ( - id, slug, category_id, category_slug, title, title_narrator, text, + id, slug, category_id, category_slug, source_type, sect_type, categories_json, + title, title_narrator, text, translation_json, address, share_link, hadis_status_id, status_title, status_color, status_main_color_code, status_text, tags_json, references_json, reference_images_json, narrators_json, explanations_json, links_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", hadis_rows ) cursor.executemany( """INSERT OR IGNORE INTO hadith_categories ( - hadis_id, category_id, category_slug - ) VALUES (?, ?, ?)""", + hadis_id, category_id, category_slug, source_type, sect_type + ) VALUES (?, ?, ?, ?, ?)""", hadith_category_rows )