Browse Source

feat(hadis): add links_json to hadiths table and export full reference/edition details in offline SQLite

- Add links_json column to hadiths table and standardize link objects via format_links_output across hadiths, interpretations, and corrections
- Include complete book, author, book_authors, edition/edition_info (title, language, edition_number, publisher, publication_place, contributors with role order and volume scope, researchers, editors), and volume/citation fields in references_json for all exported tables
master
Mohsen Taba 2 weeks ago
parent
commit
7b7179c7b8
  1. 241
      apps/hadis/management/commands/export_hadis_sqlite.py

241
apps/hadis/management/commands/export_hadis_sqlite.py

@ -17,7 +17,9 @@ from apps.hadis.models import (
from apps.hadis.serializers.hadis import ( from apps.hadis.serializers.hadis import (
DetailedCorrectionReferenceSerializer, DetailedCorrectionReferenceSerializer,
DetailedInterpretationReferenceSerializer, DetailedInterpretationReferenceSerializer,
get_cached_edition_metadata,
) )
from apps.hadis.utils.links import format_links_output
class Command(BaseCommand): class Command(BaseCommand):
@ -192,6 +194,7 @@ class Command(BaseCommand):
reference_images_json TEXT, reference_images_json TEXT,
narrators_json TEXT, narrators_json TEXT,
explanations_json TEXT, explanations_json TEXT,
links_json TEXT,
FOREIGN KEY (category_id) REFERENCES categories (id) FOREIGN KEY (category_id) REFERENCES categories (id)
); );
@ -277,18 +280,141 @@ class Command(BaseCommand):
url = '/' + url url = '/' + url
return f'https://dovodi.newhorizonco.uk{url}' return f'https://dovodi.newhorizonco.uk{url}'
def _serialize_reference(self, ref):
def _resolve_edition_for_ref(self, ref, edition_meta):
by_id = edition_meta.get('by_id', {})
by_book_id = edition_meta.get('by_book_id', {})
first_by_book_id = edition_meta.get('first_by_book_id', {})
ed_id = getattr(ref, 'edition_id', None)
if ed_id and ed_id in by_id:
return by_id[ed_id]
if getattr(ref, 'edition', None):
return by_id.get(ref.edition.id, ref.edition)
vol_obj = getattr(ref, 'book_volume', None)
if vol_obj:
bv_ed_id = getattr(vol_obj, 'edition_id', None)
if bv_ed_id and bv_ed_id in by_id:
return by_id[bv_ed_id]
if getattr(vol_obj, 'edition', None):
return by_id.get(vol_obj.edition.id, vol_obj.edition)
book_id = getattr(ref, 'book_reference_id', None)
if book_id:
if book_id in first_by_book_id:
return first_by_book_id[book_id]
book_eds = by_book_id.get(book_id, [])
if book_eds:
return book_eds[0]
return None
def _serialize_contributor_obj(self, c):
author_obj = getattr(c, 'author', None)
author_data = {
'id': author_obj.id,
'slug': author_obj.slug,
'name': author_obj.name,
} if author_obj else None
role_obj = getattr(c, 'role', None)
role_data = {
'id': role_obj.id,
'slug': role_obj.slug,
'title': role_obj.title,
'order': getattr(role_obj, 'order', 0),
} if role_obj else None
ret = {
'id': c.id,
'name': getattr(c, 'name', None) or (author_obj.name if author_obj else ''),
'slug': author_obj.slug if (author_obj and author_obj.slug) else getattr(c, 'slug', None),
'author_id': getattr(c, 'author_id', None),
'author': author_data,
}
if role_data is not None:
ret['role_id'] = role_obj.id
ret['role'] = role_data
if hasattr(c, 'volumes'):
vols = list(c.volumes.all())
ret['volume_ids'] = [v.id for v in vols]
ret['volumes'] = [{'id': v.id, 'title': v.title} for v in vols]
return ret
def _serialize_edition(self, ed):
if not ed:
return None
pub = (
ed.publisher_item.title
if (getattr(ed, 'publisher_item_id', None) and getattr(ed, 'publisher_item', None) and ed.publisher_item.title)
else getattr(ed, 'publisher', None)
)
city = (
ed.city_of_publication_city.title
if (getattr(ed, 'city_of_publication_city_id', None) and getattr(ed, 'city_of_publication_city', None) and ed.city_of_publication_city.title)
else getattr(ed, 'city_of_publication', None)
)
country = (
ed.country_of_publication_country.title
if (getattr(ed, 'country_of_publication_country_id', None) and getattr(ed, 'country_of_publication_country', None) and ed.country_of_publication_country.title)
else getattr(ed, 'country_of_publication', None)
)
if hasattr(ed, '_cached_vol_count'):
vol_count = ed._cached_vol_count
elif hasattr(ed, '_prefetched_objects_cache') and 'volumes' in ed._prefetched_objects_cache:
vol_count = len(ed.volumes.all())
elif hasattr(ed, 'volumes'):
vol_count = ed.volumes.count()
else:
vol_count = 0
num_vols = vol_count if vol_count > 0 else getattr(ed, 'number_of_volumes', None)
raw_researchers = ed._cached_researchers if hasattr(ed, '_cached_researchers') else (ed.researchers.all() if hasattr(ed, 'researchers') else [])
raw_editors = ed._cached_editors if hasattr(ed, '_cached_editors') else (ed.editors.all() if hasattr(ed, 'editors') else [])
raw_contributors = list(ed._cached_contributors if hasattr(ed, '_cached_contributors') else (ed.contributors.all() if hasattr(ed, 'contributors') else []))
raw_contributors.sort(
key=lambda c: (
getattr(c.role, 'order', 999999) if getattr(c, 'role_id', None) and getattr(c, 'role', None) else 999999,
c.id or 0,
)
)
return {
'id': ed.id,
'title': getattr(ed, 'title', None) or '',
'language': getattr(ed, 'language', 'ar') or 'ar',
'edition_number': ed.edition_number,
'publisher': pub or '',
'publication_place': {
'city': city or '',
'country': country or '',
},
'year_of_publication': getattr(ed, 'year_of_publication', None) or '',
'number_of_volumes': num_vols,
'source_url': getattr(ed, 'source_url', None) or '',
'researchers': [self._serialize_contributor_obj(r) for r in raw_researchers],
'editors': [self._serialize_contributor_obj(e) for e in raw_editors],
'contributors': [self._serialize_contributor_obj(c) for c in raw_contributors],
}
def _serialize_reference(self, ref, edition_meta=None):
if edition_meta is None:
edition_meta = get_cached_edition_metadata()
book = getattr(ref, 'book_reference', None) book = getattr(ref, 'book_reference', None)
author = book.author if book and getattr(book, 'author', None) else None author = book.author if book and getattr(book, 'author', None) else None
ed = getattr(ref, 'edition', None)
ed = self._resolve_reference_edition(ref, edition_meta) if hasattr(self, '_resolve_reference_edition') else self._resolve_edition_for_ref(ref, edition_meta)
vol_obj = getattr(ref, 'book_volume', None) vol_obj = getattr(ref, 'book_volume', None)
images_list = [] images_list = []
if hasattr(ref, 'images'): if hasattr(ref, 'images'):
for img in ref.images.all():
if img.image:
img_url = self._make_full_url(img.image.url)
thumb_url = self._make_full_url(img.thumbnail.url if getattr(img, 'thumbnail', None) else img.image.url)
for img in sorted(ref.images.all(), key=lambda x: getattr(x, 'priority', 0)):
img_file = getattr(img, 'image', None) or getattr(img, 'thumbnail', None)
if img_file:
img_url = self._make_full_url(img_file.url)
thumb_file = getattr(img, 'thumbnail', None) or img_file
thumb_url = self._make_full_url(thumb_file.url)
images_list.append({ images_list.append({
'id': img.id, 'id': img.id,
'image': img_url, 'image': img_url,
@ -296,14 +422,30 @@ class Command(BaseCommand):
'priority': getattr(img, 'priority', 0), 'priority': getattr(img, 'priority', 0),
}) })
book_dict = {
'id': book.id,
'slug': book.slug,
'title': book.title,
'publisher': getattr(book, 'publisher', '') or '',
'year_of_publication': getattr(book, 'year_of_publication', '') or '',
'number_of_volumes': getattr(book, 'number_of_volumes', None),
} if book else None
edition_dict = self._serialize_edition(ed)
book_dict = None
if book:
book_pub = getattr(book, 'publisher', None)
if not book_pub and edition_dict:
book_pub = edition_dict.get('publisher')
book_year = getattr(book, 'year_of_publication', None)
if not book_year and edition_dict:
book_year = edition_dict.get('year_of_publication')
book_num_vols = getattr(book, 'number_of_volumes', None)
if book_num_vols is None and edition_dict:
book_num_vols = edition_dict.get('number_of_volumes')
if book_num_vols is None:
book_num_vols = edition_meta.get('book_volumes_count', {}).get(book.id)
book_dict = {
'id': book.id,
'slug': book.slug,
'title': book.title,
'publisher': book_pub or '',
'year_of_publication': book_year or '',
'number_of_volumes': book_num_vols,
}
author_dict = { author_dict = {
'id': author.id, 'id': author.id,
@ -311,13 +453,7 @@ class Command(BaseCommand):
'name': author.name, 'name': author.name,
} if author else None } if author else None
edition_dict = {
'id': ed.id,
'edition_number': ed.edition_number,
'publisher': getattr(ed, 'publisher', '') or '',
'year_of_publication': getattr(ed, 'year_of_publication', '') or '',
'number_of_volumes': getattr(ed, 'number_of_volumes', None),
} if ed else None
book_authors = [{'id': author.id, 'slug': author.slug, 'name': author.name}] if author else []
volume_info = { volume_info = {
'id': vol_obj.id, 'id': vol_obj.id,
@ -327,17 +463,29 @@ class Command(BaseCommand):
open_book_url = None open_book_url = None
if vol_obj and getattr(vol_obj, 'file', None): if vol_obj and getattr(vol_obj, 'file', None):
open_book_url = self._make_full_url(vol_obj.file.url) open_book_url = self._make_full_url(vol_obj.file.url)
elif getattr(ref, 'url', None):
open_book_url = getattr(ref, 'url', None)
elif ed and getattr(ed, 'source_url', None):
open_book_url = ed.source_url
raw_vol = getattr(ref, 'volume', None)
vol_val = (
raw_vol
if (raw_vol and str(raw_vol).strip().lower() != 'none')
else (vol_obj.title if (vol_obj and vol_obj.title) else None)
)
raw_pg = getattr(ref, 'pages', None)
pg_val = raw_pg if (raw_pg and str(raw_pg).strip().lower() != 'none') else None
raw_num = getattr(ref, 'hadith_number', None)
num_val = raw_num if (raw_num and str(raw_num).strip().lower() != 'none') else None
parts = [] parts = []
vol_val = getattr(ref, 'volume', None) or (vol_obj.title if vol_obj else None)
if vol_val: if vol_val:
v_str = str(vol_val).strip() v_str = str(vol_val).strip()
parts.append(v_str if v_str.startswith("جلد") else f"جلد {v_str}") parts.append(v_str if v_str.startswith("جلد") else f"جلد {v_str}")
pg_val = getattr(ref, 'pages', None)
if pg_val: if pg_val:
p_str = str(pg_val).strip() p_str = str(pg_val).strip()
parts.append(p_str if (p_str.startswith("ص") or p_str.startswith("صفحه")) else f"صفحه {p_str}") parts.append(p_str if (p_str.startswith("ص") or p_str.startswith("صفحه")) else f"صفحه {p_str}")
num_val = getattr(ref, 'hadith_number', None)
if num_val: if num_val:
n_str = str(num_val).strip() n_str = str(num_val).strip()
parts.append(n_str if (n_str.startswith("ح") or n_str.startswith("حدیث") or n_str.startswith("شماره")) else f"شماره حدیث {n_str}") parts.append(n_str if (n_str.startswith("ح") or n_str.startswith("حدیث") or n_str.startswith("شماره")) else f"شماره حدیث {n_str}")
@ -345,11 +493,15 @@ class Command(BaseCommand):
return { return {
'id': ref.id, 'id': ref.id,
'title': getattr(book, 'title', None) if book else None,
'authors': book_authors,
'share_link': getattr(book, 'share_link', None) if book else None,
'book': book_dict, 'book': book_dict,
'author': author_dict, 'author': author_dict,
'edition': edition_dict, 'edition': edition_dict,
'edition_info': edition_dict,
'volume_info': volume_info, 'volume_info': volume_info,
'hadith_number': getattr(ref, 'hadith_number', None),
'hadith_number': num_val,
'volume': vol_val, 'volume': vol_val,
'pages': pg_val, 'pages': pg_val,
'address': getattr(ref, 'address', None), 'address': getattr(ref, 'address', None),
@ -359,11 +511,11 @@ class Command(BaseCommand):
'images': images_list, 'images': images_list,
'images_count': len(images_list), 'images_count': len(images_list),
'book_title': getattr(book, 'title', None) if book else None, 'book_title': getattr(book, 'title', None) if book else None,
'book_authors': [{'id': author.id, 'name': author.name, 'slug': author.slug}] if author else [],
'edition_info': getattr(ed, 'publisher', None) if ed else None,
'book_authors': book_authors,
} }
def _export_category_interpretations(self, cursor): def _export_category_interpretations(self, cursor):
edition_meta = get_cached_edition_metadata()
interps = HadisInterpretation.objects.select_related('category').prefetch_related( interps = HadisInterpretation.objects.select_related('category').prefetch_related(
'references__book_reference__author', 'references__book_reference__author',
'references__images', 'references__images',
@ -373,7 +525,7 @@ class Command(BaseCommand):
rows = [] rows = []
for inp in interps: for inp in interps:
refs_list = [self._serialize_reference(ref) for ref in inp.references.all()]
refs_list = [self._serialize_reference(ref, edition_meta) for ref in inp.references.all()]
rows.append(( rows.append((
inp.id, inp.id,
@ -384,7 +536,7 @@ class Command(BaseCommand):
inp.text or '', inp.text or '',
json.dumps(inp.translation, ensure_ascii=False) if isinstance(inp.translation, (dict, list)) else (inp.translation or ''), json.dumps(inp.translation, ensure_ascii=False) if isinstance(inp.translation, (dict, list)) else (inp.translation or ''),
json.dumps(refs_list, ensure_ascii=False), json.dumps(refs_list, ensure_ascii=False),
json.dumps(inp.links or [], ensure_ascii=False),
json.dumps(format_links_output(inp.links), ensure_ascii=False),
)) ))
cursor.executemany( cursor.executemany(
@ -396,6 +548,7 @@ class Command(BaseCommand):
return len(rows) return len(rows)
def _export_hadiths_and_corrections(self, cursor): def _export_hadiths_and_corrections(self, cursor):
edition_meta = get_cached_edition_metadata()
hadiths = ( hadiths = (
Hadis.objects Hadis.objects
.filter(status=True) .filter(status=True)
@ -404,6 +557,8 @@ class Command(BaseCommand):
'categories', 'categories',
'tags', 'tags',
'references__book_reference__author', 'references__book_reference__author',
'references__edition',
'references__book_volume',
'references__images', 'references__images',
'transmitters__transmitter__reliability', 'transmitters__transmitter__reliability',
'transmitters__uncertain_transmitters__reliability', 'transmitters__uncertain_transmitters__reliability',
@ -434,20 +589,10 @@ class Command(BaseCommand):
refs_list = [] refs_list = []
ref_imgs_list = [] ref_imgs_list = []
for ref in h.references.all(): for ref in h.references.all():
book = ref.book_reference
authors = [{'id': book.author.id, 'name': book.author.name}] if book and book.author else []
refs_list.append({
'id': ref.id,
'title': book.title if book else None,
'authors': authors,
'share_link': getattr(book, 'share_link', None) if book else None,
})
for img in ref.images.all():
ref_imgs_list.append({
'id': img.id,
'thumbnail': self._make_full_url(img.thumbnail.url if img.thumbnail else None),
'priority': img.priority,
})
ref_data = self._serialize_reference(ref, edition_meta)
refs_list.append(ref_data)
if ref_data.get('images'):
ref_imgs_list.extend(ref_data['images'])
# 4. Narrators # 4. Narrators
transmitters_data = [] transmitters_data = []
@ -506,6 +651,7 @@ class Command(BaseCommand):
json.dumps(ref_imgs_list, ensure_ascii=False), json.dumps(ref_imgs_list, ensure_ascii=False),
json.dumps({'transmitters': transmitters_data}, ensure_ascii=False), json.dumps({'transmitters': transmitters_data}, ensure_ascii=False),
json.dumps(explanations_data, ensure_ascii=False), json.dumps(explanations_data, ensure_ascii=False),
json.dumps(format_links_output(h.links), ensure_ascii=False),
)) ))
# 5.5 Multi-category M2M relations (plus primary category) # 5.5 Multi-category M2M relations (plus primary category)
@ -531,7 +677,7 @@ class Command(BaseCommand):
corr_imgs = [] corr_imgs = []
corr_refs = [] corr_refs = []
for cref in corr.references.all(): for cref in corr.references.all():
ref_data = self._serialize_reference(cref)
ref_data = self._serialize_reference(cref, edition_meta)
corr_refs.append(ref_data) corr_refs.append(ref_data)
if ref_data.get('images'): if ref_data.get('images'):
corr_imgs.extend(ref_data['images']) corr_imgs.extend(ref_data['images'])
@ -550,7 +696,7 @@ class Command(BaseCommand):
corr_address, corr_address,
json.dumps(corr_imgs, ensure_ascii=False), json.dumps(corr_imgs, ensure_ascii=False),
json.dumps(corr_refs, ensure_ascii=False), json.dumps(corr_refs, ensure_ascii=False),
json.dumps(corr.links or [], ensure_ascii=False),
json.dumps(format_links_output(corr.links), ensure_ascii=False),
)) ))
cursor.executemany( cursor.executemany(
@ -558,8 +704,9 @@ class Command(BaseCommand):
id, slug, category_id, category_slug, title, title_narrator, text, id, slug, category_id, category_slug, title, title_narrator, text,
translation_json, address, share_link, hadis_status_id, status_title, translation_json, address, share_link, hadis_status_id, status_title,
status_color, status_main_color_code, status_text, tags_json, status_color, status_main_color_code, status_text, tags_json,
references_json, reference_images_json, narrators_json, explanations_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
references_json, reference_images_json, narrators_json, explanations_json,
links_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
hadis_rows hadis_rows
) )

Loading…
Cancel
Save