dovodi
/
Dovodi_Backend
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

data script enhancement and deta schema update 

book creation is now only creates if a edition or folder mentioned, not blindly all the volumes !

script deletes the old books

some models updated with new values
master
Mohsen Taba 2 weeks ago
parent
1a56bc8a40
commit
f790b3252e
9 changed files with 345 additions and 104 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      apps/course/admin/course.py
  2. 249
      apps/hadis/management/commands/import_legacy_data.py
  3. 58
      apps/hadis/migrations/0011_bookreference_city_of_publication_and_more.py
  4. 68
      apps/hadis/migrations/0012_correctionreferenceimage_pages_and_more.py
  5. 40
      apps/hadis/migrations/0013_remove_correctionreferenceimage_pages_and_more.py
  6. 9
      apps/hadis/models/hadis.py
  7. 5
      apps/hadis/models/reference.py
  8. 9
      apps/hadis/models/transmitter.py
  9. 8
      apps/hadis/serializers/hadis.py

1
apps/course/admin/course.py
View File

@ -1,3 +1,4 @@
from utils.admin import admin_url_generator
import os import os
import hashlib import hashlib

249
apps/hadis/management/commands/import_legacy_data.py
View File

@ -1,12 +1,12 @@
import os import os
import json import json
import csv import csv
import re
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.core.files import File from django.core.files import File
from django.db import transaction from django.db import transaction
from django.conf import settings from django.conf import settings
# Import all necessary models
from apps.hadis.models import ( from apps.hadis.models import (
HadisCategory, HadisSect, HadisStatus, HadisTag, Hadis, HadisCategory, HadisSect, HadisStatus, HadisTag, Hadis,
HadisCorrection, HadisReference, ReferenceImage, HadisTransmitter, HadisCorrection, HadisReference, ReferenceImage, HadisTransmitter,
@ -23,16 +23,12 @@ class Command(BaseCommand):
parser.add_argument('base_dir', type=str, help='Absolute path to the "тестовая база данных" directory') parser.add_argument('base_dir', type=str, help='Absolute path to the "тестовая база данных" directory')
def wrap_lang(self, text, lang="ru"): def wrap_lang(self, text, lang="ru"):
"""Helper to format strings into the [ {'language_code': lang, 'text': text} ] schema.
Always returns a valid dictionary to bypass Django's blank=False validators."""
if text is None:
text = ""
if text is None: text = ""
return [{"language_code": lang, "text": str(text).strip()}] return [{"language_code": lang, "text": str(text).strip()}]
@transaction.atomic @transaction.atomic
def handle(self, *args, **kwargs): def handle(self, *args, **kwargs):
base_dir = kwargs['base_dir'] base_dir = kwargs['base_dir']
if not os.path.exists(base_dir): if not os.path.exists(base_dir):
self.stderr.write(self.style.ERROR(f'Directory not found: {base_dir}')) self.stderr.write(self.style.ERROR(f'Directory not found: {base_dir}'))
return return
@ -45,6 +41,28 @@ class Command(BaseCommand):
narrators_path = os.path.join(base_dir, 'narrators.json') narrators_path = os.path.join(base_dir, 'narrators.json')
tathir_path = os.path.join(base_dir, 'tathir.json') tathir_path = os.path.join(base_dir, 'tathir.json')
# --- PRE-SCAN TATHIR.JSON FOR CITED VOLUMES ---
cited_book_volumes = {}
if os.path.exists(tathir_path):
with open(tathir_path, 'r', encoding='utf-8') as f:
t_data = json.load(f).get('materials', [])
for item in t_data:
for ed in item.get('editions', []):
b_id = ed.get('book_id')
b_vol = str(ed.get('volume')).strip() if ed.get('volume') is not None else ''
if b_vol.lower() == 'none': b_vol = ''
if b_id:
if b_id not in cited_book_volumes:
cited_book_volumes[b_id] = set()
if b_vol:
try: cited_book_volumes[b_id].add(str(int(b_vol)))
except ValueError: cited_book_volumes[b_id].add(b_vol)
# --- PRE-FLIGHT CLEANUP ---
self.stdout.write(self.style.WARNING('\n--- PRE-FLIGHT: Cleaning up old legacy books ---'))
BookReference.objects.exclude(legacy_id__isnull=True).exclude(legacy_id__exact='').delete()
# --- PHASE 1: SCHOLARS & BOOKS --- # --- PHASE 1: SCHOLARS & BOOKS ---
self.stdout.write(self.style.WARNING('\n--- PHASE 1: Loading Scholars & Books ---')) self.stdout.write(self.style.WARNING('\n--- PHASE 1: Loading Scholars & Books ---'))
@ -54,11 +72,7 @@ class Command(BaseCommand):
reader = csv.reader(f) reader = csv.reader(f)
for row in reader: for row in reader:
if len(row) >= 3: if len(row) >= 3:
scholars_map[row[0].strip()] = {
"ar": row[1].strip(),
"ru": row[2].strip()
}
self.stdout.write(f'Loaded {len(scholars_map)} scholars into memory.')
scholars_map[row[0].strip()] = {"ar": row[1].strip(), "ru": row[2].strip()}
if os.path.exists(bib_path): if os.path.exists(bib_path):
with open(bib_path, 'r', encoding='utf-8') as f: with open(bib_path, 'r', encoding='utf-8') as f:
@ -70,67 +84,83 @@ class Command(BaseCommand):
author_name = row[2].strip() author_name = row[2].strip()
base_title = row[3].strip() base_title = row[3].strip()
# Extract total volumes (Column 11 / Index 10)
vol_str = row[10].strip() if len(row) > 10 else '' vol_str = row[10].strip() if len(row) > 10 else ''
try:
total_vols = int(vol_str) if vol_str.isdigit() else 1
except ValueError:
total_vols = 1
# Create a BookReference for EVERY volume
for v in range(1, total_vols + 1):
# Generate unique ID and Title for multi-volume books
is_multi_vol = total_vols > 1
legacy_id = f"{base_legacy_id}-v{v}" if is_multi_vol else base_legacy_id
title_text = f"{base_title} (Vol {v})" if is_multi_vol else base_title
try: total_vols_int = int(vol_str) if vol_str.isdigit() else 1
except ValueError: total_vols_int = 1
existing_vols = set()
book_folder = os.path.join(base_dir, 'books', base_legacy_id)
if os.path.exists(book_folder):
for item in os.listdir(book_folder):
if os.path.isdir(os.path.join(book_folder, item)):
try: existing_vols.add(str(int(item)))
except ValueError: existing_vols.add(item)
volumes_to_create = existing_vols.union(cited_book_volumes.get(base_legacy_id, set()))
if not volumes_to_create: volumes_to_create = {''}
for v in volumes_to_create:
legacy_id = f"{base_legacy_id}-v{v}" if v else base_legacy_id
title_text = f"{base_title} (Vol {v})" if v else base_title
book, _ = BookReference.objects.update_or_create( book, _ = BookReference.objects.update_or_create(
legacy_id=legacy_id, legacy_id=legacy_id,
defaults={ defaults={
'title': self.wrap_lang(title_text), 'title': self.wrap_lang(title_text),
'number_of_volumes': total_vols,
'volume': str(v),
'order': int(row[1]) if len(row) > 1 and row[1].isdigit() else 0,
'researcher': self.wrap_lang(row[4].strip() if len(row) > 4 else ''),
'publisher': self.wrap_lang(row[5].strip() if len(row) > 5 else ''),
'city_of_publication': self.wrap_lang(row[6].strip() if len(row) > 6 else ''),
'country_of_publication': self.wrap_lang(row[7].strip() if len(row) > 7 else ''),
'edition_number': row[8].strip() if len(row) > 8 else '',
'year_of_publication': row[9].strip() if len(row) > 9 else '', 'year_of_publication': row[9].strip() if len(row) > 9 else '',
'number_of_volumes': total_vols_int,
'volume': v,
'source_url': row[11].strip() if len(row) > 11 else '', 'source_url': row[11].strip() if len(row) > 11 else '',
'description': self.wrap_lang(row[12].strip() if len(row) > 12 else ''), 'description': self.wrap_lang(row[12].strip() if len(row) > 12 else ''),
'publisher': self.wrap_lang(row[5].strip() if len(row) > 5 else ''),
'language': self.wrap_lang('') 'language': self.wrap_lang('')
} }
) )
# Author
if author_name: if author_name:
author, _ = BookAuthor.objects.get_or_create(name=self.wrap_lang(author_name)) author, _ = BookAuthor.objects.get_or_create(name=self.wrap_lang(author_name))
book.authors.add(author) book.authors.add(author)
# Scan Book Folder for Specific Volume Images and PDFs
book_folder = os.path.join(base_dir, 'books', base_legacy_id)
if os.path.exists(book_folder):
vol_num_str = str(v)
vol_padded_str = str(v).zfill(2) # "1" -> "01"
# Map Book Tags
if len(row) > 13 and row[13].strip():
for t in row[13].split(','):
if t.strip():
btag, _ = BookSubjectArea.objects.get_or_create(title=self.wrap_lang(t.strip()))
book.subject_area.add(btag)
# Attach Media
if os.path.exists(book_folder):
for root, _, files in os.walk(book_folder): for root, _, files in os.walk(book_folder):
folder_name = os.path.basename(root) folder_name = os.path.basename(root)
is_root = (root == book_folder)
for file in files: for file in files:
file_path = os.path.join(root, file)
file_lower = file.lower() file_lower = file.lower()
# Attach PDF if it matches "1.pdf" or "01.pdf"
file_path = os.path.join(root, file)
if file_lower.endswith('.pdf'): if file_lower.endswith('.pdf'):
if file_lower in [f"{vol_num_str}.pdf", f"{vol_padded_str}.pdf"] or not is_multi_vol:
if v and file_lower in [f"{v}.pdf", f"{v.zfill(2)}.pdf"]:
with open(file_path, 'rb') as doc_f: with open(file_path, 'rb') as doc_f:
doc = BookReferenceDocument(book_reference=book, volume=vol_num_str, title=file)
doc = BookReferenceDocument(book_reference=book, volume=v, title=file)
doc.file.save(file, File(doc_f), save=True)
elif not v and is_root and not file_lower[0].isdigit():
with open(file_path, 'rb') as doc_f:
doc = BookReferenceDocument(book_reference=book, volume=v, title=file)
doc.file.save(file, File(doc_f), save=True) doc.file.save(file, File(doc_f), save=True)
# Attach Images if they are in folder "1" or "01"
elif file_lower.endswith(('.png', '.jpg', '.jpeg', '.gif')): elif file_lower.endswith(('.png', '.jpg', '.jpeg', '.gif')):
if folder_name in [vol_num_str, vol_padded_str] or not is_multi_vol:
if v and not is_root and folder_name.lstrip('0') == v.lstrip('0'):
with open(file_path, 'rb') as img_f: with open(file_path, 'rb') as img_f:
img = BookReferenceImage(book_reference=book, volume=vol_num_str)
img = BookReferenceImage(book_reference=book, volume=v)
img.image.save(file, File(img_f), save=True)
elif not v and is_root:
with open(file_path, 'rb') as img_f:
img = BookReferenceImage(book_reference=book, volume=v)
img.image.save(file, File(img_f), save=True) img.image.save(file, File(img_f), save=True)
self.stdout.write(self.style.SUCCESS('Books (split by volumes) loaded successfully.'))
self.stdout.write(self.style.SUCCESS('Books loaded successfully.'))
# --- PHASE 2: NARRATORS --- # --- PHASE 2: NARRATORS ---
@ -142,25 +172,29 @@ class Command(BaseCommand):
for n_data in n_data_list: for n_data in n_data_list:
legacy_id = n_data.get('id') legacy_id = n_data.get('id')
legacy_number = int(n_data.get('narrator_number')) if str(n_data.get('narrator_number')).isdigit() else None legacy_number = int(n_data.get('narrator_number')) if str(n_data.get('narrator_number')).isdigit() else None
info = n_data.get('info', {}) info = n_data.get('info', {})
ar_info = info.get('arabic', {}) ar_info = info.get('arabic', {})
reliability, _ = TransmitterReliability.objects.get_or_create(
title=self.wrap_lang(n_data.get('reliability', 'Unknown'))
)
reliability, _ = TransmitterReliability.objects.get_or_create(title=self.wrap_lang(n_data.get('reliability', 'Unknown')))
generation = int(n_data.get('generation')) if str(n_data.get('generation')).isdigit() else None generation = int(n_data.get('generation')) if str(n_data.get('generation')).isdigit() else None
if generation: if generation:
NarratorLayer.objects.get_or_create(
number=generation,
defaults={
'name': self.wrap_lang(f'Layer {generation}'),
'description': self.wrap_lang('')
}
)
NarratorLayer.objects.get_or_create(number=generation, defaults={'name': self.wrap_lang(f'Layer {generation}'), 'description': self.wrap_lang('')})
# Safe Age Extraction
age_str = info.get('age', '')
age_nums = re.findall(r'\d+', str(age_str))
age_val = int(age_nums[0]) if age_nums else None
# Madhhab Translation
madhhab_list = n_data.get('madhab', [])
madhhab_val = Transmitters.MadhhabChoices.UNKNOWN
if madhhab_list:
m_str = str(madhhab_list[0]).lower()
if 'шиит' in m_str: madhhab_val = Transmitters.MadhhabChoices.SHIA
elif 'суннит' in m_str: madhhab_val = Transmitters.MadhhabChoices.SUNNI
else: madhhab_val = Transmitters.MadhhabChoices.OTHER
# Create Transmitter
transmitter, _ = Transmitters.objects.update_or_create( transmitter, _ = Transmitters.objects.update_or_create(
legacy_id=legacy_id, legacy_id=legacy_id,
defaults={ defaults={
@ -177,11 +211,20 @@ class Command(BaseCommand):
'reliability': reliability, 'reliability': reliability,
'in_sahih_bukhari': n_data.get('transmitted_to_bukhari', False), 'in_sahih_bukhari': n_data.get('transmitted_to_bukhari', False),
'in_sahih_muslim': n_data.get('transmitted_to_muslim', False), 'in_sahih_muslim': n_data.get('transmitted_to_muslim', False),
'relatives_raw': info.get('relatives', {})
'relatives_raw': info.get('relatives', {}),
# NEW FIELDS MAPPED
'freed_slave_of': self.wrap_lang(info.get('freed_slave_of', ''), 'ru') + self.wrap_lang(ar_info.get('freed_slave_of', ''), 'ar'),
'occupation': self.wrap_lang(info.get('occupation', ''), 'ru') + self.wrap_lang(ar_info.get('occupation', ''), 'ar'),
'features': self.wrap_lang(info.get('features', ''), 'ru') + self.wrap_lang(ar_info.get('features', ''), 'ar'),
'birth_year_hijri': str(info.get('birth_year', '')),
'death_year_hijri': str(info.get('death_year', '')),
'age_at_death': age_val,
'tags': n_data.get('tags', []),
'madhhab': madhhab_val,
} }
) )
# Opinions
for op in n_data.get('strengthened_weakened', {}).get('review', []): for op in n_data.get('strengthened_weakened', {}).get('review', []):
author_ui = op.get('author_ui') author_ui = op.get('author_ui')
scholar_data = scholars_map.get(author_ui, {"ar": author_ui, "ru": author_ui}) scholar_data = scholars_map.get(author_ui, {"ar": author_ui, "ru": author_ui})
@ -191,7 +234,6 @@ class Command(BaseCommand):
scholar_name=self.wrap_lang(scholar_data['ar'], 'ar') + self.wrap_lang(scholar_data['ru'], 'ru') scholar_name=self.wrap_lang(scholar_data['ar'], 'ar') + self.wrap_lang(scholar_data['ru'], 'ru')
) )
# Original Texts
for text_data in n_data.get('excerpts', []): for text_data in n_data.get('excerpts', []):
orig_text, _ = TransmitterOriginalText.objects.get_or_create( orig_text, _ = TransmitterOriginalText.objects.get_or_create(
transmitter=transmitter, transmitter=transmitter,
@ -199,14 +241,16 @@ class Command(BaseCommand):
text=self.wrap_lang(text_data.get('text'), 'ar'), text=self.wrap_lang(text_data.get('text'), 'ar'),
translation=self.wrap_lang(text_data.get('translation'), 'ru') translation=self.wrap_lang(text_data.get('translation'), 'ru')
) )
for ed in text_data.get('editions', []): for ed in text_data.get('editions', []):
book_ref = self._get_book_volume(ed.get('book_id'), ed.get('volume')) book_ref = self._get_book_volume(ed.get('book_id'), ed.get('volume'))
ref_obj, _ = OriginalTextReference.objects.get_or_create(
ref_obj, _ = OriginalTextReference.objects.update_or_create(
original_text=orig_text, book_reference=book_ref, original_text=orig_text, book_reference=book_ref,
volume=ed.get('volume'), page=ed.get('page'), url=ed.get('url')
defaults={
'volume': str(ed.get('volume', '')),
'page': str(ed.get('pages', '')), # Fixed from 'page'
'url': ed.get('url', '')
}
) )
folder = ed.get('screenshots_folder') folder = ed.get('screenshots_folder')
if folder: if folder:
self._attach_images(os.path.join(base_dir, 'screens_trx', legacy_id, folder), OriginalTextReferenceImage, ref_obj) self._attach_images(os.path.join(base_dir, 'screens_trx', legacy_id, folder), OriginalTextReferenceImage, ref_obj)
@ -214,21 +258,17 @@ class Command(BaseCommand):
self.stdout.write(self.style.SUCCESS('Narrators loaded successfully.')) self.stdout.write(self.style.SUCCESS('Narrators loaded successfully.'))
# --- PHASE 3: HADITHS (Arguments, Corrections, Interpretations) ---
# --- PHASE 3: HADITHS ---
self.stdout.write(self.style.WARNING('\n--- PHASE 3: Loading Hadiths ---')) self.stdout.write(self.style.WARNING('\n--- PHASE 3: Loading Hadiths ---'))
default_sect, _ = HadisSect.objects.get_or_create( default_sect, _ = HadisSect.objects.get_or_create(
sect_type='sunni', sect_type='sunni',
defaults={
'title': self.wrap_lang('Sunni'),
'description': self.wrap_lang('')
}
defaults={'title': self.wrap_lang('Sunni'), 'description': self.wrap_lang('')}
) )
if os.path.exists(tathir_path): if os.path.exists(tathir_path):
with open(tathir_path, 'r', encoding='utf-8') as f: with open(tathir_path, 'r', encoding='utf-8') as f:
materials = json.load(f).get('materials', []) materials = json.load(f).get('materials', [])
# Map corrections to their parent hadiths
correction_to_hadith_map = {} correction_to_hadith_map = {}
for item in materials: for item in materials:
if item.get('type') == 'arguments': if item.get('type') == 'arguments':
@ -243,11 +283,7 @@ class Command(BaseCommand):
cat_str = item.get('category', [''])[0] cat_str = item.get('category', [''])[0]
category, _ = HadisCategory.objects.get_or_create( category, _ = HadisCategory.objects.get_or_create(
title=self.wrap_lang(cat_str), title=self.wrap_lang(cat_str),
defaults={
'sect': default_sect,
'source_type': item.get('subtype', 'hadith') or 'hadith',
'description': self.wrap_lang('')
}
defaults={'sect': default_sect, 'source_type': item.get('subtype', 'hadith') or 'hadith', 'description': self.wrap_lang('')}
) )
status, _ = HadisStatus.objects.get_or_create( status, _ = HadisStatus.objects.get_or_create(
title=self.wrap_lang(item.get('authenticity', '')), title=self.wrap_lang(item.get('authenticity', '')),
@ -257,8 +293,7 @@ class Command(BaseCommand):
hadis, _ = Hadis.objects.update_or_create( hadis, _ = Hadis.objects.update_or_create(
legacy_id=item.get('id'), legacy_id=item.get('id'),
defaults={ defaults={
'category': category,
'hadis_status': status,
'category': category, 'hadis_status': status,
'title': self.wrap_lang(item.get('aliases', [''])[0] if item.get('aliases') else ''), 'title': self.wrap_lang(item.get('aliases', [''])[0] if item.get('aliases') else ''),
'title_narrator': self.wrap_lang(item.get('aliases', [''])[0] if item.get('aliases') else ''), 'title_narrator': self.wrap_lang(item.get('aliases', [''])[0] if item.get('aliases') else ''),
'description': self.wrap_lang(''), 'description': self.wrap_lang(''),
@ -270,15 +305,17 @@ class Command(BaseCommand):
} }
) )
# Map Hadith Tags
hadis.tags.clear()
for tag_str in item.get('tags', []):
htag, _ = HadisTag.objects.get_or_create(title=self.wrap_lang(tag_str))
hadis.tags.add(htag)
raw_chain = item.get('chain', []) raw_chain = item.get('chain', [])
chain_arrays = [] chain_arrays = []
if raw_chain: if raw_chain:
# Normalize: If it's a flat list of ints, wrap it in a list so it's a 2D array
if isinstance(raw_chain[0], int):
chain_arrays = [raw_chain]
else:
chain_arrays = raw_chain
if isinstance(raw_chain[0], int): chain_arrays = [raw_chain]
else: chain_arrays = raw_chain
for chain_idx, narrator_ids in enumerate(chain_arrays): for chain_idx, narrator_ids in enumerate(chain_arrays):
for order_idx, n_id in enumerate(narrator_ids): for order_idx, n_id in enumerate(narrator_ids):
@ -289,12 +326,18 @@ class Command(BaseCommand):
hadis=hadis, transmitter=transmitter, chain_index=chain_idx, order=order_idx, hadis=hadis, transmitter=transmitter, chain_index=chain_idx, order=order_idx,
defaults={'narrator_layer': layer, 'status': transmitter.reliability} defaults={'narrator_layer': layer, 'status': transmitter.reliability}
) )
# Editions & Images
for ed in item.get('editions', []): for ed in item.get('editions', []):
book = self._get_book_volume(ed.get('book_id'), ed.get('volume')) book = self._get_book_volume(ed.get('book_id'), ed.get('volume'))
href, _ = HadisReference.objects.get_or_create(
href, _ = HadisReference.objects.update_or_create(
hadis=hadis, book_reference=book, hadis=hadis, book_reference=book,
defaults={'hadith_number': str(ed.get('hadith_number', '')), 'description': self.wrap_lang('')}
defaults={
'hadith_number': str(ed.get('hadith_number', '')),
'description': self.wrap_lang(''),
'volume': str(ed.get('volume', '')),
'pages': str(ed.get('pages', '')),
'url': ed.get('url', '')
}
) )
if ed.get('screenshots_folder'): if ed.get('screenshots_folder'):
self._attach_images(os.path.join(base_dir, 'screens', item.get('id'), ed.get('screenshots_folder')), ReferenceImage, href, field_name='thumbnail') self._attach_images(os.path.join(base_dir, 'screens', item.get('id'), ed.get('screenshots_folder')), ReferenceImage, href, field_name='thumbnail')
@ -303,19 +346,27 @@ class Command(BaseCommand):
elif i_type == 'authenticity_analysis': elif i_type == 'authenticity_analysis':
parent_id = correction_to_hadith_map.get(item.get('id')) parent_id = correction_to_hadith_map.get(item.get('id'))
parent_hadith = Hadis.objects.filter(legacy_id=parent_id).first() parent_hadith = Hadis.objects.filter(legacy_id=parent_id).first()
if parent_hadith: if parent_hadith:
corr, _ = HadisCorrection.objects.get_or_create(
# CHANGE TO update_or_create HERE:
corr, _ = HadisCorrection.objects.update_or_create(
hadis=parent_hadith, legacy_id=item.get('id'), hadis=parent_hadith, legacy_id=item.get('id'),
defaults={ defaults={
'title': self.wrap_lang(''), 'title': self.wrap_lang(''),
'text': item.get('original_text', ''), # Directly mapped to TextField
'text': item.get('original_text', ''),
'translation': self.wrap_lang(item.get('translation', ''), 'ru') 'translation': self.wrap_lang(item.get('translation', ''), 'ru')
} }
) )
for ed in item.get('editions', []): for ed in item.get('editions', []):
book = self._get_book_volume(ed.get('book_id'), ed.get('volume')) book = self._get_book_volume(ed.get('book_id'), ed.get('volume'))
cref, _ = CorrectionReference.objects.get_or_create(correction=corr, book_reference=book, defaults={'hadith_number': str(ed.get('hadith_number', ''))})
cref, _ = CorrectionReference.objects.update_or_create(
correction=corr, book_reference=book,
defaults={
'hadith_number': str(ed.get('hadith_number', '')),
'volume': str(ed.get('volume', '')),
'pages': str(ed.get('pages', '')),
'url': ed.get('url', '')
}
)
if ed.get('screenshots_folder'): if ed.get('screenshots_folder'):
self._attach_images(os.path.join(base_dir, 'screens', item.get('id'), ed.get('screenshots_folder')), CorrectionReferenceImage, cref) self._attach_images(os.path.join(base_dir, 'screens', item.get('id'), ed.get('screenshots_folder')), CorrectionReferenceImage, cref)
@ -323,9 +374,9 @@ class Command(BaseCommand):
elif i_type == 'interpretation': elif i_type == 'interpretation':
cat_str = item.get('category', [''])[0] if item.get('category') else '' cat_str = item.get('category', [''])[0] if item.get('category') else ''
category = HadisCategory.objects.filter(title__contains=[{'text': cat_str}]).first() category = HadisCategory.objects.filter(title__contains=[{'text': cat_str}]).first()
if category: if category:
interp, _ = HadisInterpretation.objects.get_or_create(
# CHANGE TO update_or_create HERE:
interp, _ = HadisInterpretation.objects.update_or_create(
category=category, legacy_id=item.get('id'), category=category, legacy_id=item.get('id'),
defaults={ defaults={
'title': self.wrap_lang(''), 'title': self.wrap_lang(''),
@ -335,7 +386,15 @@ class Command(BaseCommand):
) )
for ed in item.get('editions', []): for ed in item.get('editions', []):
book = self._get_book_volume(ed.get('book_id'), ed.get('volume')) book = self._get_book_volume(ed.get('book_id'), ed.get('volume'))
iref, _ = InterpretationReference.objects.get_or_create(interpretation=interp, book_reference=book, defaults={'hadith_number': str(ed.get('hadith_number', ''))})
iref, _ = InterpretationReference.objects.update_or_create(
interpretation=interp, book_reference=book,
defaults={
'hadith_number': str(ed.get('hadith_number', '')),
'volume': str(ed.get('volume', '')),
'pages': str(ed.get('pages', '')),
'url': ed.get('url', '')
}
)
if ed.get('screenshots_folder'): if ed.get('screenshots_folder'):
self._attach_images(os.path.join(base_dir, 'screens', item.get('id'), ed.get('screenshots_folder')), InterpretationReferenceImage, iref) self._attach_images(os.path.join(base_dir, 'screens', item.get('id'), ed.get('screenshots_folder')), InterpretationReferenceImage, iref)
@ -344,15 +403,11 @@ class Command(BaseCommand):
def _get_book_volume(self, book_id, volume_str): def _get_book_volume(self, book_id, volume_str):
"""Finds the specific volume of a book, with fallbacks.""" """Finds the specific volume of a book, with fallbacks."""
if not book_id: return None if not book_id: return None
# 1. Try to find specific volume (e.g., uuid-v2)
if volume_str: if volume_str:
vol_clean = ''.join(filter(str.isdigit, str(volume_str))) # extracts "2" from "Vol 2"
vol_clean = ''.join(filter(str.isdigit, str(volume_str)))
if vol_clean: if vol_clean:
book = BookReference.objects.filter(legacy_id=f"{book_id}-v{vol_clean}").first() book = BookReference.objects.filter(legacy_id=f"{book_id}-v{vol_clean}").first()
if book: return book if book: return book
# 2. Fallback: Find the base book (single volume) or the first volume available
return BookReference.objects.filter(legacy_id__startswith=book_id).first() return BookReference.objects.filter(legacy_id__startswith=book_id).first()
def _attach_images(self, folder_path, ImageModelClass, reference_instance, field_name='image'): def _attach_images(self, folder_path, ImageModelClass, reference_instance, field_name='image'):
@ -363,7 +418,5 @@ class Command(BaseCommand):
file_path = os.path.join(folder_path, filename) file_path = os.path.join(folder_path, filename)
with open(file_path, 'rb') as f: with open(file_path, 'rb') as f:
img_obj = ImageModelClass(reference=reference_instance, priority=i) img_obj = ImageModelClass(reference=reference_instance, priority=i)
# Dynamically grab the correct field ('image' or 'thumbnail')
image_field = getattr(img_obj, field_name) image_field = getattr(img_obj, field_name)
image_field.save(filename, File(f), save=True) image_field.save(filename, File(f), save=True)

58
apps/hadis/migrations/0011_bookreference_city_of_publication_and_more.py
View File

@ -0,0 +1,58 @@
# Generated by Django 5.2.12 on 2026-05-10 10:43
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('hadis', '0010_remove_hadiscorrection_description_and_more'),
]
operations = [
migrations.AddField(
model_name='bookreference',
name='city_of_publication',
field=models.JSONField(blank=True, default=list, verbose_name='City of Publication'),
),
migrations.AddField(
model_name='bookreference',
name='country_of_publication',
field=models.JSONField(blank=True, default=list, verbose_name='Country of Publication'),
),
migrations.AddField(
model_name='bookreference',
name='edition_number',
field=models.CharField(blank=True, max_length=100, null=True, verbose_name='Edition Number'),
),
migrations.AddField(
model_name='bookreference',
name='order',
field=models.IntegerField(default=0, verbose_name='Order'),
),
migrations.AddField(
model_name='bookreference',
name='researcher',
field=models.JSONField(blank=True, default=list, verbose_name='Researcher'),
),
migrations.AddField(
model_name='transmitters',
name='features',
field=models.JSONField(blank=True, default=list, verbose_name='Features'),
),
migrations.AddField(
model_name='transmitters',
name='freed_slave_of',
field=models.JSONField(blank=True, default=list, verbose_name='Freed Slave Of'),
),
migrations.AddField(
model_name='transmitters',
name='occupation',
field=models.JSONField(blank=True, default=list, verbose_name='Occupation'),
),
migrations.AddField(
model_name='transmitters',
name='tags',
field=models.JSONField(blank=True, default=list, verbose_name='Tags'),
),
]

68
apps/hadis/migrations/0012_correctionreferenceimage_pages_and_more.py
View File

@ -0,0 +1,68 @@
# Generated by Django 5.2.12 on 2026-05-10 10:59
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('hadis', '0011_bookreference_city_of_publication_and_more'),
]
operations = [
migrations.AddField(
model_name='correctionreferenceimage',
name='pages',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='correctionreferenceimage',
name='url',
field=models.URLField(blank=True, max_length=1000, null=True),
),
migrations.AddField(
model_name='correctionreferenceimage',
name='volume',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='hadisreference',
name='pages',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='hadisreference',
name='url',
field=models.URLField(blank=True, max_length=1000, null=True),
),
migrations.AddField(
model_name='hadisreference',
name='volume',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='interpretationreference',
name='pages',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='interpretationreference',
name='url',
field=models.URLField(blank=True, max_length=1000, null=True),
),
migrations.AddField(
model_name='interpretationreference',
name='volume',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AlterField(
model_name='transmitters',
name='birth_year_hijri',
field=models.CharField(blank=True, max_length=50, null=True, verbose_name='Birth Year (Hijri)'),
),
migrations.AlterField(
model_name='transmitters',
name='death_year_hijri',
field=models.CharField(blank=True, max_length=50, null=True, verbose_name='Death Year (Hijri)'),
),
]

40
apps/hadis/migrations/0013_remove_correctionreferenceimage_pages_and_more.py
View File

@ -0,0 +1,40 @@
# Generated by Django 5.2.12 on 2026-05-10 11:06
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('hadis', '0012_correctionreferenceimage_pages_and_more'),
]
operations = [
migrations.RemoveField(
model_name='correctionreferenceimage',
name='pages',
),
migrations.RemoveField(
model_name='correctionreferenceimage',
name='url',
),
migrations.RemoveField(
model_name='correctionreferenceimage',
name='volume',
),
migrations.AddField(
model_name='correctionreference',
name='pages',
field=models.CharField(blank=True, max_length=100, null=True),
),
migrations.AddField(
model_name='correctionreference',
name='url',
field=models.URLField(blank=True, max_length=1000, null=True),
),
migrations.AddField(
model_name='correctionreference',
name='volume',
field=models.CharField(blank=True, max_length=100, null=True),
),
]

9
apps/hadis/models/hadis.py
View File

@ -421,6 +421,9 @@ class HadisReference(models.Model):
verbose_name=_('book reference'), verbose_name=_('book reference'),
related_name='hadis_references' related_name='hadis_references'
) )
volume = models.CharField(max_length=100, null=True, blank=True)
pages = models.CharField(max_length=100, null=True, blank=True)
url = models.URLField(max_length=1000, null=True, blank=True)
hadith_number = models.CharField(max_length=100, null=True, blank=True, verbose_name=_('Hadith Number in Book')) hadith_number = models.CharField(max_length=100, null=True, blank=True, verbose_name=_('Hadith Number in Book'))
created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at')) created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at'))
@ -582,6 +585,9 @@ class CorrectionReference(models.Model):
correction = models.ForeignKey(HadisCorrection, on_delete=models.CASCADE, related_name='references') correction = models.ForeignKey(HadisCorrection, on_delete=models.CASCADE, related_name='references')
book_reference = models.ForeignKey(BookReference, on_delete=models.SET_NULL, null=True, blank=True) book_reference = models.ForeignKey(BookReference, on_delete=models.SET_NULL, null=True, blank=True)
hadith_number = models.CharField(max_length=100, null=True, blank=True) hadith_number = models.CharField(max_length=100, null=True, blank=True)
volume = models.CharField(max_length=100, null=True, blank=True)
pages = models.CharField(max_length=100, null=True, blank=True)
url = models.URLField(max_length=1000, null=True, blank=True)
class CorrectionReferenceImage(models.Model): class CorrectionReferenceImage(models.Model):
reference = models.ForeignKey(CorrectionReference, related_name='images', on_delete=models.CASCADE) reference = models.ForeignKey(CorrectionReference, related_name='images', on_delete=models.CASCADE)
@ -594,6 +600,9 @@ class InterpretationReference(models.Model):
interpretation = models.ForeignKey(HadisInterpretation, on_delete=models.CASCADE, related_name='references') interpretation = models.ForeignKey(HadisInterpretation, on_delete=models.CASCADE, related_name='references')
book_reference = models.ForeignKey(BookReference, on_delete=models.SET_NULL, null=True, blank=True) book_reference = models.ForeignKey(BookReference, on_delete=models.SET_NULL, null=True, blank=True)
hadith_number = models.CharField(max_length=100, null=True, blank=True) hadith_number = models.CharField(max_length=100, null=True, blank=True)
volume = models.CharField(max_length=100, null=True, blank=True)
pages = models.CharField(max_length=100, null=True, blank=True)
url = models.URLField(max_length=1000, null=True, blank=True)
class InterpretationReferenceImage(models.Model): class InterpretationReferenceImage(models.Model):
reference = models.ForeignKey(InterpretationReference, related_name='images', on_delete=models.CASCADE) reference = models.ForeignKey(InterpretationReference, related_name='images', on_delete=models.CASCADE)

5
apps/hadis/models/reference.py
View File

@ -74,6 +74,11 @@ class BookReference(models.Model):
publisher = models.JSONField(default = list , verbose_name=_('Publisher')) publisher = models.JSONField(default = list , verbose_name=_('Publisher'))
subject_area = models.ManyToManyField(BookSubjectArea, related_name="book_subjects", verbose_name=_('subject area'), blank=True) subject_area = models.ManyToManyField(BookSubjectArea, related_name="book_subjects", verbose_name=_('subject area'), blank=True)
type = models.ForeignKey(BookType, on_delete=models.SET_NULL, null=True, blank=True, verbose_name=_('type')) type = models.ForeignKey(BookType, on_delete=models.SET_NULL, null=True, blank=True, verbose_name=_('type'))
order = models.IntegerField(default=0, verbose_name=_('Order'))
researcher = models.JSONField(default=list, blank=True, verbose_name=_('Researcher'))
city_of_publication = models.JSONField(default=list, blank=True, verbose_name=_('City of Publication'))
country_of_publication = models.JSONField(default=list, blank=True, verbose_name=_('Country of Publication'))
edition_number = models.CharField(max_length=100, blank=True, null=True, verbose_name=_('Edition Number'))
rate = models.DecimalField( rate = models.DecimalField(
max_digits=3, max_digits=3,
decimal_places=2, decimal_places=2,

9
apps/hadis/models/transmitter.py
View File

@ -243,14 +243,19 @@ class Transmitters(models.Model):
# ManyToMany for siblings allows bi-directional sibling relationships # ManyToMany for siblings allows bi-directional sibling relationships
siblings = models.ManyToManyField('self', blank=True, verbose_name=_('Siblings')) siblings = models.ManyToManyField('self', blank=True, verbose_name=_('Siblings'))
freed_slave_of = models.JSONField(default=list, blank=True, verbose_name=_('Freed Slave Of'))
occupation = models.JSONField(default=list, blank=True, verbose_name=_('Occupation'))
features = models.JSONField(default=list, blank=True, verbose_name=_('Features'))
tags = models.JSONField(default=list, blank=True, verbose_name=_('Tags'))
# Geographic Information # Geographic Information
origin = models.JSONField(default = list , verbose_name=_('Origin')) origin = models.JSONField(default = list , verbose_name=_('Origin'))
lived_in = models.JSONField(default = list , verbose_name=_('Lived in')) lived_in = models.JSONField(default = list , verbose_name=_('Lived in'))
died_in = models.JSONField(default = list , verbose_name=_('Died in')) died_in = models.JSONField(default = list , verbose_name=_('Died in'))
# Date Information # Date Information
birth_year_hijri = models.IntegerField(verbose_name=_("Birth Year (Hijri)"), null=True, blank=True)
death_year_hijri = models.IntegerField(verbose_name=_("Death Year (Hijri)"), null=True, blank=True)
birth_year_hijri = models.CharField(max_length=50, verbose_name=_("Birth Year (Hijri)"), null=True, blank=True)
death_year_hijri = models.CharField(max_length=50, verbose_name=_("Death Year (Hijri)"), null=True, blank=True)
age_at_death = models.PositiveIntegerField(verbose_name=_('Age at Death'), blank=True, null=True) age_at_death = models.PositiveIntegerField(verbose_name=_('Age at Death'), blank=True, null=True)
generation = models.PositiveIntegerField(verbose_name=_('Generation'), blank=True, null=True) generation = models.PositiveIntegerField(verbose_name=_('Generation'), blank=True, null=True)
# Religious & Academic Information # Religious & Academic Information

8
apps/hadis/serializers/hadis.py
View File

@ -205,7 +205,7 @@ class HadisSyncSerializer(serializers.ModelSerializer):
corrections_data.append({ corrections_data.append({
'id': correction.id, 'id': correction.id,
'title': get_localized_text(correction.title, request), 'title': get_localized_text(correction.title, request),
'text': correction.text,
'description': correction.text,
'translation': get_localized_text(correction.translation, request), 'translation': get_localized_text(correction.translation, request),
'share_link': correction.share_link, 'share_link': correction.share_link,
}) })
@ -706,12 +706,14 @@ class HadisReferenceSerializer(serializers.ModelSerializer):
class HadisCorrectionSerializer(serializers.ModelSerializer): class HadisCorrectionSerializer(serializers.ModelSerializer):
"""Serializer for HadisCorrection""" """Serializer for HadisCorrection"""
title = LocalizedField() title = LocalizedField()
text = LocalizedField()
description = serializers.SerializerMethodField()
translation = LocalizedField() translation = LocalizedField()
bookmark = serializers.SerializerMethodField() bookmark = serializers.SerializerMethodField()
class Meta: class Meta:
model = HadisCorrection model = HadisCorrection
fields = ['id', 'title','slug', 'text', 'translation','share_link','bookmark']
fields = ['id', 'title','slug','description', 'translation','share_link','bookmark']
def get_description(self, obj):
return obj.text
def get_bookmark(self, obj): def get_bookmark(self, obj):
"""Get bookmark information for this correction.""" """Get bookmark information for this correction."""

Write Preview
Loading…
Cancel
Save