Browse Source

Enhance error handling and transaction management in hadis data seeding command

- Implemented inner atomic blocks to ensure that failures during the creation of NarratorLayer, TransmitterReliability, OpinionStatus, Transmitters, and related models do not affect the overall transaction.
- Improved logging to provide warnings for skipped records due to errors, enhancing feedback during the seeding process.
- Streamlined the creation process for authors and books, ensuring that existing records are utilized effectively while maintaining data integrity.
master
mortezaei 4 months ago
parent
commit
c0d2a56fee
  1. 24
      apps/hadis/management/commands/seed_complete_hadis_data.py

24
apps/hadis/management/commands/seed_complete_hadis_data.py

@ -490,6 +490,8 @@ class Command(BaseCommand):
else: else:
# Create new layer - but be defensive # Create new layer - but be defensive
try: try:
# Use an inner atomic block so a single failure doesn't poison the outer transaction
with transaction.atomic():
layer = NarratorLayer.objects.create( layer = NarratorLayer.objects.create(
number=layer_data['number'], number=layer_data['number'],
name=[{'language_code': 'ru', 'text': layer_data['name']}], name=[{'language_code': 'ru', 'text': layer_data['name']}],
@ -514,6 +516,8 @@ class Command(BaseCommand):
# Create or get reliability statuses - using filter().first() to avoid MultipleObjectsReturned # Create or get reliability statuses - using filter().first() to avoid MultipleObjectsReturned
for reliability_data in RUSSIAN_RELIABILITY_LEVELS: for reliability_data in RUSSIAN_RELIABILITY_LEVELS:
try: try:
# Wrap in a savepoint so failures don't break the outer atomic transaction
with transaction.atomic():
# Try to get by slug first # Try to get by slug first
reliability = TransmitterReliability.objects.filter(slug=reliability_data['slug']).first() reliability = TransmitterReliability.objects.filter(slug=reliability_data['slug']).first()
if reliability: if reliability:
@ -540,6 +544,8 @@ class Command(BaseCommand):
# Create or get opinion statuses - using filter().first() to avoid MultipleObjectsReturned # Create or get opinion statuses - using filter().first() to avoid MultipleObjectsReturned
for opinion_data in RUSSIAN_OPINION_STATUSES: for opinion_data in RUSSIAN_OPINION_STATUSES:
try: try:
# Wrap in a savepoint so failures don't break the outer atomic transaction
with transaction.atomic():
# Try to get by slug first # Try to get by slug first
opinion_status = OpinionStatus.objects.filter(slug=opinion_data['slug']).first() opinion_status = OpinionStatus.objects.filter(slug=opinion_data['slug']).first()
if opinion_status: if opinion_status:
@ -577,6 +583,9 @@ class Command(BaseCommand):
random_father_full_name = random.choice(RUSSIAN_TRANSMITTER_NAMES) random_father_full_name = random.choice(RUSSIAN_TRANSMITTER_NAMES)
father_name = random_father_full_name.split()[0] if ' ' in random_father_full_name else 'Абдуллах' father_name = random_father_full_name.split()[0] if ' ' in random_father_full_name else 'Абдуллах'
try:
# Savepoint per transmitter so a single bad row doesn't poison the outer atomic transaction
with transaction.atomic():
transmitter = Transmitters.objects.create( transmitter = Transmitters.objects.create(
full_name=[{'language_code': 'ru', 'text': f"{name} ибн {father_name}"}], full_name=[{'language_code': 'ru', 'text': f"{name} ибн {father_name}"}],
kunya=[{'language_code': 'ru', 'text': kunya}], kunya=[{'language_code': 'ru', 'text': kunya}],
@ -599,25 +608,36 @@ class Command(BaseCommand):
}] }]
) )
self.transmitters.append(transmitter) self.transmitters.append(transmitter)
except Exception as e:
self.stdout.write(self.style.WARNING(f"Skipping transmitter create due to error: {str(e)}"))
continue
# Add opinions for some transmitters # Add opinions for some transmitters
if random.random() > 0.5: if random.random() > 0.5:
for _ in range(random.randint(1, 3)): for _ in range(random.randint(1, 3)):
try:
with transaction.atomic():
TransmitterOpinion.objects.create( TransmitterOpinion.objects.create(
transmitter=transmitter, transmitter=transmitter,
scholar_name=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_SCHOLAR_NAMES)}], scholar_name=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_SCHOLAR_NAMES)}],
opinion_text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_OPINIONS)}], opinion_text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_OPINIONS)}],
status=random.choice(self.opinion_statuses) if self.opinion_statuses else None status=random.choice(self.opinion_statuses) if self.opinion_statuses else None
) )
except Exception as e:
self.stdout.write(self.style.WARNING(f"Skipping TransmitterOpinion due to error: {str(e)}"))
# Add original texts for some transmitters # Add original texts for some transmitters
if random.random() > 0.7: if random.random() > 0.7:
try:
with transaction.atomic():
TransmitterOriginalText.objects.create( TransmitterOriginalText.objects.create(
transmitter=transmitter, transmitter=transmitter,
title=[{'language_code': 'ru', 'text': f"Текст от {name}"}], title=[{'language_code': 'ru', 'text': f"Текст от {name}"}],
text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}], text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}],
translation=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}] translation=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}]
) )
except Exception as e:
self.stdout.write(self.style.WARNING(f"Skipping TransmitterOriginalText due to error: {str(e)}"))
self.created_counts['transmitters'] = count self.created_counts['transmitters'] = count
@ -628,6 +648,8 @@ class Command(BaseCommand):
# Create authors - check for existing ones first # Create authors - check for existing ones first
for author_name in RUSSIAN_AUTHOR_NAMES[:15]: for author_name in RUSSIAN_AUTHOR_NAMES[:15]:
try: try:
# Savepoint: do not poison the outer atomic transaction on a single failure
with transaction.atomic():
# Check if author with this name already exists # Check if author with this name already exists
existing_author = None existing_author = None
all_authors = BookAuthor.objects.all() all_authors = BookAuthor.objects.all()
@ -655,6 +677,8 @@ class Command(BaseCommand):
# Create books - check for existing ones first # Create books - check for existing ones first
for book_title in RUSSIAN_BOOK_TITLES[:20]: for book_title in RUSSIAN_BOOK_TITLES[:20]:
try: try:
# Savepoint: do not poison the outer atomic transaction on a single failure
with transaction.atomic():
# Generate slug to check for existing book # Generate slug to check for existing book
expected_slug = slugify(book_title, allow_unicode=True) expected_slug = slugify(book_title, allow_unicode=True)

Loading…
Cancel
Save