Browse Source

Refactor data integrity checks in hadis seeding command

- Improved the process for identifying and fixing empty slugs in NarratorLayer by utilizing raw SQL for more reliable detection.
- Enhanced logging to provide clear feedback on the number of records found and fixed, including error handling for any issues encountered during the slug update.
- Streamlined the slug fixing logic to directly address problematic entries, improving overall data integrity in the seeding process.
master
mortezaei 4 months ago
parent
commit
5766d87034
  1. 39
      apps/hadis/management/commands/seed_complete_hadis_data.py

39
apps/hadis/management/commands/seed_complete_hadis_data.py

@ -268,23 +268,30 @@ class Command(BaseCommand):
"""Fix all existing data integrity issues before starting main transaction""" """Fix all existing data integrity issues before starting main transaction"""
from django.utils.text import slugify from django.utils.text import slugify
from django.db.models import Count, Q from django.db.models import Count, Q
from django.db import connection
self.stdout.write('Fixing NarratorLayer empty slugs...')
# Fix NarratorLayer empty slugs - be more aggressive
# First, let's see all layers
all_layers = NarratorLayer.objects.all()
self.stdout.write(f'Total layers in database: {all_layers.count()}')
self.stdout.write('=' * 60)
self.stdout.write('PHASE 0: FIXING EXISTING DATA INTEGRITY ISSUES')
self.stdout.write('=' * 60)
fixed_count = 0
for layer in all_layers:
needs_fix = False
# Use raw SQL to find ALL records with empty slugs - more reliable
with connection.cursor() as cursor:
cursor.execute("""
SELECT id, number
FROM hadis_narratorlayer
WHERE slug IS NULL OR slug = '' OR TRIM(slug) = ''
""")
empty_slug_rows = cursor.fetchall()
# Check if slug is problematic
if not layer.slug or layer.slug.strip() == '':
needs_fix = True
self.stdout.write(f' Layer {layer.pk} (number={layer.number}) has empty slug')
self.stdout.write(f'Found {len(empty_slug_rows)} NarratorLayer records with empty slugs (via raw SQL)')
if needs_fix:
# Fix each one
fixed_count = 0
for row_id, row_number in empty_slug_rows:
try:
layer = NarratorLayer.objects.get(pk=row_id)
# Generate slug
try: try:
if layer.name and isinstance(layer.name, list) and len(layer.name) > 0: if layer.name and isinstance(layer.name, list) and len(layer.name) > 0:
text = layer.name[0].get('text', '').strip() text = layer.name[0].get('text', '').strip()
@ -307,9 +314,11 @@ class Command(BaseCommand):
layer.slug = new_slug layer.slug = new_slug
layer.save(update_fields=['slug']) layer.save(update_fields=['slug'])
fixed_count += 1 fixed_count += 1
self.stdout.write(f' Fixed layer {layer.pk} (number={layer.number}): "{new_slug}"')
self.stdout.write(f' ✓ Fixed layer {layer.pk} (number={layer.number}): "{new_slug}"')
except Exception as e:
self.stdout.write(self.style.ERROR(f' ✗ Error fixing layer {row_id}: {str(e)}'))
self.stdout.write(f'Fixed {fixed_count} layers with empty slugs')
self.stdout.write(self.style.SUCCESS(f'Fixed {fixed_count} NarratorLayer records'))
self.stdout.write('Fixing TransmitterReliability duplicates...') self.stdout.write('Fixing TransmitterReliability duplicates...')
# Fix TransmitterReliability empty and duplicate slugs # Fix TransmitterReliability empty and duplicate slugs

Loading…
Cancel
Save