Enhance error handling and transaction management in hadis data seeding command

- Implemented inner atomic blocks to ensure that failures during the creation of NarratorLayer, TransmitterReliability, OpinionStatus, Transmitters, and related models do not affect the overall transaction. - Improved logging to provide warnings for skipped records due to errors, enhancing feedback during the seeding process. - Streamlined the creation process for authors and books, ensuring that existing records are utilized effectively while maintaining data integrity.
4 months ago · c0d2a56fee
1 changed files with 132 additions and 108 deletions
--- a/apps/hadis/management/commands/seed_complete_hadis_data.py
+++ b/apps/hadis/management/commands/seed_complete_hadis_data.py
@ -490,11 +490,13 @@ class Command(BaseCommand):
            else:
                # Create new layer - but be defensive
                try:
-                    layer = NarratorLayer.objects.create(
-                        number=layer_data['number'],
-                        name=[{'language_code': 'ru', 'text': layer_data['name']}],
-                        description=[{'language_code': 'ru', 'text': layer_data['description']}]
-                    )
+                    # Use an inner atomic block so a single failure doesn't poison the outer transaction
+                    with transaction.atomic():
+                        layer = NarratorLayer.objects.create(
+                            number=layer_data['number'],
+                            name=[{'language_code': 'ru', 'text': layer_data['name']}],
+                            description=[{'language_code': 'ru', 'text': layer_data['description']}]
+                        )
                    self.stdout.write(f"Created new layer {layer_data['number']}")
                    self.narrator_layers.append(layer)
                except Exception as e:
@ -514,18 +516,20 @@ class Command(BaseCommand):
        # Create or get reliability statuses - using filter().first() to avoid MultipleObjectsReturned
        for reliability_data in RUSSIAN_RELIABILITY_LEVELS:
            try:
-                # Try to get by slug first
-                reliability = TransmitterReliability.objects.filter(slug=reliability_data['slug']).first()
-                if reliability:
-                    self.stdout.write(f"Using existing reliability: {reliability_data['slug']}")
-                else:
-                    # Create new one
-                    reliability = TransmitterReliability.objects.create(
-                        slug=reliability_data['slug'],
-                        title=[{'language_code': 'ru', 'text': reliability_data['title']}],
-                        color=reliability_data['color']
-                    )
-                    self.stdout.write(f"Created new reliability: {reliability_data['slug']}")
+                # Wrap in a savepoint so failures don't break the outer atomic transaction
+                with transaction.atomic():
+                    # Try to get by slug first
+                    reliability = TransmitterReliability.objects.filter(slug=reliability_data['slug']).first()
+                    if reliability:
+                        self.stdout.write(f"Using existing reliability: {reliability_data['slug']}")
+                    else:
+                        # Create new one
+                        reliability = TransmitterReliability.objects.create(
+                            slug=reliability_data['slug'],
+                            title=[{'language_code': 'ru', 'text': reliability_data['title']}],
+                            color=reliability_data['color']
+                        )
+                        self.stdout.write(f"Created new reliability: {reliability_data['slug']}")
                
                self.reliability_statuses.append(reliability)
            except Exception as e:
@ -540,18 +544,20 @@ class Command(BaseCommand):
        # Create or get opinion statuses - using filter().first() to avoid MultipleObjectsReturned
        for opinion_data in RUSSIAN_OPINION_STATUSES:
            try:
-                # Try to get by slug first
-                opinion_status = OpinionStatus.objects.filter(slug=opinion_data['slug']).first()
-                if opinion_status:
-                    self.stdout.write(f"Using existing opinion status: {opinion_data['slug']}")
-                else:
-                    # Create new one
-                    opinion_status = OpinionStatus.objects.create(
-                        slug=opinion_data['slug'],
-                        title=[{'language_code': 'ru', 'text': opinion_data['title']}],
-                        color=opinion_data['color']
-                    )
-                    self.stdout.write(f"Created new opinion status: {opinion_data['slug']}")
+                # Wrap in a savepoint so failures don't break the outer atomic transaction
+                with transaction.atomic():
+                    # Try to get by slug first
+                    opinion_status = OpinionStatus.objects.filter(slug=opinion_data['slug']).first()
+                    if opinion_status:
+                        self.stdout.write(f"Using existing opinion status: {opinion_data['slug']}")
+                    else:
+                        # Create new one
+                        opinion_status = OpinionStatus.objects.create(
+                            slug=opinion_data['slug'],
+                            title=[{'language_code': 'ru', 'text': opinion_data['title']}],
+                            color=opinion_data['color']
+                        )
+                        self.stdout.write(f"Created new opinion status: {opinion_data['slug']}")
                
                self.opinion_statuses.append(opinion_status)
            except Exception as e:
@ -577,47 +583,61 @@ class Command(BaseCommand):
            random_father_full_name = random.choice(RUSSIAN_TRANSMITTER_NAMES)
            father_name = random_father_full_name.split()[0] if ' ' in random_father_full_name else 'Абдуллах'
            
-            transmitter = Transmitters.objects.create(
-                full_name=[{'language_code': 'ru', 'text': f"{name} ибн {father_name}"}],
-                kunya=[{'language_code': 'ru', 'text': kunya}],
-                known_as=[{'language_code': 'ru', 'text': f"{name} {origin}ский"}],
-                nickname=[{'language_code': 'ru', 'text': f"{random.choice(['аль-Муфассир', 'аль-Хафиз', 'аль-Факих', 'аль-Мухаддис'])}"}],
-                origin=[{'language_code': 'ru', 'text': origin}],
-                lived_in=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_ORIGINS)}],
-                died_in=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_ORIGINS)}],
-                birth_year_hijri=birth_year,
-                death_year_hijri=death_year,
-                age_at_death=age,
-                generation=random.randint(1, 8),
-                reliability=random.choice(self.reliability_statuses) if self.reliability_statuses else None,
-                madhhab=random.choice(['shia', 'sunni', 'hanafi', 'maliki', 'shafii']),
-                in_sahih_muslim=random.choice([True, False]),
-                in_sahih_bukhari=random.choice([True, False]),
-                description=[{
-                    'language_code': 'ru',
-                    'text': f"{name} был известным передатчиком хадисов из {origin}. Он изучал знания у великих ученых своего времени и передал множество достоверных хадисов."
-                }]
-            )
-            self.transmitters.append(transmitter)
+            try:
+                # Savepoint per transmitter so a single bad row doesn't poison the outer atomic transaction
+                with transaction.atomic():
+                    transmitter = Transmitters.objects.create(
+                        full_name=[{'language_code': 'ru', 'text': f"{name} ибн {father_name}"}],
+                        kunya=[{'language_code': 'ru', 'text': kunya}],
+                        known_as=[{'language_code': 'ru', 'text': f"{name} {origin}ский"}],
+                        nickname=[{'language_code': 'ru', 'text': f"{random.choice(['аль-Муфассир', 'аль-Хафиз', 'аль-Факих', 'аль-Мухаддис'])}"}],
+                        origin=[{'language_code': 'ru', 'text': origin}],
+                        lived_in=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_ORIGINS)}],
+                        died_in=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_ORIGINS)}],
+                        birth_year_hijri=birth_year,
+                        death_year_hijri=death_year,
+                        age_at_death=age,
+                        generation=random.randint(1, 8),
+                        reliability=random.choice(self.reliability_statuses) if self.reliability_statuses else None,
+                        madhhab=random.choice(['shia', 'sunni', 'hanafi', 'maliki', 'shafii']),
+                        in_sahih_muslim=random.choice([True, False]),
+                        in_sahih_bukhari=random.choice([True, False]),
+                        description=[{
+                            'language_code': 'ru',
+                            'text': f"{name} был известным передатчиком хадисов из {origin}. Он изучал знания у великих ученых своего времени и передал множество достоверных хадисов."
+                        }]
+                    )
+                    self.transmitters.append(transmitter)
+            except Exception as e:
+                self.stdout.write(self.style.WARNING(f"Skipping transmitter create due to error: {str(e)}"))
+                continue
            
            # Add opinions for some transmitters
            if random.random() > 0.5:
                for _ in range(random.randint(1, 3)):
-                    TransmitterOpinion.objects.create(
-                        transmitter=transmitter,
-                        scholar_name=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_SCHOLAR_NAMES)}],
-                        opinion_text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_OPINIONS)}],
-                        status=random.choice(self.opinion_statuses) if self.opinion_statuses else None
-                    )
+                    try:
+                        with transaction.atomic():
+                            TransmitterOpinion.objects.create(
+                                transmitter=transmitter,
+                                scholar_name=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_SCHOLAR_NAMES)}],
+                                opinion_text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_OPINIONS)}],
+                                status=random.choice(self.opinion_statuses) if self.opinion_statuses else None
+                            )
+                    except Exception as e:
+                        self.stdout.write(self.style.WARNING(f"Skipping TransmitterOpinion due to error: {str(e)}"))
            
            # Add original texts for some transmitters
            if random.random() > 0.7:
-                TransmitterOriginalText.objects.create(
-                    transmitter=transmitter,
-                    title=[{'language_code': 'ru', 'text': f"Текст от {name}"}],
-                    text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}],
-                    translation=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}]
-                )
+                try:
+                    with transaction.atomic():
+                        TransmitterOriginalText.objects.create(
+                            transmitter=transmitter,
+                            title=[{'language_code': 'ru', 'text': f"Текст от {name}"}],
+                            text=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}],
+                            translation=[{'language_code': 'ru', 'text': random.choice(RUSSIAN_HADIS_BODIES)}]
+                        )
+                except Exception as e:
+                    self.stdout.write(self.style.WARNING(f"Skipping TransmitterOriginalText due to error: {str(e)}"))
        
        self.created_counts['transmitters'] = count

@ -628,23 +648,25 @@ class Command(BaseCommand):
        # Create authors - check for existing ones first
        for author_name in RUSSIAN_AUTHOR_NAMES[:15]:
            try:
-                # Check if author with this name already exists
-                existing_author = None
-                all_authors = BookAuthor.objects.all()
-                for auth in all_authors:
-                    if auth.name and isinstance(auth.name, list) and len(auth.name) > 0:
-                        if auth.name[0].get('text', '') == author_name:
-                            existing_author = auth
-                            break
-                
-                if existing_author:
-                    author = existing_author
-                    self.stdout.write(f"Using existing author: {author_name}")
-                else:
-                    author = BookAuthor.objects.create(
-                        name=[{'language_code': 'ru', 'text': author_name}]
-                    )
-                    self.stdout.write(f"Created new author: {author_name}")
+                # Savepoint: do not poison the outer atomic transaction on a single failure
+                with transaction.atomic():
+                    # Check if author with this name already exists
+                    existing_author = None
+                    all_authors = BookAuthor.objects.all()
+                    for auth in all_authors:
+                        if auth.name and isinstance(auth.name, list) and len(auth.name) > 0:
+                            if auth.name[0].get('text', '') == author_name:
+                                existing_author = auth
+                                break
+                    
+                    if existing_author:
+                        author = existing_author
+                        self.stdout.write(f"Using existing author: {author_name}")
+                    else:
+                        author = BookAuthor.objects.create(
+                            name=[{'language_code': 'ru', 'text': author_name}]
+                        )
+                        self.stdout.write(f"Created new author: {author_name}")
                
                self.authors.append(author)
            except Exception as e:
@ -655,34 +677,36 @@ class Command(BaseCommand):
        # Create books - check for existing ones first
        for book_title in RUSSIAN_BOOK_TITLES[:20]:
            try:
-                # Generate slug to check for existing book
-                expected_slug = slugify(book_title, allow_unicode=True)
-                
-                # Try to find existing book by slug or title
-                book = BookReference.objects.filter(slug=expected_slug).first()
-                
-                if not book:
-                    # Check by title as fallback
-                    all_books = BookReference.objects.all()
-                    for bk in all_books:
-                        if bk.title and isinstance(bk.title, list) and len(bk.title) > 0:
-                            if bk.title[0].get('text', '') == book_title:
-                                book = bk
-                                break
-                
-                if book:
-                    self.stdout.write(f"Using existing book: {book_title}")
-                else:
-                    book = BookReference.objects.create(
-                        title=[{'language_code': 'ru', 'text': book_title}],
-                        description=[{'language_code': 'ru', 'text': f"Классическое исламское произведение - {book_title}"}]
-                    )
-                    self.stdout.write(f"Created new book: {book_title}")
-                
-                # Add random authors if we have any
-                if self.authors and book.authors.count() == 0:
-                    num_authors = min(random.randint(1, 2), len(self.authors))
-                    book.authors.add(*random.sample(self.authors, num_authors))
+                # Savepoint: do not poison the outer atomic transaction on a single failure
+                with transaction.atomic():
+                    # Generate slug to check for existing book
+                    expected_slug = slugify(book_title, allow_unicode=True)
+                    
+                    # Try to find existing book by slug or title
+                    book = BookReference.objects.filter(slug=expected_slug).first()
+                    
+                    if not book:
+                        # Check by title as fallback
+                        all_books = BookReference.objects.all()
+                        for bk in all_books:
+                            if bk.title and isinstance(bk.title, list) and len(bk.title) > 0:
+                                if bk.title[0].get('text', '') == book_title:
+                                    book = bk
+                                    break
+                    
+                    if book:
+                        self.stdout.write(f"Using existing book: {book_title}")
+                    else:
+                        book = BookReference.objects.create(
+                            title=[{'language_code': 'ru', 'text': book_title}],
+                            description=[{'language_code': 'ru', 'text': f"Классическое исламское произведение - {book_title}"}]
+                        )
+                        self.stdout.write(f"Created new book: {book_title}")
+                    
+                    # Add random authors if we have any
+                    if self.authors and book.authors.count() == 0:
+                        num_authors = min(random.randint(1, 2), len(self.authors))
+                        book.authors.add(*random.sample(self.authors, num_authors))
                
                self.books.append(book)
            except Exception as e: