Browse Source
feat(hadis): add management commands for topic migration and source diversification
feat(hadis): add management commands for topic migration and source diversification
- Add migrate_leaf_categories_to_topics command to migrate legacy leaf categories to topics - Add diversify_topic_sources command to balance and distribute distinct source types (hadith, quran, history, fatwa, quote) under parent categories - Support --dry-run flag and automatic cache invalidationmaster
2 changed files with 232 additions and 0 deletions
-
106apps/hadis/management/commands/diversify_topic_sources.py
-
126apps/hadis/management/commands/migrate_leaf_categories_to_topics.py
@ -0,0 +1,106 @@ |
|||
import time |
|||
from django.core.management.base import BaseCommand |
|||
from django.db import transaction, connection |
|||
from django.core.cache import cache |
|||
from apps.hadis.models import HadisCategory |
|||
|
|||
|
|||
class Command(BaseCommand): |
|||
help = ( |
|||
'Diversify topic source_types across parent categories so that ' |
|||
'sub-topics under each category have distinct, varied source types ' |
|||
'(hadith, quran, history, fatwa, quote).' |
|||
) |
|||
|
|||
def add_arguments(self, parser): |
|||
parser.add_argument( |
|||
'--dry-run', |
|||
action='store_true', |
|||
help='Simulate diversification without modifying the database.' |
|||
) |
|||
|
|||
def handle(self, *args, **options): |
|||
dry_run = options['dry_run'] |
|||
|
|||
if dry_run: |
|||
self.stdout.write(self.style.WARNING("=== RUNNING IN DRY-RUN MODE (No changes will be saved) ===")) |
|||
else: |
|||
self.stdout.write(self.style.NOTICE("=== STARTING TOPIC SOURCE DIVERSIFICATION ===")) |
|||
|
|||
# Ensure database connection with retry |
|||
for attempt in range(5): |
|||
try: |
|||
connection.ensure_connection() |
|||
break |
|||
except Exception as e: |
|||
self.stdout.write(f"Connection attempt {attempt+1} failed: {e}. Retrying in 2s...") |
|||
time.sleep(2) |
|||
|
|||
source_choices = [ |
|||
HadisCategory.SourceType.HADITH, |
|||
HadisCategory.SourceType.QURAN, |
|||
HadisCategory.SourceType.HISTORY, |
|||
HadisCategory.SourceType.FATWA, |
|||
HadisCategory.SourceType.QUOTE, |
|||
] |
|||
num_choices = len(source_choices) |
|||
|
|||
parents = HadisCategory.objects.filter(is_topic=False).select_related('sect').order_by('sect_id', 'tree_id', 'lft', 'id') |
|||
parents_with_topics = [p for p in parents if p.children.filter(is_topic=True).exists()] |
|||
|
|||
self.stdout.write(f"Total parent categories with topics: {len(parents_with_topics)}") |
|||
|
|||
updated_topics_count = 0 |
|||
total_topics_count = 0 |
|||
global_rotation_idx = 0 |
|||
|
|||
with transaction.atomic(): |
|||
for parent in parents_with_topics: |
|||
topics = list(parent.children.filter(is_topic=True).order_by('order', 'id')) |
|||
topic_count = len(topics) |
|||
total_topics_count += topic_count |
|||
|
|||
sect_name = getattr(parent.sect, 'sect_type', 'unknown') |
|||
safe_slug = parent.slug.encode('ascii', 'replace').decode('ascii') |
|||
self.stdout.write( |
|||
f"\nParent [{sect_name}] ID={parent.id} ({safe_slug}) - {topic_count} topics:" |
|||
) |
|||
|
|||
# For this parent's topics, pick distinct source types from the rotation |
|||
for i, topic in enumerate(topics): |
|||
new_source = source_choices[(global_rotation_idx + i) % num_choices] |
|||
old_source = topic.source_type |
|||
safe_topic_slug = topic.slug.encode('ascii', 'replace').decode('ascii') if topic.slug else 'no-slug' |
|||
|
|||
if old_source != new_source: |
|||
updated_topics_count += 1 |
|||
self.stdout.write( |
|||
f" -> Topic ID={topic.id:4d} | '{safe_topic_slug}' : {old_source} -> {new_source}" |
|||
) |
|||
if not dry_run: |
|||
topic.source_type = new_source |
|||
topic.save(update_fields=['source_type']) |
|||
else: |
|||
self.stdout.write( |
|||
f" -> Topic ID={topic.id:4d} | '{safe_topic_slug}' : {new_source} (unchanged)" |
|||
) |
|||
|
|||
# Advance global rotation index |
|||
global_rotation_idx = (global_rotation_idx + topic_count) % num_choices |
|||
|
|||
if dry_run: |
|||
transaction.set_rollback(True) |
|||
|
|||
self.stdout.write("") |
|||
self.stdout.write(self.style.SUCCESS("=== DIVERSIFICATION SUMMARY ===")) |
|||
self.stdout.write(f" Parent categories processed: {len(parents_with_topics)}") |
|||
self.stdout.write(f" Total topics inspected: {total_topics_count}") |
|||
self.stdout.write(f" Topics updated: {updated_topics_count}") |
|||
|
|||
if not dry_run: |
|||
# Clear cache so new source types are immediately served |
|||
cache.clear() |
|||
self.stdout.write(self.style.SUCCESS("Django cache cleared successfully!")) |
|||
self.stdout.write(self.style.SUCCESS("All topic source types updated successfully!")) |
|||
else: |
|||
self.stdout.write(self.style.WARNING("Dry run finished. Run without --dry-run to commit changes.")) |
|||
@ -0,0 +1,126 @@ |
|||
from django.core.management.base import BaseCommand |
|||
from django.db import transaction |
|||
from apps.hadis.models import HadisCategory |
|||
|
|||
|
|||
class Command(BaseCommand): |
|||
help = ( |
|||
'Migrate legacy categories to topic architecture: ' |
|||
'Leaf categories (and categories with hadiths) become topics (is_topic=True) ' |
|||
'and parent categories become structural categories (is_topic=False, source_type=None).' |
|||
) |
|||
|
|||
def add_arguments(self, parser): |
|||
parser.add_argument( |
|||
'--dry-run', |
|||
action='store_true', |
|||
help='Simulate migration without modifying the database.' |
|||
) |
|||
parser.add_argument( |
|||
'--default-source-type', |
|||
type=str, |
|||
default='hadith', |
|||
help='Default source_type to assign to topics if empty (default: hadith).' |
|||
) |
|||
|
|||
def handle(self, *args, **options): |
|||
dry_run = options['dry_run'] |
|||
default_source_type = options['default_source_type'] |
|||
|
|||
valid_sources = [choice[0] for choice in HadisCategory.SourceType.choices] |
|||
if default_source_type not in valid_sources: |
|||
self.stdout.write( |
|||
self.style.ERROR( |
|||
f"Invalid default-source-type: '{default_source_type}'. " |
|||
f"Must be one of: {valid_sources}" |
|||
) |
|||
) |
|||
return |
|||
|
|||
if dry_run: |
|||
self.stdout.write(self.style.WARNING("=== RUNNING IN DRY-RUN MODE (No database changes will be saved) ===")) |
|||
else: |
|||
self.stdout.write(self.style.NOTICE("=== STARTING CATEGORY-TO-TOPIC MIGRATION ===")) |
|||
|
|||
categories = HadisCategory.objects.all().order_by('tree_id', 'level', 'id') |
|||
total_count = categories.count() |
|||
|
|||
if total_count == 0: |
|||
self.stdout.write(self.style.WARNING("No categories found in database.")) |
|||
return |
|||
|
|||
to_topic_count = 0 |
|||
to_structural_count = 0 |
|||
already_correct_count = 0 |
|||
warnings = [] |
|||
|
|||
with transaction.atomic(): |
|||
for category in categories: |
|||
has_children = category.children.exists() |
|||
has_hadis = category.hadis_set.exists() or ( |
|||
hasattr(category, 'hadis_items') and category.hadis_items.exists() |
|||
) |
|||
|
|||
if has_children and has_hadis: |
|||
warnings.append( |
|||
f"Category ID={category.id} ('{category.slug}') has both children and hadiths! " |
|||
f"Setting as structural category (is_topic=False) to preserve tree hierarchy." |
|||
) |
|||
|
|||
# Determine target state |
|||
if not has_children: |
|||
# Leaf node -> Topic |
|||
target_is_topic = True |
|||
target_source_type = category.source_type if category.source_type in valid_sources else default_source_type |
|||
else: |
|||
# Parent node with children -> Structural Category |
|||
target_is_topic = False |
|||
target_source_type = None |
|||
|
|||
needs_update = ( |
|||
category.is_topic != target_is_topic or |
|||
category.source_type != target_source_type |
|||
) |
|||
|
|||
if needs_update: |
|||
if target_is_topic: |
|||
to_topic_count += 1 |
|||
action_label = f"Convert to TOPIC (source={target_source_type})" |
|||
else: |
|||
to_structural_count += 1 |
|||
action_label = "Convert to CATEGORY (source=None)" |
|||
|
|||
self.stdout.write( |
|||
f" [UPDATE] ID={category.id:4d} | slug='{category.slug}' | {action_label}" |
|||
) |
|||
|
|||
if not dry_run: |
|||
category.is_topic = target_is_topic |
|||
category.source_type = target_source_type |
|||
category.save(update_fields=['is_topic', 'source_type']) |
|||
else: |
|||
already_correct_count += 1 |
|||
|
|||
if dry_run: |
|||
# Rollback transaction in dry run mode |
|||
transaction.set_rollback(True) |
|||
|
|||
self.stdout.write("") |
|||
self.stdout.write(self.style.SUCCESS("=== MIGRATION SUMMARY ===")) |
|||
self.stdout.write(f" Total categories inspected: {total_count}") |
|||
self.stdout.write(f" Categories converted to TOPICS: {to_topic_count}") |
|||
self.stdout.write(f" Categories kept as STRUCTURAL: {to_structural_count}") |
|||
self.stdout.write(f" Already correct: {already_correct_count}") |
|||
|
|||
if warnings: |
|||
self.stdout.write("") |
|||
self.stdout.write(self.style.WARNING(f"=== WARNINGS ({len(warnings)}) ===")) |
|||
for w in warnings: |
|||
self.stdout.write(self.style.WARNING(f" ! {w}")) |
|||
|
|||
if dry_run: |
|||
self.stdout.write("") |
|||
self.stdout.write(self.style.WARNING("Dry run finished. Run without --dry-run to apply changes to database.")) |
|||
else: |
|||
self.stdout.write("") |
|||
self.stdout.write(self.style.SUCCESS("✓ All database changes committed successfully!")) |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue