From a0ca5a645e6466dbaea707b0a5b6e0926d5f958d Mon Sep 17 00:00:00 2001 From: mohsentaba Date: Mon, 22 Dec 2025 13:59:44 +0330 Subject: [PATCH] Sync category tree fixed n+1 and indexing --- ...hadis_hadis_parent__e7a217_idx_and_more.py | 24 +++ apps/hadis/models/category.py | 4 + apps/hadis/serializers/category.py | 94 +--------- apps/hadis/views/category.py | 166 ++++++++++-------- 4 files changed, 123 insertions(+), 165 deletions(-) create mode 100644 apps/hadis/migrations/0065_hadiscategory_hadis_hadis_parent__e7a217_idx_and_more.py diff --git a/apps/hadis/migrations/0065_hadiscategory_hadis_hadis_parent__e7a217_idx_and_more.py b/apps/hadis/migrations/0065_hadiscategory_hadis_hadis_parent__e7a217_idx_and_more.py new file mode 100644 index 0000000..04940b3 --- /dev/null +++ b/apps/hadis/migrations/0065_hadiscategory_hadis_hadis_parent__e7a217_idx_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.27 on 2025-12-22 13:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("hadis", "0064_hadis_hadis_hadis_status_5e0de5_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="hadiscategory", + index=models.Index( + fields=["parent", "sect"], name="hadis_hadis_parent__e7a217_idx" + ), + ), + migrations.AddIndex( + model_name="hadiscategory", + index=models.Index( + fields=["sect", "order"], name="hadis_hadis_sect_id_b57c1d_idx" + ), + ), + ] diff --git a/apps/hadis/models/category.py b/apps/hadis/models/category.py index 202ad4d..9bd70fb 100644 --- a/apps/hadis/models/category.py +++ b/apps/hadis/models/category.py @@ -104,6 +104,10 @@ class HadisCategory(MPTTModel): class Meta: + indexes = [ + models.Index(fields=['parent', 'sect']), + models.Index(fields=['sect', 'order']) + ] verbose_name = _('Hadis Category') verbose_name_plural = _('Hadis Categories') ordering = ('order',) diff --git a/apps/hadis/serializers/category.py b/apps/hadis/serializers/category.py index fb012d8..97d0951 100644 --- a/apps/hadis/serializers/category.py +++ b/apps/hadis/serializers/category.py @@ -93,102 +93,12 @@ class HadisCategorySectListSerializer(serializers.ModelSerializer): class HadisCategoryTreeSerializer(serializers.ModelSerializer): title = LocalizedField() - """Serializer for HadisCategory tree structure""" - sect_id = serializers.IntegerField(source='sect.id', read_only=True) - sect_type = serializers.CharField(source='sect.sect_type', read_only=True) - children = serializers.SerializerMethodField() - + class Meta: model = HadisCategory - fields = ['id', 'title', 'source_type', 'sect_id', 'sect_type', 'children'] - - def get_name(self, obj): - """Get category name based on request language""" - request = self.context.get('request') - language_code = getattr(request, 'LANGUAGE_CODE', 'en') - return obj.get_translation(language_code) if hasattr(obj, 'get_translation') else obj.title - - def get_children(self, obj): - """Get all active children categories (no filtering by hadis/children)""" - children = obj.get_children().filter(sect=obj.sect).order_by('order') - return [self.to_dict(cat) for cat in children] - - def get_hadis_count(self, obj): - """Get direct hadis count for this category only""" - return Hadis.objects.filter(category=obj, status=True).count() - - def get_has_hadis(self, obj): - """Check if category can have hadis (no active children) and has hadis""" - # Check if category has active children - has_active_children = obj.get_children().filter(sect=obj.sect).exists() - - # If has active children, cannot have hadis - if has_active_children: - return False - - # If no active children, check if has hadis - return Hadis.objects.filter(category=obj, status=True).exists() - - def get_xmind_file(self, obj): - """Get absolute URL for xmind file""" - if obj.xmind_file: - request = self.context.get('request') - if request: - return request.build_absolute_uri(obj.xmind_file.url) - return obj.xmind_file.url - return None + fields = ['id', 'title', 'source_type'] - def get_has_xmind_file(self, obj): - """Check if category has xmind file""" - return bool(obj.xmind_file) - - def get_thumbnail(self, obj): - """Get absolute URL for thumbnail""" - if hasattr(obj, 'thumbnail') and obj.thumbnail: - request = self.context.get('request') - if request: - return request.build_absolute_uri(obj.thumbnail.url) - return obj.thumbnail.url - return None - - def get_hadis_index(self, obj): - """Get list of hadis numbers in this category (not including children)""" - return list( - Hadis.objects.filter( - category=obj, - status=True - ).order_by('number').values_list('number', flat=True) - ) - - def get_children_count(self, obj): - """Get count of active children categories""" - return obj.get_children().filter(sect=obj.sect).count() - - - def to_dict(self, category, request=None): - """Convert category to dict, applying localization""" - if request is None: - request = self.context.get('request') - return { - 'id': category.id, - 'title': get_localized_text(category.title, request), # <-- use helper - 'description': get_localized_text(category.description, request), # <-- use helper - 'slug': category.slug, - 'source_type': category.source_type, - 'hadis_count': self.get_hadis_count(category), - 'has_hadis': self.get_has_hadis(category), - 'children_count': self.get_children_count(category), - 'order': category.order, - 'thumbnail': self.get_thumbnail(category), - 'xmind_file': self.get_xmind_file(category), - 'has_xmind_file': self.get_has_xmind_file(category), - 'children': [ - self.to_dict(child, request) # recursively apply - for child in category.children.all() - ] - } - class HadisCategorySelectSerializer(serializers.ModelSerializer): """Serializer for HadisCategory Selection Flow""" sect_id = serializers.IntegerField(source='sect.id', read_only=True) diff --git a/apps/hadis/views/category.py b/apps/hadis/views/category.py index fcae763..23e92d9 100644 --- a/apps/hadis/views/category.py +++ b/apps/hadis/views/category.py @@ -2,6 +2,7 @@ from rest_framework.generics import ListAPIView from rest_framework.response import Response from django.shortcuts import get_object_or_404 from utils.pagination import NoPagination +from django.db.models import Q from ..models import HadisSect, HadisCategory from ..serializers import ( @@ -68,7 +69,6 @@ class HadisCategorySectListView(ListAPIView): class HadisCategoryTreeView(ListAPIView): """ API view to get all HadisCategory tree structure grouped by sect - Returns all categories in a tree structure (source_type grouping removed for mobile filtering) """ serializer_class = HadisCategoryTreeSerializer pagination_class = NoPagination @@ -78,60 +78,72 @@ class HadisCategoryTreeView(ListAPIView): return self.list(request, *args, **kwargs) def get_queryset(self): - return HadisCategory.objects.filter( - parent__isnull=True, - sect__is_active=True - ).select_related('sect').prefetch_related( - 'children', - 'children__children' - ).order_by('sect__order', 'order') + """ + Prefetch ALL data at once to avoid N+1 + """ + from django.db.models import Prefetch, Count, Q + + # Create a queryset for children that also has annotations + children_queryset = ( + HadisCategory.objects + .select_related('sect') + .prefetch_related('children') + .annotate( + hadis_count=Count('hadis', filter=Q(hadis__status=True)), + children_count=Count('children', distinct=True) + ) + .order_by('order') + ) + + return ( + HadisCategory.objects + .filter(parent__isnull=True, sect__is_active=True) + .select_related('sect') + .prefetch_related( + Prefetch( + 'children', + queryset=children_queryset + ), + ) + .annotate( + hadis_count=Count('hadis', filter=Q(hadis__status=True)), + children_count=Count('children', distinct=True) + ) + .order_by('sect__order', 'order') + ) + def list(self, request, *args, **kwargs): queryset = self.get_queryset() - grouped_data = {} - serializer_instance = HadisCategoryTreeSerializer(context={'request': request}) - - # گروه‌بندی بر اساس sect_type (shia/sunni) + # Single pass through prefetched data (NO QUERIES) for category in queryset: sect_type = category.sect.sect_type if sect_type not in grouped_data: - # ایجاد گروه برای هر sect_type grouped_data[sect_type] = { 'sects': {}, 'categories': [] } - # اضافه کردن اطلاعات sect به گروه sect_type + # Add sect info sect_id = str(category.sect.id) if sect_id not in grouped_data[sect_type]['sects']: grouped_data[sect_type]['sects'][sect_id] = { 'id': category.sect.id, 'sect_type': category.sect.sect_type, - 'title': get_localized_text(category.sect.title,self.request), + 'title': get_localized_text(category.sect.title, request), 'description': category.sect.description, 'order': category.sect.order } - category_data = self.build_enhanced_category_tree(category, serializer_instance) + # Build tree using prefetched data + category_data = self._build_tree(category, request) grouped_data[sect_type]['categories'].append(category_data) - def count_children(children_list): - count = 0 - for item in children_list: - count += 1 - if 'children' in item and item['children']: - count += count_children(item['children']) - return count - - total_count = 0 - for sect_type_data in grouped_data.values(): - for item in sect_type_data['categories']: - total_count += 1 - if 'children' in item and item['children']: - total_count += count_children(item['children']) + # Count total categories (simple loop, no queries) + total_count = self._count_tree_items(grouped_data) response_data = { 'count': total_count, @@ -140,51 +152,59 @@ class HadisCategoryTreeView(ListAPIView): return Response(response_data) - def build_enhanced_category_tree(self, category, serializer_instance): - """Build enhanced category tree with father category info and hadis details""" - # serializer_instance already has context={'request': request} - base_data = serializer_instance.to_dict(category, request=self.request) - - # Rest of the code... - enhanced_children = [] - for child_data in base_data.get('children', []): - enhanced_child = self.enhance_child_data(child_data, category, serializer_instance) - enhanced_children.append(enhanced_child) - - base_data['children'] = enhanced_children - return base_data - - def enhance_child_data(self, child_data, parent_category, serializer_instance): - """Enhance child data with father category info (no hadis payload for sync tree)""" - - # Add father category information - child_data['father_category'] = { - 'id': parent_category.id, - 'title': get_localized_text(parent_category.title,self.request), - 'sect_id': parent_category.sect.id, - 'sect_type': parent_category.sect.sect_type, - 'source_type': parent_category.source_type + def _build_tree(self, category, request): + """ + Build tree from ALREADY PREFETCHED data + No database queries here! + """ + return { + 'id': category.id, + 'title': get_localized_text(category.title, request), + 'description': get_localized_text(category.description, request), + 'slug': category.slug, + 'source_type': category.source_type, + 'hadis_count': category.hadis_count, # ← Use annotated value + 'children_count': category.children_count, # ← Use annotated value + 'has_hadis': category.hadis_count > 0, # ← Simple calculation + 'order': category.order, + 'thumbnail': self._get_thumbnail_url(category, request), + 'xmind_file': self._get_xmind_url(category, request), + 'has_xmind_file': bool(getattr(category, 'xmind_file', None)), + 'children': [ + self._build_tree(child, request) + for child in category.children.all() # Already prefetched! + ] } - # Note: we intentionally DO NOT load or attach hadis details here for performance. - - # Recursively enhance children's children - if child_data.get('children', []): - enhanced_grandchildren = [] - try: - from ..models import HadisCategory - child_category = HadisCategory.objects.get(id=child_data['id']) - - for grandchild_data in child_data['children']: - enhanced_grandchild = self.enhance_child_data(grandchild_data, child_category, serializer_instance) - enhanced_grandchildren.append(enhanced_grandchild) - except: - # If there's an error, keep original children - enhanced_grandchildren = child_data['children'] - - child_data['children'] = enhanced_grandchildren - - return child_data + def _count_tree_items(self, grouped_data): + """Count total items in tree""" + total = 0 + for sect_type_data in grouped_data.values(): + for item in sect_type_data['categories']: + total += 1 + total += self._count_children(item.get('children', [])) + return total + + def _count_children(self, children_list): + """Recursively count children""" + count = 0 + for item in children_list: + count += 1 + if item.get('children'): + count += self._count_children(item['children']) + return count + + def _get_thumbnail_url(self, category, request): + """Get absolute thumbnail URL""" + if hasattr(category, 'thumbnail') and category.thumbnail: + return request.build_absolute_uri(category.thumbnail.url) if request else category.thumbnail.url + return None + + def _get_xmind_url(self, category, request): + """Get absolute xmind URL""" + if getattr(category, 'xmind_file', None): + return request.build_absolute_uri(category.xmind_file.url) if request else category.xmind_file.url + return None class HadisCategoryTreeNormalView(ListAPIView):