From 89a1418d76e1e521a28a139d17435e43daa78874 Mon Sep 17 00:00:00 2001 From: mortezaei Date: Wed, 11 Jun 2025 23:59:29 +0330 Subject: [PATCH] fix: optimize database queries and add indexes across course and lesson models --- OPTIMIZATION_PLAN.md | 134 ++++++++++++++++++ .../migrations/0005_add_database_indexes.py | 122 ++++++++++++++++ apps/course/models/course.py | 15 +- apps/course/models/lesson.py | 28 +++- apps/course/models/participant.py | 8 +- apps/course/serializers/course.py | 50 ++++++- apps/course/serializers/lesson.py | 12 +- apps/course/views/course.py | 58 +++++++- apps/course/views/lesson.py | 36 +++-- apps/course/views/participant.py | 7 +- apps/video/views.py | 7 +- 11 files changed, 450 insertions(+), 27 deletions(-) create mode 100644 OPTIMIZATION_PLAN.md create mode 100644 apps/course/migrations/0005_add_database_indexes.py diff --git a/OPTIMIZATION_PLAN.md b/OPTIMIZATION_PLAN.md new file mode 100644 index 0000000..ec3e538 --- /dev/null +++ b/OPTIMIZATION_PLAN.md @@ -0,0 +1,134 @@ +# Django Database Query Optimization Plan + +## Phase 1: Analysis Complete ✅ + +### Current Issues Identified: +1. **N+1 Query Problems** in course, video, library, article, podcast views +2. **Missing select_related/prefetch_related** optimizations +3. **Inefficient serializer methods** with individual database queries +4. **Missing database indexes** on frequently queried fields +5. **Suboptimal queryset patterns** in views and admin + +## Phase 2: Query Optimization Implementation Plan + +### Step 1: Course App Optimization +**Priority: HIGH** (Core functionality) + +#### 1.1 Course Views Optimization +- **CourseListAPIView**: Add select_related for professor, category +- **CourseDetailAPIView**: Add prefetch_related for lessons, attachments, glossaries, participants +- **MyCourseListAPIView**: Optimize participant and completion queries + +#### 1.2 Course Serializers Optimization +- **CourseListSerializer**: Optimize professor and category access +- **CourseDetailSerializer**: Optimize all related object access +- **CourseLessonSerializer**: Optimize lesson completion and quiz queries + +#### 1.3 Course Admin Optimization +- **CourseAdmin**: Add select_related/prefetch_related to get_queryset +- **ParticipantAdmin**: Optimize student and course queries + +### Step 2: Video App Optimization +**Priority: HIGH** (Heavy content usage) + +#### 2.1 Video Views Optimization +- **VideoListAPIView**: Add prefetch_related for categories, collections +- **VideoDetailAPIView**: Optimize playlist and bookmark queries +- **VideoCollectionViews**: Optimize video relationships + +#### 2.2 Video Serializers Optimization +- **VideoDetailSerializer**: Optimize bookmark, rate, and playlist queries +- **VideoCollectionSerializer**: Optimize video access + +### Step 3: Library App Optimization +**Priority: HIGH** (Heavy content usage) + +#### 3.1 Library Views Optimization +- **BookListView**: Add prefetch_related for categories, collections +- **BookDetailView**: Optimize bookmark and rate queries +- **BookCollectionViews**: Optimize book relationships + +#### 3.2 Library Serializers Optimization +- **BookSerializer**: Optimize bookmark and rate queries +- **BookCollectionSerializer**: Optimize book access + +### Step 4: Article & Podcast Apps Optimization +**Priority: MEDIUM** + +#### 4.1 Similar patterns to Video/Library apps +- Apply same optimization patterns +- Focus on category and collection relationships +- Optimize bookmark and rate queries + +### Step 5: Account App Optimization +**Priority: MEDIUM** + +#### 5.1 User Admin Optimization +- **UserAdmin**: Already has some prefetch_related, enhance further +- **StudentUserAdmin**: Optimize course participation queries + +### Step 6: Chat App Optimization +**Priority: MEDIUM** + +#### 6.1 Chat Views Optimization +- **RoomMessage queries**: Add select_related for initiator, recipient, course +- **ChatMessage queries**: Add select_related for sender, room + +### Step 7: Bookmark & Rate System Optimization +**Priority: HIGH** (Used across all content types) + +#### 7.1 Bookmark Queries Optimization +- Optimize bookmark status checks in serializers +- Add bulk bookmark queries where possible + +## Phase 3: Database Indexing Plan + +### Step 1: Primary Indexes +- Add indexes on status fields (all models) +- Add indexes on created_at, updated_at fields +- Add indexes on slug fields + +### Step 2: Foreign Key Indexes +- Ensure all ForeignKey fields have indexes +- Add composite indexes for common query patterns + +### Step 3: Composite Indexes +- (user_id, service, status) for Bookmark model +- (course_id, student_id) for Participant model +- (status, created_at) for content models + +## Phase 4: Implementation Order + +### Week 1: Course App (Core functionality) +1. Course views optimization +2. Course serializers optimization +3. Course admin optimization +4. Add course-related indexes + +### Week 2: Content Apps (Video, Library) +1. Video app optimization +2. Library app optimization +3. Add content-related indexes + +### Week 3: Remaining Apps +1. Article and Podcast apps +2. Account app enhancements +3. Chat app optimization +4. Bookmark system optimization + +### Week 4: Final Optimizations +1. Remaining indexes +2. Performance testing +3. Query analysis and fine-tuning + +## Success Metrics +- Reduce average response time by 50-70% +- Reduce database query count per request by 60-80% +- Maintain exact same API response format +- Zero breaking changes to existing functionality + +## Implementation Strategy +- One optimization at a time +- Test each change individually +- Maintain backward compatibility +- Monitor performance improvements diff --git a/apps/course/migrations/0005_add_database_indexes.py b/apps/course/migrations/0005_add_database_indexes.py new file mode 100644 index 0000000..9723b80 --- /dev/null +++ b/apps/course/migrations/0005_add_database_indexes.py @@ -0,0 +1,122 @@ +# Generated by Django 5.1.8 on 2025-06-11 19:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0003_locationhistory'), + ('course', '0004_alter_attachment_options_alter_glossary_options_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='courselesson', + options={'verbose_name': 'Course Lesson', 'verbose_name_plural': 'Course Lessons'}, + ), + migrations.AlterModelOptions( + name='lesson', + options={'verbose_name': 'Lesson', 'verbose_name_plural': 'Lessons'}, + ), + migrations.AddIndex( + model_name='course', + index=models.Index(fields=['status'], name='course_cour_status_57ffd9_idx'), + ), + migrations.AddIndex( + model_name='course', + index=models.Index(fields=['is_free'], name='course_cour_is_free_9453a1_idx'), + ), + migrations.AddIndex( + model_name='course', + index=models.Index(fields=['created_at'], name='course_cour_created_49f06e_idx'), + ), + migrations.AddIndex( + model_name='course', + index=models.Index(fields=['slug'], name='course_cour_slug_235a66_idx'), + ), + migrations.AddIndex( + model_name='course', + index=models.Index(fields=['status', 'created_at'], name='course_cour_status_bfcd24_idx'), + ), + migrations.AddIndex( + model_name='course', + index=models.Index(fields=['category', 'status'], name='course_cour_categor_26bb4d_idx'), + ), + migrations.AddIndex( + model_name='course', + index=models.Index(fields=['professor', 'status'], name='course_cour_profess_5eae9a_idx'), + ), + migrations.AddIndex( + model_name='courseattachment', + index=models.Index(fields=['course'], name='course_cour_course__106cc8_idx'), + ), + migrations.AddIndex( + model_name='courseattachment', + index=models.Index(fields=['attachment'], name='course_cour_attachm_2da12a_idx'), + ), + migrations.AddIndex( + model_name='courselesson', + index=models.Index(fields=['course'], name='course_cour_course__4afa4c_idx'), + ), + migrations.AddIndex( + model_name='courselesson', + index=models.Index(fields=['lesson'], name='course_cour_lesson__e5c835_idx'), + ), + migrations.AddIndex( + model_name='courselesson', + index=models.Index(fields=['priority'], name='course_cour_priorit_dedac7_idx'), + ), + migrations.AddIndex( + model_name='courselesson', + index=models.Index(fields=['is_active'], name='course_cour_is_acti_490c61_idx'), + ), + migrations.AddIndex( + model_name='courselesson', + index=models.Index(fields=['course', 'priority'], name='course_cour_course__192d2c_idx'), + ), + migrations.AddIndex( + model_name='courselesson', + index=models.Index(fields=['course', 'is_active'], name='course_cour_course__7c6f06_idx'), + ), + migrations.AddIndex( + model_name='lesson', + index=models.Index(fields=['content_type'], name='course_less_content_e1cf57_idx'), + ), + migrations.AddIndex( + model_name='lesson', + index=models.Index(fields=['created_at'], name='course_less_created_4efb58_idx'), + ), + migrations.AddIndex( + model_name='lessoncompletion', + index=models.Index(fields=['student'], name='course_less_student_f3c9b8_idx'), + ), + migrations.AddIndex( + model_name='lessoncompletion', + index=models.Index(fields=['course_lesson'], name='course_less_course__1f3841_idx'), + ), + migrations.AddIndex( + model_name='lessoncompletion', + index=models.Index(fields=['completed_at'], name='course_less_complet_8d2220_idx'), + ), + migrations.AddIndex( + model_name='lessoncompletion', + index=models.Index(fields=['student', 'course_lesson'], name='course_less_student_3b6367_idx'), + ), + migrations.AddIndex( + model_name='participant', + index=models.Index(fields=['student'], name='course_part_student_566b08_idx'), + ), + migrations.AddIndex( + model_name='participant', + index=models.Index(fields=['course'], name='course_part_course__7cbf7c_idx'), + ), + migrations.AddIndex( + model_name='participant', + index=models.Index(fields=['joined_date'], name='course_part_joined__27eaa0_idx'), + ), + migrations.AddIndex( + model_name='participant', + index=models.Index(fields=['student', 'course'], name='course_part_student_c97a97_idx'), + ), + ] diff --git a/apps/course/models/course.py b/apps/course/models/course.py index 7fa1fd1..5a3a228 100644 --- a/apps/course/models/course.py +++ b/apps/course/models/course.py @@ -141,6 +141,15 @@ class Course(models.Model): class Meta: verbose_name = "Course" verbose_name_plural = "Courses" + indexes = [ + models.Index(fields=['status']), + models.Index(fields=['is_free']), + models.Index(fields=['created_at']), + models.Index(fields=['slug']), + models.Index(fields=['status', 'created_at']), + models.Index(fields=['category', 'status']), + models.Index(fields=['professor', 'status']), + ] class Glossary(models.Model): @@ -240,4 +249,8 @@ class CourseAttachment(models.Model): class Meta: ordering = ("-id",) verbose_name = "Course Attachment" - verbose_name_plural = "Course Attachments" \ No newline at end of file + verbose_name_plural = "Course Attachments" + indexes = [ + models.Index(fields=['course']), + models.Index(fields=['attachment']), + ] \ No newline at end of file diff --git a/apps/course/models/lesson.py b/apps/course/models/lesson.py index 1e636d3..c3f12cb 100644 --- a/apps/course/models/lesson.py +++ b/apps/course/models/lesson.py @@ -36,6 +36,14 @@ class Lesson(models.Model): def __str__(self): return self.title + class Meta: + verbose_name = "Lesson" + verbose_name_plural = "Lessons" + indexes = [ + models.Index(fields=['content_type']), + models.Index(fields=['created_at']), + ] + class CourseLesson(models.Model): """ @@ -91,6 +99,18 @@ class CourseLesson(models.Model): # Shift priorities for lessons with the same or higher priority lessons.filter(priority__gte=self.priority).update(priority=models.F('priority') + 1) + class Meta: + verbose_name = "Course Lesson" + verbose_name_plural = "Course Lessons" + indexes = [ + models.Index(fields=['course']), + models.Index(fields=['lesson']), + models.Index(fields=['priority']), + models.Index(fields=['is_active']), + models.Index(fields=['course', 'priority']), + models.Index(fields=['course', 'is_active']), + ] + class LessonCompletion(models.Model): student = models.ForeignKey( @@ -107,7 +127,13 @@ class LessonCompletion(models.Model): created_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Created at")) class Meta: - unique_together = ('student', 'course_lesson') + unique_together = ('student', 'course_lesson') + indexes = [ + models.Index(fields=['student']), + models.Index(fields=['course_lesson']), + models.Index(fields=['completed_at']), + models.Index(fields=['student', 'course_lesson']), + ] def __str__(self): return f"{self.student.fullname} - {self.course_lesson.title} - Completed" diff --git a/apps/course/models/participant.py b/apps/course/models/participant.py index eee680a..2c2abd8 100644 --- a/apps/course/models/participant.py +++ b/apps/course/models/participant.py @@ -21,4 +21,10 @@ class Participant(models.Model): unread_messages_count = models.IntegerField(default=0) class Meta: - unique_together = ('student', 'course') \ No newline at end of file + unique_together = ('student', 'course') + indexes = [ + models.Index(fields=['student']), + models.Index(fields=['course']), + models.Index(fields=['joined_date']), + models.Index(fields=['student', 'course']), + ] \ No newline at end of file diff --git a/apps/course/serializers/course.py b/apps/course/serializers/course.py index e1e8958..1ab64ca 100644 --- a/apps/course/serializers/course.py +++ b/apps/course/serializers/course.py @@ -58,6 +58,11 @@ class CourseListSerializer(serializers.ModelSerializer): return obj.participants.count() def get_lessons_count(self, obj): + # Use prefetched lessons if available + if hasattr(obj, 'lessons') and obj.lessons.all(): + lessons_count = sum(1 for lesson in obj.lessons.all() if lesson.is_active) + return max(lessons_count, obj.lessons_count) + # Fallback to direct query lessons_count = obj.lessons.filter(is_active=True).count() return max(lessons_count, obj.lessons_count) @@ -131,6 +136,10 @@ class CourseDetailSerializer(serializers.ModelSerializer): ] def get_room_id(self, obj): + # Use prefetched room_messages if available + if hasattr(obj, 'room_messages') and obj.room_messages.all(): + return obj.room_messages.first().id + # Fallback to direct query if not prefetched room_message = RoomMessage.objects.filter(course=obj).first() if room_message: return room_message.id @@ -152,15 +161,37 @@ class CourseDetailSerializer(serializers.ModelSerializer): request = self.context.get('request') if request and request.user.is_authenticated: user = request.user - - # آخرین درس تکمیل‌شده توسط کاربر + + # Use prefetched lessons if available + if hasattr(obj, 'lessons') and obj.lessons.all(): + lessons = [lesson for lesson in obj.lessons.all() if lesson.is_active] + completed_lessons = [] + + # Check which lessons are completed using prefetched data + for lesson in lessons: + if hasattr(lesson, 'completions') and lesson.completions.all(): + if any(completion.student_id == user.id for completion in lesson.completions.all()): + completed_lessons.append(lesson) + + if completed_lessons: + # Find the last completed lesson by priority + last_completed = max(completed_lessons, key=lambda x: x.priority) + # Find next lesson + next_lessons = [l for l in lessons if l.priority > last_completed.priority] + if next_lessons: + return min(next_lessons, key=lambda x: x.priority).id + + # If no completed lessons or no next lesson, return first lesson + if lessons: + return min(lessons, key=lambda x: x.priority).id + + # Fallback to direct queries if not prefetched last_completed_lesson = LessonCompletion.objects.filter( student=user, course_lesson__course=obj ).order_by('-completed_at').first() - + if last_completed_lesson: - # پیدا کردن درس بعدی بر اساس priority next_lesson = CourseLesson.objects.filter( course=obj, priority__gt=last_completed_lesson.course_lesson.priority, @@ -172,7 +203,7 @@ class CourseDetailSerializer(serializers.ModelSerializer): is_active=True ).order_by('priority').first() if next_lesson: - return next_lesson.id + return next_lesson.id return None @@ -190,6 +221,11 @@ class CourseDetailSerializer(serializers.ModelSerializer): return False def get_lessons_count(self, obj): + # Use prefetched lessons if available + if hasattr(obj, 'lessons') and obj.lessons.all(): + lessons_count = sum(1 for lesson in obj.lessons.all() if lesson.is_active) + return max(lessons_count, obj.lessons_count) + # Fallback to direct query lessons_count = obj.lessons.filter(is_active=True).count() return max(lessons_count, obj.lessons_count) @@ -222,6 +258,10 @@ class CourseDetailSerializer(serializers.ModelSerializer): return get_thumbs(obj.thumbnail, self.context.get('request')) def get_participant_count(self, obj): + # Use prefetched participants if available + if hasattr(obj, 'participants') and obj.participants.all(): + return len(obj.participants.all()) + # Fallback to direct query return obj.participants.count() def get_price(self, obj): diff --git a/apps/course/serializers/lesson.py b/apps/course/serializers/lesson.py index 133f5a7..f545f16 100644 --- a/apps/course/serializers/lesson.py +++ b/apps/course/serializers/lesson.py @@ -41,16 +41,22 @@ class CourseLessonSerializer(serializers.ModelSerializer): def get_is_complated(self, obj): request = self.context.get('request') if not request or not request.user.is_authenticated: - return False + return False user = request.user + + # Use prefetched completions if available + if hasattr(obj, 'completions') and obj.completions.all(): + return any(completion.student_id == user.id for completion in obj.completions.all()) + + # Fallback to direct queries is_participant = Participant.objects.filter( student=user, course=obj.course ).exists() - + if not is_participant: return False - + return LessonCompletion.objects.filter( student=user, course_lesson=obj diff --git a/apps/course/views/course.py b/apps/course/views/course.py index 648d009..3b245a6 100644 --- a/apps/course/views/course.py +++ b/apps/course/views/course.py @@ -29,11 +29,19 @@ class CourseCategoryAPIView(ListAPIView): class CourseListAPIView(ListAPIView): - queryset = Course.objects.all().exclude(status=Course.StatusChoices.INACTIVE) serializer_class = CourseListSerializer - filter_backends = [SearchFilter] + filter_backends = [SearchFilter] search_fields = ['title', 'category__name', 'professor__fullname'] + def get_queryset(self): + """ + Optimized queryset with select_related for ForeignKey relationships + """ + return Course.objects.select_related( + 'category', + 'professor' + ).exclude(status=Course.StatusChoices.INACTIVE) + @swagger_auto_schema( operation_description=doc_course_list(), manual_parameters=[ @@ -104,10 +112,25 @@ class CourseListAPIView(ListAPIView): class CourseDetailAPIView(RetrieveAPIView): - queryset = Course.objects.all() serializer_class = CourseDetailSerializer lookup_field = "slug" + def get_queryset(self): + """ + Optimized queryset with select_related and prefetch_related for all relationships + """ + return Course.objects.select_related( + 'category', + 'professor' + ).prefetch_related( + 'lessons__lesson', + 'lessons__completions', + 'attachments__attachment', + 'glossaries__glossary', + 'participants__student', + 'room_messages' + ) + @swagger_auto_schema( operation_description=doc_course_detail(), ) @@ -139,7 +162,18 @@ class MyCourseListAPIView(ListAPIView): return super().get(request, *args, **kwargs) def get_queryset(self): - queryset = Course.objects.exclude(status=Course.StatusChoices.INACTIVE) + """ + Optimized queryset for user's courses with select_related and prefetch_related + """ + queryset = Course.objects.select_related( + 'category', + 'professor' + ).prefetch_related( + 'lessons__lesson', + 'lessons__completions', + 'participants__student' + ).exclude(status=Course.StatusChoices.INACTIVE) + request = self.request filters = request.query_params student = self.request.user @@ -197,12 +231,18 @@ class AttachmentListAPIView(ListAPIView): return super().get(request, *args, **kwargs) def get_queryset(self): + """ + Optimized queryset with select_related for attachment relationship + """ course_slug = self.kwargs.get('slug') try: course = Course.objects.get(slug=course_slug) except Course.DoesNotExist: raise NotFound("Course not found") - return CourseAttachment.objects.filter(course=course) + return CourseAttachment.objects.select_related( + 'course', + 'attachment' + ).filter(course=course) @@ -213,13 +253,19 @@ class GlossaryListAPIView(ListAPIView): search_fields = ['glossary__title', 'glossary__description'] def get_queryset(self): + """ + Optimized queryset with select_related for glossary relationship + """ course_slug = self.kwargs.get('slug') try: course = Course.objects.get(slug=course_slug) except Course.DoesNotExist: raise NotFound("Course not found") - return CourseGlossary.objects.filter(course=course) + return CourseGlossary.objects.select_related( + 'course', + 'glossary' + ).filter(course=course) \ No newline at end of file diff --git a/apps/course/views/lesson.py b/apps/course/views/lesson.py index 0ca1a9c..17f8aa4 100644 --- a/apps/course/views/lesson.py +++ b/apps/course/views/lesson.py @@ -18,7 +18,6 @@ from rest_framework.permissions import IsAuthenticated class LessonListView(ListAPIView): serializer_class = CourseLessonSerializer - queryset = CourseLesson.objects.filter(is_active=True) @swagger_auto_schema( operation_description=doc_courses_lesson(), @@ -27,13 +26,22 @@ class LessonListView(ListAPIView): return super().get(request, *args, **kwargs) def get_queryset(self): + """ + Optimized queryset with select_related and prefetch_related for lesson relationships + """ course_slug = self.kwargs.get('slug') course = get_object_or_404(Course, slug=course_slug) - course = Course.objects.filter(slug=course_slug).first() - if not course: - raise AppAPIException({"message": "course not found"}, status_code=status.HTTP_404_NOT_FOUND) - - return self.queryset.filter(course=course).order_by('priority','id') + + return CourseLesson.objects.select_related( + 'course', + 'lesson' + ).prefetch_related( + 'completions', + 'lesson__quizzes' + ).filter( + course=course, + is_active=True + ).order_by('priority', 'id') @@ -42,11 +50,23 @@ class LessonDetailView(RetrieveAPIView): serializer_class = CourseLessonSerializer def get(self, request, *args, **kwargs): + """ + Optimized lesson detail view with select_related for relationships + """ lesson_id = self.kwargs.get('id') - course_lesson = get_object_or_404(CourseLesson, id=lesson_id, is_active=True) + course_lesson = get_object_or_404( + CourseLesson.objects.select_related('course', 'lesson'), + id=lesson_id, + is_active=True + ) course = course_lesson.course - lessons = CourseLesson.objects.filter(course=course, is_active=True).order_by('priority') + lessons = CourseLesson.objects.select_related( + 'lesson' + ).filter( + course=course, + is_active=True + ).order_by('priority') total_lessons = lessons.count() current_lesson_number = list(lessons.values_list('id', flat=True)).index(course_lesson.id) + 1 diff --git a/apps/course/views/participant.py b/apps/course/views/participant.py index a89e9ae..74f257f 100644 --- a/apps/course/views/participant.py +++ b/apps/course/views/participant.py @@ -21,13 +21,18 @@ class CourseParticipantsView(generics.ListAPIView): operation_description=doc_course_participants(), ) def get_queryset(self): + """ + Optimized queryset with select_related for course relationship + """ course_slug = self.kwargs.get('slug') try: course = Course.objects.get(slug=course_slug) except Course.DoesNotExist: raise AppAPIException({'message': "Course not found"}) # Handle course not found - return StudentUser.objects.filter(participated_courses__course=course) + return StudentUser.objects.select_related().filter( + participated_courses__course=course + ) diff --git a/apps/video/views.py b/apps/video/views.py index 628d0de..a675f8e 100755 --- a/apps/video/views.py +++ b/apps/video/views.py @@ -29,7 +29,12 @@ class VideoCategoryListAPIView(generics.ListAPIView): return super().get(request, *args, **kwargs) def get_queryset(self): - return VideoCategory.objects.filter(status=True).order_by('order') + """ + Optimized queryset with prefetch_related for videos + """ + return VideoCategory.objects.filter(status=True).prefetch_related( + 'videos' + ).order_by('order')