Browse Source

Update production configuration and enhance hadis data seeding logic

- Reduced the number of Gunicorn workers in the production Docker configuration from 32 to 6 for improved resource management.
- Enhanced the hadis data seeding command to calculate and distribute hadis entries more effectively across categories, ensuring balanced creation while respecting maximum limits.
master
mortezaei 4 months ago
parent
commit
e56aeadee5
  1. 48
      apps/hadis/management/commands/seed_russian_data.py
  2. 2
      docker-compose.prod.yml

48
apps/hadis/management/commands/seed_russian_data.py

@ -319,26 +319,54 @@ class Command(BaseCommand):
def create_hadis_bulk(self, leaf_categories):
"""Create hadis records using bulk_create for maximum performance."""
self.stdout.write(f'Creating hadis entries using bulk_create (max {self.MAX_HADIS_RECORDS})...')
self.stdout.write(f'Creating hadis entries using bulk_create (target: {self.MAX_HADIS_RECORDS})...')
num_categories = len(leaf_categories)
if num_categories == 0:
self.stdout.write(self.style.WARNING('No leaf categories found!'))
return
# Calculate distribution: minimum per category, then distribute remainder
min_per_category = self.HADIS_PER_CATEGORY
base_total = min_per_category * num_categories
if base_total > self.MAX_HADIS_RECORDS:
# If minimum would exceed max, reduce proportionally
min_per_category = self.MAX_HADIS_RECORDS // num_categories
remainder = self.MAX_HADIS_RECORDS % num_categories
else:
# We can give minimum to all, distribute remainder
remainder = self.MAX_HADIS_RECORDS - base_total
# Create distribution list
distribution = [min_per_category] * num_categories
# Distribute remainder randomly across categories
if remainder > 0:
indices = random.sample(range(num_categories), min(remainder, num_categories))
for idx in indices:
distribution[idx] += 1
# If remainder is larger than num_categories, distribute multiple times
remaining = remainder - len(indices)
while remaining > 0:
for idx in range(num_categories):
if remaining <= 0:
break
distribution[idx] += 1
remaining -= 1
self.stdout.write(f' Distribution: {num_categories} categories, '
f'min={min(distribution)}, max={max(distribution)}, '
f'total={sum(distribution)}')
# Pre-generate all hadis objects
hadis_list = []
hadis_number = 1
hadis_per_cat = max(self.HADIS_PER_CATEGORY, self.MAX_HADIS_RECORDS // num_categories)
for category in leaf_categories:
if len(hadis_list) >= self.MAX_HADIS_RECORDS:
break
for _ in range(hadis_per_cat):
if len(hadis_list) >= self.MAX_HADIS_RECORDS:
break
for idx, category in enumerate(leaf_categories):
count_for_this_category = distribution[idx]
for _ in range(count_for_this_category):
title = random.choice(RUSSIAN_HADIS_TITLES)
text = generate_hadis_text()
opening = random.choice(RUSSIAN_HADIS_OPENINGS).rstrip(':')

2
docker-compose.prod.yml

@ -7,7 +7,7 @@ services:
build:
context: .
dockerfile: Dockerfile.prod
command: gunicorn config.wsgi:application --bind 0.0.0.0:8000 --workers=32 --timeout 560
command: gunicorn config.wsgi:application --bind 0.0.0.0:8000 --workers=6 --timeout 560
volumes:
# - static_volume:/usr/src/app/static
- media_volume:/usr/src/app/media

Loading…
Cancel
Save