Browse Source

Update production configuration and enhance hadis data seeding logic

- Reduced the number of Gunicorn workers in the production Docker configuration from 32 to 6 for improved resource management.
- Enhanced the hadis data seeding command to calculate and distribute hadis entries more effectively across categories, ensuring balanced creation while respecting maximum limits.
shokhmgar
mortezaei 10 months ago
parent
commit
e56aeadee5
  1. 48
      apps/hadis/management/commands/seed_russian_data.py
  2. 2
      docker-compose.prod.yml

48
apps/hadis/management/commands/seed_russian_data.py

@ -319,26 +319,54 @@ class Command(BaseCommand):
def create_hadis_bulk(self, leaf_categories): def create_hadis_bulk(self, leaf_categories):
"""Create hadis records using bulk_create for maximum performance.""" """Create hadis records using bulk_create for maximum performance."""
self.stdout.write(f'Creating hadis entries using bulk_create (max {self.MAX_HADIS_RECORDS})...')
self.stdout.write(f'Creating hadis entries using bulk_create (target: {self.MAX_HADIS_RECORDS})...')
num_categories = len(leaf_categories) num_categories = len(leaf_categories)
if num_categories == 0: if num_categories == 0:
self.stdout.write(self.style.WARNING('No leaf categories found!')) self.stdout.write(self.style.WARNING('No leaf categories found!'))
return return
# Calculate distribution: minimum per category, then distribute remainder
min_per_category = self.HADIS_PER_CATEGORY
base_total = min_per_category * num_categories
if base_total > self.MAX_HADIS_RECORDS:
# If minimum would exceed max, reduce proportionally
min_per_category = self.MAX_HADIS_RECORDS // num_categories
remainder = self.MAX_HADIS_RECORDS % num_categories
else:
# We can give minimum to all, distribute remainder
remainder = self.MAX_HADIS_RECORDS - base_total
# Create distribution list
distribution = [min_per_category] * num_categories
# Distribute remainder randomly across categories
if remainder > 0:
indices = random.sample(range(num_categories), min(remainder, num_categories))
for idx in indices:
distribution[idx] += 1
# If remainder is larger than num_categories, distribute multiple times
remaining = remainder - len(indices)
while remaining > 0:
for idx in range(num_categories):
if remaining <= 0:
break
distribution[idx] += 1
remaining -= 1
self.stdout.write(f' Distribution: {num_categories} categories, '
f'min={min(distribution)}, max={max(distribution)}, '
f'total={sum(distribution)}')
# Pre-generate all hadis objects # Pre-generate all hadis objects
hadis_list = [] hadis_list = []
hadis_number = 1 hadis_number = 1
hadis_per_cat = max(self.HADIS_PER_CATEGORY, self.MAX_HADIS_RECORDS // num_categories)
for category in leaf_categories:
if len(hadis_list) >= self.MAX_HADIS_RECORDS:
break
for _ in range(hadis_per_cat):
if len(hadis_list) >= self.MAX_HADIS_RECORDS:
break
for idx, category in enumerate(leaf_categories):
count_for_this_category = distribution[idx]
for _ in range(count_for_this_category):
title = random.choice(RUSSIAN_HADIS_TITLES) title = random.choice(RUSSIAN_HADIS_TITLES)
text = generate_hadis_text() text = generate_hadis_text()
opening = random.choice(RUSSIAN_HADIS_OPENINGS).rstrip(':') opening = random.choice(RUSSIAN_HADIS_OPENINGS).rstrip(':')

2
docker-compose.prod.yml

@ -7,7 +7,7 @@ services:
build: build:
context: . context: .
dockerfile: Dockerfile.prod dockerfile: Dockerfile.prod
command: gunicorn config.wsgi:application --bind 0.0.0.0:8000 --workers=32 --timeout 560
command: gunicorn config.wsgi:application --bind 0.0.0.0:8000 --workers=6 --timeout 560
volumes: volumes:
# - static_volume:/usr/src/app/static # - static_volume:/usr/src/app/static
- media_volume:/usr/src/app/media - media_volume:/usr/src/app/media

Loading…
Cancel
Save