Add EmbeddingSession model and admin interface for background processing

- Introduced the `EmbeddingSession` model to track the status and progress of embedding tasks, including fields for status, progress, processed items, and error messages.
- Created `EmbeddingSessionAdmin` to manage embedding sessions in the admin interface, featuring custom display methods for status badges and progress bars.
- Implemented background processing to trigger a FastAPI agent upon creating a new embedding session, enhancing the system's responsiveness and user experience.
- Added migrations to establish the new model in the database.

apps/agent/admin.py

@@ -5,7 +5,10 @@ from django.urls import reverse
 # 1. Change TabularInline to StackedInline
 from unfold.admin import ModelAdmin, TabularInline
 from utils.admin import dovoodi_admin_site, project_admin_site
-from .models import AgentSettings, AgentPrompt
+from .models import AgentSettings, AgentPrompt, EmbeddingSession
+from django.utils.html import format_html
+import requests
+import threading
 
 class AgentPromptInline(TabularInline): 
     model = AgentPrompt
@@ -56,5 +59,102 @@ class AgentSettingsAdmin(ModelAdmin):
     # 2. Add the Stacked Inline
     inlines = [AgentPromptInline]
 
+
+
+class EmbeddingSessionAdmin(ModelAdmin):
+    # What to show in the table view
+    list_display = ['id', 'status_badge', 'progress_bar', 'created_at']
+    
+    # We make everything read-only so admins can't fake the progress
+    readonly_fields = ['status', 'progress', 'processed_items', 'total_items', 'error_message']
+    
+    # Optional: If you want to customize how the error message text area looks in Unfold
+    formfield_overrides = {
+        models.TextField: {
+            'widget': admin.widgets.AdminTextareaWidget(attrs={
+                'rows': 4,
+                'class': (
+                    'border border-gray-300 rounded-md shadow-sm '
+                    'w-full block sm:text-sm '
+                    'bg-white text-gray-900 '
+                    'dark:bg-gray-900 dark:text-white dark:border-gray-700 '
+                    'focus:ring-primary-500 focus:border-primary-500'
+                ),
+                'style': 'background-color: inherit; color: inherit; width: 100%;'
+            })
+        },
+    }
+
+    @admin.display(description="Status")
+    def status_badge(self, obj):
+        """Creates a beautiful Unfold/Tailwind badge based on the status."""
+        colors = {
+            'PENDING': 'bg-gray-100 text-gray-800 dark:bg-gray-800 dark:text-gray-300',
+            'PROCESSING': 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-300',
+            'COMPLETED': 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-300',
+            'FAILED': 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-300',
+        }
+        color_classes = colors.get(obj.status, colors['PENDING'])
+        
+        return format_html(
+            '<span class="px-2.5 py-1 inline-flex text-xs leading-5 font-semibold rounded-full {}">'
+            '{}'
+            '</span>',
+            color_classes,
+            obj.status
+        )
+
+    @admin.display(description="Progress")
+    def progress_bar(self, obj):
+        """Creates a Tailwind progress bar to show sync status with percentages."""
+        if obj.status == 'PENDING':
+            return format_html('<span class="text-sm text-gray-500 dark:text-gray-400">Waiting to start...</span>')
+            
+        return format_html(
+            '''
+            <div class="flex flex-col gap-1 w-64">
+                <div class="w-full bg-gray-200 rounded-full h-2 dark:bg-gray-700">
+                  <div class="bg-primary-600 h-2 rounded-full transition-all duration-500" style="width: {}%"></div>
+                </div>
+                
+                <div class="flex justify-between text-xs text-gray-500 dark:text-gray-400">
+                    <span class="font-bold text-gray-700 dark:text-gray-300">{}%</span>
+                    <span>{} / {} items</span>
+                </div>
+            </div>
+            ''',
+            obj.progress,        # For the CSS width inside the style="" tag
+            obj.progress,        # For the text percentage display
+            obj.processed_items, # For the items count
+            obj.total_items      # For the total count
+        )
+        
+    def save_model(self, request, obj, form, change):
+        """
+        Intercept the save. If it's a new record, tell the FastAPI agent to start working.
+        """
+        is_new = obj.pk is None
+        super().save_model(request, obj, form, change)
+        
+        if is_new:
+            # 🟢 Trigger the FastAPI Agent in the background
+            def fire_and_forget():
+                try:
+                    # Point this to your FastAPI Agent URL
+                    # Make sure host.docker.internal works, or use the agent's container name
+                    requests.post(
+                        "http://127.0.0.1:8081/api/sync-knowledge", 
+                        json={"session_id": obj.id}, 
+                        timeout=5
+                    )
+                except Exception as e:
+                    print(f"Failed to trigger agent: {e}")
+                    
+            threading.Thread(target=fire_and_forget).start()
+
+# Register to your custom Unfold admin sites
+dovoodi_admin_site.register(EmbeddingSession, EmbeddingSessionAdmin)
+project_admin_site.register(EmbeddingSession, EmbeddingSessionAdmin)
+
 dovoodi_admin_site.register(AgentSettings, AgentSettingsAdmin)
 project_admin_site.register(AgentSettings, AgentSettingsAdmin)

apps/agent/migrations/0004_embeddingsession.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.27 on 2026-02-18 11:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('agent', '0003_alter_agentprompt_options_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EmbeddingSession',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('status', models.CharField(choices=[('PENDING', 'Pending'), ('PROCESSING', 'Processing'), ('COMPLETED', 'Completed'), ('FAILED', 'Failed')], default='PENDING', max_length=20)),
+                ('progress', models.IntegerField(default=0)),
+                ('processed_items', models.IntegerField(default=0)),
+                ('total_items', models.IntegerField(default=0)),
+                ('error_message', models.TextField(blank=True, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+            ],
+        ),
+    ]

apps/agent/models.py

@@ -43,3 +43,25 @@ class AgentPrompt(models.Model):
         # else:
         #     return f"Inactive Prompt: {self.content[:50]}..." if self.content else "Empty Prompt"
         return ""
+
+class EmbeddingSession(models.Model):
+    STATUS_CHOICES = (
+        ('PENDING', 'Pending'),
+        ('PROCESSING', 'Processing'),
+        ('COMPLETED', 'Completed'),
+        ('FAILED', 'Failed'),
+    )
+    
+    # # E.g., 'jina-embeddings-v4' or 'openai-small'
+    # target_embedder = models.CharField(max_length=100) 
+    # # E.g., 'hadith' or 'article'
+    # data_type = models.CharField(max_length=50) 
+    status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='PENDING')
+    progress = models.IntegerField(default=0)
+    processed_items = models.IntegerField(default=0)
+    total_items = models.IntegerField(default=0)
+    error_message = models.TextField(blank=True, null=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    def __str__(self):
+        return f"Global Sync #{self.id} - {self.status}"

apps/article/migrations/0002_article_embedded_in.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.27 on 2026-02-18 11:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('article', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='article',
+            name='embedded_in',
+            field=models.JSONField(blank=True, default=list),
+        ),
+    ]

apps/article/models.py

@@ -108,6 +108,7 @@ class Article(models.Model):
 
     created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at'))
     updated_at = models.DateTimeField(auto_now=True, verbose_name=_('updated at'))
+    embedded_in = models.JSONField(default=list, blank=True)
 
     def __str__(self):
         return self.title
@@ -126,6 +127,14 @@ class Article(models.Model):
     def save(self, *args, **kwargs):
         if not self.slug:
             self.slug = generate_slug_for_model(Article, self.title)
+        
+        # Reset embedded_in if text or translation changes
+        if self.pk:
+            old_instance = Article.objects.get(pk=self.pk)
+            if (old_instance.description != self.description or 
+                old_instance.content != self.content):
+                self.embedded_in = [] # Reset!
+
         super().save(*args, **kwargs)
 
     class Meta:

apps/hadis/migrations/0006_hadis_embedded_in_hadiscorrection_embedded_in_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.27 on 2026-02-18 11:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hadis', '0005_alter_hadiscategory_slug'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='hadis',
+            name='embedded_in',
+            field=models.JSONField(blank=True, default=list),
+        ),
+        migrations.AddField(
+            model_name='hadiscorrection',
+            name='embedded_in',
+            field=models.JSONField(blank=True, default=list),
+        ),
+        migrations.AddField(
+            model_name='transmitteroriginaltext',
+            name='embedded_in',
+            field=models.JSONField(blank=True, default=list),
+        ),
+    ]

apps/hadis/models/hadis.py

@@ -314,6 +314,7 @@ class Hadis(models.Model):
 
     created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('created at'))
     updated_at = models.DateTimeField(auto_now=True, verbose_name=_('updated at'))
+    embedded_in = models.JSONField(default=list, blank=True)
 
     def __str__(self):
         return f"{self.number} - {self.title[0]['text']}" if self.title else f"Hadis {self.number}"
@@ -359,6 +360,13 @@ class Hadis(models.Model):
             category_slug = self.category.slug if self.category and self.category.slug else 'uncategorized'
             self.share_link = f"{settings.DOVODI_DOMAIN}/arguments/hadith/{category_slug}/{self.slug}"
 
+                # Reset embedded_in if text or translation changes
+        if self.pk:
+            old_instance = Hadis.objects.get(pk=self.pk)
+            if (old_instance.text != self.text or 
+                old_instance.translation != self.translation):
+                self.embedded_in = [] # Reset!
+        
         super().save(*args, **kwargs)
 
     def _get_json_field(self, field_name: str, lang:  Optional[str]=None , fallback: str = "en"):
@@ -512,6 +520,7 @@ class HadisCorrection(models.Model):
     created_at = models.DateTimeField(auto_now_add=True, verbose_name=_("created at"))
     updated_at = models.DateTimeField(auto_now=True, verbose_name=_("updated at"))
     share_link = models.CharField(max_length=255, verbose_name=_('share link'), null=True, blank=True)
+    embedded_in = models.JSONField(default=list, blank=True)
 
     class Meta:
         verbose_name = _("Hadis Correction")
@@ -561,6 +570,13 @@ class HadisCorrection(models.Model):
             category_slug = self.hadis.category.slug if self.hadis.category and self.hadis.category.slug else 'uncategorized'
             self.share_link = f"{settings.DOVODI_DOMAIN}/arguments/hadith/{category_slug}/{self.hadis.slug}/corrections/{self.slug}"
         
+        # Reset embedded_in if text or translation changes
+        if self.pk:
+            old_instance = HadisCorrection.objects.get(pk=self.pk)
+            if (old_instance.description != self.description or 
+                old_instance.translation != self.translation):
+                self.embedded_in = [] # Reset!
+
         super().save(*args, **kwargs)
 
     def get_title(self,lang):

apps/hadis/models/transmitter.py

@@ -654,6 +654,7 @@ class TransmitterOriginalText(models.Model):
     text = models.JSONField(default = list  , verbose_name=_('Text'))
     translation = models.JSONField(verbose_name=_('translation'), default=list)
     share_link = models.CharField(max_length=255, verbose_name=_('share link'), null=True, blank=True)
+    embedded_in = models.JSONField(default=list, blank=True)
 
     class Meta:
         indexes = [
@@ -706,6 +707,13 @@ class TransmitterOriginalText(models.Model):
         if self.slug and self.transmitter and self.transmitter.slug:
             self.share_link = f"{settings.DOVODI_DOMAIN}/arguments/narrators/{self.transmitter.slug}/original-texts/{self.slug}"
             
+        # Reset embedded_in if text or translation changes
+        if self.pk:
+            old_instance = TransmitterOriginalText.objects.get(pk=self.pk)
+            if (old_instance.text != self.text or 
+                old_instance.translation != self.translation):
+                self.embedded_in = [] # Reset!
+
         super().save(*args, **kwargs)