Add preprocessing time to response

cope2n-ai-fi/api/sdsap_sbt/prediction_sbt.py

@@ -5,6 +5,7 @@ import random
 import numpy as np
 from pathlib import Path
 import uuid
+from copy import deepcopy
 import sys, os
 cur_dir = str(Path(__file__).parents[2])
 sys.path.append(cur_dir)

cope2n-ai-fi/configs/sdsap_sbt/configs.py

@@ -6,7 +6,7 @@ ocr_engine = {
         "device": device
     },
     "recognizer": {
-        "version": "/workspace/cope2n-ai-fi/weights/models/sdsvtr/hub/sbt_20231210_sdsrv.pth",
+        "version": "/workspace/cope2n-ai-fi/weights/models/sdsvtr/hub/sbt_20231218_e116_sdstr.pth",
         "device": device
     },
     "deskew": {

cope2n-api/fwd_api/api/ctel_view.py

@@ -11,6 +11,7 @@ from rest_framework.response import Response
 from typing import List
 from rest_framework.renderers import JSONRenderer
 from rest_framework_xml.renderers import XMLRenderer
+from multiprocessing.pool import ThreadPool
 
 from fwd import settings
 from ..celery_worker.client_connector import c_connector
@@ -31,6 +32,9 @@ class CtelViewSet(viewsets.ViewSet):
         'multipart/form-data': {
             'type': 'object',
             'properties': {
+                'is_test_request': {
+                    'type': 'boolean',
+                },
                 'file': {
                     'type': 'string',
                     'format': 'binary'
@@ -43,7 +47,6 @@ class CtelViewSet(viewsets.ViewSet):
         }
     }, responses=None, tags=['OCR'])
     @action(detail=False, url_path="image/process", methods=["POST"])
-    # @transaction.atomic
     def process(self, request):
         s_time = time.time()
         user_info = ProcessUtil.get_user(request)
@@ -59,15 +62,19 @@ class CtelViewSet(viewsets.ViewSet):
         file_extension = file_obj.name.split(".")[-1].lower()
         p_type = validated_data['type']
         file_name = f"temp_{rq_id}.{file_extension}"
+        is_test_request = validated_data.get("is_test_request", False)
 
         total_page = 1
 
-        new_request: SubscriptionRequest = SubscriptionRequest(pages=total_page,
+        new_request: SubscriptionRequest = SubscriptionRequest(
-                                                            pages_left=total_page,
+            pages=total_page,
-                                                            doc_type="all",
+            pages_left=total_page,
-                                                            process_type=p_type, status=1, request_id=rq_id,
+            doc_type="all",
-                                                            provider_code=provider_code,
+            process_type=p_type, status=1, request_id=rq_id,
-                                                            subscription=sub)
+            provider_code=provider_code,
+            subscription=sub,
+            is_test_request=is_test_request
+        )
         new_request.save()
         from ..celery_worker.client_connector import c_connector
         file_obj.seek(0)
@@ -82,7 +89,6 @@ class CtelViewSet(viewsets.ViewSet):
 
         if file_extension in pdf_extensions:
             c_connector.do_pdf((rq_id, sub.id, p_type, user.id, files))
-        #     b_url = ProcessUtil.process_pdf_file(file_name, file_obj, new_request, user)
         elif file_extension in image_extensions:
             b_url = ProcessUtil.process_image_file(file_name, file_obj, new_request, user)
             j_time = time.time()
@@ -216,26 +222,39 @@ class CtelViewSet(viewsets.ViewSet):
                                                             provider_code=provider_code,
                                                             subscription=sub)
         new_request.save()
+
         count = 0
-        compact_files = []
+        doc_files_with_type = []
         for doc_type, doc_files in files.items():
             for i, doc_file in enumerate(doc_files):
                 _ext = doc_file.name.split(".")[-1]
                 if _ext not in allowed_file_extensions:
                     return JsonResponse(status=status.HTTP_406_NOT_ACCEPTABLE, data={"request_id": rq_id, "message": f"File {_ext} is now allowed"})
-                _name = f"temp_{doc_type}_{rq_id}_{i}.{_ext}"
+                tmp_file_name = f"temp_{doc_type}_{rq_id}_{i}.{_ext}"
-                doc_file.seek(0)
+                doc_files_with_type.append((
-                file_path = FileUtils.resize_and_save_file(_name, new_request, doc_file, 100)
+                    count, doc_type, doc_file, tmp_file_name
-                FileUtils.save_to_S3(_name, new_request, file_path)
+                ))
                 count += 1
-                this_file = {
-                    "file_name": _name,
-                    "file_path": file_path,
-                    "file_type": doc_type
-                }
-                compact_files.append(this_file)
-        c_connector.do_pdf((rq_id, sub.id, p_type, user.id, compact_files))
 
+        # Run file processing in a pool of 2 threads. TODO: Convert to Celery worker when possible
+        compact_files = [None] * len(doc_files_with_type)
+        pool = ThreadPool(processes=2)
+        def process_file(data):
+            idx, doc_type, doc_file, tmp_file_name = data
+            doc_file.seek(0)
+            file_path = FileUtils.resize_and_save_file(tmp_file_name, new_request, doc_file, 100)
+            FileUtils.save_to_S3(tmp_file_name, new_request, file_path)
+            return {
+                "idx": idx,
+                "file_name": tmp_file_name,
+                "file_path": file_path,
+                "file_type": doc_type
+            }
+        for result in pool.map(process_file, doc_files_with_type):
+            compact_files[result["idx"]] = result
+
+        # Send to AI queue
+        c_connector.do_pdf((rq_id, sub.id, p_type, user.id, compact_files))
 
         time_limit = 120
         start_time = time.time()

cope2n-api/fwd_api/api_router.py

@@ -13,7 +13,6 @@ else:
 
 router.register("ctel", CtelViewSet, basename="CtelAPI")
 router.register("ctel", CtelUserViewSet, basename="CtelUserAPI")
-# router.register("ctel", CtelTemplateViewSet, basename="CtelTemplateAPI")
 
 app_name = "api"
 urlpatterns = router.urls

cope2n-api/fwd_api/celery_worker/internal_task.py

@@ -3,6 +3,7 @@ import uuid
 import os
 import base64
 import traceback
+from multiprocessing.pool import ThreadPool
 
 from fwd_api.models import SubscriptionRequest, UserProfile
 from fwd_api.celery_worker.worker import app
@@ -66,14 +67,13 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files):
             "file_type": ""
         },]
     """
-    start = time.time()
     new_request = SubscriptionRequest.objects.filter(request_id=rq_id)[0]
     user = UserProfile.objects.filter(id=user_id).first()
-    b_urls = []
     new_request.pages = len(files)
     new_request.pages_left = len(files)
 
-    for i, file in enumerate(files):
+    def process_and_save_file(data):
+        idx, file = data
         extension = file["file_name"].split(".")[-1].lower()
         if extension == "pdf":
             _b_urls = process_pdf_file(file["file_name"], file["file_path"], new_request, user)
@@ -83,28 +83,39 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files):
                 raise FileContentInvalidException
             for j in range(len(_b_urls)):
                 _b_urls[j]["doc_type"] = file["file_type"]
-                _b_urls[j]["page_number"] = j + len(b_urls)
+                _b_urls[j]["page_number"] = idx            
-            # b_urls += _b_urls # TODO: Client may request all images in a file, for now, extract the first page only                
+            return idx, _b_urls[0]
-            b_urls.append(_b_urls[0])
         elif extension in image_extensions:
             this_url = ProcessUtil.process_image_local_file(file["file_name"], file["file_path"], new_request, user)[0]
-            this_url["page_number"] = len(b_urls)
+            this_url["page_number"] = idx
             if file["file_type"]:   
                 this_url["doc_type"] = file["file_type"]
-            b_urls.append(this_url)
+            return idx, this_url
+
+    # Run file processing in a pool of 2 threads. TODO: Convert to Celery worker when possible
+    start_time = time.time()
+    b_urls = [None] * len(files)
+    pool = ThreadPool(processes=2)
+    files_with_idx = [(idx, file) for idx, file in enumerate(files)]
+    for idx, url in pool.map(process_and_save_file, files_with_idx):
+        b_urls[idx] = url
+    new_request.preprocessing_time = time.time() - start_time
 
-    start_process = time.time()
-    logger.info(f"BE proccessing time: {start_process - start}")
     # TODO: send to queue with different request_ids
     doc_type_string = ""
+    to_queue = []
     for i, b_url in enumerate(b_urls):
         fractorized_request_id = rq_id + f"_sub_{i}"
-        ProcessUtil.send_to_queue2(fractorized_request_id, sub_id, [b_url], user_id, p_type)
+        to_queue.append((fractorized_request_id, sub_id, [b_url], user_id, p_type))
         doc_type_string += "{},".format(b_url["doc_type"])
     doc_type_string = doc_type_string[:-1]
     new_request.doc_type = doc_type_string
     new_request.save()
 
+    # Send to next queue
+    for sub_rq_id, sub_id, urls, user_id, p_type in to_queue:
+        ProcessUtil.send_to_queue2(sub_rq_id, sub_id, urls, user_id, p_type)
+
 
 @app.task(name='upload_file_to_s3')
 def upload_file_to_s3(local_file_path, s3_key):

cope2n-api/fwd_api/migrations/0158_subscriptionrequest_is_test_request.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.3 on 2023-12-21 04:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0157_alter_subscriptionrequest_created_at'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='is_test_request',
+            field=models.BooleanField(default=False),
+        ),
+    ]

cope2n-api/fwd_api/migrations/0159_subscriptionrequest_ai_inference_time_and_more.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.1.3 on 2023-12-21 06:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0158_subscriptionrequest_is_test_request'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='ai_inference_time',
+            field=models.FloatField(default=-1),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='cpu_percent',
+            field=models.FloatField(default=-1),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='gpu_percent',
+            field=models.FloatField(default=-1),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='gpu_stats',
+            field=models.CharField(max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='is_bad_image_quality',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='is_reviewed',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='preprocessing_time',
+            field=models.FloatField(default=-1),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='total_memory',
+            field=models.FloatField(default=-1),
+        ),
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='used_memory',
+            field=models.FloatField(default=-1),
+        ),
+    ]

cope2n-api/fwd_api/models/SubscriptionRequest.py

@@ -19,3 +19,14 @@ class SubscriptionRequest(models.Model):
     subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
     created_at = models.DateTimeField(default=timezone.now, db_index=True)
     updated_at = models.DateTimeField(auto_now=True)
+    is_test_request = models.BooleanField(default=False)
+
+    preprocessing_time = models.FloatField(default=-1)
+    ai_inference_time = models.FloatField(default=-1)
+    cpu_percent = models.FloatField(default=-1)
+    gpu_percent = models.FloatField(default=-1)
+    used_memory = models.FloatField(default=-1)
+    total_memory = models.FloatField(default=-1)
+    gpu_stats = models.CharField(max_length=100, null=True)
+    is_reviewed = models.BooleanField(default=False)
+    is_bad_image_quality = models.BooleanField(default=False)

cope2n-api/fwd_api/utils/health.py

@@ -5,7 +5,7 @@ from datetime import timedelta
 from ..models import SubscriptionRequest
 
 def get_latest_requests(limit=50):
-    requests = SubscriptionRequest.objects.order_by("-created_at")[:limit]
+    requests = SubscriptionRequest.objects.filter(is_test_request=False).order_by("-created_at")[:limit]
     requests_dict = []
     for request in requests:
         requests_dict.append({
@@ -14,13 +14,23 @@ def get_latest_requests(limit=50):
             "doc_type": request.doc_type,
             # "predict_result": request.predict_result,
             "created_at": request.created_at,
+            "updated_at": request.updated_at,
+            "preprocessing_time": request.preprocessing_time,
+            "ai_inference_time": request.ai_inference_time,
+            "cpu_percent": request.cpu_percent,
+            "gpu_percent": request.gpu_percent,
+            "used_memory": request.used_memory,
+            "total_memory": request.total_memory,
+            "gpu_stats": request.gpu_stats,
+            "is_reviewed": request.is_reviewed,
+            "is_bad_image_quality": request.is_bad_image_quality,
         })
     return requests_dict
 
 def count_requests_by_date(days_limit=5):
     today = timezone.now().date()
     start_date = today - timedelta(days=days_limit)
-    requests_by_date = SubscriptionRequest.objects.filter(created_at__gte=start_date).values('created_at__date').annotate(count=Count('id')).values('created_at__date', 'count').order_by('created_at__date')
+    requests_by_date = SubscriptionRequest.objects.filter(is_test_request=False).filter(created_at__gte=start_date).values('created_at__date').annotate(count=Count('id')).values('created_at__date', 'count').order_by('created_at__date')
     count_dict = []
     for rbd in requests_by_date:
         count_dict.append({

cope2n-api/fwd_api/utils/process.py

@@ -104,6 +104,8 @@ def validate_ocr_request_and_get(request, subscription):
     FileUtils.validate_list_file(list_file)
     validated_data['file'] = list_file[0]
 
+    validated_data['is_test_request'] = request.data.get('is_test_request', False)
+
     return validated_data
 
 def sbt_validate_ocr_request_and_get(request, subscription):

docker-compose.yml

@@ -33,7 +33,8 @@ services:
     command: bash run.sh
     deploy:
       mode: replicated
-      replicas: 3
+      replicas: 2
+
   # Back-end services
   be-ctel-sbt:
     build:
@@ -71,7 +72,7 @@ services:
       - ctel-sbt
     volumes:
       - ${HOST_MEDIA_FOLDER}:${MEDIA_ROOT}
-      - ./data/BE_static:/app/static
+      - ./data/static:/app/static
       - ./cope2n-api:/app
     working_dir: /app
     depends_on:
@@ -160,7 +161,7 @@ services:
       - ./cope2n-api:/app
 
     working_dir: /app
-    command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
+    command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 8"
 
   # Back-end persistent
   db-sbt:
@@ -197,7 +198,7 @@ services:
     shm_size: 10gb
     privileged: true
     ports:
-      - 80:80
+      - ${SIDP_SERVICE_PORT:-9881}:80
     depends_on:
       be-ctel-sbt:
         condition: service_started
@@ -207,7 +208,7 @@ services:
       - VITE_PROXY=http://be-ctel-sbt:${BASE_PORT}
       - VITE_API_BASE_URL=http://fe-sbt:80
     volumes:
-      - ./data/BE_static:/backend-static
+      - ./data/static:/backend-static
     networks:
       - ctel-sbt
 

speedtest_sync.py

@@ -94,27 +94,26 @@ def process_file(data):
 
 
 
-invoice_files = [
-    ('invoice_file', ('invoice.pdf', open("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", "rb").read())),
-]
 # invoice_files = [
-#     ('invoice_file', ('invoice.jpg', open("test_samples/sbt/invoice.jpg", "rb").read())),
+#     ('invoice_file', ('invoice.pdf', open("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", "rb").read())),
 # ]
+invoice_files = [
+    ('invoice_file', ('invoice.jpg', open("test_samples/sbt/invoice.jpg", "rb").read())),
+]
 imei_files = [
     ('imei_files', ("test_samples/sbt/imei1.jpg", open("test_samples/sbt/imei1.jpg", "rb").read())),
-    ('imei_files', ("test_samples/sbt/imei2.jpg", open("test_samples/sbt/imei2.jpg", "rb").read())),
+    # ('imei_files', ("test_samples/sbt/imei2.jpg", open("test_samples/sbt/imei2.jpg", "rb").read())),
-    ('imei_files', ("test_samples/sbt/imei3.jpg", open("test_samples/sbt/imei3.jpg", "rb").read())),
+    # ('imei_files', ("test_samples/sbt/imei3.jpg", open("test_samples/sbt/imei3.jpg", "rb").read())),
-    ('imei_files', ("test_samples/sbt/imei4.jpeg", open("test_samples/sbt/imei4.jpeg", "rb").read())),
+    # ('imei_files', ("test_samples/sbt/imei4.jpeg", open("test_samples/sbt/imei4.jpeg", "rb").read())),
-    ('imei_files', ("test_samples/sbt/imei5.jpg", open("test_samples/sbt/imei5.jpg", "rb").read())),
+    # ('imei_files', ("test_samples/sbt/imei5.jpg", open("test_samples/sbt/imei5.jpg", "rb").read())),
 ]
 def get_imei_files():
     num_files = random.randint(1, len(imei_files) + 1)
-    print("Num imeis", num_files)
     files = imei_files[:num_files]
     # print("Num of imei files:", len(files))
     return files
 def get_files():
-    return invoice_files + get_imei_files()
+    return invoice_files + imei_files # get_imei_files()
 def gen_input(num_input):
     for _ in range(num_input):
         yield (get_files(), token)