Merge pull request #17 from dx-tan/feature/add_health_check

Refactor + Add usage report API

cope2n-ai-fi/modules/sdsvkvu

@@ -1 +1 @@
-Subproject commit 97218a499a6375e943c1c59c592e1b1780d43477
+Subproject commit bdba044bb2eacac7c7cfe0e0f321196d03b681f6

cope2n-api/fwd/settings.py

@@ -175,8 +175,8 @@ REST_FRAMEWORK = {
 }
 
 SPECTACULAR_SETTINGS = {
-    'TITLE': 'SDS C2open',
-    'DESCRIPTION': 'AI powered by SamsungSDS VietNam',
+    'TITLE': 'OCR Engine for Invoices',
+    'DESCRIPTION': 'AI Powered by Samsung SDS Vietnam',
     'VERSION': '1.0.0',
     'SERVE_INCLUDE_SCHEMA': True,
     # OTHER SETTINGS
@@ -188,6 +188,13 @@ SPECTACULAR_SETTINGS = {
     # Custom Spectacular Settings
     "EXCLUDE_PATH": [reverse_lazy("schema")],
     "EXCLUDE_RELATIVE_PATH": ["/rsa", '/gen-token', '/app/'],
+    "TAGS": [
+        "Login",
+        "OCR",
+        "Data",
+        "System",
+    ],
+    "TAGS_SORTER": "alpha"
 }
 
 FILE_UPLOAD_HANDLERS = [

cope2n-api/fwd_api/api/ctel_template_view.py

@@ -17,8 +17,9 @@ from ..constant.common import EntityStatus, TEMPLATE_ID
 from ..exception.exceptions import InvalidException
 from ..request.UpdateTemplateRequest import UpdateTemplateRequest
 from ..response.TemplateResponse import TemplateResponse
-from ..utils import FileUtils, ProcessUtil
-from ..utils.ProcessUtil import UserData
+from ..utils import file as FileUtils
+from ..utils import process as ProcessUtil
+from ..utils.process import UserData
 from drf_spectacular.utils import extend_schema
 
 

cope2n-api/fwd_api/api/ctel_user_view.py

@@ -15,13 +15,13 @@ from ..exception.exceptions import InvalidException, NotFoundException, LockedEn
 from ..models import UserProfile, PricingPlan, Subscription
 from ..request.UpsertUserRequest import UpsertUserRequest
 from ..response.SubscriptionResponse import SubscriptionResponse
-from ..utils import ProcessUtil, DateUtil
-from ..utils.CryptoUtils import sds_authenticator, admin_sds_authenticator, SdsAuthentication
+from ..utils.health import get_health_report
+from ..utils import date as DateUtil
+from ..utils.crypto import sds_authenticator, admin_sds_authenticator, SdsAuthentication
 import datetime
 from ..request.LoginRequest import LoginRequest
-from ..request.HealcheckSerializer import HealthCheckSerializer
+from ..utils.date import default_zone
 
-from ..utils.DateUtil import default_zone
 from fwd import settings
 
 class CtelUserViewSet(viewsets.ViewSet):
@@ -41,7 +41,7 @@ class CtelUserViewSet(viewsets.ViewSet):
             },
             'required': {'username', 'password'}
         }
-    }, responses=None, tags=['ocr'])
+    }, responses=None, tags=['Login'])
     @action(detail=False, url_path="login", methods=["POST"])
     def login(self, request):
         serializer = LoginRequest(data=request.data)
@@ -89,49 +89,55 @@ class CtelUserViewSet(viewsets.ViewSet):
             'token': sds_authenticator.generate_token(user_id=settings.FI_USER_NAME, internal_id=user.id, status=EntityStatus.ACTIVE.value, sub_id=sub.id)
         })
     
-    @extend_schema(request=None, responses=None, tags=['users'])
-    @action(detail=False, url_path="users/info", methods=["GET"])
-    @throw_on_failure(InvalidException(excArgs='data'))
-    def get_be_user(self, request):
-        user_data = ProcessUtil.get_user(request)
 
-        return Response(status=status.HTTP_200_OK, data={
-            'userId': user_data.user.sync_id,
-            'message': 'User is valid'
-        })
+    # TODO (vietanhdev): Make this optional. Comment out first
+    # NOTE: Don't remove this code
+    # @extend_schema(request=None, responses=None, tags=['users'])
+    # @action(detail=False, url_path="users/info", methods=["GET"])
+    # @throw_on_failure(InvalidException(excArgs='data'))
+    # def get_be_user(self, request):
+    #     user_data = ProcessUtil.get_user(request)
 
-    @extend_schema(request=None, responses=None, tags=['users'], methods=['GET'], parameters=[
-        OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"),
-        OpenApiParameter("subscription_id", OpenApiTypes.STR, OpenApiParameter.QUERY, description="Subscription id"),
+    #     return Response(status=status.HTTP_200_OK, data={
+    #         'userId': user_data.user.sync_id,
+    #         'message': 'User is valid'
+    #     })
 
-    ])
-    @extend_schema(request=UpsertUserRequest, responses=None, tags=['users'], methods=['POST'], parameters=[
-        OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"),
-    ], examples=[
-        OpenApiExample(
-            'Example',
-            summary='Request Sample',
-            description='Status : 0 ( active ) / 1 (inactive). Default is 0. <br>'
-                        'Datetime Format: <b>dd/mm/YYYY HH:MM:SS</b> <br>'
-                        'Plan code ( TRIAL / BASIC / ADVANCED ) ( TRIAL apply only one-time per user account )',
-            value={
-                'plan_code': 'A03',
-                'plan_start_at': '01/04/2023 00:00:00',
-                'status': 1,
-                'email': 'abc@mamil.vn',
-                'name': 'Pham Van A'
-            },
-            request_only=True,
-            response_only=False
-        ),
-    ])
-    @action(detail=False, url_path="users", methods=["POST", "GET"])
-    @throw_on_failure(InvalidException(excArgs='data'))
-    def user_view(self, request):
-        if request.method == "POST":
-            return self.upsert_user(request)
-        else:
-            return self.get_user(request)
+
+    # TODO (vietanhdev): Make this optional. Comment out first
+    # NOTE: Don't remove this code
+    # @extend_schema(request=None, responses=None, tags=['users'], methods=['GET'], parameters=[
+    #     OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"),
+    #     OpenApiParameter("subscription_id", OpenApiTypes.STR, OpenApiParameter.QUERY, description="Subscription id"),
+
+    # ])
+    # @extend_schema(request=UpsertUserRequest, responses=None, tags=['users'], methods=['POST'], parameters=[
+    #     OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"),
+    # ], examples=[
+    #     OpenApiExample(
+    #         'Example',
+    #         summary='Request Sample',
+    #         description='Status : 0 ( active ) / 1 (inactive). Default is 0. <br>'
+    #                     'Datetime Format: <b>dd/mm/YYYY HH:MM:SS</b> <br>'
+    #                     'Plan code ( TRIAL / BASIC / ADVANCED ) ( TRIAL apply only one-time per user account )',
+    #         value={
+    #             'plan_code': 'A03',
+    #             'plan_start_at': '01/04/2023 00:00:00',
+    #             'status': 1,
+    #             'email': 'abc@mamil.vn',
+    #             'name': 'Pham Van A'
+    #         },
+    #         request_only=True,
+    #         response_only=False
+    #     ),
+    # ])
+    # @action(detail=False, url_path="users", methods=["POST", "GET"])
+    # @throw_on_failure(InvalidException(excArgs='data'))
+    # def user_view(self, request):
+    #     if request.method == "POST":
+    #         return self.upsert_user(request)
+    #     else:
+    #         return self.get_user(request)
 
     def upsert_user(self, request):
         if not hasattr(request, 'user_data'):
@@ -241,56 +247,48 @@ class CtelUserViewSet(viewsets.ViewSet):
             'token': gen_x.generate_token(user.sync_id, user.id, user.status, sub['id']),
         })
 
-    # @extend_schema(request={
-    #     'application/json': {
-    #         'type': 'object',
-    #         'properties': {
-    #             'userId': {
-    #                 'type': 'string',
-    #                 'example': "C2H5OH"
-    #             },
-    #             'expiredAt': {
-    #                 'type': 'string',
-    #                 'example': "21/12/2023 00:00:00"
-    #             }
-    #         },
-    #         'required': {'collection_id', 'source', 'files'}
+    # TODO (vietanhdev): Make this optional. Comment out first
+    # NOTE: Don't remove this code
+    # @action(detail=False, url_path="users/gen-token", methods=["POST"])
+    # @throw_on_failure(InvalidException(excArgs='data'))
+    # def gen_cmc_token(self, request):
+    #     data = request.data
+
+    #     if "expiredAt" not in request.data:
+    #         raise InvalidException(excArgs='expiredAt')
+    #     uid = ProcessUtil.get_random_string(10) if "userId" not in data or data['userId'] is None else data['userId']
+    #     m_text = {
+    #         'userId': uid,
+    #         "expiredAt": data['expiredAt']
     #     }
+    #     import os
+    #     from ..utils.crypto import ctel_cryptor
+    #     import json
+    #     iv = os.urandom(16).hex()
+    #     e_text = ctel_cryptor.encrypt_ctel(json.dumps(m_text), iv)
+    #     e_data = {
+    #         'content': e_text,
+    #         'iv': iv
+    #     }
+    #     return Response(status=status.HTTP_200_OK, data={
+    #         'token': base64.b64encode(json.dumps(e_data).encode())
+    #     })
 
-    # }, responses=None, tags=['users'],
-    #     description="API mô phỏng token CMC truyền sang SDS. Dùng để truyền vào header user")
-    @action(detail=False, url_path="users/gen-token", methods=["POST"])
-    @throw_on_failure(InvalidException(excArgs='data'))
-    def gen_cmc_token(self, request):
-        data = request.data
-
-        if "expiredAt" not in request.data:
-            raise InvalidException(excArgs='expiredAt')
-        uid = ProcessUtil.get_random_string(10) if "userId" not in data or data['userId'] is None else data['userId']
-        m_text = {
-            'userId': uid,
-            "expiredAt": data['expiredAt']
-        }
-        import os
-        from ..utils.CryptoUtils import ctel_cryptor
-        import json
-        iv = os.urandom(16).hex()
-        e_text = ctel_cryptor.encrypt_ctel(json.dumps(m_text), iv)
-        e_data = {
-            'content': e_text,
-            'iv': iv
-        }
-        return Response(status=status.HTTP_200_OK, data={
-            'token': base64.b64encode(json.dumps(e_data).encode())
-        })
-
+    @extend_schema(responses=None, tags=['System'])
     @action(detail=False, url_path="healthcheck", methods=["GET"], authentication_classes=[], permission_classes=[])
     def health_check(self, request):
-        # Perform any checks to determine the health status of the application
-        # TODO: check database connectivity, S3 service availability, etc.
-        serializer = HealthCheckSerializer(data={
-            'status': "OK",
-            'message': 'Application is running smoothly.'
-        })
-        serializer.is_valid(raise_exception=True)
-        return JsonResponse(status=status.HTTP_200_OK, data=serializer.data)
+        response = {
+            "status": "OK",
+        }
+        return JsonResponse(status=status.HTTP_200_OK, data=response)
+    
+    @extend_schema(responses=None, tags=['System'])
+    @action(detail=False, url_path="system_usage", methods=["GET"])
+    def usage(self, request):
+        data = get_health_report()
+        response = {
+            "status": "OK",
+            "data": data,
+        }
+        return JsonResponse(status=status.HTTP_200_OK, data=response)
+

cope2n-api/fwd_api/api/ctel_view.py

@@ -20,7 +20,8 @@ from ..exception.exceptions import RequiredFieldException, InvalidException, Not
     PermissionDeniedException, LockedEntityException, FileContentInvalidException, ServiceTimeoutException
 from ..models import SubscriptionRequest, SubscriptionRequestFile, OcrTemplate
 from ..response.ReportSerializer import ReportSerializer
-from ..utils import FileUtils, ProcessUtil
+from ..utils import file as FileUtils
+from ..utils import process as ProcessUtil
 
 class CtelViewSet(viewsets.ViewSet):
     lookup_field = "username"
@@ -40,14 +41,11 @@ class CtelViewSet(viewsets.ViewSet):
             },
             'required': {'file', 'processType'}
         }
-    }, responses=None, tags=['ocr'])
+    }, responses=None, tags=['OCR'])
     @action(detail=False, url_path="image/process", methods=["POST"])
     # @transaction.atomic
     def process(self, request):
         s_time = time.time()
-        # print(30*"=")
-        # print(f"[DEBUG]: request: {request}")
-        # print(30*"=")
         user_info = ProcessUtil.get_user(request)
         user = user_info.user
         sub = user_info.current_sub
@@ -120,7 +118,7 @@ class CtelViewSet(viewsets.ViewSet):
             },
             'required': {'imei_files'}
         }
-    }, responses=None, tags=['ocr'])
+    }, responses=None, tags=['OCR'])
     @action(detail=False, url_path="images/process", methods=["POST"])
     def processes(self, request):
         user_info = ProcessUtil.get_user(request)
@@ -191,7 +189,7 @@ class CtelViewSet(viewsets.ViewSet):
             },
             'required': {'imei_files'}
         }
-    }, responses=None, tags=['ocr'])
+    }, responses=None, tags=['OCR'])
     @action(detail=False, url_path="images/process_sync", methods=["POST"])
     def processes_sync(self, request):
         user_info = ProcessUtil.get_user(request)
@@ -306,7 +304,7 @@ class CtelViewSet(viewsets.ViewSet):
             },
             'required': ['request_id', 'retailername', 'sold_to_party', 'purchase_date', 'imei_number']
         }
-    }, responses=None, tags=['ocr'])
+    }, responses=None, tags=['OCR'])
     @action(detail=False, url_path="images/feedback", methods=["POST"])
     def feedback(self, request):
         validated_data = ProcessUtil.sbt_validate_feedback(request)
@@ -329,7 +327,7 @@ class CtelViewSet(viewsets.ViewSet):
         return JsonResponse(status=status.HTTP_200_OK, data={"request_id": rq_id})
 
 
-    @extend_schema(request=None, responses=None, tags=['data'])
+    @extend_schema(request=None, responses=None, tags=['Data'])
     @extend_schema(request=None, responses=None, tags=['templates'], methods=['GET'])
     @action(detail=False, url_path=r"media/(?P<folder_type>\w+)/(?P<uq_id>\w+)", methods=["GET"])
     def get_file_v2(self, request, uq_id=None, folder_type=None):
@@ -378,7 +376,7 @@ class CtelViewSet(viewsets.ViewSet):
         else:
             raise InvalidException(excArgs='type')
 
-    @extend_schema(request=None, responses=None, tags=['data'])
+    @extend_schema(request=None, responses=None, tags=['Data'])
     @action(detail=False, url_path=r"v2/media/request/(?P<media_id>\w+)", methods=["GET"])
     def get_file_v3(self, request, media_id=None):
 
@@ -406,7 +404,7 @@ class CtelViewSet(viewsets.ViewSet):
                             headers={'Content-Disposition': 'filename={fn}'.format(fn=file_name)},
                             content_type=content_type)
 
-    @extend_schema(request=None, responses=None, tags=['data'])
+    @extend_schema(request=None, responses=None, tags=['Data'])
     @throw_on_failure(InvalidException(excArgs='data'))
     @action(detail=False, url_path=r"result/(?P<request_id>\w+)", methods=["GET"], renderer_classes=[JSONRenderer, XMLRenderer])
     def get_result(self, request, request_id=None):

cope2n-api/fwd_api/api_router.py

@@ -13,7 +13,7 @@ else:
 
 router.register("ctel", CtelViewSet, basename="CtelAPI")
 router.register("ctel", CtelUserViewSet, basename="CtelUserAPI")
-router.register("ctel", CtelTemplateViewSet, basename="CtelTemplateAPI")
+# router.register("ctel", CtelTemplateViewSet, basename="CtelTemplateAPI")
 
 app_name = "api"
 urlpatterns = router.urls

cope2n-api/fwd_api/celery_worker/internal_task.py

@@ -9,14 +9,17 @@ from fwd_api.celery_worker.worker import app
 from ..constant.common import FolderFileType, image_extensions
 from ..exception.exceptions import FileContentInvalidException
 from fwd_api.models import SubscriptionRequestFile
-from ..utils import FileUtils, ProcessUtil, S3_process
+from ..utils import file as FileUtils
+from ..utils import process as ProcessUtil
+from ..utils import s3 as S3Util
+
 from celery.utils.log import get_task_logger
 from fwd import settings
 
 
 logger = get_task_logger(__name__)
 
-s3_client = S3_process.MinioS3Client(
+s3_client = S3Util.MinioS3Client(
     endpoint=settings.S3_ENDPOINT,
     access_key=settings.S3_ACCESS_KEY,
     secret_key=settings.S3_SECRET_KEY,
@@ -93,7 +96,7 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files):
     start_process = time.time()
     logger.info(f"BE proccessing time: {start_process - start}")
     # TODO: send to queue with different request_ids
-    doc_type_string =""
+    doc_type_string = ""
     for i, b_url in enumerate(b_urls):
         fractorized_request_id = rq_id + f"_sub_{i}"
         ProcessUtil.send_to_queue2(fractorized_request_id, sub_id, [b_url], user_id, p_type)

cope2n-api/fwd_api/celery_worker/process_result_tasks.py

@@ -7,35 +7,42 @@ from fwd_api.models import SubscriptionRequest
 from fwd_api.exception.exceptions import InvalidException
 from fwd_api.models import SubscriptionRequest
 from fwd_api.constant.common import ProcessType
-from fwd_api.utils.RedisUtils import RedisUtils
+from fwd_api.utils.redis import RedisUtils
+from fwd_api.utils import process as ProcessUtil
 
 redis_client = RedisUtils()
 
-def aggregate_result(resutls, doc_types):
+def aggregate_result(results, doc_types):
     doc_types = doc_types.split(',')
 
-    des_result = deepcopy(list(resutls.values()))[0]
+    des_result = deepcopy(list(results.values()))[0]
     des_result["content"]["total_pages"] = 0
     des_result["content"]["ocr_num_pages"] = 0 
     des_result["content"]["document"][0]["end_page"] = 0
     des_result["content"]["document"][0]["content"][3]["value"] = [None for _ in range(doc_types.count("imei"))]
     des_result["content"]["document"][0]["content"][2]["value"] = []
 
-    for index, resutl in resutls.items():
+    sorted_results = [None] * len(doc_types)
+    for index, result in results.items():
         index = int(index)
         doc_type = doc_types[index]
+        sorted_results[index] = ((doc_type, result))
 
+
+    imei_count = 0
+    for doc_type, result in sorted_results:
         des_result["content"]["total_pages"] += 1
         des_result["content"]["ocr_num_pages"] += 1
         des_result["content"]["document"][0]["end_page"] += 1
         if doc_type == "imei":
-            des_result["content"]["document"][0]["content"][3]["value"][index] = resutl["content"]["document"][0]["content"][3]["value"][0]
+            des_result["content"]["document"][0]["content"][3]["value"][imei_count] = result["content"]["document"][0]["content"][3]["value"][0]
+            imei_count += 1
         elif doc_type == "invoice":
-            des_result["content"]["document"][0]["content"][0]["value"] = resutl["content"]["document"][0]["content"][0]["value"]
-            des_result["content"]["document"][0]["content"][1]["value"] = resutl["content"]["document"][0]["content"][1]["value"]
-            des_result["content"]["document"][0]["content"][2]["value"] += resutl["content"]["document"][0]["content"][2]["value"]
+            des_result["content"]["document"][0]["content"][0]["value"] = result["content"]["document"][0]["content"][0]["value"]
+            des_result["content"]["document"][0]["content"][1]["value"] = result["content"]["document"][0]["content"][1]["value"]
+            des_result["content"]["document"][0]["content"][2]["value"] += result["content"]["document"][0]["content"][2]["value"]
         elif doc_type == "all":
-            des_result.update(resutl)
+            des_result.update(result)
         else:
             raise InvalidException(f"doc_type: {doc_type}")
 
@@ -56,7 +63,6 @@ def update_user(rq: SubscriptionRequest):
     sub = rq.subscription
     predict_status = rq.status
     if predict_status == 3:
-        from fwd_api.utils import ProcessUtil
         sub.current_token += ProcessUtil.token_value(int(rq.process_type))
         sub.save()
 

cope2n-api/fwd_api/filter/AuthFilter.py

@@ -6,8 +6,8 @@ from rest_framework import authentication
 from rest_framework.exceptions import NotAuthenticated
 
 from fwd_api.annotation.api import throw_on_failure
-from fwd_api.utils import DateUtil
-from fwd_api.utils.CryptoUtils import ctel_cryptor, sds_authenticator, image_authenticator
+from fwd_api.utils import date as DateUtil
+from fwd_api.utils.crypto import ctel_cryptor, sds_authenticator, image_authenticator
 from fwd_api.exception.exceptions import InvalidException, TokenExpiredException, PermissionDeniedException
 
 

cope2n-api/fwd_api/migrations/0157_alter_subscriptionrequest_created_at.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.1.3 on 2023-12-15 05:10
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0156_alter_subscriptionrequestfile_code'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='subscriptionrequest',
+            name='created_at',
+            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
+        ),
+    ]

cope2n-api/fwd_api/models/SubscriptionRequest.py

@@ -17,5 +17,5 @@ class SubscriptionRequest(models.Model):
     feedback_result = models.JSONField(null=True)
     status = models.IntegerField()  # 1: Processing(Pending) 2: PredictCompleted 3: ReturnCompleted
     subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
-    created_at = models.DateTimeField(default=timezone.now)
+    created_at = models.DateTimeField(default=timezone.now, db_index=True)
     updated_at = models.DateTimeField(auto_now=True)

cope2n-api/fwd_api/models/fields/EncryptedCharField.py

@@ -1,6 +1,6 @@
 from django.db import models
 
-from fwd_api.utils.CryptoUtils import sds_db_encryptor
+from fwd_api.utils.crypto import sds_db_encryptor
 
 
 class EncryptedCharField(models.CharField):

cope2n-api/fwd_api/request/CreateTemplateRequest.py

@@ -3,7 +3,7 @@ from rest_framework import serializers
 from fwd_api.constant.common import MAX_NUMBER_OF_TEMPLATE_DATA_BOX, MAX_NUMBER_OF_TEMPLATE_ANCHOR_BOX, \
     NUMBER_OF_ITEM_IN_A_BOX, ESCAPE_VALUE
 from fwd_api.exception.exceptions import InvalidException
-from fwd_api.utils import ProcessUtil
+from fwd_api.utils import process as ProcessUtil
 
 
 class CreateTemplateRequest(serializers.Serializer):

cope2n-api/fwd_api/request/UpdateTemplateRequest.py

@@ -3,7 +3,7 @@ from rest_framework import serializers
 from fwd_api.constant.common import MAX_NUMBER_OF_TEMPLATE_DATA_BOX, NUMBER_OF_ITEM_IN_A_BOX, \
     MAX_NUMBER_OF_TEMPLATE_ANCHOR_BOX, ESCAPE_VALUE
 from fwd_api.exception.exceptions import InvalidException
-from fwd_api.utils import ProcessUtil
+from fwd_api.utils import process as ProcessUtil
 
 
 class UpdateTemplateRequest(serializers.Serializer):

cope2n-api/fwd_api/response/ReportFileSerializer.py

@@ -3,7 +3,7 @@ from rest_framework import serializers
 from fwd import settings
 from fwd_api.constant.common import FileCategory, FolderFileType
 from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest
-from fwd_api.utils import FileUtils
+from fwd_api.utils import file as FileUtils
 
 
 class ReportFileSerializer(serializers.Serializer):

cope2n-api/fwd_api/response/ReportSerializer.py

@@ -3,9 +3,9 @@ from rest_framework import serializers
 from fwd_api.constant.common import ProcessType, FileCategory
 from fwd_api.models import SubscriptionRequest, SubscriptionRequestFile
 from fwd_api.response.ReportFileSerializer import ReportFileSerializer
-from fwd_api.utils.DateUtil import FORMAT
+from fwd_api.utils.date import FORMAT
 from django.utils.translation import gettext as _
-from fwd_api.utils import FileUtils
+from fwd_api.utils import file as FileUtils
 
 def i18n_for_label(data) -> list:
     if 'fields' in data and isinstance(data['fields'], list):

cope2n-api/fwd_api/response/SubscriptionResponse.py

@@ -2,7 +2,7 @@ from rest_framework import serializers
 
 from fwd_api.models import SubscriptionRequest, Subscription
 
-from fwd_api.utils.DateUtil import FORMAT
+from fwd_api.utils.date import FORMAT
 
 
 class SubscriptionResponse(serializers.Serializer):

cope2n-api/fwd_api/response/TemplateResponse.py

@@ -4,7 +4,7 @@ from fwd_api.constant.common import TEMPLATE_BOX_TYPE, FolderFileType
 
 from fwd_api.models.OcrTemplate import OcrTemplate
 from fwd_api.models.OcrTemplateBox import OcrTemplateBox
-from fwd_api.utils import FileUtils
+from fwd_api.utils import file as FileUtils
 
 
 class TemplateBoxResponse(serializers.Serializer):

cope2n-api/fwd_api/utils/crypto.py

@@ -11,7 +11,9 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from fwd import settings
 from fwd_api.annotation.api import throw_on_failure
 from fwd_api.exception.exceptions import InvalidException
-from fwd_api.utils.DateUtil import default_zone, get_date_time_now
+from fwd_api.utils.date import default_zone, get_date_time_now, FORMAT
+from fwd_api.utils import date as DateUtil
+
 
 class InternalCryptor:
 
@@ -56,8 +58,6 @@ class SdsAuthentication:
     def generate_token(self, user_id, internal_id, status, sub_id=-1) -> str:
         c_time = get_date_time_now() + datetime.timedelta(hours=self.duration_of_token)
         c_time.replace(tzinfo=ZoneInfo(default_zone))
-        from fwd_api.utils import DateUtil
-        from fwd_api.utils.DateUtil import FORMAT
         payload = {"id": user_id, "expired_at": DateUtil.to_str(c_time, FORMAT.DD_MM_YYYY_HHMMSS.value),
                    'internal_id': internal_id, 'status': status, 'subscription_id': sub_id}
         return self.encode_data(payload)
@@ -65,16 +65,12 @@ class SdsAuthentication:
     def generate_img_token(self, user_id) -> str:
         c_time = get_date_time_now() + datetime.timedelta(hours=self.duration_of_token)
         c_time.replace(tzinfo=ZoneInfo(default_zone))
-        from fwd_api.utils import DateUtil
-        from fwd_api.utils.DateUtil import FORMAT
         payload = {"expired_at": DateUtil.to_str(c_time, FORMAT.DD_MM_YYYY_HHMMSS.value), "internal_id": user_id}
         return self.encode_data(payload)
     
     def generate_img_token_v2(self, user_id, sub_id=None, user_sync_id=None) -> str:
         c_time = get_date_time_now() + datetime.timedelta(hours=self.duration_of_token)
         c_time.replace(tzinfo=ZoneInfo(default_zone))
-        from fwd_api.utils import DateUtil
-        from fwd_api.utils.DateUtil import FORMAT
         payload = {"expired_at": DateUtil.to_str(c_time, FORMAT.DD_MM_YYYY_HHMMSS.value), "internal_id": user_id, 'subscription_id': sub_id, "id": user_sync_id}
         return self.encode_data(payload)
     

cope2n-api/fwd_api/utils/file.py

@@ -12,8 +12,8 @@ from fwd_api.constant.common import allowed_file_extensions
 from fwd_api.exception.exceptions import GeneralException, RequiredFieldException, InvalidException, \
     ServiceUnavailableException, FileFormatInvalidException, LimitReachedException, InvalidDecompressedSizeException
 from fwd_api.models import SubscriptionRequest, OcrTemplate
-from fwd_api.utils import ProcessUtil
-from fwd_api.utils.CryptoUtils import image_authenticator
+from fwd_api.utils import process as ProcessUtil
+from fwd_api.utils.crypto import image_authenticator
 from fwd_api.utils.image import resize
 from ..celery_worker.client_connector import c_connector
 import imagesize

cope2n-api/fwd_api/utils/health.py

@@ -0,0 +1,37 @@
+from django.db.models import Count
+from django.utils import timezone
+from datetime import timedelta
+
+from ..models import SubscriptionRequest
+
+def get_latest_requests(limit=50):
+    requests = SubscriptionRequest.objects.order_by("-created_at")[:limit]
+    requests_dict = []
+    for request in requests:
+        requests_dict.append({
+            "request_id": request.request_id,
+            "pages": request.pages,
+            "doc_type": request.doc_type,
+            # "predict_result": request.predict_result,
+            "created_at": request.created_at,
+        })
+    return requests_dict
+
+def count_requests_by_date(days_limit=5):
+    today = timezone.now().date()
+    start_date = today - timedelta(days=days_limit)
+    requests_by_date = SubscriptionRequest.objects.filter(created_at__gte=start_date).values('created_at__date').annotate(count=Count('id')).values('created_at__date', 'count').order_by('created_at__date')
+    count_dict = []
+    for rbd in requests_by_date:
+        count_dict.append({
+            "date": rbd["created_at__date"],
+            "count": rbd["count"],
+        })
+    return count_dict
+
+
+def get_health_report():
+    return {
+        "latest_requests": get_latest_requests(),
+        "count_by_date": count_requests_by_date(),
+    }

cope2n-api/fwd_api/utils/process.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 import random
 import string
 import tempfile
@@ -15,14 +16,14 @@ from fwd_api.constant.common import LIST_BOX_MESSAGE, pattern, NAME_MESSAGE, all
     FolderFileType, FileCategory
 from fwd_api.exception.exceptions import NumberOfBoxLimitReachedException, \
     ServiceUnavailableException, DuplicateEntityException, LimitReachedException, BadGatewayException
-from fwd_api.utils import DateUtil, FileUtils
+from fwd_api.utils import date as DateUtil
+from fwd_api.utils import file as FileUtils
 from ..constant.common import ProcessType, TEMPLATE_BOX_TYPE, EntityStatus
 from ..exception.exceptions import InvalidException, NotFoundException, \
     PermissionDeniedException, RequiredFieldException, InvalidException, InvalidDecompressedSizeException
 from ..models import UserProfile, OcrTemplate, OcrTemplateBox, \
     Subscription, SubscriptionRequestFile, SubscriptionRequest
 from ..celery_worker.client_connector import c_connector
-import uuid
 
 from celery.utils.log import get_task_logger
 
@@ -132,9 +133,6 @@ def sbt_validate_ocr_request_and_get(request, subscription):
 
     FileUtils.validate_list_file(imei_files, file_field="imei_file")
     FileUtils.validate_list_file(invoice_file, max_file_num=1, min_file_num=0, file_field="invoice_file")
-    # if not isinstance(redemption_ID, str):
-    #     # raise RequiredFieldException(excArgs="redemption_ID")
-    #     raise InvalidException(excArgs="redemption_ID")
 
     validated_data['imei_file'] = imei_files
     validated_data['invoice_file'] = invoice_file
@@ -413,7 +411,7 @@ def process_image_local_file(file_name: str, file_path: str, request: Subscripti
 def pdf_to_images_urls(doc_path, request: SubscriptionRequest, user, dpi: int = 300) -> list:
     pdf_extracted = []
     saving_path = FileUtils.get_folder_path(request)
-    break_file_name = f'break_0.jpg'
+    break_file_name = f'{os.path.basename(doc_path.name)}_page_0.jpg'
     saving_path = os.path.join(saving_path, break_file_name)
 
     image = get_first_page_pdf(doc_path, 300)

cope2n-fe/src/layouts/main-layout.tsx

@@ -110,12 +110,15 @@ export function MainLayout() {
               style={{
                 marginBottom: 0,
                 color: gray[0],
-                textAlign: 'center',
+                textAlign: 'left',
               }}
               ellipsis={{
-                rows: 2,
+                rows: 10,
               }}
             >
+              <b>NOTE: </b> This UI is only for reference. Please use swagger UI for full-option API testing.
+              <br />
+              <br />
               Powered by SAMSUNG SDS.
               <br />
               All rights reserved.
@@ -142,9 +145,11 @@ export function MainLayout() {
             style={{
               marginBottom: 0,
               color: gray[0],
-              textAlign: 'center',
+              textAlign: 'left',
             }}
           >
+            <b>NOTE: </b> This UI is only for reference. Please use swagger UI for full-option API testing.
+            <br />
             Powered by SAMSUNG SDS.
           </Typography.Paragraph>
         </Layout.Header>

cope2n-fe/src/locales/en/messages.json

@@ -51,9 +51,7 @@
   "Key Information Extraction": "Key Information Extraction",
   "Label": "Label",
   "Label of extracted field must not have more than 255 characters": "Label of extracted field must not have more than 255 characters",
-  "Language": "Language",
   "Login": "Login",
-  "Login with  <0>CMC Account</0>?": "Login with  <0>CMC Account</0>?",
   "Name": "Name",
   "Nation": "Nation",
   "Nationality": "Nationality",

cope2n-fe/src/locales/vi/messages.json

@@ -51,9 +51,7 @@
   "Key Information Extraction": "Trích xuất thông tin",
   "Label": "Nhãn",
   "Label of extracted field must not have more than 255 characters": "Độ dài nhãn của trường thông tin trích xuất không được vượt quá 255 kí tự",
-  "Language": "",
   "Login": "",
-  "Login with  <0>CMC Account</0>?": "",
   "Name": "Tên",
   "Nation": "Dân tộc",
   "Nationality": "Quốc tịch",

docker-compose.yml

@@ -93,6 +93,7 @@ services:
     - ./data/minio_data:/data
     networks:
     - ctel-sbt
+    restart: always
     command: server --address :9884 --console-address :9885 /data
     profiles: ["local"]
 
@@ -149,11 +150,9 @@ services:
       - BASE_URL=http://be-ctel-sbt:${BASE_PORT}
       - REDIS_HOST=result-cache
       - REDIS_PORT=6379
-      
-
+    restart: always
     networks:
       - ctel-sbt
-    # restart: always
     depends_on:
       db-sbt:
         condition: service_started
@@ -168,6 +167,7 @@ services:
 
   # Back-end persistent
   db-sbt:
+    restart: always
     mem_reservation: 500m
     # mem_limit: 1g
     # container_name: sidp-cope2n-be-sbt-db
@@ -200,6 +200,7 @@ services:
 
   # Front-end services
   fe-sbt:
+    restart: always
     build:
       context: cope2n-fe
       shm_size: 10gb

speedtest_sync.py

@@ -94,12 +94,12 @@ def process_file(data):
 
 
 
-# invoice_files = [
-#     ('invoice_file', ('invoice.pdf', open("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", "rb").read())),
-# ]
 invoice_files = [
-    ('invoice_file', ('invoice.jpg', open("test_samples/sbt/invoice.jpg", "rb").read())),
+    ('invoice_file', ('invoice.pdf', open("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", "rb").read())),
 ]
+# invoice_files = [
+#     ('invoice_file', ('invoice.jpg', open("test_samples/sbt/invoice.jpg", "rb").read())),
+# ]
 imei_files = [
     ('imei_files', ("test_samples/sbt/imei1.jpg", open("test_samples/sbt/imei1.jpg", "rb").read())),
     ('imei_files', ("test_samples/sbt/imei2.jpg", open("test_samples/sbt/imei2.jpg", "rb").read())),
@@ -127,7 +127,6 @@ print("## TEST REPORT #################################")
 print("Number of requests: {}".format(args.num_requests))
 print("Number of concurrent requests: {}".format(args.num_workers))
 print("Number of files: 1 invoice, 1-5 imei files (random)")
-print("Query time interval for result: {:.3f}s ".format(args.checking_interval))
 print("--------------------------------------")
 print("SUCCESS RATE")
 counter = {}
@@ -143,14 +142,12 @@ if len(uploading_time) == 0:
     print("No valid uploading time")
     print("Check the results!")
 processing_time = [x["process_time"] for x in results if x["success"]]
-print("Uploading time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(uploading_time) / len(uploading_time), min(uploading_time), max(uploading_time)))
-print("Processing time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(processing_time) / len(processing_time), min(processing_time), max(processing_time)))
+print("Uploading + Processing time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(processing_time) / len(processing_time), min(processing_time), max(processing_time)))
 print("--------------------------------------")
 print("TIME BY IMAGE")
 uploading_time = [x["upload_time"] for x in results if x["success"]]
 processing_time = [x["process_time"] for x in results if x["success"]]
 num_images = sum(x["num_files"] for x in results if x["success"])
 print("Total images:", num_images)
-print("Uploading time: {:.3f}s".format(sum(uploading_time) / num_images))
-print("Processing time: {:.3f}s".format(sum(processing_time) / num_images))
+print("Uploading + Processing time: {:.3f}s".format(sum(processing_time) / num_images))
 print("--------------------------------------")