diff --git a/cope2n-ai-fi/modules/sdsvkvu b/cope2n-ai-fi/modules/sdsvkvu index 97218a4..bdba044 160000 --- a/cope2n-ai-fi/modules/sdsvkvu +++ b/cope2n-ai-fi/modules/sdsvkvu @@ -1 +1 @@ -Subproject commit 97218a499a6375e943c1c59c592e1b1780d43477 +Subproject commit bdba044bb2eacac7c7cfe0e0f321196d03b681f6 diff --git a/cope2n-api/fwd/settings.py b/cope2n-api/fwd/settings.py index 08ef761..514ff37 100755 --- a/cope2n-api/fwd/settings.py +++ b/cope2n-api/fwd/settings.py @@ -175,8 +175,8 @@ REST_FRAMEWORK = { } SPECTACULAR_SETTINGS = { - 'TITLE': 'SDS C2open', - 'DESCRIPTION': 'AI powered by SamsungSDS VietNam', + 'TITLE': 'OCR Engine for Invoices', + 'DESCRIPTION': 'AI Powered by Samsung SDS Vietnam', 'VERSION': '1.0.0', 'SERVE_INCLUDE_SCHEMA': True, # OTHER SETTINGS @@ -188,6 +188,13 @@ SPECTACULAR_SETTINGS = { # Custom Spectacular Settings "EXCLUDE_PATH": [reverse_lazy("schema")], "EXCLUDE_RELATIVE_PATH": ["/rsa", '/gen-token', '/app/'], + "TAGS": [ + "Login", + "OCR", + "Data", + "System", + ], + "TAGS_SORTER": "alpha" } FILE_UPLOAD_HANDLERS = [ diff --git a/cope2n-api/fwd_api/api/ctel_template_view.py b/cope2n-api/fwd_api/api/ctel_template_view.py index e90e126..ffbb216 100755 --- a/cope2n-api/fwd_api/api/ctel_template_view.py +++ b/cope2n-api/fwd_api/api/ctel_template_view.py @@ -17,8 +17,9 @@ from ..constant.common import EntityStatus, TEMPLATE_ID from ..exception.exceptions import InvalidException from ..request.UpdateTemplateRequest import UpdateTemplateRequest from ..response.TemplateResponse import TemplateResponse -from ..utils import FileUtils, ProcessUtil -from ..utils.ProcessUtil import UserData +from ..utils import file as FileUtils +from ..utils import process as ProcessUtil +from ..utils.process import UserData from drf_spectacular.utils import extend_schema diff --git a/cope2n-api/fwd_api/api/ctel_user_view.py b/cope2n-api/fwd_api/api/ctel_user_view.py index 541a16a..64dcb78 100755 --- a/cope2n-api/fwd_api/api/ctel_user_view.py +++ b/cope2n-api/fwd_api/api/ctel_user_view.py @@ -15,13 +15,13 @@ from ..exception.exceptions import InvalidException, NotFoundException, LockedEn from ..models import UserProfile, PricingPlan, Subscription from ..request.UpsertUserRequest import UpsertUserRequest from ..response.SubscriptionResponse import SubscriptionResponse -from ..utils import ProcessUtil, DateUtil -from ..utils.CryptoUtils import sds_authenticator, admin_sds_authenticator, SdsAuthentication +from ..utils.health import get_health_report +from ..utils import date as DateUtil +from ..utils.crypto import sds_authenticator, admin_sds_authenticator, SdsAuthentication import datetime from ..request.LoginRequest import LoginRequest -from ..request.HealcheckSerializer import HealthCheckSerializer +from ..utils.date import default_zone -from ..utils.DateUtil import default_zone from fwd import settings class CtelUserViewSet(viewsets.ViewSet): @@ -41,7 +41,7 @@ class CtelUserViewSet(viewsets.ViewSet): }, 'required': {'username', 'password'} } - }, responses=None, tags=['ocr']) + }, responses=None, tags=['Login']) @action(detail=False, url_path="login", methods=["POST"]) def login(self, request): serializer = LoginRequest(data=request.data) @@ -89,49 +89,55 @@ class CtelUserViewSet(viewsets.ViewSet): 'token': sds_authenticator.generate_token(user_id=settings.FI_USER_NAME, internal_id=user.id, status=EntityStatus.ACTIVE.value, sub_id=sub.id) }) - @extend_schema(request=None, responses=None, tags=['users']) - @action(detail=False, url_path="users/info", methods=["GET"]) - @throw_on_failure(InvalidException(excArgs='data')) - def get_be_user(self, request): - user_data = ProcessUtil.get_user(request) - return Response(status=status.HTTP_200_OK, data={ - 'userId': user_data.user.sync_id, - 'message': 'User is valid' - }) + # TODO (vietanhdev): Make this optional. Comment out first + # NOTE: Don't remove this code + # @extend_schema(request=None, responses=None, tags=['users']) + # @action(detail=False, url_path="users/info", methods=["GET"]) + # @throw_on_failure(InvalidException(excArgs='data')) + # def get_be_user(self, request): + # user_data = ProcessUtil.get_user(request) - @extend_schema(request=None, responses=None, tags=['users'], methods=['GET'], parameters=[ - OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"), - OpenApiParameter("subscription_id", OpenApiTypes.STR, OpenApiParameter.QUERY, description="Subscription id"), + # return Response(status=status.HTTP_200_OK, data={ + # 'userId': user_data.user.sync_id, + # 'message': 'User is valid' + # }) - ]) - @extend_schema(request=UpsertUserRequest, responses=None, tags=['users'], methods=['POST'], parameters=[ - OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"), - ], examples=[ - OpenApiExample( - 'Example', - summary='Request Sample', - description='Status : 0 ( active ) / 1 (inactive). Default is 0.
' - 'Datetime Format: dd/mm/YYYY HH:MM:SS
' - 'Plan code ( TRIAL / BASIC / ADVANCED ) ( TRIAL apply only one-time per user account )', - value={ - 'plan_code': 'A03', - 'plan_start_at': '01/04/2023 00:00:00', - 'status': 1, - 'email': 'abc@mamil.vn', - 'name': 'Pham Van A' - }, - request_only=True, - response_only=False - ), - ]) - @action(detail=False, url_path="users", methods=["POST", "GET"]) - @throw_on_failure(InvalidException(excArgs='data')) - def user_view(self, request): - if request.method == "POST": - return self.upsert_user(request) - else: - return self.get_user(request) + + # TODO (vietanhdev): Make this optional. Comment out first + # NOTE: Don't remove this code + # @extend_schema(request=None, responses=None, tags=['users'], methods=['GET'], parameters=[ + # OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"), + # OpenApiParameter("subscription_id", OpenApiTypes.STR, OpenApiParameter.QUERY, description="Subscription id"), + + # ]) + # @extend_schema(request=UpsertUserRequest, responses=None, tags=['users'], methods=['POST'], parameters=[ + # OpenApiParameter("user", OpenApiTypes.STR, OpenApiParameter.HEADER, description="CMC/SDS encrypted token"), + # ], examples=[ + # OpenApiExample( + # 'Example', + # summary='Request Sample', + # description='Status : 0 ( active ) / 1 (inactive). Default is 0.
' + # 'Datetime Format: dd/mm/YYYY HH:MM:SS
' + # 'Plan code ( TRIAL / BASIC / ADVANCED ) ( TRIAL apply only one-time per user account )', + # value={ + # 'plan_code': 'A03', + # 'plan_start_at': '01/04/2023 00:00:00', + # 'status': 1, + # 'email': 'abc@mamil.vn', + # 'name': 'Pham Van A' + # }, + # request_only=True, + # response_only=False + # ), + # ]) + # @action(detail=False, url_path="users", methods=["POST", "GET"]) + # @throw_on_failure(InvalidException(excArgs='data')) + # def user_view(self, request): + # if request.method == "POST": + # return self.upsert_user(request) + # else: + # return self.get_user(request) def upsert_user(self, request): if not hasattr(request, 'user_data'): @@ -241,56 +247,48 @@ class CtelUserViewSet(viewsets.ViewSet): 'token': gen_x.generate_token(user.sync_id, user.id, user.status, sub['id']), }) - # @extend_schema(request={ - # 'application/json': { - # 'type': 'object', - # 'properties': { - # 'userId': { - # 'type': 'string', - # 'example': "C2H5OH" - # }, - # 'expiredAt': { - # 'type': 'string', - # 'example': "21/12/2023 00:00:00" - # } - # }, - # 'required': {'collection_id', 'source', 'files'} + # TODO (vietanhdev): Make this optional. Comment out first + # NOTE: Don't remove this code + # @action(detail=False, url_path="users/gen-token", methods=["POST"]) + # @throw_on_failure(InvalidException(excArgs='data')) + # def gen_cmc_token(self, request): + # data = request.data + + # if "expiredAt" not in request.data: + # raise InvalidException(excArgs='expiredAt') + # uid = ProcessUtil.get_random_string(10) if "userId" not in data or data['userId'] is None else data['userId'] + # m_text = { + # 'userId': uid, + # "expiredAt": data['expiredAt'] # } + # import os + # from ..utils.crypto import ctel_cryptor + # import json + # iv = os.urandom(16).hex() + # e_text = ctel_cryptor.encrypt_ctel(json.dumps(m_text), iv) + # e_data = { + # 'content': e_text, + # 'iv': iv + # } + # return Response(status=status.HTTP_200_OK, data={ + # 'token': base64.b64encode(json.dumps(e_data).encode()) + # }) - # }, responses=None, tags=['users'], - # description="API mô phỏng token CMC truyền sang SDS. Dùng để truyền vào header user") - @action(detail=False, url_path="users/gen-token", methods=["POST"]) - @throw_on_failure(InvalidException(excArgs='data')) - def gen_cmc_token(self, request): - data = request.data - - if "expiredAt" not in request.data: - raise InvalidException(excArgs='expiredAt') - uid = ProcessUtil.get_random_string(10) if "userId" not in data or data['userId'] is None else data['userId'] - m_text = { - 'userId': uid, - "expiredAt": data['expiredAt'] - } - import os - from ..utils.CryptoUtils import ctel_cryptor - import json - iv = os.urandom(16).hex() - e_text = ctel_cryptor.encrypt_ctel(json.dumps(m_text), iv) - e_data = { - 'content': e_text, - 'iv': iv - } - return Response(status=status.HTTP_200_OK, data={ - 'token': base64.b64encode(json.dumps(e_data).encode()) - }) - + @extend_schema(responses=None, tags=['System']) @action(detail=False, url_path="healthcheck", methods=["GET"], authentication_classes=[], permission_classes=[]) def health_check(self, request): - # Perform any checks to determine the health status of the application - # TODO: check database connectivity, S3 service availability, etc. - serializer = HealthCheckSerializer(data={ - 'status': "OK", - 'message': 'Application is running smoothly.' - }) - serializer.is_valid(raise_exception=True) - return JsonResponse(status=status.HTTP_200_OK, data=serializer.data) + response = { + "status": "OK", + } + return JsonResponse(status=status.HTTP_200_OK, data=response) + + @extend_schema(responses=None, tags=['System']) + @action(detail=False, url_path="system_usage", methods=["GET"]) + def usage(self, request): + data = get_health_report() + response = { + "status": "OK", + "data": data, + } + return JsonResponse(status=status.HTTP_200_OK, data=response) + diff --git a/cope2n-api/fwd_api/api/ctel_view.py b/cope2n-api/fwd_api/api/ctel_view.py index c044e4d..27f0847 100755 --- a/cope2n-api/fwd_api/api/ctel_view.py +++ b/cope2n-api/fwd_api/api/ctel_view.py @@ -20,7 +20,8 @@ from ..exception.exceptions import RequiredFieldException, InvalidException, Not PermissionDeniedException, LockedEntityException, FileContentInvalidException, ServiceTimeoutException from ..models import SubscriptionRequest, SubscriptionRequestFile, OcrTemplate from ..response.ReportSerializer import ReportSerializer -from ..utils import FileUtils, ProcessUtil +from ..utils import file as FileUtils +from ..utils import process as ProcessUtil class CtelViewSet(viewsets.ViewSet): lookup_field = "username" @@ -40,14 +41,11 @@ class CtelViewSet(viewsets.ViewSet): }, 'required': {'file', 'processType'} } - }, responses=None, tags=['ocr']) + }, responses=None, tags=['OCR']) @action(detail=False, url_path="image/process", methods=["POST"]) # @transaction.atomic def process(self, request): s_time = time.time() - # print(30*"=") - # print(f"[DEBUG]: request: {request}") - # print(30*"=") user_info = ProcessUtil.get_user(request) user = user_info.user sub = user_info.current_sub @@ -120,7 +118,7 @@ class CtelViewSet(viewsets.ViewSet): }, 'required': {'imei_files'} } - }, responses=None, tags=['ocr']) + }, responses=None, tags=['OCR']) @action(detail=False, url_path="images/process", methods=["POST"]) def processes(self, request): user_info = ProcessUtil.get_user(request) @@ -191,7 +189,7 @@ class CtelViewSet(viewsets.ViewSet): }, 'required': {'imei_files'} } - }, responses=None, tags=['ocr']) + }, responses=None, tags=['OCR']) @action(detail=False, url_path="images/process_sync", methods=["POST"]) def processes_sync(self, request): user_info = ProcessUtil.get_user(request) @@ -306,7 +304,7 @@ class CtelViewSet(viewsets.ViewSet): }, 'required': ['request_id', 'retailername', 'sold_to_party', 'purchase_date', 'imei_number'] } - }, responses=None, tags=['ocr']) + }, responses=None, tags=['OCR']) @action(detail=False, url_path="images/feedback", methods=["POST"]) def feedback(self, request): validated_data = ProcessUtil.sbt_validate_feedback(request) @@ -329,7 +327,7 @@ class CtelViewSet(viewsets.ViewSet): return JsonResponse(status=status.HTTP_200_OK, data={"request_id": rq_id}) - @extend_schema(request=None, responses=None, tags=['data']) + @extend_schema(request=None, responses=None, tags=['Data']) @extend_schema(request=None, responses=None, tags=['templates'], methods=['GET']) @action(detail=False, url_path=r"media/(?P\w+)/(?P\w+)", methods=["GET"]) def get_file_v2(self, request, uq_id=None, folder_type=None): @@ -378,7 +376,7 @@ class CtelViewSet(viewsets.ViewSet): else: raise InvalidException(excArgs='type') - @extend_schema(request=None, responses=None, tags=['data']) + @extend_schema(request=None, responses=None, tags=['Data']) @action(detail=False, url_path=r"v2/media/request/(?P\w+)", methods=["GET"]) def get_file_v3(self, request, media_id=None): @@ -406,7 +404,7 @@ class CtelViewSet(viewsets.ViewSet): headers={'Content-Disposition': 'filename={fn}'.format(fn=file_name)}, content_type=content_type) - @extend_schema(request=None, responses=None, tags=['data']) + @extend_schema(request=None, responses=None, tags=['Data']) @throw_on_failure(InvalidException(excArgs='data')) @action(detail=False, url_path=r"result/(?P\w+)", methods=["GET"], renderer_classes=[JSONRenderer, XMLRenderer]) def get_result(self, request, request_id=None): diff --git a/cope2n-api/fwd_api/api_router.py b/cope2n-api/fwd_api/api_router.py index bb8b815..6a8ac73 100755 --- a/cope2n-api/fwd_api/api_router.py +++ b/cope2n-api/fwd_api/api_router.py @@ -13,7 +13,7 @@ else: router.register("ctel", CtelViewSet, basename="CtelAPI") router.register("ctel", CtelUserViewSet, basename="CtelUserAPI") -router.register("ctel", CtelTemplateViewSet, basename="CtelTemplateAPI") +# router.register("ctel", CtelTemplateViewSet, basename="CtelTemplateAPI") app_name = "api" urlpatterns = router.urls diff --git a/cope2n-api/fwd_api/celery_worker/internal_task.py b/cope2n-api/fwd_api/celery_worker/internal_task.py index a908e2e..e3615f5 100755 --- a/cope2n-api/fwd_api/celery_worker/internal_task.py +++ b/cope2n-api/fwd_api/celery_worker/internal_task.py @@ -9,14 +9,17 @@ from fwd_api.celery_worker.worker import app from ..constant.common import FolderFileType, image_extensions from ..exception.exceptions import FileContentInvalidException from fwd_api.models import SubscriptionRequestFile -from ..utils import FileUtils, ProcessUtil, S3_process +from ..utils import file as FileUtils +from ..utils import process as ProcessUtil +from ..utils import s3 as S3Util + from celery.utils.log import get_task_logger from fwd import settings logger = get_task_logger(__name__) -s3_client = S3_process.MinioS3Client( +s3_client = S3Util.MinioS3Client( endpoint=settings.S3_ENDPOINT, access_key=settings.S3_ACCESS_KEY, secret_key=settings.S3_SECRET_KEY, @@ -93,7 +96,7 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files): start_process = time.time() logger.info(f"BE proccessing time: {start_process - start}") # TODO: send to queue with different request_ids - doc_type_string ="" + doc_type_string = "" for i, b_url in enumerate(b_urls): fractorized_request_id = rq_id + f"_sub_{i}" ProcessUtil.send_to_queue2(fractorized_request_id, sub_id, [b_url], user_id, p_type) diff --git a/cope2n-api/fwd_api/celery_worker/process_result_tasks.py b/cope2n-api/fwd_api/celery_worker/process_result_tasks.py index e634995..5dc3b3f 100755 --- a/cope2n-api/fwd_api/celery_worker/process_result_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_result_tasks.py @@ -7,35 +7,42 @@ from fwd_api.models import SubscriptionRequest from fwd_api.exception.exceptions import InvalidException from fwd_api.models import SubscriptionRequest from fwd_api.constant.common import ProcessType -from fwd_api.utils.RedisUtils import RedisUtils +from fwd_api.utils.redis import RedisUtils +from fwd_api.utils import process as ProcessUtil redis_client = RedisUtils() -def aggregate_result(resutls, doc_types): +def aggregate_result(results, doc_types): doc_types = doc_types.split(',') - des_result = deepcopy(list(resutls.values()))[0] + des_result = deepcopy(list(results.values()))[0] des_result["content"]["total_pages"] = 0 des_result["content"]["ocr_num_pages"] = 0 des_result["content"]["document"][0]["end_page"] = 0 des_result["content"]["document"][0]["content"][3]["value"] = [None for _ in range(doc_types.count("imei"))] des_result["content"]["document"][0]["content"][2]["value"] = [] - for index, resutl in resutls.items(): + sorted_results = [None] * len(doc_types) + for index, result in results.items(): index = int(index) doc_type = doc_types[index] + sorted_results[index] = ((doc_type, result)) + + imei_count = 0 + for doc_type, result in sorted_results: des_result["content"]["total_pages"] += 1 des_result["content"]["ocr_num_pages"] += 1 des_result["content"]["document"][0]["end_page"] += 1 if doc_type == "imei": - des_result["content"]["document"][0]["content"][3]["value"][index] = resutl["content"]["document"][0]["content"][3]["value"][0] + des_result["content"]["document"][0]["content"][3]["value"][imei_count] = result["content"]["document"][0]["content"][3]["value"][0] + imei_count += 1 elif doc_type == "invoice": - des_result["content"]["document"][0]["content"][0]["value"] = resutl["content"]["document"][0]["content"][0]["value"] - des_result["content"]["document"][0]["content"][1]["value"] = resutl["content"]["document"][0]["content"][1]["value"] - des_result["content"]["document"][0]["content"][2]["value"] += resutl["content"]["document"][0]["content"][2]["value"] + des_result["content"]["document"][0]["content"][0]["value"] = result["content"]["document"][0]["content"][0]["value"] + des_result["content"]["document"][0]["content"][1]["value"] = result["content"]["document"][0]["content"][1]["value"] + des_result["content"]["document"][0]["content"][2]["value"] += result["content"]["document"][0]["content"][2]["value"] elif doc_type == "all": - des_result.update(resutl) + des_result.update(result) else: raise InvalidException(f"doc_type: {doc_type}") @@ -56,7 +63,6 @@ def update_user(rq: SubscriptionRequest): sub = rq.subscription predict_status = rq.status if predict_status == 3: - from fwd_api.utils import ProcessUtil sub.current_token += ProcessUtil.token_value(int(rq.process_type)) sub.save() diff --git a/cope2n-api/fwd_api/filter/AuthFilter.py b/cope2n-api/fwd_api/filter/AuthFilter.py index d537b08..fc7de47 100755 --- a/cope2n-api/fwd_api/filter/AuthFilter.py +++ b/cope2n-api/fwd_api/filter/AuthFilter.py @@ -6,8 +6,8 @@ from rest_framework import authentication from rest_framework.exceptions import NotAuthenticated from fwd_api.annotation.api import throw_on_failure -from fwd_api.utils import DateUtil -from fwd_api.utils.CryptoUtils import ctel_cryptor, sds_authenticator, image_authenticator +from fwd_api.utils import date as DateUtil +from fwd_api.utils.crypto import ctel_cryptor, sds_authenticator, image_authenticator from fwd_api.exception.exceptions import InvalidException, TokenExpiredException, PermissionDeniedException diff --git a/cope2n-api/fwd_api/migrations/0157_alter_subscriptionrequest_created_at.py b/cope2n-api/fwd_api/migrations/0157_alter_subscriptionrequest_created_at.py new file mode 100644 index 0000000..cc4b0a7 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0157_alter_subscriptionrequest_created_at.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1.3 on 2023-12-15 05:10 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0156_alter_subscriptionrequestfile_code'), + ] + + operations = [ + migrations.AlterField( + model_name='subscriptionrequest', + name='created_at', + field=models.DateTimeField(db_index=True, default=django.utils.timezone.now), + ), + ] diff --git a/cope2n-api/fwd_api/models/SubscriptionRequest.py b/cope2n-api/fwd_api/models/SubscriptionRequest.py index 47b2728..ea6d44c 100755 --- a/cope2n-api/fwd_api/models/SubscriptionRequest.py +++ b/cope2n-api/fwd_api/models/SubscriptionRequest.py @@ -17,5 +17,5 @@ class SubscriptionRequest(models.Model): feedback_result = models.JSONField(null=True) status = models.IntegerField() # 1: Processing(Pending) 2: PredictCompleted 3: ReturnCompleted subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) - created_at = models.DateTimeField(default=timezone.now) + created_at = models.DateTimeField(default=timezone.now, db_index=True) updated_at = models.DateTimeField(auto_now=True) \ No newline at end of file diff --git a/cope2n-api/fwd_api/models/fields/EncryptedCharField.py b/cope2n-api/fwd_api/models/fields/EncryptedCharField.py index ad2dc51..1c79e27 100755 --- a/cope2n-api/fwd_api/models/fields/EncryptedCharField.py +++ b/cope2n-api/fwd_api/models/fields/EncryptedCharField.py @@ -1,6 +1,6 @@ from django.db import models -from fwd_api.utils.CryptoUtils import sds_db_encryptor +from fwd_api.utils.crypto import sds_db_encryptor class EncryptedCharField(models.CharField): diff --git a/cope2n-api/fwd_api/request/CreateTemplateRequest.py b/cope2n-api/fwd_api/request/CreateTemplateRequest.py index 59a08e7..dc51f3f 100755 --- a/cope2n-api/fwd_api/request/CreateTemplateRequest.py +++ b/cope2n-api/fwd_api/request/CreateTemplateRequest.py @@ -3,7 +3,7 @@ from rest_framework import serializers from fwd_api.constant.common import MAX_NUMBER_OF_TEMPLATE_DATA_BOX, MAX_NUMBER_OF_TEMPLATE_ANCHOR_BOX, \ NUMBER_OF_ITEM_IN_A_BOX, ESCAPE_VALUE from fwd_api.exception.exceptions import InvalidException -from fwd_api.utils import ProcessUtil +from fwd_api.utils import process as ProcessUtil class CreateTemplateRequest(serializers.Serializer): diff --git a/cope2n-api/fwd_api/request/UpdateTemplateRequest.py b/cope2n-api/fwd_api/request/UpdateTemplateRequest.py index cf6f1dd..914ff6d 100755 --- a/cope2n-api/fwd_api/request/UpdateTemplateRequest.py +++ b/cope2n-api/fwd_api/request/UpdateTemplateRequest.py @@ -3,7 +3,7 @@ from rest_framework import serializers from fwd_api.constant.common import MAX_NUMBER_OF_TEMPLATE_DATA_BOX, NUMBER_OF_ITEM_IN_A_BOX, \ MAX_NUMBER_OF_TEMPLATE_ANCHOR_BOX, ESCAPE_VALUE from fwd_api.exception.exceptions import InvalidException -from fwd_api.utils import ProcessUtil +from fwd_api.utils import process as ProcessUtil class UpdateTemplateRequest(serializers.Serializer): diff --git a/cope2n-api/fwd_api/response/ReportFileSerializer.py b/cope2n-api/fwd_api/response/ReportFileSerializer.py index f6f12fb..7bbd119 100755 --- a/cope2n-api/fwd_api/response/ReportFileSerializer.py +++ b/cope2n-api/fwd_api/response/ReportFileSerializer.py @@ -3,7 +3,7 @@ from rest_framework import serializers from fwd import settings from fwd_api.constant.common import FileCategory, FolderFileType from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest -from fwd_api.utils import FileUtils +from fwd_api.utils import file as FileUtils class ReportFileSerializer(serializers.Serializer): diff --git a/cope2n-api/fwd_api/response/ReportSerializer.py b/cope2n-api/fwd_api/response/ReportSerializer.py index ba6d04a..21dbe53 100755 --- a/cope2n-api/fwd_api/response/ReportSerializer.py +++ b/cope2n-api/fwd_api/response/ReportSerializer.py @@ -3,9 +3,9 @@ from rest_framework import serializers from fwd_api.constant.common import ProcessType, FileCategory from fwd_api.models import SubscriptionRequest, SubscriptionRequestFile from fwd_api.response.ReportFileSerializer import ReportFileSerializer -from fwd_api.utils.DateUtil import FORMAT +from fwd_api.utils.date import FORMAT from django.utils.translation import gettext as _ -from fwd_api.utils import FileUtils +from fwd_api.utils import file as FileUtils def i18n_for_label(data) -> list: if 'fields' in data and isinstance(data['fields'], list): diff --git a/cope2n-api/fwd_api/response/SubscriptionResponse.py b/cope2n-api/fwd_api/response/SubscriptionResponse.py index 608e5ca..7657054 100755 --- a/cope2n-api/fwd_api/response/SubscriptionResponse.py +++ b/cope2n-api/fwd_api/response/SubscriptionResponse.py @@ -2,7 +2,7 @@ from rest_framework import serializers from fwd_api.models import SubscriptionRequest, Subscription -from fwd_api.utils.DateUtil import FORMAT +from fwd_api.utils.date import FORMAT class SubscriptionResponse(serializers.Serializer): diff --git a/cope2n-api/fwd_api/response/TemplateResponse.py b/cope2n-api/fwd_api/response/TemplateResponse.py index 826dd17..b07ed95 100755 --- a/cope2n-api/fwd_api/response/TemplateResponse.py +++ b/cope2n-api/fwd_api/response/TemplateResponse.py @@ -4,7 +4,7 @@ from fwd_api.constant.common import TEMPLATE_BOX_TYPE, FolderFileType from fwd_api.models.OcrTemplate import OcrTemplate from fwd_api.models.OcrTemplateBox import OcrTemplateBox -from fwd_api.utils import FileUtils +from fwd_api.utils import file as FileUtils class TemplateBoxResponse(serializers.Serializer): diff --git a/cope2n-api/fwd_api/utils/CryptoUtils.py b/cope2n-api/fwd_api/utils/crypto.py old mode 100755 new mode 100644 similarity index 93% rename from cope2n-api/fwd_api/utils/CryptoUtils.py rename to cope2n-api/fwd_api/utils/crypto.py index 9865c20..d2e5d9c --- a/cope2n-api/fwd_api/utils/CryptoUtils.py +++ b/cope2n-api/fwd_api/utils/crypto.py @@ -11,7 +11,9 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from fwd import settings from fwd_api.annotation.api import throw_on_failure from fwd_api.exception.exceptions import InvalidException -from fwd_api.utils.DateUtil import default_zone, get_date_time_now +from fwd_api.utils.date import default_zone, get_date_time_now, FORMAT +from fwd_api.utils import date as DateUtil + class InternalCryptor: @@ -56,8 +58,6 @@ class SdsAuthentication: def generate_token(self, user_id, internal_id, status, sub_id=-1) -> str: c_time = get_date_time_now() + datetime.timedelta(hours=self.duration_of_token) c_time.replace(tzinfo=ZoneInfo(default_zone)) - from fwd_api.utils import DateUtil - from fwd_api.utils.DateUtil import FORMAT payload = {"id": user_id, "expired_at": DateUtil.to_str(c_time, FORMAT.DD_MM_YYYY_HHMMSS.value), 'internal_id': internal_id, 'status': status, 'subscription_id': sub_id} return self.encode_data(payload) @@ -65,16 +65,12 @@ class SdsAuthentication: def generate_img_token(self, user_id) -> str: c_time = get_date_time_now() + datetime.timedelta(hours=self.duration_of_token) c_time.replace(tzinfo=ZoneInfo(default_zone)) - from fwd_api.utils import DateUtil - from fwd_api.utils.DateUtil import FORMAT payload = {"expired_at": DateUtil.to_str(c_time, FORMAT.DD_MM_YYYY_HHMMSS.value), "internal_id": user_id} return self.encode_data(payload) def generate_img_token_v2(self, user_id, sub_id=None, user_sync_id=None) -> str: c_time = get_date_time_now() + datetime.timedelta(hours=self.duration_of_token) c_time.replace(tzinfo=ZoneInfo(default_zone)) - from fwd_api.utils import DateUtil - from fwd_api.utils.DateUtil import FORMAT payload = {"expired_at": DateUtil.to_str(c_time, FORMAT.DD_MM_YYYY_HHMMSS.value), "internal_id": user_id, 'subscription_id': sub_id, "id": user_sync_id} return self.encode_data(payload) diff --git a/cope2n-api/fwd_api/utils/DateUtil.py b/cope2n-api/fwd_api/utils/date.py old mode 100755 new mode 100644 similarity index 100% rename from cope2n-api/fwd_api/utils/DateUtil.py rename to cope2n-api/fwd_api/utils/date.py diff --git a/cope2n-api/fwd_api/utils/FileUtils.py b/cope2n-api/fwd_api/utils/file.py old mode 100755 new mode 100644 similarity index 99% rename from cope2n-api/fwd_api/utils/FileUtils.py rename to cope2n-api/fwd_api/utils/file.py index 5178ff6..29c15b9 --- a/cope2n-api/fwd_api/utils/FileUtils.py +++ b/cope2n-api/fwd_api/utils/file.py @@ -12,8 +12,8 @@ from fwd_api.constant.common import allowed_file_extensions from fwd_api.exception.exceptions import GeneralException, RequiredFieldException, InvalidException, \ ServiceUnavailableException, FileFormatInvalidException, LimitReachedException, InvalidDecompressedSizeException from fwd_api.models import SubscriptionRequest, OcrTemplate -from fwd_api.utils import ProcessUtil -from fwd_api.utils.CryptoUtils import image_authenticator +from fwd_api.utils import process as ProcessUtil +from fwd_api.utils.crypto import image_authenticator from fwd_api.utils.image import resize from ..celery_worker.client_connector import c_connector import imagesize diff --git a/cope2n-api/fwd_api/utils/health.py b/cope2n-api/fwd_api/utils/health.py new file mode 100644 index 0000000..9a69ca0 --- /dev/null +++ b/cope2n-api/fwd_api/utils/health.py @@ -0,0 +1,37 @@ +from django.db.models import Count +from django.utils import timezone +from datetime import timedelta + +from ..models import SubscriptionRequest + +def get_latest_requests(limit=50): + requests = SubscriptionRequest.objects.order_by("-created_at")[:limit] + requests_dict = [] + for request in requests: + requests_dict.append({ + "request_id": request.request_id, + "pages": request.pages, + "doc_type": request.doc_type, + # "predict_result": request.predict_result, + "created_at": request.created_at, + }) + return requests_dict + +def count_requests_by_date(days_limit=5): + today = timezone.now().date() + start_date = today - timedelta(days=days_limit) + requests_by_date = SubscriptionRequest.objects.filter(created_at__gte=start_date).values('created_at__date').annotate(count=Count('id')).values('created_at__date', 'count').order_by('created_at__date') + count_dict = [] + for rbd in requests_by_date: + count_dict.append({ + "date": rbd["created_at__date"], + "count": rbd["count"], + }) + return count_dict + + +def get_health_report(): + return { + "latest_requests": get_latest_requests(), + "count_by_date": count_requests_by_date(), + } \ No newline at end of file diff --git a/cope2n-api/fwd_api/utils/NumberUtils.py b/cope2n-api/fwd_api/utils/number.py old mode 100755 new mode 100644 similarity index 100% rename from cope2n-api/fwd_api/utils/NumberUtils.py rename to cope2n-api/fwd_api/utils/number.py diff --git a/cope2n-api/fwd_api/utils/ProcessUtil.py b/cope2n-api/fwd_api/utils/process.py old mode 100755 new mode 100644 similarity index 98% rename from cope2n-api/fwd_api/utils/ProcessUtil.py rename to cope2n-api/fwd_api/utils/process.py index 097d60c..7a833c9 --- a/cope2n-api/fwd_api/utils/ProcessUtil.py +++ b/cope2n-api/fwd_api/utils/process.py @@ -1,4 +1,5 @@ import os +import uuid import random import string import tempfile @@ -15,14 +16,14 @@ from fwd_api.constant.common import LIST_BOX_MESSAGE, pattern, NAME_MESSAGE, all FolderFileType, FileCategory from fwd_api.exception.exceptions import NumberOfBoxLimitReachedException, \ ServiceUnavailableException, DuplicateEntityException, LimitReachedException, BadGatewayException -from fwd_api.utils import DateUtil, FileUtils +from fwd_api.utils import date as DateUtil +from fwd_api.utils import file as FileUtils from ..constant.common import ProcessType, TEMPLATE_BOX_TYPE, EntityStatus from ..exception.exceptions import InvalidException, NotFoundException, \ PermissionDeniedException, RequiredFieldException, InvalidException, InvalidDecompressedSizeException from ..models import UserProfile, OcrTemplate, OcrTemplateBox, \ Subscription, SubscriptionRequestFile, SubscriptionRequest from ..celery_worker.client_connector import c_connector -import uuid from celery.utils.log import get_task_logger @@ -132,9 +133,6 @@ def sbt_validate_ocr_request_and_get(request, subscription): FileUtils.validate_list_file(imei_files, file_field="imei_file") FileUtils.validate_list_file(invoice_file, max_file_num=1, min_file_num=0, file_field="invoice_file") - # if not isinstance(redemption_ID, str): - # # raise RequiredFieldException(excArgs="redemption_ID") - # raise InvalidException(excArgs="redemption_ID") validated_data['imei_file'] = imei_files validated_data['invoice_file'] = invoice_file @@ -413,7 +411,7 @@ def process_image_local_file(file_name: str, file_path: str, request: Subscripti def pdf_to_images_urls(doc_path, request: SubscriptionRequest, user, dpi: int = 300) -> list: pdf_extracted = [] saving_path = FileUtils.get_folder_path(request) - break_file_name = f'break_0.jpg' + break_file_name = f'{os.path.basename(doc_path.name)}_page_0.jpg' saving_path = os.path.join(saving_path, break_file_name) image = get_first_page_pdf(doc_path, 300) diff --git a/cope2n-api/fwd_api/utils/RedisUtils.py b/cope2n-api/fwd_api/utils/redis.py similarity index 100% rename from cope2n-api/fwd_api/utils/RedisUtils.py rename to cope2n-api/fwd_api/utils/redis.py diff --git a/cope2n-api/fwd_api/utils/S3_process.py b/cope2n-api/fwd_api/utils/s3.py old mode 100755 new mode 100644 similarity index 100% rename from cope2n-api/fwd_api/utils/S3_process.py rename to cope2n-api/fwd_api/utils/s3.py diff --git a/cope2n-fe/src/layouts/main-layout.tsx b/cope2n-fe/src/layouts/main-layout.tsx index 12aaed8..7dbc4de 100644 --- a/cope2n-fe/src/layouts/main-layout.tsx +++ b/cope2n-fe/src/layouts/main-layout.tsx @@ -110,12 +110,15 @@ export function MainLayout() { style={{ marginBottom: 0, color: gray[0], - textAlign: 'center', + textAlign: 'left', }} ellipsis={{ - rows: 2, + rows: 10, }} > + NOTE: This UI is only for reference. Please use swagger UI for full-option API testing. +
+
Powered by SAMSUNG SDS.
All rights reserved. @@ -142,9 +145,11 @@ export function MainLayout() { style={{ marginBottom: 0, color: gray[0], - textAlign: 'center', + textAlign: 'left', }} > + NOTE: This UI is only for reference. Please use swagger UI for full-option API testing. +
Powered by SAMSUNG SDS. diff --git a/cope2n-fe/src/locales/en/messages.json b/cope2n-fe/src/locales/en/messages.json index f0a583b..4ff4f87 100644 --- a/cope2n-fe/src/locales/en/messages.json +++ b/cope2n-fe/src/locales/en/messages.json @@ -51,9 +51,7 @@ "Key Information Extraction": "Key Information Extraction", "Label": "Label", "Label of extracted field must not have more than 255 characters": "Label of extracted field must not have more than 255 characters", - "Language": "Language", "Login": "Login", - "Login with  <0>CMC Account?": "Login with  <0>CMC Account?", "Name": "Name", "Nation": "Nation", "Nationality": "Nationality", diff --git a/cope2n-fe/src/locales/vi/messages.json b/cope2n-fe/src/locales/vi/messages.json index d2fce32..4ca257e 100644 --- a/cope2n-fe/src/locales/vi/messages.json +++ b/cope2n-fe/src/locales/vi/messages.json @@ -51,9 +51,7 @@ "Key Information Extraction": "Trích xuất thông tin", "Label": "Nhãn", "Label of extracted field must not have more than 255 characters": "Độ dài nhãn của trường thông tin trích xuất không được vượt quá 255 kí tự", - "Language": "", "Login": "", - "Login with  <0>CMC Account?": "", "Name": "Tên", "Nation": "Dân tộc", "Nationality": "Quốc tịch", diff --git a/docker-compose.yml b/docker-compose.yml index 7290533..d56930f 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,6 +93,7 @@ services: - ./data/minio_data:/data networks: - ctel-sbt + restart: always command: server --address :9884 --console-address :9885 /data profiles: ["local"] @@ -149,11 +150,9 @@ services: - BASE_URL=http://be-ctel-sbt:${BASE_PORT} - REDIS_HOST=result-cache - REDIS_PORT=6379 - - + restart: always networks: - ctel-sbt - # restart: always depends_on: db-sbt: condition: service_started @@ -168,6 +167,7 @@ services: # Back-end persistent db-sbt: + restart: always mem_reservation: 500m # mem_limit: 1g # container_name: sidp-cope2n-be-sbt-db @@ -200,6 +200,7 @@ services: # Front-end services fe-sbt: + restart: always build: context: cope2n-fe shm_size: 10gb diff --git a/speedtest_sync.py b/speedtest_sync.py index 0a4a629..449f451 100644 --- a/speedtest_sync.py +++ b/speedtest_sync.py @@ -94,12 +94,12 @@ def process_file(data): -# invoice_files = [ -# ('invoice_file', ('invoice.pdf', open("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", "rb").read())), -# ] invoice_files = [ - ('invoice_file', ('invoice.jpg', open("test_samples/sbt/invoice.jpg", "rb").read())), + ('invoice_file', ('invoice.pdf', open("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", "rb").read())), ] +# invoice_files = [ +# ('invoice_file', ('invoice.jpg', open("test_samples/sbt/invoice.jpg", "rb").read())), +# ] imei_files = [ ('imei_files', ("test_samples/sbt/imei1.jpg", open("test_samples/sbt/imei1.jpg", "rb").read())), ('imei_files', ("test_samples/sbt/imei2.jpg", open("test_samples/sbt/imei2.jpg", "rb").read())), @@ -127,7 +127,6 @@ print("## TEST REPORT #################################") print("Number of requests: {}".format(args.num_requests)) print("Number of concurrent requests: {}".format(args.num_workers)) print("Number of files: 1 invoice, 1-5 imei files (random)") -print("Query time interval for result: {:.3f}s ".format(args.checking_interval)) print("--------------------------------------") print("SUCCESS RATE") counter = {} @@ -143,14 +142,12 @@ if len(uploading_time) == 0: print("No valid uploading time") print("Check the results!") processing_time = [x["process_time"] for x in results if x["success"]] -print("Uploading time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(uploading_time) / len(uploading_time), min(uploading_time), max(uploading_time))) -print("Processing time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(processing_time) / len(processing_time), min(processing_time), max(processing_time))) +print("Uploading + Processing time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(processing_time) / len(processing_time), min(processing_time), max(processing_time))) print("--------------------------------------") print("TIME BY IMAGE") uploading_time = [x["upload_time"] for x in results if x["success"]] processing_time = [x["process_time"] for x in results if x["success"]] num_images = sum(x["num_files"] for x in results if x["success"]) print("Total images:", num_images) -print("Uploading time: {:.3f}s".format(sum(uploading_time) / num_images)) -print("Processing time: {:.3f}s".format(sum(processing_time) / num_images)) +print("Uploading + Processing time: {:.3f}s".format(sum(processing_time) / num_images)) print("--------------------------------------")