From 0159d8d90c6db7080d6f5629ea76f3344cdc9cf9 Mon Sep 17 00:00:00 2001 From: Viet Anh Nguyen Date: Thu, 14 Dec 2023 10:54:03 +0700 Subject: [PATCH] Clean up code --- .../celery_worker/client_connector.py | 2 +- .../celery_worker/client_connector_fi.py | 2 +- cope2n-ai-fi/celery_worker/worker.py | 2 +- cope2n-ai-fi/celery_worker/worker_fi.py | 2 +- cope2n-ai-fi/modules/sdsvkie | 1 + cope2n-ai-fi/modules/sdsvtd | 1 + cope2n-ai-fi/modules/sdsvtr | 1 + cope2n-api/fwd/settings.py | 23 ++++++------- .../celery_worker/process_result_tasks.py | 5 ++- ...0156_alter_subscriptionrequestfile_code.py | 19 +++++++++++ .../fwd_api/models/SubscriptionRequestFile.py | 5 ++- cope2n-api/fwd_api/utils/FileUtils.py | 1 - docker-compose.yml | 27 ++++++++++++--- test_pdf_reader.py | 34 ------------------- 14 files changed, 66 insertions(+), 59 deletions(-) create mode 160000 cope2n-ai-fi/modules/sdsvkie create mode 160000 cope2n-ai-fi/modules/sdsvtd create mode 160000 cope2n-ai-fi/modules/sdsvtr create mode 100644 cope2n-api/fwd_api/migrations/0156_alter_subscriptionrequestfile_code.py delete mode 100644 test_pdf_reader.py diff --git a/cope2n-ai-fi/celery_worker/client_connector.py b/cope2n-ai-fi/celery_worker/client_connector.py index c9f7082..01b5247 100755 --- a/cope2n-ai-fi/celery_worker/client_connector.py +++ b/cope2n-ai-fi/celery_worker/client_connector.py @@ -2,7 +2,7 @@ from celery import Celery import base64 import environ env = environ.Env( - DEBUG=(bool, True) + DEBUG=(bool, False) ) class CeleryConnector: diff --git a/cope2n-ai-fi/celery_worker/client_connector_fi.py b/cope2n-ai-fi/celery_worker/client_connector_fi.py index 43b205d..dfbc5ab 100755 --- a/cope2n-ai-fi/celery_worker/client_connector_fi.py +++ b/cope2n-ai-fi/celery_worker/client_connector_fi.py @@ -1,7 +1,7 @@ from celery import Celery import environ env = environ.Env( - DEBUG=(bool, True) + DEBUG=(bool, False) ) class CeleryConnector: diff --git a/cope2n-ai-fi/celery_worker/worker.py b/cope2n-ai-fi/celery_worker/worker.py index a8025b8..0be7421 100755 --- a/cope2n-ai-fi/celery_worker/worker.py +++ b/cope2n-ai-fi/celery_worker/worker.py @@ -2,7 +2,7 @@ from celery import Celery from kombu import Queue, Exchange import environ env = environ.Env( - DEBUG=(bool, True) + DEBUG=(bool, False) ) app: Celery = Celery( diff --git a/cope2n-ai-fi/celery_worker/worker_fi.py b/cope2n-ai-fi/celery_worker/worker_fi.py index 7e9b906..d1c8eec 100755 --- a/cope2n-ai-fi/celery_worker/worker_fi.py +++ b/cope2n-ai-fi/celery_worker/worker_fi.py @@ -2,7 +2,7 @@ from celery import Celery from kombu import Queue, Exchange import environ env = environ.Env( - DEBUG=(bool, True) + DEBUG=(bool, False) ) app: Celery = Celery( diff --git a/cope2n-ai-fi/modules/sdsvkie b/cope2n-ai-fi/modules/sdsvkie new file mode 160000 index 0000000..8349a89 --- /dev/null +++ b/cope2n-ai-fi/modules/sdsvkie @@ -0,0 +1 @@ +Subproject commit 8349a89de7fd8c9e6958907047d16bdc23a77adf diff --git a/cope2n-ai-fi/modules/sdsvtd b/cope2n-ai-fi/modules/sdsvtd new file mode 160000 index 0000000..a9a796f --- /dev/null +++ b/cope2n-ai-fi/modules/sdsvtd @@ -0,0 +1 @@ +Subproject commit a9a796f843f6ad99977a8dcba02d65fe75225574 diff --git a/cope2n-ai-fi/modules/sdsvtr b/cope2n-ai-fi/modules/sdsvtr new file mode 160000 index 0000000..2c788e9 --- /dev/null +++ b/cope2n-ai-fi/modules/sdsvtr @@ -0,0 +1 @@ +Subproject commit 2c788e9e2d7fe369869d474fbb22426a4ca84590 diff --git a/cope2n-api/fwd/settings.py b/cope2n-api/fwd/settings.py index 15f8d18..08ef761 100755 --- a/cope2n-api/fwd/settings.py +++ b/cope2n-api/fwd/settings.py @@ -19,30 +19,29 @@ from django.urls import reverse_lazy BASE_DIR = Path(__file__).resolve().parent.parent env = environ.Env( - DEBUG=(bool, True) + DEBUG=(bool, False) ) -DEBUG = env("DEBUG") -if DEBUG: - environ.Env.read_env(os.path.join(BASE_DIR, '.env')) +DEBUG = False +environ.Env.read_env(os.path.join(BASE_DIR, '.env')) ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=['*'] + ['107.120.{}.{}'.format(i, j) for i in range(256) for j in range(256)]) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env.str("SECRET_KEY", '999999999999999999999999999999999999999999999999999999999999999999') -CTEL_KEY = env.str("CTEL_KEY", 'fTjWnZr4u7x!A%D*G-KaPdRgUkXp2s5v') -INTERNAL_SDS_KEY = env.str('SDS_SECRET_KEY', '2a5c21b593e0ec84c5ad68e175f75a2b2f2c47c387d9adfc9c8d42e16ec848f8e75de10dbcb3abdaf375420e3023fd7c05446a8a9521100038a750d312ab0005') -DB_ENCRYPT_KEY = env.str('DB_INTERNAL_KEY', '7LYk-iaWTFPqsZHIE5GHuv41S0c_Vlb0ZVc-BnsEZqQ=') +SECRET_KEY = env.str("SECRET_KEY", '') +CTEL_KEY = env.str("CTEL_KEY", '') +INTERNAL_SDS_KEY = env.str('SDS_SECRET_KEY', '') +DB_ENCRYPT_KEY = env.str('DB_INTERNAL_KEY', '') BASE_URL = env.str("BASE_URL", "") -BASE_UI_URL = env.str("BASE_UI_URL", "http://107.120.70.136:8080/") +BASE_UI_URL = env.str("BASE_UI_URL", "") AUTH_TOKEN_LIFE_TIME = env.int("AUTH_TOKEN_LIFE_TIME", 0) IMAGE_TOKEN_LIFE_TIME = env.int("IMAGE_TOKEN_LIFE_TIME", 0) -FI_USER_NAME = env.str("FI_USER_NAME", "Manulife") +FI_USER_NAME = env.str("FI_USER_NAME", "secret_username") FI_PASSWORD = env.str("FI_PASSWORD", 'admin')# SECURITY WARNING: don't run with debug turned on in production! # Application definition S3_ENDPOINT = env.str("S3_ENDPOINT", "") -S3_ACCESS_KEY = env.str("S3_ACCESS_KEY", "TannedCung") -S3_SECRET_KEY = env.str("S3_SECRET_KEY", "TannedCung") +S3_ACCESS_KEY = env.str("S3_ACCESS_KEY", "") +S3_SECRET_KEY = env.str("S3_SECRET_KEY", "") S3_BUCKET_NAME = env.str("S3_BUCKET_NAME", "ocr-data") REDIS_HOST = env.str("REDIS_HOST", "result-cache") REDIS_PORT = env.int("REDIS_PORT", 6379) diff --git a/cope2n-api/fwd_api/celery_worker/process_result_tasks.py b/cope2n-api/fwd_api/celery_worker/process_result_tasks.py index a8dfa40..e634995 100755 --- a/cope2n-api/fwd_api/celery_worker/process_result_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_result_tasks.py @@ -20,8 +20,7 @@ def aggregate_result(resutls, doc_types): des_result["content"]["document"][0]["end_page"] = 0 des_result["content"]["document"][0]["content"][3]["value"] = [None for _ in range(doc_types.count("imei"))] des_result["content"]["document"][0]["content"][2]["value"] = [] - - print(f"[INFO]: resutls: {resutls}") + for index, resutl in resutls.items(): index = int(index) doc_type = doc_types[index] @@ -142,7 +141,7 @@ def process_invoice_sbt_result(rq_id, result): rq.status = 200 # stop waiting results = redis_client.get_all_cache(rq_id) rq.predict_result = aggregate_result(results, rq.doc_type) - print(f"[DEBUG]: rq.predict_result: {rq.predict_result}") + # print(f"[DEBUG]: rq.predict_result: {rq.predict_result}") redis_client.remove_cache(rq_id) rq.save() diff --git a/cope2n-api/fwd_api/migrations/0156_alter_subscriptionrequestfile_code.py b/cope2n-api/fwd_api/migrations/0156_alter_subscriptionrequestfile_code.py new file mode 100644 index 0000000..3b55359 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0156_alter_subscriptionrequestfile_code.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1.3 on 2023-12-14 03:13 + +from django.db import migrations, models +import fwd_api.models.SubscriptionRequestFile + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0155_alter_subscriptionrequestfile_code'), + ] + + operations = [ + migrations.AlterField( + model_name='subscriptionrequestfile', + name='code', + field=models.CharField(default=fwd_api.models.SubscriptionRequestFile.gen_random_code, max_length=300), + ), + ] diff --git a/cope2n-api/fwd_api/models/SubscriptionRequestFile.py b/cope2n-api/fwd_api/models/SubscriptionRequestFile.py index bb25657..ef3c788 100755 --- a/cope2n-api/fwd_api/models/SubscriptionRequestFile.py +++ b/cope2n-api/fwd_api/models/SubscriptionRequestFile.py @@ -8,7 +8,10 @@ import uuid class SubscriptionRequestFile(models.Model): - code = models.CharField(max_length=300, default=f'FIL{uuid.uuid4().hex}') + def gen_random_code(): + return f"FIL{uuid.uuid4().hex}" + + code = models.CharField(max_length=300, default=gen_random_code) file_name = models.CharField(max_length=300, default=None) file_path = EncryptedCharField(max_length=500, default=None) file_category = models.CharField(max_length=200, default=FileCategory.Origin.value) diff --git a/cope2n-api/fwd_api/utils/FileUtils.py b/cope2n-api/fwd_api/utils/FileUtils.py index 9d0f56c..bce9e54 100755 --- a/cope2n-api/fwd_api/utils/FileUtils.py +++ b/cope2n-api/fwd_api/utils/FileUtils.py @@ -26,7 +26,6 @@ def validate_list_file(files, max_file_num=settings.MAX_UPLOAD_FILES_IN_A_REQUES raise LimitReachedException(excArgs=(f'Number of {file_field}', str(max_file_num), '')) for f in files: - print(f'dafile {f} is file type{type(f)}') if not isinstance(f, TemporaryUploadedFile): # print(f'[DEBUG]: {f.name}') raise InvalidException(excArgs="files") diff --git a/docker-compose.yml b/docker-compose.yml index 08d1670..a6374c1 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,7 @@ services: - BASE_PORT=${BASE_PORT} - CTEL_KEY=${CTEL_KEY} - SECRET_KEY=${SECRET_KEY} + - DB_INTERNAL_KEY=${DB_INTERNAL_KEY} - ALLOWED_HOSTS=${ALLOWED_HOSTS} - BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672 - BASE_URL=http://be-ctel-sbt:${BASE_PORT} @@ -86,14 +87,15 @@ services: python manage.py migrate && python manage.py compilemessages && gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker -b 0.0.0.0:9000" # pre-makemigrations on prod - # command: sh -c "tail -f > /dev/null" minio: image: minio/minio - # ports: - # - 9884:9884 - # - 9885:9885 + ports: + - 9886:9884 + - 9887:9885 environment: + - MINIO_ROOT_USER=${S3_ACCESS_KEY} + - MINIO_ROOT_PASSWORD=${S3_SECRET_KEY} - MINIO_ACCESS_KEY=${S3_ACCESS_KEY} - MINIO_SECRET_KEY=${S3_SECRET_KEY} volumes: @@ -101,6 +103,22 @@ services: networks: - ctel-sbt command: server --address :9884 --console-address :9885 /data + profiles: ["local"] + + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9884 ${S3_ACCESS_KEY} ${S3_SECRET_KEY}; + /usr/bin/mc mb myminio/${S3_BUCKET_NAME}; + /usr/bin/mc policy set public myminio/${S3_BUCKET_NAME}; + exit 0; + " + networks: + - ctel-sbt + profiles: ["local"] result-cache: image: redis:6.2-alpine @@ -134,6 +152,7 @@ services: - BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672 - BASE_UI_URL=http://fe:${VITE_PORT} - DEBUG=${DEBUG} + - DB_INTERNAL_KEY=${DB_INTERNAL_KEY} - IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME} - CTEL_KEY=${CTEL_KEY} - SECRET_KEY=${SECRET_KEY} diff --git a/test_pdf_reader.py b/test_pdf_reader.py deleted file mode 100644 index 1fdcb57..0000000 --- a/test_pdf_reader.py +++ /dev/null @@ -1,34 +0,0 @@ -from PyPDF2 import PdfReader, PdfWriter -from PIL import Image -from pdf2image import convert_from_bytes - - -def get_first_page_pdf(filename, max_size=2048): - def pdf_scale_page(page, size=297): - """Scale page to specified size mm""" - (w, h) = page.mediabox[2:] - # Units of measurement are not "points". The units of measurement are user defined and default to 1/72 inch. See section 4.2.1 Coordinate spaces of the PDF specification. - # If we have a page width of 297 mm and at points 1 inch = 25.4 mm - pmm = (1/72*25.4) - ks = size / (float(max((w, h))) * pmm) - page.scale_by(ks) - return page - - reader = PdfReader(filename) - page = reader.pages[0] - scaled_page = pdf_scale_page(page, max_size) - - # Create BytesIO - pdf_bytes = io.BytesIO() - dst_pdf = PdfWriter() - dst_pdf.add_page(scaled_page) - dst_pdf.write(pdf_bytes) - pdf_bytes.seek(0) - - image = convert_from_bytes(pdf_bytes.read()) - if isinstance(image, list): - return image[0] - return image - -img = get_first_page_pdf("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", max_size=300) -img.save("invoice.jpg", "JPEG")