Clean up code

cope2n-ai-fi/celery_worker/client_connector.py

@@ -2,7 +2,7 @@ from celery import Celery
 import base64
 import environ
 env = environ.Env(
-    DEBUG=(bool, True)
+    DEBUG=(bool, False)
 )
 
 class CeleryConnector:

cope2n-ai-fi/celery_worker/client_connector_fi.py

@@ -1,7 +1,7 @@
 from celery import Celery
 import environ
 env = environ.Env(
-    DEBUG=(bool, True)
+    DEBUG=(bool, False)
 )
 
 class CeleryConnector:

cope2n-ai-fi/celery_worker/worker.py

@@ -2,7 +2,7 @@ from celery import Celery
 from kombu import Queue, Exchange
 import environ
 env = environ.Env(
-    DEBUG=(bool, True)
+    DEBUG=(bool, False)
 )
 
 app: Celery = Celery(

cope2n-ai-fi/celery_worker/worker_fi.py

@@ -2,7 +2,7 @@ from celery import Celery
 from kombu import Queue, Exchange
 import environ
 env = environ.Env(
-    DEBUG=(bool, True)
+    DEBUG=(bool, False)
 )
 
 app: Celery = Celery(

cope2n-ai-fi/modules/sdsvkie

@@ -0,0 +1 @@
+Subproject commit 8349a89de7fd8c9e6958907047d16bdc23a77adf

cope2n-ai-fi/modules/sdsvtd

@@ -0,0 +1 @@
+Subproject commit a9a796f843f6ad99977a8dcba02d65fe75225574

cope2n-ai-fi/modules/sdsvtr

@@ -0,0 +1 @@
+Subproject commit 2c788e9e2d7fe369869d474fbb22426a4ca84590

cope2n-api/fwd/settings.py

@@ -19,30 +19,29 @@ from django.urls import reverse_lazy
 BASE_DIR = Path(__file__).resolve().parent.parent
 
 env = environ.Env(
-    DEBUG=(bool, True)
+    DEBUG=(bool, False)
 )
-DEBUG = env("DEBUG")
+DEBUG = False
-if DEBUG:
+environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
-    environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
 ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=['*'] + ['107.120.{}.{}'.format(i, j) for i in range(256) for j in range(256)])
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = env.str("SECRET_KEY", '999999999999999999999999999999999999999999999999999999999999999999')
+SECRET_KEY = env.str("SECRET_KEY", '')
-CTEL_KEY = env.str("CTEL_KEY", 'fTjWnZr4u7x!A%D*G-KaPdRgUkXp2s5v')
+CTEL_KEY = env.str("CTEL_KEY", '')
-INTERNAL_SDS_KEY = env.str('SDS_SECRET_KEY', '2a5c21b593e0ec84c5ad68e175f75a2b2f2c47c387d9adfc9c8d42e16ec848f8e75de10dbcb3abdaf375420e3023fd7c05446a8a9521100038a750d312ab0005')
+INTERNAL_SDS_KEY = env.str('SDS_SECRET_KEY', '')
-DB_ENCRYPT_KEY = env.str('DB_INTERNAL_KEY', '7LYk-iaWTFPqsZHIE5GHuv41S0c_Vlb0ZVc-BnsEZqQ=')
+DB_ENCRYPT_KEY = env.str('DB_INTERNAL_KEY', '')
 BASE_URL = env.str("BASE_URL", "")
-BASE_UI_URL = env.str("BASE_UI_URL", "http://107.120.70.136:8080/")
+BASE_UI_URL = env.str("BASE_UI_URL", "")
 AUTH_TOKEN_LIFE_TIME = env.int("AUTH_TOKEN_LIFE_TIME", 0)
 IMAGE_TOKEN_LIFE_TIME = env.int("IMAGE_TOKEN_LIFE_TIME", 0)
-FI_USER_NAME = env.str("FI_USER_NAME", "Manulife")
+FI_USER_NAME = env.str("FI_USER_NAME", "secret_username")
 FI_PASSWORD = env.str("FI_PASSWORD", 'admin')# SECURITY WARNING: don't run with debug turned on in production!
 # Application definition
 S3_ENDPOINT = env.str("S3_ENDPOINT", "")
-S3_ACCESS_KEY = env.str("S3_ACCESS_KEY", "TannedCung")
+S3_ACCESS_KEY = env.str("S3_ACCESS_KEY", "")
-S3_SECRET_KEY = env.str("S3_SECRET_KEY", "TannedCung")
+S3_SECRET_KEY = env.str("S3_SECRET_KEY", "")
 S3_BUCKET_NAME = env.str("S3_BUCKET_NAME", "ocr-data")
 REDIS_HOST = env.str("REDIS_HOST", "result-cache")
 REDIS_PORT = env.int("REDIS_PORT", 6379)

cope2n-api/fwd_api/celery_worker/process_result_tasks.py

@@ -20,8 +20,7 @@ def aggregate_result(resutls, doc_types):
     des_result["content"]["document"][0]["end_page"] = 0
     des_result["content"]["document"][0]["content"][3]["value"] = [None for _ in range(doc_types.count("imei"))]
     des_result["content"]["document"][0]["content"][2]["value"] = []
-    
+
-    print(f"[INFO]: resutls: {resutls}")
     for index, resutl in resutls.items():
         index = int(index)
         doc_type = doc_types[index]
@@ -142,7 +141,7 @@ def process_invoice_sbt_result(rq_id, result):
                 rq.status = 200 # stop waiting
                 results = redis_client.get_all_cache(rq_id)
                 rq.predict_result = aggregate_result(results, rq.doc_type)
-                print(f"[DEBUG]: rq.predict_result: {rq.predict_result}")
+                # print(f"[DEBUG]: rq.predict_result: {rq.predict_result}")
                 redis_client.remove_cache(rq_id)
                 rq.save()
                 

cope2n-api/fwd_api/migrations/0156_alter_subscriptionrequestfile_code.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.1.3 on 2023-12-14 03:13
+
+from django.db import migrations, models
+import fwd_api.models.SubscriptionRequestFile
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0155_alter_subscriptionrequestfile_code'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='subscriptionrequestfile',
+            name='code',
+            field=models.CharField(default=fwd_api.models.SubscriptionRequestFile.gen_random_code, max_length=300),
+        ),
+    ]

cope2n-api/fwd_api/models/SubscriptionRequestFile.py

@@ -8,7 +8,10 @@ import uuid
 
 
 class SubscriptionRequestFile(models.Model):
-    code = models.CharField(max_length=300, default=f'FIL{uuid.uuid4().hex}')
+    def gen_random_code():
+        return f"FIL{uuid.uuid4().hex}"
+
+    code = models.CharField(max_length=300, default=gen_random_code)
     file_name = models.CharField(max_length=300, default=None)
     file_path = EncryptedCharField(max_length=500, default=None)
     file_category = models.CharField(max_length=200, default=FileCategory.Origin.value)

cope2n-api/fwd_api/utils/FileUtils.py

@@ -26,7 +26,6 @@ def validate_list_file(files, max_file_num=settings.MAX_UPLOAD_FILES_IN_A_REQUES
         raise LimitReachedException(excArgs=(f'Number of {file_field}', str(max_file_num), ''))
 
     for f in files:
-        print(f'dafile {f} is file type{type(f)}')
         if not isinstance(f, TemporaryUploadedFile):
             # print(f'[DEBUG]: {f.name}')
             raise InvalidException(excArgs="files")

docker-compose.yml

@@ -56,6 +56,7 @@ services:
       - BASE_PORT=${BASE_PORT}
       - CTEL_KEY=${CTEL_KEY}
       - SECRET_KEY=${SECRET_KEY}
+      - DB_INTERNAL_KEY=${DB_INTERNAL_KEY}
       - ALLOWED_HOSTS=${ALLOWED_HOSTS}
       - BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672
       - BASE_URL=http://be-ctel-sbt:${BASE_PORT}
@@ -86,14 +87,15 @@ services:
                   python manage.py migrate &&
                   python manage.py compilemessages &&
                   gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker -b 0.0.0.0:9000" # pre-makemigrations on prod
-    # command: sh -c "tail -f > /dev/null"
 
   minio:
     image: minio/minio
-    # ports:
+    ports:
-    #  - 9884:9884
+     - 9886:9884
-    #  - 9885:9885
+     - 9887:9885
     environment:
+    - MINIO_ROOT_USER=${S3_ACCESS_KEY}
+    - MINIO_ROOT_PASSWORD=${S3_SECRET_KEY}
     - MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
     - MINIO_SECRET_KEY=${S3_SECRET_KEY}
     volumes:
@@ -101,6 +103,22 @@ services:
     networks:
     - ctel-sbt
     command: server --address :9884 --console-address :9885 /data
+    profiles: ["local"]
+
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - minio
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set myminio http://minio:9884 ${S3_ACCESS_KEY} ${S3_SECRET_KEY};
+      /usr/bin/mc mb myminio/${S3_BUCKET_NAME};
+      /usr/bin/mc policy set public myminio/${S3_BUCKET_NAME};
+      exit 0;
+      "
+    networks:
+    - ctel-sbt
+    profiles: ["local"]
 
   result-cache:
     image: redis:6.2-alpine
@@ -134,6 +152,7 @@ services:
       - BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672
       - BASE_UI_URL=http://fe:${VITE_PORT}      
       - DEBUG=${DEBUG}
+      - DB_INTERNAL_KEY=${DB_INTERNAL_KEY}
       - IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME}
       - CTEL_KEY=${CTEL_KEY}
       - SECRET_KEY=${SECRET_KEY}

test_pdf_reader.py

@@ -1,34 +0,0 @@
-from PyPDF2 import PdfReader, PdfWriter
-from PIL import Image
-from pdf2image import convert_from_bytes
-
-
-def get_first_page_pdf(filename, max_size=2048):
-    def pdf_scale_page(page, size=297):
-        """Scale page to specified size mm"""
-        (w, h) = page.mediabox[2:]
-        #  Units of measurement are not "points". The units of measurement are user defined and default to 1/72 inch. See section 4.2.1 Coordinate spaces of the PDF specification.
-        # If we have a page width of 297 mm and at points 1 inch = 25.4 mm
-        pmm = (1/72*25.4)
-        ks = size / (float(max((w, h))) * pmm)
-        page.scale_by(ks)
-        return page
-
-    reader = PdfReader(filename)
-    page = reader.pages[0]
-    scaled_page = pdf_scale_page(page, max_size)
-
-    # Create BytesIO
-    pdf_bytes = io.BytesIO()
-    dst_pdf = PdfWriter()
-    dst_pdf.add_page(scaled_page)
-    dst_pdf.write(pdf_bytes)
-    pdf_bytes.seek(0)
-
-    image = convert_from_bytes(pdf_bytes.read())
-    if isinstance(image, list):
-        return image[0]
-    return image
-
-img = get_first_page_pdf("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", max_size=300)
-img.save("invoice.jpg", "JPEG")