tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up code

Browse Source
This commit is contained in:
Viet Anh Nguyen 2023-12-14 10:54:03 +07:00
parent 6840f16ee7
commit 0159d8d90c
14 changed files with 66 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cope2n-ai-fi/celery_worker/client_connector.py
View File

@ -2,7 +2,7 @@ from celery import Celery
import base64
import environ
env = environ.Env(
DEBUG=(bool, True)
DEBUG=(bool, False)
)
class CeleryConnector:

2
cope2n-ai-fi/celery_worker/client_connector_fi.py
View File

@ -1,7 +1,7 @@
from celery import Celery
import environ
env = environ.Env(
DEBUG=(bool, True)
DEBUG=(bool, False)
)
class CeleryConnector:

2
cope2n-ai-fi/celery_worker/worker.py
View File

@ -2,7 +2,7 @@ from celery import Celery
from kombu import Queue, Exchange
import environ
env = environ.Env(
DEBUG=(bool, True)
DEBUG=(bool, False)
)
app: Celery = Celery(

2
cope2n-ai-fi/celery_worker/worker_fi.py
View File

@ -2,7 +2,7 @@ from celery import Celery
from kombu import Queue, Exchange
import environ
env = environ.Env(
DEBUG=(bool, True)
DEBUG=(bool, False)
)
app: Celery = Celery(

1
cope2n-ai-fi/modules/sdsvkie Submodule

@ -0,0 +1 @@
Subproject commit 8349a89de7fd8c9e6958907047d16bdc23a77adf

1
cope2n-ai-fi/modules/sdsvtd Submodule

@ -0,0 +1 @@
Subproject commit a9a796f843f6ad99977a8dcba02d65fe75225574

1
cope2n-ai-fi/modules/sdsvtr Submodule

@ -0,0 +1 @@
Subproject commit 2c788e9e2d7fe369869d474fbb22426a4ca84590

23
cope2n-api/fwd/settings.py
View File

@ -19,30 +19,29 @@ from django.urls import reverse_lazy
BASE_DIR = Path(__file__).resolve().parent.parent
env = environ.Env(
DEBUG=(bool, True)
DEBUG=(bool, False)
)
DEBUG = env("DEBUG")
if DEBUG:
environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
DEBUG = False
environ.Env.read_env(os.path.join(BASE_DIR, '.env'))
ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=['*'] + ['107.120.{}.{}'.format(i, j) for i in range(256) for j in range(256)])
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = env.str("SECRET_KEY", '999999999999999999999999999999999999999999999999999999999999999999')
CTEL_KEY = env.str("CTEL_KEY", 'fTjWnZr4u7x!A%D*G-KaPdRgUkXp2s5v')
INTERNAL_SDS_KEY = env.str('SDS_SECRET_KEY', '2a5c21b593e0ec84c5ad68e175f75a2b2f2c47c387d9adfc9c8d42e16ec848f8e75de10dbcb3abdaf375420e3023fd7c05446a8a9521100038a750d312ab0005')
DB_ENCRYPT_KEY = env.str('DB_INTERNAL_KEY', '7LYk-iaWTFPqsZHIE5GHuv41S0c_Vlb0ZVc-BnsEZqQ=')
SECRET_KEY = env.str("SECRET_KEY", '')
CTEL_KEY = env.str("CTEL_KEY", '')
INTERNAL_SDS_KEY = env.str('SDS_SECRET_KEY', '')
DB_ENCRYPT_KEY = env.str('DB_INTERNAL_KEY', '')
BASE_URL = env.str("BASE_URL", "")
BASE_UI_URL = env.str("BASE_UI_URL", "http://107.120.70.136:8080/")
BASE_UI_URL = env.str("BASE_UI_URL", "")
AUTH_TOKEN_LIFE_TIME = env.int("AUTH_TOKEN_LIFE_TIME", 0)
IMAGE_TOKEN_LIFE_TIME = env.int("IMAGE_TOKEN_LIFE_TIME", 0)
FI_USER_NAME = env.str("FI_USER_NAME", "Manulife")
FI_USER_NAME = env.str("FI_USER_NAME", "secret_username")
FI_PASSWORD = env.str("FI_PASSWORD", 'admin')# SECURITY WARNING: don't run with debug turned on in production!
# Application definition
S3_ENDPOINT = env.str("S3_ENDPOINT", "")
S3_ACCESS_KEY = env.str("S3_ACCESS_KEY", "TannedCung")
S3_SECRET_KEY = env.str("S3_SECRET_KEY", "TannedCung")
S3_ACCESS_KEY = env.str("S3_ACCESS_KEY", "")
S3_SECRET_KEY = env.str("S3_SECRET_KEY", "")
S3_BUCKET_NAME = env.str("S3_BUCKET_NAME", "ocr-data")
REDIS_HOST = env.str("REDIS_HOST", "result-cache")
REDIS_PORT = env.int("REDIS_PORT", 6379)

5
cope2n-api/fwd_api/celery_worker/process_result_tasks.py
View File

@ -20,8 +20,7 @@ def aggregate_result(resutls, doc_types):
des_result["content"]["document"][0]["end_page"] = 0
des_result["content"]["document"][0]["content"][3]["value"] = [None for _ in range(doc_types.count("imei"))]
des_result["content"]["document"][0]["content"][2]["value"] = []
print(f"[INFO]: resutls: {resutls}")
for index, resutl in resutls.items():
index = int(index)
doc_type = doc_types[index]
@ -142,7 +141,7 @@ def process_invoice_sbt_result(rq_id, result):
rq.status = 200 # stop waiting
results = redis_client.get_all_cache(rq_id)
rq.predict_result = aggregate_result(results, rq.doc_type)
print(f"[DEBUG]: rq.predict_result: {rq.predict_result}")
# print(f"[DEBUG]: rq.predict_result: {rq.predict_result}")
redis_client.remove_cache(rq_id)
rq.save()

19
cope2n-api/fwd_api/migrations/0156_alter_subscriptionrequestfile_code.py Normal file
View File

@ -0,0 +1,19 @@
# Generated by Django 4.1.3 on 2023-12-14 03:13
from django.db import migrations, models
import fwd_api.models.SubscriptionRequestFile
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0155_alter_subscriptionrequestfile_code'),
]
operations = [
migrations.AlterField(
model_name='subscriptionrequestfile',
name='code',
field=models.CharField(default=fwd_api.models.SubscriptionRequestFile.gen_random_code, max_length=300),
),
]

5
cope2n-api/fwd_api/models/SubscriptionRequestFile.py
View File

@ -8,7 +8,10 @@ import uuid
class SubscriptionRequestFile(models.Model):
code = models.CharField(max_length=300, default=f'FIL{uuid.uuid4().hex}')
def gen_random_code():
return f"FIL{uuid.uuid4().hex}"
code = models.CharField(max_length=300, default=gen_random_code)
file_name = models.CharField(max_length=300, default=None)
file_path = EncryptedCharField(max_length=500, default=None)
file_category = models.CharField(max_length=200, default=FileCategory.Origin.value)

1
cope2n-api/fwd_api/utils/FileUtils.py
View File

@ -26,7 +26,6 @@ def validate_list_file(files, max_file_num=settings.MAX_UPLOAD_FILES_IN_A_REQUES
raise LimitReachedException(excArgs=(f'Number of {file_field}', str(max_file_num), ''))
for f in files:
print(f'dafile {f} is file type{type(f)}')
if not isinstance(f, TemporaryUploadedFile):
# print(f'[DEBUG]: {f.name}')
raise InvalidException(excArgs="files")

27
docker-compose.yml
View File

@ -56,6 +56,7 @@ services:
- BASE_PORT=${BASE_PORT}
- CTEL_KEY=${CTEL_KEY}
- SECRET_KEY=${SECRET_KEY}
- DB_INTERNAL_KEY=${DB_INTERNAL_KEY}
- ALLOWED_HOSTS=${ALLOWED_HOSTS}
- BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672
- BASE_URL=http://be-ctel-sbt:${BASE_PORT}
@ -86,14 +87,15 @@ services:
python manage.py migrate &&
python manage.py compilemessages &&
gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker -b 0.0.0.0:9000" # pre-makemigrations on prod
# command: sh -c "tail -f > /dev/null"
minio:
image: minio/minio
# ports:
# - 9884:9884
# - 9885:9885
ports:
- 9886:9884
- 9887:9885
environment:
- MINIO_ROOT_USER=${S3_ACCESS_KEY}
- MINIO_ROOT_PASSWORD=${S3_SECRET_KEY}
- MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
- MINIO_SECRET_KEY=${S3_SECRET_KEY}
volumes:
@ -101,6 +103,22 @@ services:
networks:
- ctel-sbt
command: server --address :9884 --console-address :9885 /data
profiles: ["local"]
createbuckets:
image: minio/mc
depends_on:
- minio
entrypoint: >
/bin/sh -c "
/usr/bin/mc alias set myminio http://minio:9884 ${S3_ACCESS_KEY} ${S3_SECRET_KEY};
/usr/bin/mc mb myminio/${S3_BUCKET_NAME};
/usr/bin/mc policy set public myminio/${S3_BUCKET_NAME};
exit 0;
"
networks:
- ctel-sbt
profiles: ["local"]
result-cache:
image: redis:6.2-alpine
@ -134,6 +152,7 @@ services:
- BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672
- BASE_UI_URL=http://fe:${VITE_PORT}
- DEBUG=${DEBUG}
- DB_INTERNAL_KEY=${DB_INTERNAL_KEY}
- IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME}
- CTEL_KEY=${CTEL_KEY}
- SECRET_KEY=${SECRET_KEY}

34
test_pdf_reader.py
View File

@ -1,34 +0,0 @@
from PyPDF2 import PdfReader, PdfWriter
from PIL import Image
from pdf2image import convert_from_bytes
def get_first_page_pdf(filename, max_size=2048):
def pdf_scale_page(page, size=297):
"""Scale page to specified size mm"""
(w, h) = page.mediabox[2:]
# Units of measurement are not "points". The units of measurement are user defined and default to 1/72 inch. See section 4.2.1 Coordinate spaces of the PDF specification.
# If we have a page width of 297 mm and at points 1 inch = 25.4 mm
pmm = (1/72*25.4)
ks = size / (float(max((w, h))) * pmm)
page.scale_by(ks)
return page
reader = PdfReader(filename)
page = reader.pages[0]
scaled_page = pdf_scale_page(page, max_size)
# Create BytesIO
pdf_bytes = io.BytesIO()
dst_pdf = PdfWriter()
dst_pdf.add_page(scaled_page)
dst_pdf.write(pdf_bytes)
pdf_bytes.seek(0)
image = convert_from_bytes(pdf_bytes.read())
if isinstance(image, list):
return image[0]
return image
img = get_first_page_pdf("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", max_size=300)
img.save("invoice.jpg", "JPEG")