tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://code.sdsdev.co.kr/dx-tan/SBT-IDP into feature/tests

Browse Source
This commit is contained in:
Viet Anh Nguyen 2023-12-26 16:50:16 +07:00
commit 4444c3c96f
24 changed files with 363 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored
View File

@ -28,4 +28,10 @@ curl.md
cope2n-api/fwd_api/commands/init_database.py
/data
backup
demo-ocr/
demo-ocr/
logs/
docker-compose_.yml
cope2n-ai-fi/Dockerfile_old_work
*.sql
*.sql
.env_prod

1
cope2n-ai-fi/.dockerignore
View File

@ -1,6 +1,7 @@
.github
.git
.vscode
packages/
__pycache__
DataBase/image_temp/
DataBase/json_temp/

74
cope2n-ai-fi/Dockerfile
View File

@ -1,40 +1,64 @@
FROM thucpd2408/env-deskew
FROM pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime
COPY ./packages/cudnn-linux*.tar.xz /tmp/cudnn-linux*.tar.xz
RUN apt-get update && \
apt-get install -y git gcc g++ ffmpeg libsm6 libxext6 && \
apt-get -y autoremove && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN tar -xvf /tmp/cudnn-linux*.tar.xz -C /tmp/ \
&& cp /tmp/cudnn-*-archive/include/cudnn*.h /usr/local/cuda/include \
&& cp -P /tmp/cudnn-*-archive/lib/libcudnn* /usr/local/cuda/lib64 \
&& chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn* \
&& rm -rf /tmp/cudnn-*-archive
# RUN pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir
RUN apt-get update && apt-get install -y gcc g++ ffmpeg libsm6 libxext6 poppler-utils
RUN pip install -U openmim==0.3.7 --no-cache-dir
RUN mim install mmcv-full==1.7.1
RUN pip install mmcv==1.6.0 -f https://download.openmmlab.com/mmcv/dst/cu116/torch1.13/index.html --no-cache-dir
WORKDIR /workspace
RUN ln -s /opt/conda/lib/python3.10/site-packages/torch/lib/libcudnn.so.8 /usr/lib/libcudnn.so && \
ln -s /opt/conda/lib/libcublas.so /usr/lib/libcublas.so
RUN python -m pip install paddlepaddle-gpu==2.4.2.post116 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html --no-cache-dir
# COPY ./modules/ocr_engine/externals/ /workspace/cope2n-ai-fi/modules/ocr_engine/externals/
# COPY ./modules/ocr_engine/requirements.txt /workspace/cope2n-ai-fi/modules/ocr_engine/requirements.txt
# COPY ./modules/sdsvkvu/ /workspace/cope2n-ai-fi/modules/sdsvkvu/
# COPY ./requirements.txt /workspace/cope2n-ai-fi/requirements.txt
RUN python -m pip install 'git+https://github.com/facebookresearch/detectron2.git' --no-cache-dir
# Install SDSV packages
COPY . /workspace/cope2n-ai-fi
RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsv_dewarp && pip3 install -v -e .
RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtd && pip3 install -v -e .
RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtr && pip3 install -v -e .
RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsv_dewarp && pip3 install -v -e . --no-cache-dir
RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtd && pip3 install -v -e . --no-cache-dir
RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtr && pip3 install -v -e . --no-cache-dir
RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu && pip3 install -v -e .
RUN cd /workspace/cope2n-ai-fi && pip3 install -r requirements.txt
# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsv_dewarp /tmp/sdsv_dewarp
# RUN cd /tmp/sdsv_dewarp && pip install -v -e . --no-cache-dir
RUN rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \
rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \
rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11 && \
ln -s /usr/local/cuda-11.8/targets/x86_64-linux/lib/libcublasLt.so.11 /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \
ln -s /usr/local/cuda-11.8/targets/x86_64-linux/lib/libcublas.so.11 /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \
ln -s /usr/local/cuda-11.8/targets/x86_64-linux/lib/libnvblas.so.11 /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11
# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsvtd /tmp/sdsvtd
# RUN cd /tmp/sdsvtd && pip install -v -e . --no-cache-dir
# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsvtr /tmp/sdsvtr
# RUN cd /tmp/sdsvtr && pip install -v -e . --no-cache-dir
# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/requirements.txt /tmp/sdsvocr/requirements.txt
# RUN pip install -r /tmp/sdsvocr/requirements.txt
# ENV PIP_DEFAULT_TIMEOUT 100
# COPY ./modules/sdsvkvu /tmp/sdsvkvu
# RUN cd /tmp/sdsvkvu && pip install -v -e . --no-cache-dir
# COPY ./requirements.txt /tmp/requirements.txt
# RUN pip install -r /tmp/requirements.txt --no-cache-dir
RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu && pip3 install -v -e . --no-cache-dir
RUN cd /workspace/cope2n-ai-fi && pip3 install -r requirements.txt --no-cache-dir
# RUN rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \
# rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \
# rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11 && \
# ln -s /opt/conda/lib/libcublasLt.so /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \
# ln -s /opt/conda/lib/libcublas.so /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \
# ln -s /opt/conda/lib/libnvblas.so /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11
WORKDIR /workspace
ENV PYTHONPATH="."
ENV TZ="Asia/Ho_Chi_Minh"
CMD [ "sh", "run.sh"]
CMD [ "sh", "run.sh"]

2
cope2n-ai-fi/configs/sdsap_sbt/configs.py
View File

@ -6,7 +6,7 @@ ocr_engine = {
"device": device
},
"recognizer": {
"version": "/workspace/cope2n-ai-fi/weights/models/sdsvtr/hub/sbt_20231218_e116_sdstr.pth",
"version": "/workspace/cope2n-ai-fi/weights/models/ocr_engine/sdsvtr/hub/sbt_20231218_e116_sdstr.pth",
"device": device
},
"deskew": {

1
cope2n-ai-fi/modules/sdsvkie_

@ -1 +0,0 @@
Subproject commit 8349a89de7fd8c9e6958907047d16bdc23a77adf

1
cope2n-ai-fi/modules/sdsvkv_oldu

@ -1 +0,0 @@
Subproject commit bdba044bb2eacac7c7cfe0e0f321196d03b681f6

1
cope2n-ai-fi/modules/sdsvtd_

@ -1 +0,0 @@
Subproject commit a9a796f843f6ad99977a8dcba02d65fe75225574

1
cope2n-ai-fi/modules/sdsvtr_

@ -1 +0,0 @@
Subproject commit 2c788e9e2d7fe369869d474fbb22426a4ca84590

3
cope2n-ai-fi/requirements.txt
View File

@ -10,4 +10,5 @@ easydict
imagesize==1.4.1
pdf2image==1.16.3
redis==5.0.1
redis==5.0.1
celery==5.3.6

1
cope2n-api/fwd_api/api/ctel_view.py
View File

@ -292,6 +292,7 @@ class CtelViewSet(viewsets.ViewSet):
if serializer.data[0]["data"].get("status", 200) != 200:
continue
serializer.data[0]["request_id"] = rq_id
return Response(status=status.HTTP_200_OK, data=serializer.data[0])
raise ServiceTimeoutException(excArgs=f"{rq_id}")

11
cope2n-api/fwd_api/celery_worker/client_connector.py
View File

@ -29,9 +29,9 @@ class CeleryConnector:
'process_manulife_invoice': {'queue': "invoice_manulife"},
'process_sbt_invoice': {'queue': "invoice_sbt"},
'do_pdf': {'queue': "do_pdf"},
'upload_file_to_s3': {'queue': "upload_file_to_s3"},
'upload_file_to_s3': {'queue': "upload_file_to_s3"},
'upload_obj_to_s3': {'queue': "upload_obj_to_s3"},
'remove_local_file': {'queue': "remove_local_file"},
}
app = Celery(
@ -45,6 +45,8 @@ class CeleryConnector:
return self.send_task('upload_file_to_s3', args)
def upload_obj_to_s3(self, args):
return self.send_task('upload_obj_to_s3', args)
def remove_local_file(self, args):
return self.send_task('remove_local_file', args, countdown=280) # nearest execution of this task in 280 seconds
def process_fi(self, args):
return self.send_task('process_fi_invoice', args)
def process_fi_result(self, args):
@ -86,10 +88,9 @@ class CeleryConnector:
def process_invoice_sbt(self, args):
return self.send_task('process_sbt_invoice', args)
def send_task(self, name=None, args=None):
def send_task(self, name=None, args=None, countdown=None):
if name not in self.task_routes or 'queue' not in self.task_routes[name]:
raise GeneralException("System")
return self.app.send_task(name, args, queue=self.task_routes[name]['queue'], expires=300)
return self.app.send_task(name, args, queue=self.task_routes[name]['queue'], expires=300, countdown=countdown)
c_connector = CeleryConnector()

22
cope2n-api/fwd_api/celery_worker/internal_task.py
View File

@ -119,14 +119,28 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files):
@app.task(name='upload_file_to_s3')
def upload_file_to_s3(local_file_path, s3_key):
def upload_file_to_s3(local_file_path, s3_key, request_id):
if s3_client.s3_client is not None:
res = s3_client.upload_file(local_file_path, s3_key)
if res != None and res["ResponseMetadata"]["HTTPStatusCode"] == 200:
os.remove(local_file_path)
try:
s3_client.upload_file(local_file_path, s3_key)
sub_request = SubscriptionRequest.objects.filter(request_id=request_id)[0]
sub_request.S3_uploaded = True
sub_request.save()
except Exception as e:
logger.error(f"Unable to set S3: {e}")
print(f"Unable to set S3: {e}")
return
else:
logger.info(f"S3 is not available, skipping,...")
@app.task(name='remove_local_file')
def remove_local_file(local_file_path, request_id):
print(f"[INFO] Removing local file: {local_file_path}, ...")
try:
os.remove(local_file_path)
except Exception as e:
logger.info(f"Unable to remove local file: {e}")
@app.task(name='upload_obj_to_s3')
def upload_obj_to_s3(byte_obj, s3_key):
if s3_client.s3_client is not None:

15
cope2n-api/fwd_api/celery_worker/worker.py
View File

@ -16,6 +16,17 @@ app: Celery = Celery(
broker_transport_options={'confirm_publish': False},
)
app.config_from_object("django.conf:settings", namespace="CELERY")
app.autodiscover_tasks()
@app.on_after_finalize.connect
def setup_periodic_tasks(sender, **kwargs):
from fwd_api.bg_tasks.clean_local_files import clean_media
sender.add_periodic_task(
10.0, clean_media.s(), expires=120.0
)
app.conf.update({
'task_queues':
[
@ -26,6 +37,8 @@ app.conf.update({
Queue('do_pdf'),
Queue('upload_file_to_s3'),
Queue('upload_obj_to_s3'),
Queue('remove_local_file'),
],
'task_routes': {
@ -40,6 +53,8 @@ app.conf.update({
'do_pdf': {'queue': "do_pdf"},
'upload_file_to_s3': {'queue': "upload_file_to_s3"},
'upload_obj_to_s3': {'queue': "upload_obj_to_s3"},
'upload_file_to_s3': {'queue': "upload_file_to_s3"},
'remove_local_file': {'queue': "remove_local_file"},
}
})

18
cope2n-api/fwd_api/migrations/0158_subscriptionrequest_s3_uploaded.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.3 on 2023-12-22 10:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0157_alter_subscriptionrequest_created_at'),
]
operations = [
migrations.AddField(
model_name='subscriptionrequest',
name='S3_uploaded',
field=models.BooleanField(default=False),
),
]

14
cope2n-api/fwd_api/migrations/0162_merge_20231225_1439.py Normal file
View File

@ -0,0 +1,14 @@
# Generated by Django 4.1.3 on 2023-12-25 07:39
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0158_subscriptionrequest_s3_uploaded'),
('fwd_api', '0161_alter_subscriptionrequest_ai_inference_start_time_and_more'),
]
operations = [
]

2
cope2n-api/fwd_api/models/SubscriptionRequest.py
View File

@ -18,7 +18,9 @@ class SubscriptionRequest(models.Model):
subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE)
created_at = models.DateTimeField(default=timezone.now, db_index=True)
updated_at = models.DateTimeField(auto_now=True)
S3_uploaded = models.BooleanField(default=False)
is_test_request = models.BooleanField(default=False)
S3_uploaded = models.BooleanField(default=False)
preprocessing_time = models.FloatField(default=-1)
ai_inference_start_time = models.FloatField(default=0)

4
cope2n-api/fwd_api/utils/file.py
View File

@ -156,9 +156,11 @@ def resize_and_save_file(file_name: str, rq: SubscriptionRequest, file: Temporar
def save_to_S3(file_name, rq, local_file_path):
try:
file_path = get_folder_path(rq)
request_id = rq.request_id
assert len(file_path.split("/")) >= 2, "file_path must have at least process type and request id"
s3_key = os.path.join(file_path.split("/")[-2], file_path.split("/")[-1], file_name)
c_connector.upload_file_to_s3((local_file_path, s3_key))
c_connector.upload_file_to_s3((local_file_path, s3_key, request_id))
c_connector.remove_local_file((local_file_path, request_id))
return s3_key
except Exception as e:
print(f"[ERROR]: {e}")

3
cope2n-api/requirements.txt
View File

@ -49,4 +49,5 @@ djangorestframework-xml==2.0.0
boto3==1.29.7
imagesize==1.4.1
pdf2image==1.16.3
redis==5.0.1
redis==5.0.1
django-celery-beat==2.5.0

7
deploy_images.py
View File

@ -8,8 +8,8 @@ from datetime import datetime
BASH_FILE = './deploy_images.sh'
S3_ENDPOINT = ""
S3_ACCESS_KEY = "AKIA3AFPFVWZD77UACHE"
S3_SECRET_KEY = "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"
S3_ACCESS_KEY = "secret"
S3_SECRET_KEY = "secret"
S3_BUCKET = "ocr-deployment-config"
class MinioS3Client:
@ -77,7 +77,8 @@ def deploy():
# Define the variable
tag = str(random_hash()[:8])
now = datetime.now()
tag = tag + "_" + str(now.strftime("%d%m%y%H%M%S"))
# tag = tag + "_" + str(now.strftime("%d%m%y%H%M%S"))
tag = "4cae5134_261223123256"
print(tag)
# Execute the Bash script with the variable as a command-line argument

29
deploy_images.sh
View File

@ -1,24 +1,25 @@
#!/bin/bash
set -e
tag=$1
echo "[INFO] Tag received from Python: $tag"
echo "[INFO] Pushing AI image with tag: $tag..."
docker compose -f docker-compose-dev.yml build cope2n-fi-sbt
docker tag sidp/cope2n-ai-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag}
docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag}
# echo "[INFO] Pushing AI image with tag: $tag..."
# docker compose -f docker-compose-dev.yml build cope2n-fi-sbt
# docker tag sidp/cope2n-ai-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag}
# docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag}
echo "[INFO] Pushing BE image with tag: $tag..."
docker compose -f docker-compose-dev.yml build be-ctel-sbt
docker tag sidp/cope2n-be-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag}
docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag}
# echo "[INFO] Pushing BE image with tag: $tag..."
# docker compose -f docker-compose-dev.yml build be-ctel-sbt
# docker tag sidp/cope2n-be-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag}
# docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag}
echo "[INFO] Pushing FE image with tag: $tag..."
docker compose -f docker-compose-dev.yml build fe-sbt
docker tag sidp/cope2n-fe-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag}
docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag}
# echo "[INFO] Pushing FE image with tag: $tag..."
# docker compose -f docker-compose-dev.yml build fe-sbt
# docker tag sidp/cope2n-fe-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag}
# docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag}
cp ./docker-compose.yml ./docker-compose_${tag}.yml
cp ./docker-compose-prod.yml ./docker-compose_${tag}.yml
sed -i "s/{{tag}}/$tag/g" ./docker-compose_${tag}.yml
cp .env .env_${tag}
cp .env_prod .env_${tag}

8
docker-compose-dev.yml
View File

@ -76,7 +76,7 @@ services:
volumes:
- ${HOST_MEDIA_FOLDER}:${MEDIA_ROOT}
- BE_static:/app/static
# - ./cope2n-api:/app
- ./cope2n-api:/app
working_dir: /app
depends_on:
db-sbt:
@ -85,6 +85,8 @@ services:
python manage.py migrate &&
python manage.py compilemessages &&
gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
# command: bash -c "tail -f > /dev/null"
minio:
image: minio/minio
environment:
@ -162,7 +164,7 @@ services:
condition: service_started
volumes:
- ${HOST_MEDIA_FOLDER}:${MEDIA_ROOT}
# - ./cope2n-api:/app
- ./cope2n-api:/app
working_dir: /app
command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
@ -171,7 +173,7 @@ services:
db-sbt:
restart: always
mem_reservation: 500m
image: postgres:14.7-alpine
image: postgres:15.4-alpine
volumes:
- ./data/postgres_data:/var/lib/postgresql/data
networks:

195
docker-compose-prod.yml Normal file
View File

@ -0,0 +1,195 @@
# TODO: use docker-compose extend: for compact purpose
version: '3.0'
networks:
ctel-sbt:
driver: bridge
services:
cope2n-fi-sbt:
shm_size: 10gb
mem_limit: 10gb
restart: always
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:{{tag}}
networks:
- ctel-sbt
privileged: true
environment:
- CELERY_BROKER=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672
working_dir: /workspace/cope2n-ai-fi
command: bash run.sh
deploy:
mode: replicated
replicas: 2
# Back-end services
be-ctel-sbt:
environment:
- MEDIA_ROOT=${MEDIA_ROOT}
- DB_ENGINE=${DB_ENGINE}
- DB_SCHEMA=${DB_SCHEMA}
- DB_USER=${DB_USER}
- DB_PASSWORD=${DB_PASSWORD}
- DB_HOST=${DB_HOST}
- DB_PORT=${DB_PUBLIC_PORT}
- DEBUG=${DEBUG}
- CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS}
- BASE_PORT=${BASE_PORT}
- CTEL_KEY=${CTEL_KEY}
- SECRET_KEY=${SECRET_KEY}
- DB_INTERNAL_KEY=${DB_INTERNAL_KEY}
- ALLOWED_HOSTS=${ALLOWED_HOSTS}
- BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672
- BASE_URL=http://be-ctel-sbt:${BASE_PORT}
- BASE_UI_URL=http://fe:${VITE_PORT}
- AUTH_TOKEN_LIFE_TIME=${AUTH_TOKEN_LIFE_TIME}
- IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME}
- INTERNAL_SDS_KEY=${INTERNAL_SDS_KEY}
- FI_USER_NAME=${FI_USER_NAME}
- FI_PASSWORD=${FI_PASSWORD}
- S3_ENDPOINT=${S3_ENDPOINT}
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
- S3_SECRET_KEY=${S3_SECRET_KEY}
- S3_BUCKET_NAME=${S3_BUCKET_NAME}
restart: always
privileged: true # for chmod
mem_limit: 10gb
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}}
networks:
- ctel-sbt
volumes:
- BE_media:${MEDIA_ROOT}
- BE_static:/app/static
working_dir: /app
command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input &&
python manage.py migrate &&
python manage.py compilemessages &&
gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
minio:
image: minio/minio
environment:
- MINIO_ROOT_USER=${S3_ACCESS_KEY}
- MINIO_ROOT_PASSWORD=${S3_SECRET_KEY}
- MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
- MINIO_SECRET_KEY=${S3_SECRET_KEY}
volumes:
- ./data/minio_data:/data
networks:
- ctel-sbt
restart: always
command: server --address :9884 --console-address :9885 /data
profiles: ["local"]
createbuckets:
image: minio/mc
depends_on:
- minio
entrypoint: >
/bin/sh -c "
/usr/bin/mc alias set myminio http://minio:9884 ${S3_ACCESS_KEY} ${S3_SECRET_KEY};
/usr/bin/mc mb myminio/${S3_BUCKET_NAME};
/usr/bin/mc policy set public myminio/${S3_BUCKET_NAME};
exit 0;
"
networks:
- ctel-sbt
profiles: ["local"]
result-cache:
image: redis:6.2-alpine
restart: always
mem_limit: 10gb
command: redis-server --save 20 1 --loglevel warning
networks:
- ctel-sbt
be-celery-sbt:
environment:
- MEDIA_ROOT=${MEDIA_ROOT}
- PYTHONPATH=${PYTHONPATH}:/app # For import module
- PYTHONUNBUFFERED=1 # For show print log
- DB_ENGINE=${DB_ENGINE}
- DB_SCHEMA=${DB_SCHEMA}
- DB_USER=${DB_USER}
- DB_PASSWORD=${DB_PASSWORD}
- DB_HOST=${DB_HOST}
- DB_PORT=${DB_PUBLIC_PORT}
- BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672
- BASE_UI_URL=http://fe:${VITE_PORT}
- DEBUG=${DEBUG}
- DB_INTERNAL_KEY=${DB_INTERNAL_KEY}
- IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME}
- CTEL_KEY=${CTEL_KEY}
- SECRET_KEY=${SECRET_KEY}
- ALLOWED_HOSTS=${ALLOWED_HOSTS}
- S3_ENDPOINT=${S3_ENDPOINT}
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
- S3_SECRET_KEY=${S3_SECRET_KEY}
- S3_BUCKET_NAME=${S3_BUCKET_NAME}
- BASE_URL=http://be-ctel-sbt:${BASE_PORT}
- REDIS_HOST=result-cache
- REDIS_PORT=6379
restart: always
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}}
networks:
- ctel-sbt
depends_on:
rabbitmq-sbt:
condition: service_started
volumes:
- BE_media:${MEDIA_ROOT}
working_dir: /app
command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
# Back-end persistent
db-sbt:
restart: always
mem_reservation: 500m
image: postgres:14.7-alpine
volumes:
- ./data/postgres_data:/var/lib/postgresql/data
networks:
- ctel-sbt
environment:
- POSTGRES_USER=${DB_USER}
- POSTGRES_PASSWORD=${DB_PASSWORD}
- POSTGRES_DB=${DB_SCHEMA}
profiles: ["local"]
rabbitmq-sbt:
mem_reservation: 600m
restart: always
image: rabbitmq:3.10-alpine
working_dir: /workspace/cope2n-api
networks:
- ctel-sbt
environment:
- RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}
- RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}
# Front-end services
fe-sbt:
restart: always
mem_limit: 4gb
shm_size: 10gb
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:{{tag}}
privileged: true
ports:
- 80:80
depends_on:
be-ctel-sbt:
condition: service_started
be-celery-sbt:
condition: service_started
environment:
- VITE_PROXY=http://be-ctel-sbt:${BASE_PORT}
- VITE_API_BASE_URL=http://fe-sbt:80
volumes:
- BE_static:/backend-static
networks:
- ctel-sbt
volumes:
db_data:
BE_static:
BE_media:

13
docker-compose.yml
View File

@ -10,7 +10,7 @@ services:
shm_size: 10gb
mem_limit: 10gb
restart: always
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:{{tag}}
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt
networks:
- ctel-sbt
privileged: true
@ -53,7 +53,7 @@ services:
- S3_BUCKET_NAME=${S3_BUCKET_NAME}
restart: always
mem_limit: 10gb
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}}
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:
networks:
- ctel-sbt
volumes:
@ -62,9 +62,6 @@ services:
- ./cope2n-api:/app
working_dir: /app
depends_on:
db-sbt:
condition: service_started
command: sh -c "chmod -R 777 /app/static; sleep 5; python manage.py collectstatic --no-input &&
python manage.py migrate &&
python manage.py compilemessages &&
@ -134,12 +131,10 @@ services:
- REDIS_HOST=result-cache
- REDIS_PORT=6379
restart: always
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}}
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:
networks:
- ctel-sbt
depends_on:
db-sbt:
condition: service_started
rabbitmq-sbt:
condition: service_started
volumes:
@ -179,7 +174,7 @@ services:
restart: always
mem_limit: 4gb
shm_size: 10gb
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:{{tag}}
image: public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt
privileged: true
ports:
- ${SIDP_SERVICE_PORT:-9881}:80

1
scripts/speedtest_sync.py
View File

@ -163,3 +163,4 @@ num_images = sum(x["num_files"] for x in results if x["success"])
print("Total images:", num_images)
print("Uploading + Processing time: {:.3f}s".format(sum(processing_time) / num_images))
print("--------------------------------------")