diff --git a/.gitignore b/.gitignore index 06e3308..7cffb73 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,10 @@ curl.md cope2n-api/fwd_api/commands/init_database.py /data backup -demo-ocr/ \ No newline at end of file +demo-ocr/ +logs/ +docker-compose_.yml +cope2n-ai-fi/Dockerfile_old_work +*.sql +*.sql +.env_prod diff --git a/cope2n-ai-fi/.dockerignore b/cope2n-ai-fi/.dockerignore index d38e621..b2ac9a7 100755 --- a/cope2n-ai-fi/.dockerignore +++ b/cope2n-ai-fi/.dockerignore @@ -1,6 +1,7 @@ .github .git .vscode +packages/ __pycache__ DataBase/image_temp/ DataBase/json_temp/ diff --git a/cope2n-ai-fi/Dockerfile b/cope2n-ai-fi/Dockerfile index 405796c..13ba941 100755 --- a/cope2n-ai-fi/Dockerfile +++ b/cope2n-ai-fi/Dockerfile @@ -1,40 +1,64 @@ -FROM thucpd2408/env-deskew +FROM pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime -COPY ./packages/cudnn-linux*.tar.xz /tmp/cudnn-linux*.tar.xz +RUN apt-get update && \ + apt-get install -y git gcc g++ ffmpeg libsm6 libxext6 && \ + apt-get -y autoremove && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -RUN tar -xvf /tmp/cudnn-linux*.tar.xz -C /tmp/ \ - && cp /tmp/cudnn-*-archive/include/cudnn*.h /usr/local/cuda/include \ - && cp -P /tmp/cudnn-*-archive/lib/libcudnn* /usr/local/cuda/lib64 \ - && chmod a+r /usr/local/cuda/include/cudnn*.h /usr/local/cuda/lib64/libcudnn* \ - && rm -rf /tmp/cudnn-*-archive +# RUN pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116 --no-cache-dir -RUN apt-get update && apt-get install -y gcc g++ ffmpeg libsm6 libxext6 poppler-utils +RUN pip install -U openmim==0.3.7 --no-cache-dir +RUN mim install mmcv-full==1.7.1 +RUN pip install mmcv==1.6.0 -f https://download.openmmlab.com/mmcv/dst/cu116/torch1.13/index.html --no-cache-dir -WORKDIR /workspace +RUN ln -s /opt/conda/lib/python3.10/site-packages/torch/lib/libcudnn.so.8 /usr/lib/libcudnn.so && \ + ln -s /opt/conda/lib/libcublas.so /usr/lib/libcublas.so +RUN python -m pip install paddlepaddle-gpu==2.4.2.post116 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html --no-cache-dir -# COPY ./modules/ocr_engine/externals/ /workspace/cope2n-ai-fi/modules/ocr_engine/externals/ -# COPY ./modules/ocr_engine/requirements.txt /workspace/cope2n-ai-fi/modules/ocr_engine/requirements.txt -# COPY ./modules/sdsvkvu/ /workspace/cope2n-ai-fi/modules/sdsvkvu/ -# COPY ./requirements.txt /workspace/cope2n-ai-fi/requirements.txt +RUN python -m pip install 'git+https://github.com/facebookresearch/detectron2.git' --no-cache-dir +# Install SDSV packages COPY . /workspace/cope2n-ai-fi -RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsv_dewarp && pip3 install -v -e . -RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtd && pip3 install -v -e . -RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtr && pip3 install -v -e . +RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsv_dewarp && pip3 install -v -e . --no-cache-dir +RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtd && pip3 install -v -e . --no-cache-dir +RUN cd /workspace/cope2n-ai-fi/modules/ocr_engine/externals/sdsvtr && pip3 install -v -e . --no-cache-dir -RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu && pip3 install -v -e . -RUN cd /workspace/cope2n-ai-fi && pip3 install -r requirements.txt +# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsv_dewarp /tmp/sdsv_dewarp +# RUN cd /tmp/sdsv_dewarp && pip install -v -e . --no-cache-dir -RUN rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \ - rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \ - rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11 && \ - ln -s /usr/local/cuda-11.8/targets/x86_64-linux/lib/libcublasLt.so.11 /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \ - ln -s /usr/local/cuda-11.8/targets/x86_64-linux/lib/libcublas.so.11 /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \ - ln -s /usr/local/cuda-11.8/targets/x86_64-linux/lib/libnvblas.so.11 /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11 +# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsvtd /tmp/sdsvtd +# RUN cd /tmp/sdsvtd && pip install -v -e . --no-cache-dir + +# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsvtr /tmp/sdsvtr +# RUN cd /tmp/sdsvtr && pip install -v -e . --no-cache-dir + +# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/requirements.txt /tmp/sdsvocr/requirements.txt +# RUN pip install -r /tmp/sdsvocr/requirements.txt + +# ENV PIP_DEFAULT_TIMEOUT 100 +# COPY ./modules/sdsvkvu /tmp/sdsvkvu +# RUN cd /tmp/sdsvkvu && pip install -v -e . --no-cache-dir + +# COPY ./requirements.txt /tmp/requirements.txt +# RUN pip install -r /tmp/requirements.txt --no-cache-dir + +RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu && pip3 install -v -e . --no-cache-dir +RUN cd /workspace/cope2n-ai-fi && pip3 install -r requirements.txt --no-cache-dir + +# RUN rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \ +# rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \ +# rm -f /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11 && \ +# ln -s /opt/conda/lib/libcublasLt.so /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublasLt.so.11 && \ +# ln -s /opt/conda/lib/libcublas.so /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libcublas.so.11 && \ +# ln -s /opt/conda/lib/libnvblas.so /usr/local/lib/python3.10/dist-packages/nvidia/cublas/lib/libnvblas.so.11 + + +WORKDIR /workspace ENV PYTHONPATH="." ENV TZ="Asia/Ho_Chi_Minh" -CMD [ "sh", "run.sh"] +CMD [ "sh", "run.sh"] \ No newline at end of file diff --git a/cope2n-ai-fi/configs/sdsap_sbt/configs.py b/cope2n-ai-fi/configs/sdsap_sbt/configs.py index 8d40ba0..e2d43b2 100755 --- a/cope2n-ai-fi/configs/sdsap_sbt/configs.py +++ b/cope2n-ai-fi/configs/sdsap_sbt/configs.py @@ -6,7 +6,7 @@ ocr_engine = { "device": device }, "recognizer": { - "version": "/workspace/cope2n-ai-fi/weights/models/sdsvtr/hub/sbt_20231218_e116_sdstr.pth", + "version": "/workspace/cope2n-ai-fi/weights/models/ocr_engine/sdsvtr/hub/sbt_20231218_e116_sdstr.pth", "device": device }, "deskew": { diff --git a/cope2n-ai-fi/modules/sdsvkie_ b/cope2n-ai-fi/modules/sdsvkie_ deleted file mode 160000 index 8349a89..0000000 --- a/cope2n-ai-fi/modules/sdsvkie_ +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8349a89de7fd8c9e6958907047d16bdc23a77adf diff --git a/cope2n-ai-fi/modules/sdsvkv_oldu b/cope2n-ai-fi/modules/sdsvkv_oldu deleted file mode 160000 index bdba044..0000000 --- a/cope2n-ai-fi/modules/sdsvkv_oldu +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bdba044bb2eacac7c7cfe0e0f321196d03b681f6 diff --git a/cope2n-ai-fi/modules/sdsvtd_ b/cope2n-ai-fi/modules/sdsvtd_ deleted file mode 160000 index a9a796f..0000000 --- a/cope2n-ai-fi/modules/sdsvtd_ +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a9a796f843f6ad99977a8dcba02d65fe75225574 diff --git a/cope2n-ai-fi/modules/sdsvtr_ b/cope2n-ai-fi/modules/sdsvtr_ deleted file mode 160000 index 2c788e9..0000000 --- a/cope2n-ai-fi/modules/sdsvtr_ +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2c788e9e2d7fe369869d474fbb22426a4ca84590 diff --git a/cope2n-ai-fi/requirements.txt b/cope2n-ai-fi/requirements.txt index 7c388a4..faf39a9 100755 --- a/cope2n-ai-fi/requirements.txt +++ b/cope2n-ai-fi/requirements.txt @@ -10,4 +10,5 @@ easydict imagesize==1.4.1 pdf2image==1.16.3 -redis==5.0.1 \ No newline at end of file +redis==5.0.1 +celery==5.3.6 \ No newline at end of file diff --git a/cope2n-api/fwd_api/api/ctel_view.py b/cope2n-api/fwd_api/api/ctel_view.py index e262cba..e2664e0 100755 --- a/cope2n-api/fwd_api/api/ctel_view.py +++ b/cope2n-api/fwd_api/api/ctel_view.py @@ -292,6 +292,7 @@ class CtelViewSet(viewsets.ViewSet): if serializer.data[0]["data"].get("status", 200) != 200: continue + serializer.data[0]["request_id"] = rq_id return Response(status=status.HTTP_200_OK, data=serializer.data[0]) raise ServiceTimeoutException(excArgs=f"{rq_id}") diff --git a/cope2n-api/fwd_api/celery_worker/client_connector.py b/cope2n-api/fwd_api/celery_worker/client_connector.py index 10557ed..3ff8f88 100755 --- a/cope2n-api/fwd_api/celery_worker/client_connector.py +++ b/cope2n-api/fwd_api/celery_worker/client_connector.py @@ -29,9 +29,9 @@ class CeleryConnector: 'process_manulife_invoice': {'queue': "invoice_manulife"}, 'process_sbt_invoice': {'queue': "invoice_sbt"}, 'do_pdf': {'queue': "do_pdf"}, - 'upload_file_to_s3': {'queue': "upload_file_to_s3"}, + 'upload_file_to_s3': {'queue': "upload_file_to_s3"}, 'upload_obj_to_s3': {'queue': "upload_obj_to_s3"}, - + 'remove_local_file': {'queue': "remove_local_file"}, } app = Celery( @@ -45,6 +45,8 @@ class CeleryConnector: return self.send_task('upload_file_to_s3', args) def upload_obj_to_s3(self, args): return self.send_task('upload_obj_to_s3', args) + def remove_local_file(self, args): + return self.send_task('remove_local_file', args, countdown=280) # nearest execution of this task in 280 seconds def process_fi(self, args): return self.send_task('process_fi_invoice', args) def process_fi_result(self, args): @@ -86,10 +88,9 @@ class CeleryConnector: def process_invoice_sbt(self, args): return self.send_task('process_sbt_invoice', args) - def send_task(self, name=None, args=None): + def send_task(self, name=None, args=None, countdown=None): if name not in self.task_routes or 'queue' not in self.task_routes[name]: raise GeneralException("System") - return self.app.send_task(name, args, queue=self.task_routes[name]['queue'], expires=300) - + return self.app.send_task(name, args, queue=self.task_routes[name]['queue'], expires=300, countdown=countdown) c_connector = CeleryConnector() diff --git a/cope2n-api/fwd_api/celery_worker/internal_task.py b/cope2n-api/fwd_api/celery_worker/internal_task.py index 2dd96bb..d1376d5 100755 --- a/cope2n-api/fwd_api/celery_worker/internal_task.py +++ b/cope2n-api/fwd_api/celery_worker/internal_task.py @@ -119,14 +119,28 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files): @app.task(name='upload_file_to_s3') -def upload_file_to_s3(local_file_path, s3_key): +def upload_file_to_s3(local_file_path, s3_key, request_id): if s3_client.s3_client is not None: - res = s3_client.upload_file(local_file_path, s3_key) - if res != None and res["ResponseMetadata"]["HTTPStatusCode"] == 200: - os.remove(local_file_path) + try: + s3_client.upload_file(local_file_path, s3_key) + sub_request = SubscriptionRequest.objects.filter(request_id=request_id)[0] + sub_request.S3_uploaded = True + sub_request.save() + except Exception as e: + logger.error(f"Unable to set S3: {e}") + print(f"Unable to set S3: {e}") + return else: logger.info(f"S3 is not available, skipping,...") +@app.task(name='remove_local_file') +def remove_local_file(local_file_path, request_id): + print(f"[INFO] Removing local file: {local_file_path}, ...") + try: + os.remove(local_file_path) + except Exception as e: + logger.info(f"Unable to remove local file: {e}") + @app.task(name='upload_obj_to_s3') def upload_obj_to_s3(byte_obj, s3_key): if s3_client.s3_client is not None: diff --git a/cope2n-api/fwd_api/celery_worker/worker.py b/cope2n-api/fwd_api/celery_worker/worker.py index 70d302c..ee497cd 100755 --- a/cope2n-api/fwd_api/celery_worker/worker.py +++ b/cope2n-api/fwd_api/celery_worker/worker.py @@ -16,6 +16,17 @@ app: Celery = Celery( broker_transport_options={'confirm_publish': False}, ) +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() + +@app.on_after_finalize.connect +def setup_periodic_tasks(sender, **kwargs): + from fwd_api.bg_tasks.clean_local_files import clean_media + sender.add_periodic_task( + 10.0, clean_media.s(), expires=120.0 + ) + + app.conf.update({ 'task_queues': [ @@ -26,6 +37,8 @@ app.conf.update({ Queue('do_pdf'), Queue('upload_file_to_s3'), Queue('upload_obj_to_s3'), + Queue('remove_local_file'), + ], 'task_routes': { @@ -40,6 +53,8 @@ app.conf.update({ 'do_pdf': {'queue': "do_pdf"}, 'upload_file_to_s3': {'queue': "upload_file_to_s3"}, 'upload_obj_to_s3': {'queue': "upload_obj_to_s3"}, + 'upload_file_to_s3': {'queue': "upload_file_to_s3"}, + 'remove_local_file': {'queue': "remove_local_file"}, } }) diff --git a/cope2n-api/fwd_api/migrations/0158_subscriptionrequest_s3_uploaded.py b/cope2n-api/fwd_api/migrations/0158_subscriptionrequest_s3_uploaded.py new file mode 100644 index 0000000..97b3810 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0158_subscriptionrequest_s3_uploaded.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2023-12-22 10:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0157_alter_subscriptionrequest_created_at'), + ] + + operations = [ + migrations.AddField( + model_name='subscriptionrequest', + name='S3_uploaded', + field=models.BooleanField(default=False), + ), + ] diff --git a/cope2n-api/fwd_api/migrations/0162_merge_20231225_1439.py b/cope2n-api/fwd_api/migrations/0162_merge_20231225_1439.py new file mode 100644 index 0000000..b8d5f4b --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0162_merge_20231225_1439.py @@ -0,0 +1,14 @@ +# Generated by Django 4.1.3 on 2023-12-25 07:39 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0158_subscriptionrequest_s3_uploaded'), + ('fwd_api', '0161_alter_subscriptionrequest_ai_inference_start_time_and_more'), + ] + + operations = [ + ] diff --git a/cope2n-api/fwd_api/models/SubscriptionRequest.py b/cope2n-api/fwd_api/models/SubscriptionRequest.py index e8a4456..780df18 100755 --- a/cope2n-api/fwd_api/models/SubscriptionRequest.py +++ b/cope2n-api/fwd_api/models/SubscriptionRequest.py @@ -18,7 +18,9 @@ class SubscriptionRequest(models.Model): subscription = models.ForeignKey(Subscription, on_delete=models.CASCADE) created_at = models.DateTimeField(default=timezone.now, db_index=True) updated_at = models.DateTimeField(auto_now=True) + S3_uploaded = models.BooleanField(default=False) is_test_request = models.BooleanField(default=False) + S3_uploaded = models.BooleanField(default=False) preprocessing_time = models.FloatField(default=-1) ai_inference_start_time = models.FloatField(default=0) diff --git a/cope2n-api/fwd_api/utils/file.py b/cope2n-api/fwd_api/utils/file.py index 29c15b9..4a94b39 100644 --- a/cope2n-api/fwd_api/utils/file.py +++ b/cope2n-api/fwd_api/utils/file.py @@ -156,9 +156,11 @@ def resize_and_save_file(file_name: str, rq: SubscriptionRequest, file: Temporar def save_to_S3(file_name, rq, local_file_path): try: file_path = get_folder_path(rq) + request_id = rq.request_id assert len(file_path.split("/")) >= 2, "file_path must have at least process type and request id" s3_key = os.path.join(file_path.split("/")[-2], file_path.split("/")[-1], file_name) - c_connector.upload_file_to_s3((local_file_path, s3_key)) + c_connector.upload_file_to_s3((local_file_path, s3_key, request_id)) + c_connector.remove_local_file((local_file_path, request_id)) return s3_key except Exception as e: print(f"[ERROR]: {e}") diff --git a/cope2n-api/requirements.txt b/cope2n-api/requirements.txt index 327bea3..c204228 100755 --- a/cope2n-api/requirements.txt +++ b/cope2n-api/requirements.txt @@ -49,4 +49,5 @@ djangorestframework-xml==2.0.0 boto3==1.29.7 imagesize==1.4.1 pdf2image==1.16.3 -redis==5.0.1 \ No newline at end of file +redis==5.0.1 +django-celery-beat==2.5.0 \ No newline at end of file diff --git a/deploy_images.py b/deploy_images.py index 5f1804f..2de1774 100644 --- a/deploy_images.py +++ b/deploy_images.py @@ -8,8 +8,8 @@ from datetime import datetime BASH_FILE = './deploy_images.sh' S3_ENDPOINT = "" -S3_ACCESS_KEY = "AKIA3AFPFVWZD77UACHE" -S3_SECRET_KEY = "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" +S3_ACCESS_KEY = "secret" +S3_SECRET_KEY = "secret" S3_BUCKET = "ocr-deployment-config" class MinioS3Client: @@ -77,7 +77,8 @@ def deploy(): # Define the variable tag = str(random_hash()[:8]) now = datetime.now() - tag = tag + "_" + str(now.strftime("%d%m%y%H%M%S")) + # tag = tag + "_" + str(now.strftime("%d%m%y%H%M%S")) + tag = "4cae5134_261223123256" print(tag) # Execute the Bash script with the variable as a command-line argument diff --git a/deploy_images.sh b/deploy_images.sh index 688822a..b0b8195 100755 --- a/deploy_images.sh +++ b/deploy_images.sh @@ -1,24 +1,25 @@ #!/bin/bash +set -e tag=$1 echo "[INFO] Tag received from Python: $tag" -echo "[INFO] Pushing AI image with tag: $tag..." -docker compose -f docker-compose-dev.yml build cope2n-fi-sbt -docker tag sidp/cope2n-ai-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag} -docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag} +# echo "[INFO] Pushing AI image with tag: $tag..." +# docker compose -f docker-compose-dev.yml build cope2n-fi-sbt +# docker tag sidp/cope2n-ai-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag} +# docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:${tag} -echo "[INFO] Pushing BE image with tag: $tag..." -docker compose -f docker-compose-dev.yml build be-ctel-sbt -docker tag sidp/cope2n-be-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag} -docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag} +# echo "[INFO] Pushing BE image with tag: $tag..." +# docker compose -f docker-compose-dev.yml build be-ctel-sbt +# docker tag sidp/cope2n-be-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag} +# docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:${tag} -echo "[INFO] Pushing FE image with tag: $tag..." -docker compose -f docker-compose-dev.yml build fe-sbt -docker tag sidp/cope2n-fe-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag} -docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag} +# echo "[INFO] Pushing FE image with tag: $tag..." +# docker compose -f docker-compose-dev.yml build fe-sbt +# docker tag sidp/cope2n-fe-fi-sbt:latest public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag} +# docker push public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:${tag} -cp ./docker-compose.yml ./docker-compose_${tag}.yml +cp ./docker-compose-prod.yml ./docker-compose_${tag}.yml sed -i "s/{{tag}}/$tag/g" ./docker-compose_${tag}.yml -cp .env .env_${tag} +cp .env_prod .env_${tag} diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 96e890a..45b7e4f 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -76,7 +76,7 @@ services: volumes: - ${HOST_MEDIA_FOLDER}:${MEDIA_ROOT} - BE_static:/app/static - # - ./cope2n-api:/app + - ./cope2n-api:/app working_dir: /app depends_on: db-sbt: @@ -85,6 +85,8 @@ services: python manage.py migrate && python manage.py compilemessages && gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod + # command: bash -c "tail -f > /dev/null" + minio: image: minio/minio environment: @@ -162,7 +164,7 @@ services: condition: service_started volumes: - ${HOST_MEDIA_FOLDER}:${MEDIA_ROOT} - # - ./cope2n-api:/app + - ./cope2n-api:/app working_dir: /app command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" @@ -171,7 +173,7 @@ services: db-sbt: restart: always mem_reservation: 500m - image: postgres:14.7-alpine + image: postgres:15.4-alpine volumes: - ./data/postgres_data:/var/lib/postgresql/data networks: diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml new file mode 100644 index 0000000..35106c7 --- /dev/null +++ b/docker-compose-prod.yml @@ -0,0 +1,195 @@ + +# TODO: use docker-compose extend: for compact purpose +version: '3.0' +networks: + ctel-sbt: + driver: bridge + +services: + cope2n-fi-sbt: + shm_size: 10gb + mem_limit: 10gb + restart: always + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:{{tag}} + networks: + - ctel-sbt + privileged: true + environment: + - CELERY_BROKER=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672 + working_dir: /workspace/cope2n-ai-fi + command: bash run.sh + deploy: + mode: replicated + replicas: 2 + # Back-end services + be-ctel-sbt: + environment: + - MEDIA_ROOT=${MEDIA_ROOT} + - DB_ENGINE=${DB_ENGINE} + - DB_SCHEMA=${DB_SCHEMA} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=${DB_HOST} + - DB_PORT=${DB_PUBLIC_PORT} + - DEBUG=${DEBUG} + - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS} + - BASE_PORT=${BASE_PORT} + - CTEL_KEY=${CTEL_KEY} + - SECRET_KEY=${SECRET_KEY} + - DB_INTERNAL_KEY=${DB_INTERNAL_KEY} + - ALLOWED_HOSTS=${ALLOWED_HOSTS} + - BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672 + - BASE_URL=http://be-ctel-sbt:${BASE_PORT} + - BASE_UI_URL=http://fe:${VITE_PORT} + - AUTH_TOKEN_LIFE_TIME=${AUTH_TOKEN_LIFE_TIME} + - IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME} + - INTERNAL_SDS_KEY=${INTERNAL_SDS_KEY} + - FI_USER_NAME=${FI_USER_NAME} + - FI_PASSWORD=${FI_PASSWORD} + - S3_ENDPOINT=${S3_ENDPOINT} + - S3_ACCESS_KEY=${S3_ACCESS_KEY} + - S3_SECRET_KEY=${S3_SECRET_KEY} + - S3_BUCKET_NAME=${S3_BUCKET_NAME} + restart: always + privileged: true # for chmod + mem_limit: 10gb + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}} + networks: + - ctel-sbt + volumes: + - BE_media:${MEDIA_ROOT} + - BE_static:/app/static + working_dir: /app + command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && + python manage.py migrate && + python manage.py compilemessages && + gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod + minio: + image: minio/minio + environment: + - MINIO_ROOT_USER=${S3_ACCESS_KEY} + - MINIO_ROOT_PASSWORD=${S3_SECRET_KEY} + - MINIO_ACCESS_KEY=${S3_ACCESS_KEY} + - MINIO_SECRET_KEY=${S3_SECRET_KEY} + volumes: + - ./data/minio_data:/data + networks: + - ctel-sbt + restart: always + command: server --address :9884 --console-address :9885 /data + profiles: ["local"] + + createbuckets: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + /usr/bin/mc alias set myminio http://minio:9884 ${S3_ACCESS_KEY} ${S3_SECRET_KEY}; + /usr/bin/mc mb myminio/${S3_BUCKET_NAME}; + /usr/bin/mc policy set public myminio/${S3_BUCKET_NAME}; + exit 0; + " + networks: + - ctel-sbt + profiles: ["local"] + + result-cache: + image: redis:6.2-alpine + restart: always + mem_limit: 10gb + command: redis-server --save 20 1 --loglevel warning + networks: + - ctel-sbt + + be-celery-sbt: + environment: + - MEDIA_ROOT=${MEDIA_ROOT} + - PYTHONPATH=${PYTHONPATH}:/app # For import module + - PYTHONUNBUFFERED=1 # For show print log + - DB_ENGINE=${DB_ENGINE} + - DB_SCHEMA=${DB_SCHEMA} + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=${DB_HOST} + - DB_PORT=${DB_PUBLIC_PORT} + - BROKER_URL=amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@rabbitmq-sbt:5672 + - BASE_UI_URL=http://fe:${VITE_PORT} + - DEBUG=${DEBUG} + - DB_INTERNAL_KEY=${DB_INTERNAL_KEY} + - IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME} + - CTEL_KEY=${CTEL_KEY} + - SECRET_KEY=${SECRET_KEY} + - ALLOWED_HOSTS=${ALLOWED_HOSTS} + - S3_ENDPOINT=${S3_ENDPOINT} + - S3_ACCESS_KEY=${S3_ACCESS_KEY} + - S3_SECRET_KEY=${S3_SECRET_KEY} + - S3_BUCKET_NAME=${S3_BUCKET_NAME} + - BASE_URL=http://be-ctel-sbt:${BASE_PORT} + - REDIS_HOST=result-cache + - REDIS_PORT=6379 + restart: always + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}} + networks: + - ctel-sbt + depends_on: + rabbitmq-sbt: + condition: service_started + volumes: + - BE_media:${MEDIA_ROOT} + + working_dir: /app + command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" + + # Back-end persistent + db-sbt: + restart: always + mem_reservation: 500m + image: postgres:14.7-alpine + volumes: + - ./data/postgres_data:/var/lib/postgresql/data + networks: + - ctel-sbt + environment: + - POSTGRES_USER=${DB_USER} + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=${DB_SCHEMA} + profiles: ["local"] + + rabbitmq-sbt: + mem_reservation: 600m + restart: always + image: rabbitmq:3.10-alpine + working_dir: /workspace/cope2n-api + networks: + - ctel-sbt + environment: + - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} + - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} + + # Front-end services + fe-sbt: + restart: always + mem_limit: 4gb + shm_size: 10gb + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:{{tag}} + privileged: true + ports: + - 80:80 + depends_on: + be-ctel-sbt: + condition: service_started + be-celery-sbt: + condition: service_started + environment: + - VITE_PROXY=http://be-ctel-sbt:${BASE_PORT} + - VITE_API_BASE_URL=http://fe-sbt:80 + volumes: + - BE_static:/backend-static + networks: + - ctel-sbt + +volumes: + db_data: + BE_static: + BE_media: diff --git a/docker-compose.yml b/docker-compose.yml index b677be1..50a9ef7 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: shm_size: 10gb mem_limit: 10gb restart: always - image: public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt:{{tag}} + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-ai-fi-sbt networks: - ctel-sbt privileged: true @@ -53,7 +53,7 @@ services: - S3_BUCKET_NAME=${S3_BUCKET_NAME} restart: always mem_limit: 10gb - image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}} + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt: networks: - ctel-sbt volumes: @@ -62,9 +62,6 @@ services: - ./cope2n-api:/app working_dir: /app - depends_on: - db-sbt: - condition: service_started command: sh -c "chmod -R 777 /app/static; sleep 5; python manage.py collectstatic --no-input && python manage.py migrate && python manage.py compilemessages && @@ -134,12 +131,10 @@ services: - REDIS_HOST=result-cache - REDIS_PORT=6379 restart: always - image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}} + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt: networks: - ctel-sbt depends_on: - db-sbt: - condition: service_started rabbitmq-sbt: condition: service_started volumes: @@ -179,7 +174,7 @@ services: restart: always mem_limit: 4gb shm_size: 10gb - image: public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt:{{tag}} + image: public.ecr.aws/v4n9y6r8/sidp/cope2n-fe-fi-sbt privileged: true ports: - ${SIDP_SERVICE_PORT:-9881}:80 diff --git a/scripts/speedtest_sync.py b/scripts/speedtest_sync.py index 38cbffa..1f5d84b 100644 --- a/scripts/speedtest_sync.py +++ b/scripts/speedtest_sync.py @@ -163,3 +163,4 @@ num_images = sum(x["num_files"] for x in results if x["success"]) print("Total images:", num_images) print("Uploading + Processing time: {:.3f}s".format(sum(processing_time) / num_images)) print("--------------------------------------") +