From bceb842cad0d69fc762832055d09950b9617298e Mon Sep 17 00:00:00 2001 From: dx-tan Date: Mon, 4 Mar 2024 15:38:38 +0700 Subject: [PATCH] Fix: #61 --- api-cronjob/Dockerfile | 5 ++- api-cronjob/run.py | 71 +++++++++++++++++++++++++++++++ cope2n-api/scripts/script.py | 2 +- deploy_images.sh | 5 +++ docker-compose-dev.yml | 22 ++++++++++ docker-compose-prod.yml | 21 +++++++++ scripts/crawl_database_by_time.py | 6 +-- 7 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 api-cronjob/run.py diff --git a/api-cronjob/Dockerfile b/api-cronjob/Dockerfile index 606072c..e2bbccc 100644 --- a/api-cronjob/Dockerfile +++ b/api-cronjob/Dockerfile @@ -2,8 +2,9 @@ FROM python:3.9-slim WORKDIR /app -COPY script.py . +COPY run.py . RUN apt-get update && apt-get -y install curl +RUN pip install requests -CMD [ "python", "script.py" ] \ No newline at end of file +CMD [ "python", "-u", "run.py" ] \ No newline at end of file diff --git a/api-cronjob/run.py b/api-cronjob/run.py new file mode 100644 index 0000000..3a2f225 --- /dev/null +++ b/api-cronjob/run.py @@ -0,0 +1,71 @@ +import os +import time +import requests +from datetime import datetime + +# Get the proxy URL from the environment variable +interval = 60*60*1 # 1 minute +update_cost = int(60*2) +proxy_url = os.getenv('PROXY', "localhost") +user = os.getenv('ADMIN_USER_NAME', "") +password = os.getenv('ADMIN_PASSWORD', "") + +# Define the login API URL +login_url = f'{proxy_url}/api/ctel/login/' +login_token = None + +# Define the login credentials +login_credentials = { + 'username': user, + 'password': password +} + +# Define the command to call the update API +update_url = f'{proxy_url}/api/ctel/make_report/' +update_data = { + 'is_daily_report': True, + 'report_overview_duration': '', + 'subsidiary': None +} + +"report_overview_duration" + +# def update_report(login_token, report_overview_duration=["30d", "7d"], subsidiary=["all", "SEAU", "SESP", "SME", "SEPCO", "TSE", "SEIN"]): +def update_report(login_token, report_overview_duration=["7d", "30d"], subsidiary=["SEAO", "SEAU", "SESP", "SME", "SEPCO", "TSE", "SEIN"]): + headers = {'Authorization': login_token} + for dur in report_overview_duration: + for sub in subsidiary: + update_data["report_overview_duration"] = dur + update_data["subsidiary"] = sub + update_response = requests.post(update_url, data=update_data, headers=headers) + print("[INFO]: update_response at {} by {} - {} with status {}".format(datetime.now(), dur, sub, update_response.status_code)) + update_response.raise_for_status() + time.sleep(update_cost) + +# Define the interval in seconds between API calls +# time.sleep(60) + +while True: + # Call the login API and retrieve the login token + if not login_token: + login_response = requests.post(login_url, data=login_credentials) + # login_response.raise_for_status() + if login_response.status_code == 200: + login_token = login_response.json()['token'] + print("[INFO] relogged in at {}".format(datetime.now())) + + # Call the update API + try: + update_report(login_token) + except Exception as e: + print(f"[ERROR]: {e}") + print(f"[ERROR]: Failed to update_response, retrying...") + login_response = requests.post(login_url, data=login_credentials) + # login_response.raise_for_status() + if login_response.status_code == 200: + login_token = login_response.json()['token'] + print("[INFO] relogged in at {}".format(datetime.now())) + update_report(login_token) + + # Wait for the specified interval + time.sleep(interval) \ No newline at end of file diff --git a/cope2n-api/scripts/script.py b/cope2n-api/scripts/script.py index f6f0c64..e255257 100644 --- a/cope2n-api/scripts/script.py +++ b/cope2n-api/scripts/script.py @@ -30,7 +30,7 @@ update_data = { "report_overview_duration" # def update_report(login_token, report_overview_duration=["30d", "7d"], subsidiary=["all", "SEAU", "SESP", "SME", "SEPCO", "TSE", "SEIN"]): -def update_report(login_token, report_overview_duration=["7d", "30d"], subsidiary=["SEAO", "SEAU", "SESP", "SME", "SEPCO", "TSE", "SEIN"]): +def update_report(login_token, report_overview_duration=["30d", "7d"], subsidiary=["SEAO", "SEAU", "SESP", "SME", "SEPCO", "TSE", "SEIN"]): headers = {'Authorization': login_token} for dur in report_overview_duration: for sub in subsidiary: diff --git a/deploy_images.sh b/deploy_images.sh index 3b57e42..3266429 100755 --- a/deploy_images.sh +++ b/deploy_images.sh @@ -30,6 +30,11 @@ docker tag sidp/cope2n-fe-fi-sbt:latest public.ecr.aws/sdsrv/sidp/cope2n-fe-fi-s docker push public.ecr.aws/sdsrv/sidp/cope2n-fe-fi-sbt:${tag} # docker push public.ecr.aws/sdsrv/sidp/cope2n-fe-fi-sbt:production +echo "[INFO] Pushing API caller image with tag: $tag..." +docker compose -f docker-compose-dev.yml build dashboard_refresh +docker tag sidp/api-caller-sbt:latest public.ecr.aws/sdsrv/sidp/api-caller-sbt:${tag} +docker push public.ecr.aws/sdsrv/sidp/api-caller-sbt:${tag} + cp ./docker-compose-prod.yml ./docker-compose_${tag}.yml sed -i "s/{{tag}}/$tag/g" ./docker-compose_${tag}.yml cp .env_prod .env_${tag} \ No newline at end of file diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 4de33ee..57d8a25 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -72,6 +72,11 @@ services: - S3_SECRET_KEY=${S3_SECRET_KEY} - S3_BUCKET_NAME=${S3_BUCKET_NAME} restart: always + healthcheck: + test: ["CMD-SHELL", "curl --fail http://localhost:9000/api/ctel/healthcheck || exit 1"] + interval: 10s + timeout: 5s + retries: 50 ports: - 9000:9000 networks: @@ -228,6 +233,23 @@ services: - BE_static:/backend-static networks: - ctel-sbt + + dashboard_refresh: + build: + context: api-cronjob + dockerfile: Dockerfile + image: sidp/api-caller-sbt:latest + environment: + - PROXY=http://be-ctel-sbt:9000 + - ADMIN_USER_NAME=${ADMIN_USER_NAME} + - ADMIN_PASSWORD=${ADMIN_PASSWORD} + depends_on: + be-ctel-sbt: + condition: service_healthy + restart: always + networks: + - ctel-sbt + volumes: db_data: diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index cbc1e68..a482a89 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -53,6 +53,11 @@ services: - S3_SECRET_KEY=${S3_SECRET_KEY} - S3_BUCKET_NAME=${S3_BUCKET_NAME} restart: always + healthcheck: + test: ["CMD-SHELL", "curl --fail http://localhost:9000/api/ctel/healthcheck || exit 1"] + interval: 10s + timeout: 5s + retries: 50 privileged: true # for chmod mem_limit: 10gb image: public.ecr.aws/v4n9y6r8/sidp/cope2n-be-fi-sbt:{{tag}} @@ -189,6 +194,22 @@ services: - BE_static:/backend-static networks: - ctel-sbt + + dashboard_refresh: + build: + context: api-cronjob + dockerfile: Dockerfile + image: sidp/api-caller-sbt:{{tag}} + environment: + - PROXY=http://be-ctel-sbt:9000 + - ADMIN_USER_NAME=${ADMIN_USER_NAME} + - ADMIN_PASSWORD=${ADMIN_PASSWORD} + depends_on: + be-ctel-sbt: + condition: service_healthy + restart: always + networks: + - ctel-sbt volumes: db_data: diff --git a/scripts/crawl_database_by_time.py b/scripts/crawl_database_by_time.py index 2671181..449e3f6 100644 --- a/scripts/crawl_database_by_time.py +++ b/scripts/crawl_database_by_time.py @@ -10,9 +10,9 @@ from dotenv import load_dotenv load_dotenv("../.env_prod") -OUTPUT_NAME = "all_0219_0226" -START_DATE = datetime(2024, 2, 19, tzinfo=timezone('Asia/Ho_Chi_Minh')) -END_DATE = datetime(2024, 2, 27, tzinfo=timezone('Asia/Ho_Chi_Minh')) +OUTPUT_NAME = "all_0226_0304" +START_DATE = datetime(2024, 2, 26, tzinfo=timezone('Asia/Ho_Chi_Minh')) +END_DATE = datetime(2024, 3, 4, tzinfo=timezone('Asia/Ho_Chi_Minh')) # Database connection details db_host = os.environ.get('DB_HOST', "")