From d1f0da4493dda89cc317afd05a241e312e8c2011 Mon Sep 17 00:00:00 2001 From: TannedCung Date: Fri, 5 Jul 2024 15:22:49 +0700 Subject: [PATCH 1/2] Add: filebeat --- Deploy.md | 4 ++++ deploy_images.py | 6 ++++++ docker-compose-filebeat.yml | 22 ++++++++++++++++++++++ filebeat.yml | 23 +++++++++++++++++++++++ pull.sh | 13 +++++++++++++ push.sh | 7 +++++++ 6 files changed, 75 insertions(+) create mode 100644 Deploy.md create mode 100644 docker-compose-filebeat.yml create mode 100644 filebeat.yml create mode 100644 pull.sh create mode 100644 push.sh diff --git a/Deploy.md b/Deploy.md new file mode 100644 index 0000000..ca9b5cc --- /dev/null +++ b/Deploy.md @@ -0,0 +1,4 @@ +### Push code and model to the build server +rsync -rv -e "ssh -i dev-sbt.pem" sbt-idp ubuntu@ec2-13-212-141-4.ap-southeast-1.compute.amazonaws.com:/home/ubuntu/TannedCung + + diff --git a/deploy_images.py b/deploy_images.py index bf99177..24f6e04 100644 --- a/deploy_images.py +++ b/deploy_images.py @@ -96,10 +96,16 @@ def deploy(): s3_compose_file = f"docker-yaml-files/docker-compose_{tag}.yml" s3_env_file = f"env-files/.env_{tag}" + s3_filebeat_compose = f"docker_compose_filebeat_file/docker-compose-filebeat.yml" + local_filebeat_compose = f"./docker-compose-filebeat.yml" + s3_filebeat_config = f"filebeat_file/filebeat.yml" + local_filebeat_config = f"./filebeat.yml" print(f"[INFO]: Uploading compose and env file to s3...") s3_client.upload_file(local_compose_file, s3_compose_file) s3_client.upload_file(local_env_file, s3_env_file) + s3_client.upload_file(local_filebeat_compose, s3_filebeat_compose) + s3_client.upload_file(local_filebeat_config, s3_filebeat_config) print(f"[INFO]: Deleting files...") os.remove(local_compose_file) diff --git a/docker-compose-filebeat.yml b/docker-compose-filebeat.yml new file mode 100644 index 0000000..e0ec42b --- /dev/null +++ b/docker-compose-filebeat.yml @@ -0,0 +1,22 @@ +version: "3" +services: + filebeat: + image: "docker.elastic.co/beats/filebeat:7.2.0" + container_name: filebeat + user: root + volumes: + - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /data/docker-data:/var/lib/docker:ro + - /var/run/docker.sock:/var/run/docker.sock + + networks: + - elastic + +volumes: + filebeat-data: + name: filebeat-data + driver: local + +networks: + elastic: + driver: bridge \ No newline at end of file diff --git a/filebeat.yml b/filebeat.yml new file mode 100644 index 0000000..ced5758 --- /dev/null +++ b/filebeat.yml @@ -0,0 +1,23 @@ +filebeat.inputs: +- type: container + paths: + - '/var/lib/docker/containers/*/*.log' + +processors: +- add_docker_metadata: + host: "unix:///var/run/docker.sock" + +- decode_json_fields: + fields: ["message"] + target: "json" + overwrite_keys: true + +output.elasticsearch: + hosts: ["18.141.135.47:9200"] + username: "elastic" + password: "changeme" + indices: + - index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}" + +logging.json: true +logging.metrics.enabled: false \ No newline at end of file diff --git a/pull.sh b/pull.sh new file mode 100644 index 0000000..ff53cb0 --- /dev/null +++ b/pull.sh @@ -0,0 +1,13 @@ +git config --global http.proxy http://107.120.133.64:8080 +git config --global https.proxy http://107.120.133.64:8080 + +git pull origin main + +# Update submodules to remote +git submodule update # update to origin +ORIGINAL_CWD=$(pwd) +cd ./cope2n-ai-fi/modules/sdsvkvu && git submodule update && cd $ORIGINAL_CWD +cd ./cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr && git submodule update && cd $ORIGINAL_CWD +cd ./cope2n-api/fwd_api/utils/sdsvkvu && git submodule update && cd $ORIGINAL_CWD +cd ./cope2n-api/fwd_api/utils/sdsvkvu/sdsvkvu/externals/sdsvocr && git submodule update && cd $ORIGINAL_CWD + diff --git a/push.sh b/push.sh new file mode 100644 index 0000000..e84333f --- /dev/null +++ b/push.sh @@ -0,0 +1,7 @@ +git config --global --unset https.proxy +git config --global --unset http.proxy + +git push prod main + +git config --global http.proxy http://107.120.133.64:8080 +git config --global https.proxy http://107.120.133.64:8080 \ No newline at end of file From 6a4d8c7ff6a7dc73f8af7502d184e68fd1c83be2 Mon Sep 17 00:00:00 2001 From: TannedCung Date: Fri, 5 Jul 2024 15:25:54 +0700 Subject: [PATCH 2/2] Remove: secret host --- Deploy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Deploy.md b/Deploy.md index ca9b5cc..c328108 100644 --- a/Deploy.md +++ b/Deploy.md @@ -1,4 +1,4 @@ ### Push code and model to the build server -rsync -rv -e "ssh -i dev-sbt.pem" sbt-idp ubuntu@ec2-13-212-141-4.ap-southeast-1.compute.amazonaws.com:/home/ubuntu/TannedCung +rsync -rv -e "ssh -i dev-sbt.pem" sbt-idp ubuntu@:/home/ubuntu/TannedCung