Add: filebeat

Deploy.md

@@ -0,0 +1,4 @@
+### Push code and model to the build server
+rsync -rv -e "ssh -i dev-sbt.pem" sbt-idp ubuntu@ec2-13-212-141-4.ap-southeast-1.compute.amazonaws.com:/home/ubuntu/TannedCung
+
+

deploy_images.py

@@ -96,10 +96,16 @@ def deploy():
     s3_compose_file = f"docker-yaml-files/docker-compose_{tag}.yml"
     s3_env_file = f"env-files/.env_{tag}"
 
+    s3_filebeat_compose = f"docker_compose_filebeat_file/docker-compose-filebeat.yml"
+    local_filebeat_compose = f"./docker-compose-filebeat.yml"
+    s3_filebeat_config = f"filebeat_file/filebeat.yml"
+    local_filebeat_config = f"./filebeat.yml"
 
     print(f"[INFO]: Uploading compose and env file to s3...")
     s3_client.upload_file(local_compose_file, s3_compose_file)
     s3_client.upload_file(local_env_file, s3_env_file)
+    s3_client.upload_file(local_filebeat_compose, s3_filebeat_compose)
+    s3_client.upload_file(local_filebeat_config, s3_filebeat_config)
 
     print(f"[INFO]: Deleting files...")
     os.remove(local_compose_file)

docker-compose-filebeat.yml

@@ -0,0 +1,22 @@
+version: "3"
+services:
+  filebeat:
+    image: "docker.elastic.co/beats/filebeat:7.2.0"
+    container_name: filebeat
+    user: root
+    volumes:
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /data/docker-data:/var/lib/docker:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+    networks:
+      - elastic
+
+volumes:
+  filebeat-data:
+    name: filebeat-data
+    driver: local
+
+networks:
+  elastic:
+    driver: bridge

filebeat.yml

@@ -0,0 +1,23 @@
+filebeat.inputs:
+- type: container
+  paths: 
+    - '/var/lib/docker/containers/*/*.log'
+
+processors:
+- add_docker_metadata:
+    host: "unix:///var/run/docker.sock"
+
+- decode_json_fields:
+    fields: ["message"]
+    target: "json"
+    overwrite_keys: true
+
+output.elasticsearch:
+  hosts: ["18.141.135.47:9200"]
+  username: "elastic"
+  password: "changeme"
+  indices:
+    - index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}"
+
+logging.json: true
+logging.metrics.enabled: false

pull.sh

@@ -0,0 +1,13 @@
+git config --global http.proxy http://107.120.133.64:8080
+git config --global https.proxy http://107.120.133.64:8080
+
+git pull origin main
+
+# Update submodules to remote
+git submodule update # update to origin
+ORIGINAL_CWD=$(pwd)
+cd ./cope2n-ai-fi/modules/sdsvkvu && git submodule update && cd $ORIGINAL_CWD
+cd ./cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr && git submodule update && cd $ORIGINAL_CWD
+cd ./cope2n-api/fwd_api/utils/sdsvkvu && git submodule update && cd $ORIGINAL_CWD
+cd ./cope2n-api/fwd_api/utils/sdsvkvu/sdsvkvu/externals/sdsvocr && git submodule update && cd $ORIGINAL_CWD
+

push.sh

@@ -0,0 +1,7 @@
+git config --global --unset https.proxy
+git config --global --unset http.proxy
+
+git push prod main
+
+git config --global http.proxy http://107.120.133.64:8080
+git config --global https.proxy http://107.120.133.64:8080