Add: refill S3 image by redemption script
This commit is contained in:
parent
74431c3bc9
commit
d27a7ec9d2
@ -0,0 +1,174 @@
|
|||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
from tqdm import tqdm
|
||||||
|
from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest
|
||||||
|
from fwd_api.exception.exceptions import InvalidException
|
||||||
|
from fwd_api.utils.s3 import MinioS3Client
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import os
|
||||||
|
import glob
|
||||||
|
import traceback
|
||||||
|
import copy
|
||||||
|
import json
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
IMAGE_DIRS = ["/external_data/SGGE", "/external_data/zipsGwp1", "/external_data/zipsGwp2", "/external_data/zipsGwp3", "/external_data/zipsGwp4", "/external_data/zipsEvoucher"]
|
||||||
|
# IMAGE_DIRS = ["/external_data/SGGE"]
|
||||||
|
image_extensions = ['*.jpg', '*.jpeg', '*.png', '*.gif']
|
||||||
|
pdf_extensions = ['*.pdf']
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
help = 'Refactor database for image level'
|
||||||
|
|
||||||
|
def add_arguments(self, parser):
|
||||||
|
# Add your command-line arguments here
|
||||||
|
parser.add_argument('start', type=str, help='start date, sample: 2023-01-02T00:00:00+0700')
|
||||||
|
parser.add_argument('end', type=str, help='end date, sample: 2023-01-03T00:00:00+0700')
|
||||||
|
|
||||||
|
def _prepare_data(self, redemtion_dirs):
|
||||||
|
prepared_data = {} # {"redemption_id": {"image_paths": []}, "pages": 1}
|
||||||
|
for redemtion_dir in redemtion_dirs:
|
||||||
|
redemptions = os.listdir(redemtion_dir)
|
||||||
|
for redemption in redemptions:
|
||||||
|
files_in_dir = []
|
||||||
|
for ext in image_extensions + pdf_extensions:
|
||||||
|
files_in_dir.extend(glob.glob(os.path.join(redemtion_dir, redemption, ext)))
|
||||||
|
redemption = redemption.replace("Data", "")
|
||||||
|
if prepared_data.get(redemption, None):
|
||||||
|
prepared_data[redemption]["image_paths"] += files_in_dir
|
||||||
|
prepared_data[redemption]["pages"] += len(files_in_dir)
|
||||||
|
else:
|
||||||
|
prepared_data[redemption] = {"image_paths": files_in_dir, "pages": len(files_in_dir)}
|
||||||
|
|
||||||
|
return prepared_data
|
||||||
|
|
||||||
|
def _add_error(self, result, error, redemption_id):
|
||||||
|
if not result.get("Error", None):
|
||||||
|
result["Error"] = {}
|
||||||
|
if result["Error"].get(error, None):
|
||||||
|
result["Error"][error].add(redemption_id)
|
||||||
|
else:
|
||||||
|
result["Error"][error] = set([redemption_id])
|
||||||
|
|
||||||
|
def _add_info(self, result, info, redemption_id):
|
||||||
|
if not result.get("Info", None):
|
||||||
|
result["Info"] = {}
|
||||||
|
if result["Info"].get(info, None):
|
||||||
|
result["Info"][info].add(redemption_id)
|
||||||
|
else:
|
||||||
|
result["Info"][info] = set([redemption_id])
|
||||||
|
|
||||||
|
def _add_warning(self, result, warn, redemption_id):
|
||||||
|
if not result.get("Warning", None):
|
||||||
|
result["Warning"] = {}
|
||||||
|
if result["Warning"].get(warn, None):
|
||||||
|
result["Warning"][warn].add(redemption_id)
|
||||||
|
else:
|
||||||
|
result["Warning"][warn] = set([redemption_id])
|
||||||
|
|
||||||
|
def _try_find_doc_type(self, file_paths):
|
||||||
|
doc_types = {"invoice": [],
|
||||||
|
"imei": [],
|
||||||
|
"undefined": []}
|
||||||
|
for file_path in file_paths:
|
||||||
|
if "invoice" in os.path.basename(file_path):
|
||||||
|
doc_types["invoice"].append(file_path)
|
||||||
|
elif "imei" in os.path.basename(file_path):
|
||||||
|
doc_types["imei"].append(file_path)
|
||||||
|
else:
|
||||||
|
doc_types["undefined"].append(file_path)
|
||||||
|
return doc_types
|
||||||
|
|
||||||
|
def process_request(self, request, data, result, s3_client):
|
||||||
|
if not request.predict_result:
|
||||||
|
# self.stdout.write(self.style.WARNING(f"Key predict_result not found in {request.request_id}"))
|
||||||
|
return
|
||||||
|
if request.predict_result.get("status", 200) != 200:
|
||||||
|
# self.stdout.write(self.style.WARNING(f"Not a sucess request {request.request_id}"))
|
||||||
|
return
|
||||||
|
# Find to coresponding redemption_ID
|
||||||
|
self._add_info(result, "[OCR]: redemptions", request.redemption_id)
|
||||||
|
if request.redemption_id not in list(data.keys()):
|
||||||
|
self._add_error(result, "[OCR]: Not found redemption_ID", request.redemption_id)
|
||||||
|
return
|
||||||
|
if request.pages != data[request.redemption_id]["pages"]:
|
||||||
|
self._add_error(result, "[SBT]: Mismatch files number in a request", request.redemption_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
file_paths_by_doc_type = self._try_find_doc_type(data[request.redemption_id]["image_paths"])
|
||||||
|
if len(file_paths_by_doc_type["undefined"]) > 0:
|
||||||
|
self._add_warning(result, "[SBT]: Undefined doc type", request.redemption_id)
|
||||||
|
|
||||||
|
if len(request.request_id.split(".")[0].split("_")) < 2:
|
||||||
|
return
|
||||||
|
images = SubscriptionRequestFile.objects.filter(request=request, file_category="Origin")
|
||||||
|
|
||||||
|
for i, image in enumerate(images):
|
||||||
|
if image.doc_type not in ["imei", "invoice"]:
|
||||||
|
self._add_error(result, "[OCR]: Weird doc type", request.redemption_id)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if len(file_paths_by_doc_type[image.doc_type]) > 0:
|
||||||
|
local_file_path = file_paths_by_doc_type[image.doc_type].pop(0)
|
||||||
|
else:
|
||||||
|
local_file_path = file_paths_by_doc_type["undefined"].pop(0)
|
||||||
|
predir = "sbt_invoice"
|
||||||
|
s3_key = os.path.join(predir, request.request_id, image.file_name)
|
||||||
|
# s3_client.upload_file(local_file_path, s3_key)
|
||||||
|
result['total'] += 1
|
||||||
|
self._add_info(result, "[OCR]: Success", request.redemption_id)
|
||||||
|
except IndexError as e:
|
||||||
|
self._add_error(result, "[OCR]: Mismatch doc type", request.redemption_id)
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
self.stdout.write(self.style.ERROR(f"Request: {request.request_id} failed with {e}"))
|
||||||
|
print(traceback.format_exc())
|
||||||
|
result['failed'] += 1
|
||||||
|
self._add_info(result, "[OCR]: Failed", request.redemption_id)
|
||||||
|
continue
|
||||||
|
data.pop(request.redemption_id, None)
|
||||||
|
|
||||||
|
def handle(self, *args, **options):
|
||||||
|
start = options['start']
|
||||||
|
end = options['end']
|
||||||
|
result = {'total':0,
|
||||||
|
'failed':0}
|
||||||
|
# TODO: redemption ID is not null on filter
|
||||||
|
if start or end:
|
||||||
|
try:
|
||||||
|
start_date = timezone.datetime.strptime(start, '%Y-%m-%dT%H:%M:%S%z') # We care only about day precision only
|
||||||
|
end_date = timezone.datetime.strptime(end, '%Y-%m-%dT%H:%M:%S%z')
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[INFO]: start: {start}")
|
||||||
|
print(f"[INFO]: end: {end}")
|
||||||
|
raise InvalidException(excArgs="Date format")
|
||||||
|
subcription_iter = SubscriptionRequest.objects.filter(created_at__range=(start_date, end_date), redemption_id__isnull=False)
|
||||||
|
else:
|
||||||
|
subcription_iter = SubscriptionRequest.objects.filter(redemption_id__isnull=False)
|
||||||
|
print(f"[INFO]: Preparing data for filling up...")
|
||||||
|
prepared_data = self._prepare_data(IMAGE_DIRS)
|
||||||
|
print(f"[INFO]: Prepared data, total: {len(list(prepared_data.keys()))}")
|
||||||
|
prepared_data_copy = copy.deepcopy(prepared_data)
|
||||||
|
s3_client = MinioS3Client(
|
||||||
|
# endpoint='http://107.120.133.27:9884',
|
||||||
|
access_key='AKIA3AFPFVWZHTZHB6FW',
|
||||||
|
secret_key='qYmEkfnO8ltQ7n9GfnF8+HRcfOsbXhx0YSNOLxdW',
|
||||||
|
bucket_name='ocr-sds'
|
||||||
|
)
|
||||||
|
# file = open("modified.txt", "w")
|
||||||
|
for request in tqdm(subcription_iter.iterator()):
|
||||||
|
self.process_request(request, prepared_data_copy, result, s3_client)
|
||||||
|
# file.close()
|
||||||
|
self.stdout.write(self.style.SUCCESS('Sample Django management command executed successfully! total/failed: {}/{}'.format(result['total'], result['failed'])))
|
||||||
|
# print(f"[INFO]: result: {result}")
|
||||||
|
for err in result.get("Error", []):
|
||||||
|
print("[INFO]: Error: {}: {}".format(err, len(result["Error"][err])))
|
||||||
|
result["Error"][err] = list(result["Error"][err])
|
||||||
|
for info in result.get("Info", []):
|
||||||
|
print("[INFO]: Info: {}: {}".format(info, len(result["Info"][info])))
|
||||||
|
result["Info"][info] = list(result["Info"][info])
|
||||||
|
for warn in result.get("Warning", []):
|
||||||
|
print("[INFO]: Warning: {}: {}".format(warn, len(result["Warning"][warn])))
|
||||||
|
result["Warning"][warn] = list(result["Warning"][warn])
|
||||||
|
with open("result.json", "w") as outfile:
|
||||||
|
json.dump(result, outfile)
|
||||||
Loading…
Reference in New Issue
Block a user