From e6ec2004c617302b77f0f0401ae51052b4ff1dd5 Mon Sep 17 00:00:00 2001 From: dx-tan Date: Wed, 21 Feb 2024 12:46:41 +0700 Subject: [PATCH] Update: request_list API --- .gitignore | 1 + cope2n-api/fwd_api/api/accuracy_view.py | 43 ++++++++++---- .../fwd_api/celery_worker/internal_task.py | 3 +- .../celery_worker/process_report_tasks.py | 2 +- cope2n-api/fwd_api/utils/accuracy.py | 59 +++++++++++-------- cope2n-api/fwd_api/utils/file.py | 6 +- cope2n-api/fwd_api/utils/sdsvkvu | 2 +- docker-compose-dev.yml | 4 +- unset_env.sh | 12 ++++ 9 files changed, 87 insertions(+), 45 deletions(-) create mode 100644 unset_env.sh diff --git a/.gitignore b/.gitignore index 451c961..b5cd87e 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ Jan.csv *.csv cope2n-api/reviewed/date.xlsx cope2n-api/reviewed/retailer.xlsx +/scripts/* diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index 87b608f..afd0628 100644 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -43,7 +43,7 @@ class AccuracyViewSet(viewsets.ViewSet): default='2024-01-10T00:00:00+0700', ), OpenApiParameter( - name='include_test', + name='includes_test', location=OpenApiParameter.QUERY, description='Whether to include test record or not', type=OpenApiTypes.BOOL, @@ -53,7 +53,14 @@ class AccuracyViewSet(viewsets.ViewSet): location=OpenApiParameter.QUERY, description='Which records to be query', type=OpenApiTypes.STR, - enum=['reviewed', 'not reviewed', 'all'], + enum=['reviewed', 'not_reviewed', 'all'], + ), + OpenApiParameter( + name='subsidiary', + location=OpenApiParameter.QUERY, + description='Which subsidiary to be included', + type=OpenApiTypes.STR, + enum=list(settings.SUBS.keys()), ), OpenApiParameter( name='request_id', @@ -94,15 +101,27 @@ class AccuracyViewSet(viewsets.ViewSet): request_id = request.GET.get('request_id', None) redemption_id = request.GET.get('redemption_id', None) is_reviewed = request.GET.get('is_reviewed', None) - include_test = request.GET.get('include_test', False) + include_test = request.GET.get('includes_test', False) + subsidiary = request.data.get("subsidiary", "all") + subsidiary = map_subsidiary_long_to_short(subsidiary) - try: - start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%dT%H:%M:%S%z') - end_date = timezone.datetime.strptime(end_date_str, '%Y-%m-%dT%H:%M:%S%z') - except ValueError: - raise InvalidException(excArgs="Date format") + base_query = Q() + if start_date_str or end_date_str: + try: + start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%d') # We care only about day precision only + end_date = timezone.datetime.strptime(end_date_str, '%Y-%m-%d') + # Round: + # end_date_str to the beginning of the next day + # start_date_str to the start of the date + start_date = timezone.make_aware(start_date) + end_date = timezone.make_aware(end_date) + + start_date_str = start_date.strftime('%Y-%m-%dT%H:%M:%S%z') # inside logic will include second precision with timezone for calculation + end_date_str = (end_date + timezone.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%S%z') + base_query &= Q(created_at__range=(start_date, end_date)) + except Exception as e: + raise InvalidException(excArgs="Date format") - base_query = Q(created_at__range=(start_date, end_date)) if request_id: base_query &= Q(request_id=request_id) if redemption_id: @@ -113,17 +132,19 @@ class AccuracyViewSet(viewsets.ViewSet): if include_test: # base_query = ~base_query base_query.children = base_query.children[:-1] - elif isinstance(include_test, bool): if include_test: base_query = ~base_query if isinstance(is_reviewed, str): if is_reviewed == "reviewed": base_query &= Q(is_reviewed=True) - elif is_reviewed == "not reviewed": + elif is_reviewed == "not_reviewed": base_query &= Q(is_reviewed=False) elif is_reviewed == "all": pass + if isinstance(subsidiary, str): + if subsidiary and subsidiary.lower().replace(" ", "")!="all": + base_query &= Q(redemption_id__startswith=subsidiary) subscription_requests = SubscriptionRequest.objects.filter(base_query).order_by('created_at') diff --git a/cope2n-api/fwd_api/celery_worker/internal_task.py b/cope2n-api/fwd_api/celery_worker/internal_task.py index 53d2277..07de96a 100755 --- a/cope2n-api/fwd_api/celery_worker/internal_task.py +++ b/cope2n-api/fwd_api/celery_worker/internal_task.py @@ -259,9 +259,10 @@ def upload_feedback_to_s3(local_file_path, s3_key, feedback_id): logger.info(f"S3 is not available, skipping,...") @app.task(name='upload_report_to_s3') -def upload_report_to_s3(local_file_path, s3_key, report_id): +def upload_report_to_s3(local_file_path, s3_key, report_id, delay): if s3_client.s3_client is not None: try: + time.sleep(delay) s3_client.upload_file(local_file_path, s3_key) report = Report.objects.filter(report_id=report_id)[0] report.S3_uploaded = True diff --git a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py index 52578a1..6755432 100644 --- a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py @@ -287,7 +287,7 @@ def make_a_report_2(report_id, query_set): data = extract_report_detail_list(report_files, lower=True) data_workbook = dict2xlsx(data, _type='report_detail') local_workbook = save_workbook_file(report.report_id + ".xlsx", report, data_workbook) - s3_key=save_report_to_S3(report.report_id, local_workbook) + s3_key=save_report_to_S3(report.report_id, local_workbook, 5) if query_set["is_daily_report"]: # Save overview dashboard # multiple accuracy by 100 diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index 11b2d9f..36a1e27 100644 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -552,7 +552,7 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True) "Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])), "OCR Image Accuracy": report_file.acc, "OCR Image Speed (seconds)": report_file.time_cost, - "Reviewed?": "No", + "Is Reviewed": report_file.is_reviewed, "Bad Image Reasons": report_file.bad_image_reason, "Countermeasures": report_file.counter_measures, "IMEI_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("imei_number", [None])), @@ -801,9 +801,6 @@ def calculate_a_request(report, request): images = SubscriptionRequestFile.objects.filter(request=request, file_category=FileCategory.Origin.value) report_files = [] for image in images: - if image.reason in settings.ACC_EXCLUDE_RESEASONS: - continue - status, att = calculate_subcription_file(image) if status != 200: continue @@ -821,8 +818,6 @@ def calculate_a_request(report, request): _sub = map_subsidiary_short_to_long(request.redemption_id[:2]) else: print(f"[WARM]: empty redemption_id, check request: {request.request_id}") - if att["is_bad_image"]: - request_att["bad_image_list"].append(image.file_name) new_report_file = ReportFile(report=report, subsidiary=_sub, correspond_request_id=request.request_id, @@ -835,35 +830,47 @@ def calculate_a_request(report, request): reviewed_accuracy=acc_maximize_list_values(att["acc"]["reviewed"]), acc=att["avg_acc"], is_bad_image=att["is_bad_image"], + is_reviewed="Yes" if request.is_reviewed else "No", time_cost=image.processing_time, bad_image_reason=image.reason, counter_measures=image.counter_measures, error="|".join(att["err"]) ) report_files.append(new_report_file) + _att = copy.deepcopy(att) + if att["is_bad_image"] or image.reason in settings.ACC_EXCLUDE_RESEASONS: + request_att["bad_image_list"].append(image.file_name) + # TODO: Exclude bad image accuracy from average accuracy + _att["avg_acc"] = None + for t in ["feedback", "reviewed"]: + for k in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + _att["acc"][t][k] = [] + + if request_att["time_cost"].get(image.doc_type, None): request_att["time_cost"][image.doc_type].append(image.processing_time) else: request_att["time_cost"][image.doc_type] = [image.processing_time] + try: - request_att["acc"]["feedback"]["imei_number"] += att["acc"]["feedback"]["imei_number"] - request_att["acc"]["feedback"]["purchase_date"] += att["acc"]["feedback"]["purchase_date"] - request_att["acc"]["feedback"]["retailername"] += att["acc"]["feedback"]["retailername"] - request_att["acc"]["feedback"]["sold_to_party"] += att["acc"]["feedback"]["sold_to_party"] + request_att["acc"]["feedback"]["imei_number"] += _att["acc"]["feedback"]["imei_number"] + request_att["acc"]["feedback"]["purchase_date"] += _att["acc"]["feedback"]["purchase_date"] + request_att["acc"]["feedback"]["retailername"] += _att["acc"]["feedback"]["retailername"] + request_att["acc"]["feedback"]["sold_to_party"] += _att["acc"]["feedback"]["sold_to_party"] - request_att["acc"]["reviewed"]["imei_number"] += att["acc"]["reviewed"]["imei_number"] - request_att["acc"]["reviewed"]["purchase_date"] += att["acc"]["reviewed"]["purchase_date"] - request_att["acc"]["reviewed"]["retailername"] += att["acc"]["reviewed"]["retailername"] - request_att["acc"]["reviewed"]["sold_to_party"] += att["acc"]["reviewed"]["sold_to_party"] + request_att["acc"]["reviewed"]["imei_number"] += _att["acc"]["reviewed"]["imei_number"] + request_att["acc"]["reviewed"]["purchase_date"] += _att["acc"]["reviewed"]["purchase_date"] + request_att["acc"]["reviewed"]["retailername"] += _att["acc"]["reviewed"]["retailername"] + request_att["acc"]["reviewed"]["sold_to_party"] += _att["acc"]["reviewed"]["sold_to_party"] - request_att["acc"]["acumulated"]["imei_number"] += att["acc"]["reviewed"]["imei_number"] if att["acc"]["reviewed"]["imei_number"] else att["acc"]["feedback"]["imei_number"] - request_att["acc"]["acumulated"]["purchase_date"] += att["acc"]["reviewed"]["purchase_date"] if att["acc"]["reviewed"]["purchase_date"] else att["acc"]["feedback"]["purchase_date"] - request_att["acc"]["acumulated"]["retailername"] += att["acc"]["reviewed"]["retailername"] if att["acc"]["reviewed"]["retailername"] else att["acc"]["feedback"]["retailername"] - request_att["acc"]["acumulated"]["sold_to_party"] += att["acc"]["reviewed"]["sold_to_party"] if att["acc"]["reviewed"]["sold_to_party"] else att["acc"]["feedback"]["sold_to_party"] + request_att["acc"]["acumulated"]["imei_number"] += _att["acc"]["reviewed"]["imei_number"] if att["acc"]["reviewed"]["imei_number"] else att["acc"]["feedback"]["imei_number"] + request_att["acc"]["acumulated"]["purchase_date"] += _att["acc"]["reviewed"]["purchase_date"] if att["acc"]["reviewed"]["purchase_date"] else att["acc"]["feedback"]["purchase_date"] + request_att["acc"]["acumulated"]["retailername"] += _att["acc"]["reviewed"]["retailername"] if att["acc"]["reviewed"]["retailername"] else att["acc"]["feedback"]["retailername"] + request_att["acc"]["acumulated"]["sold_to_party"] += _att["acc"]["reviewed"]["sold_to_party"] if att["acc"]["reviewed"]["sold_to_party"] else att["acc"]["feedback"]["sold_to_party"] - request_att["bad_images"] += int(att["is_bad_image"]) + request_att["bad_images"] += int(_att["is_bad_image"]) request_att["total_images"] += 1 - request_att["err"] += att["err"] + request_att["err"] += _att["err"] except Exception as e: print(f"[ERROR]: failed to calculate request: {request.request_id} - request_file: {image.file_name} because of {e}") continue @@ -907,18 +914,18 @@ def calculate_subcription_file(subcription_request_file): if avg_reviewed is not None: avg_acc = avg_reviewed + att["avg_acc"] = avg_acc if avg_acc < BAD_THRESHOLD: att["is_bad_image"] = True # exclude bad images - for key_name in valid_keys: - att["acc"]["feedback"][key_name] = [] - att["acc"]["reviewed"][key_name] = [] - att["avg_acc"] = None - else: - att["avg_acc"] = avg_acc + # for key_name in valid_keys: + # att["acc"]["feedback"][key_name] = [] + # att["acc"]["reviewed"][key_name] = [] + # att["avg_acc"] = None return 200, att def calculate_attributions(request): # for one request, return in order + # Deprecated acc = {"feedback": {}, "reviewed": {}} # {"feedback": {"retailername": [0.1], "sold_to_party":[0.9], "purchase_date":[0.6], "imei_number":[0.8]}, # "reviewed": {"retailername": [0.1], "sold_to_party":[0.9], "purchase_date":[0.6], "imei_number":[0.8]}} diff --git a/cope2n-api/fwd_api/utils/file.py b/cope2n-api/fwd_api/utils/file.py index 5eeef41..ff613fa 100644 --- a/cope2n-api/fwd_api/utils/file.py +++ b/cope2n-api/fwd_api/utils/file.py @@ -329,10 +329,10 @@ def save_feedback_to_S3(file_name, id, local_file_path): print(f"[ERROR]: {e}") raise ServiceUnavailableException() -def save_report_to_S3(id, local_file_path): +def save_report_to_S3(id, local_file_path, delay=0): try: s3_key = os.path.join("report", local_file_path.split("/")[-2], local_file_path.split("/")[-1]) - c_connector.upload_report_to_s3((local_file_path, s3_key, id)) + c_connector.upload_report_to_s3((local_file_path, s3_key, id, delay)) c_connector.remove_local_file((local_file_path, id)) return s3_key except Exception as e: @@ -444,7 +444,7 @@ def get_value(_dict, keys): print(f"[ERROR]: value: {value}") print(f"[ERROR]: keys: {keys}") - if not value: + if value is None: return "-" elif isinstance(value, list): value = str(value) diff --git a/cope2n-api/fwd_api/utils/sdsvkvu b/cope2n-api/fwd_api/utils/sdsvkvu index b6d4fab..d01de31 160000 --- a/cope2n-api/fwd_api/utils/sdsvkvu +++ b/cope2n-api/fwd_api/utils/sdsvkvu @@ -1 +1 @@ -Subproject commit b6d4fab46f7f8689dd6b050cfbff2faa6a6f3fec +Subproject commit d01de312ab86db554ffa2f1b01396ef8d56b78ed diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 932f381..f6d8386 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -174,8 +174,8 @@ services: - ./cope2n-api:/app working_dir: /app - command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" - # command: bash -c "tail -f > /dev/null" + # command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" + command: bash -c "tail -f > /dev/null" # Back-end persistent db-sbt: diff --git a/unset_env.sh b/unset_env.sh new file mode 100644 index 0000000..aa09799 --- /dev/null +++ b/unset_env.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Load .env file into current shell session +set -a +source .env +set +a + +# Unset each variable defined in .env +while IFS= read -r line || [[ -n "$line" ]]; do + variable=$(echo "$line" | cut -d= -f1) + unset "$variable" +done < .env \ No newline at end of file