From adc8c8bbbd719cbb4cd380413357b8cb39fcd2c6 Mon Sep 17 00:00:00 2001 From: dx-tan Date: Wed, 6 Mar 2024 15:09:29 +0700 Subject: [PATCH] Fix: #67 --- cope2n-api/fwd_api/api/accuracy_view.py | 5 +- .../celery_worker/process_report_tasks.py | 4 +- cope2n-api/fwd_api/utils/accuracy.py | 142 +++++++++--------- cope2n-api/fwd_api/utils/cache.py | 1 - cope2n-fe/src/utils/metric-format.ts | 2 +- docker-compose-dev.yml | 16 +- 6 files changed, 89 insertions(+), 81 deletions(-) diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index 9d14f51..9cc6676 100644 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -20,6 +20,7 @@ from ..utils.file import download_from_S3, dict2xlsx, save_report_to_S3, build_S from ..utils.redis import RedisUtils from ..utils.process import string_to_boolean from ..utils.cache import get_cache, set_cache +from fwd_api.constant.common import FileCategory from ..request.ReportCreationSerializer import ReportCreationSerializer from ..utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long from ..utils.report import aggregate_overview @@ -110,7 +111,7 @@ class AccuracyViewSet(viewsets.ViewSet): subsidiary = request.data.get("subsidiary", "all") subsidiary = map_subsidiary_long_to_short(subsidiary) - base_query = Q() + base_query = Q(status=200) if start_date_str or end_date_str: try: start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%d') # We care only about day precision only @@ -670,7 +671,7 @@ class AccuracyViewSet(viewsets.ViewSet): data = [] files = [] - subscription_request_files = SubscriptionRequestFile.objects.filter(request=subscription_request.id) + subscription_request_files = SubscriptionRequestFile.objects.filter(request=subscription_request.id, file_category=FileCategory.Origin.value) for subscription_request_file in subscription_request_files: sub = subscription_request.subscription diff --git a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py index aeab9b1..d56f978 100644 --- a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py @@ -261,6 +261,8 @@ def make_a_report_2(report_id, query_set): report.average_OCR_time = {"invoice": time_cost["invoice"](), "imei": time_cost["imei"](), "invoice_count": time_cost["invoice"].count, "imei_count": time_cost["imei"].count} + report.average_OCR_time["invoice"] = 0 if report.average_OCR_time["invoice"] is None else report.average_OCR_time["invoice"] + report.average_OCR_time["imei"] = 0 if report.average_OCR_time["imei"] is None else report.average_OCR_time["imei"] report.average_OCR_time["avg"] = (report.average_OCR_time["invoice"]*report.average_OCR_time["invoice_count"] + report.average_OCR_time["imei"]*report.average_OCR_time["imei_count"])/(report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) if (report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) > 0 else None report.number_imei_transaction = transaction_att.get("imei", 0) @@ -311,7 +313,7 @@ def make_a_report_2(report_id, query_set): for key in keys: if report_fine_data[i][key]: for x_key in report_fine_data[i][key].keys(): - report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 + report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 if report_fine_data[i][key][x_key] is not None else None data_workbook = dict2xlsx(report_fine_data, _type='report') overview_filename = query_set["subsidiary"] + "_" + query_set["report_overview_duration"] + ".xlsx" local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT) diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index 5651de3..fb17f13 100644 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -115,84 +115,86 @@ class ReportAccumulateByRequest: @staticmethod def update_total(total, report_file): - total["total_images"] += 1 - total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 - total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 - doc_type = "imei" - if report_file.doc_type in ["imei", "invoice", "all"]: - doc_type = report_file.doc_type - else: - print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") - total["num_imei"] += 1 if doc_type == "imei" else 0 - total["num_invoice"] += 1 if doc_type == "invoice" else 0 + if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS: + total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 + total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 + total["total_images"] += 1 + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + total["num_imei"] += 1 if doc_type == "imei" else 0 + total["num_invoice"] += 1 if doc_type == "invoice" else 0 - if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : - total["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) - total["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) - total["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) - total["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) - elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: - total["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) - total["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) - total["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) - total["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) + if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : + total["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) + total["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) + total["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) + total["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) + elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: + total["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) + total["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) + total["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) + total["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - total["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - total["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + total["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + total["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) - if not total["average_processing_time"].get(report_file.doc_type, None): - print(f"[WARM]: Weird doctype: {report_file.doc_type}") - total["average_processing_time"][report_file.doc_type] = IterAvg() - total["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 + if not total["average_processing_time"].get(report_file.doc_type, None): + print(f"[WARM]: Weird doctype: {report_file.doc_type}") + total["average_processing_time"][report_file.doc_type] = IterAvg() + total["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 - doc_type = "imei" - if report_file.doc_type in ["imei", "invoice", "all"]: - doc_type = report_file.doc_type - else: - print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") - total["usage"]["imei"] += 1 if doc_type == "imei" else 0 - total["usage"]["invoice"] += 1 if doc_type == "invoice" else 0 - total["usage"]["total_images"] += 1 - total["review_progress"].append(report_file.review_status) + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + total["usage"]["imei"] += 1 if doc_type == "imei" else 0 + total["usage"]["invoice"] += 1 if doc_type == "invoice" else 0 + total["usage"]["total_images"] += 1 + total["review_progress"].append(report_file.review_status) return total @staticmethod def update_day(day_data, report_file): - day_data["total_images"] += 1 - day_data["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 - day_data["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 - doc_type = "imei" - if report_file.doc_type in ["imei", "invoice", "all"]: - doc_type = report_file.doc_type - else: - print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") - day_data["num_imei"] += 1 if doc_type == "imei" else 0 - day_data["num_invoice"] += 1 if doc_type == "invoice" else 0 - day_data["report_files"].append(report_file) - - if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0: - day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) - day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) - day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) - day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) - elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: - day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) - day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) - day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) - day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) + if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS: + day_data["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 + day_data["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 + day_data["total_images"] += 1 + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + day_data["num_imei"] += 1 if doc_type == "imei" else 0 + day_data["num_invoice"] += 1 if doc_type == "invoice" else 0 + day_data["report_files"].append(report_file) + + if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0: + day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) + day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) + day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) + day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) + elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: + day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) + day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) + day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) + day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) - if not day_data["average_processing_time"].get(report_file.doc_type, None): - print(f"[WARM]: Weird doctype: {report_file.doc_type}") - day_data["average_processing_time"][report_file.doc_type] = IterAvg() - day_data["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 - day_data["review_progress"].append(report_file.review_status) + if not day_data["average_processing_time"].get(report_file.doc_type, None): + print(f"[WARM]: Weird doctype: {report_file.doc_type}") + day_data["average_processing_time"][report_file.doc_type] = IterAvg() + day_data["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 + day_data["review_progress"].append(report_file.review_status) return day_data def add(self, request, report_files): @@ -255,6 +257,8 @@ class ReportAccumulateByRequest: _average_OCR_time = {"invoice": self.data[month][1][day]["average_processing_time"]["invoice"](), "imei": self.data[month][1][day]["average_processing_time"]["imei"](), "invoice_count": self.data[month][1][day]["average_processing_time"]["invoice"].count, "imei_count": self.data[month][1][day]["average_processing_time"]["imei"].count} + _average_OCR_time["invoice"] = 0 if _average_OCR_time["invoice"] is None else _average_OCR_time["invoice"] + _average_OCR_time["imei"] = 0 if _average_OCR_time["imei"] is None else _average_OCR_time["imei"] _average_OCR_time["avg"] = (_average_OCR_time["invoice"]*_average_OCR_time["invoice_count"] + _average_OCR_time["imei"]*_average_OCR_time["imei_count"])/(_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) if (_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) > 0 else None acumulated_acc = {"feedback_accuracy": {}, "reviewed_accuracy": {}} @@ -527,6 +531,8 @@ class IterAvg: self.avg = (self.avg*(self.count-count) + avg*count)/(self.count) def __call__(self): + if self.count == 0: + return None return self.avg def validate_feedback_file(feedback, predict): diff --git a/cope2n-api/fwd_api/utils/cache.py b/cope2n-api/fwd_api/utils/cache.py index ea68137..4885d94 100644 --- a/cope2n-api/fwd_api/utils/cache.py +++ b/cope2n-api/fwd_api/utils/cache.py @@ -10,7 +10,6 @@ def set_cache(key, value): this_cache.save() return this_cache - def get_cache(key): value = {} cache = Caching.objects.filter(key=key) diff --git a/cope2n-fe/src/utils/metric-format.ts b/cope2n-fe/src/utils/metric-format.ts index c106edc..79a6340 100644 --- a/cope2n-fe/src/utils/metric-format.ts +++ b/cope2n-fe/src/utils/metric-format.ts @@ -1,5 +1,5 @@ export const formatPercent = (value: number, floatingPoint: number = 1) => { - if (value === 0) { + if (value === null || value === undefined) { return '-'; } if (value < 100.0) { diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index ffb2d60..3426a9d 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -89,12 +89,12 @@ services: depends_on: db-sbt: condition: service_started - command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && - python manage.py makemigrations && - python manage.py migrate && - python manage.py compilemessages && - gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod - # command: bash -c "tail -f > /dev/null" + # command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && + # python manage.py makemigrations && + # python manage.py migrate && + # python manage.py compilemessages && + # gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod + command: bash -c "tail -f > /dev/null" minio: image: minio/minio @@ -179,8 +179,8 @@ services: - ./cope2n-api:/app working_dir: /app - command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" - # command: bash -c "tail -f > /dev/null" + # command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" + command: bash -c "tail -f > /dev/null" # Back-end persistent db-sbt: