From 044c668c546adeb653a4a3f81a7f6a6d7731ab4d Mon Sep 17 00:00:00 2001 From: dx-tan Date: Thu, 14 Mar 2024 10:11:20 +0700 Subject: [PATCH] Fix: #79, #80 --- cope2n-api/fwd/settings.py | 2 +- cope2n-api/fwd_api/api/accuracy_view.py | 1 + .../celery_worker/process_report_tasks.py | 21 ++++++++------- cope2n-api/fwd_api/utils/accuracy.py | 26 +++++++++---------- .../report-detail/report-overview-table.tsx | 16 ++++++++++++ .../components/report-detail/report-table.tsx | 13 ++++++++++ document-classification-kv-demo | 1 - 7 files changed, 55 insertions(+), 25 deletions(-) delete mode 160000 document-classification-kv-demo diff --git a/cope2n-api/fwd/settings.py b/cope2n-api/fwd/settings.py index 5e5ed7b..396dc38 100755 --- a/cope2n-api/fwd/settings.py +++ b/cope2n-api/fwd/settings.py @@ -114,7 +114,7 @@ DATABASES = { 'PASSWORD': env.str("DB_PASSWORD", None), 'HOST': env.str("DB_HOST", None), 'PORT': env.str("DB_PORT", None), - 'CONN_MAX_AGE': None, + 'CONN_MAX_AGE': 0, } } diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index 00b354c..6c751bf 100755 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -428,6 +428,7 @@ class AccuracyViewSet(viewsets.ViewSet): "End Date": report.end_at, "No. Requests": report.number_request, "Status": report.status, + "Invoice number Acc": acc["invoice_no"], "Purchase Date Acc": acc["purchase_date"], "Retailer Acc": acc["retailername"], "IMEI Acc": acc["imei_number"], diff --git a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py index 84f0dec..a8f1d7e 100755 --- a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py @@ -3,7 +3,7 @@ import traceback from fwd_api.models import SubscriptionRequest, Report, ReportFile from fwd_api.celery_worker.worker import app from ..utils import s3 as S3Util -from ..utils.accuracy import (update_temp_accuracy, IterAvg, calculate_and_save_subcription_file, +from ..utils.accuracy import (update_temp_accuracy, IterAvg, count_transactions, extract_report_detail_list, calculate_a_request, ReportAccumulateByRequest, create_billing_data) from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3, save_images_to_csv_briefly @@ -78,15 +78,18 @@ def create_accuracy_report(report_id, **kwargs): accuracy = {"feedback" :{"imei_number": IterAvg(), "purchase_date": IterAvg(), "retailername": IterAvg(), - "sold_to_party": IterAvg(),}, + "sold_to_party": IterAvg(), + "invoice_no": IterAvg()}, "reviewed" :{"imei_number": IterAvg(), "purchase_date": IterAvg(), "retailername": IterAvg(), - "sold_to_party": IterAvg(),}, + "sold_to_party": IterAvg(), + "invoice_no": IterAvg()}, "acumulated":{"imei_number": IterAvg(), "purchase_date": IterAvg(), "retailername": IterAvg(), - "sold_to_party": IterAvg(),} + "sold_to_party": IterAvg(), + "invoice_no": IterAvg()} } # {"imei": {"acc": 0.1, count: 1}, ...} time_cost = {"invoice": IterAvg(), "imei": IterAvg()} @@ -123,9 +126,9 @@ def create_accuracy_report(report_id, **kwargs): number_images += request_att["total_images"] number_bad_images += request_att["bad_images"] bad_image_list += request_att["bad_image_list"] - update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]) - update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]) - update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]) + update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=settings.FIELD) + update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=settings.FIELD) + update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=settings.FIELD) time_cost["imei"].add(request_att["time_cost"].get("imei", [])) time_cost["invoice"].add(request_att["time_cost"].get("invoice", [])) @@ -158,7 +161,7 @@ def create_accuracy_report(report_id, **kwargs): "acumulated": {}} for acc_type in ["feedback", "reviewed", "acumulated"]: avg_acc = IterAvg() - for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]: + for key in settings.FIELD: acumulated_acc[acc_type][key] = accuracy[acc_type][key]() acumulated_acc[acc_type][key + "_count"] = accuracy[acc_type][key].count avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"]) @@ -235,7 +238,7 @@ def create_billing_report(report_id, **kwargs): "reviewed": {}, "acumulated": {}} for acc_type in ["feedback", "reviewed", "acumulated"]: - for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]: + for key in settings.FIELD: acumulated_acc[acc_type][key] = None acumulated_acc[acc_type][key + "_count"] = None acumulated_acc[acc_type]["avg"] = None diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index 6ae00d2..4aaaa21 100755 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -38,9 +38,9 @@ class ReportAccumulateByRequest: 'bad_percent': 0 }, 'average_accuracy_rate': { - 'imei': IterAvg(), + 'imei_number': IterAvg(), 'purchase_date': IterAvg(), - 'retailer_name': IterAvg(), + 'retailername': IterAvg(), 'sold_to_party': IterAvg(), 'invoice_no': IterAvg() }, @@ -86,7 +86,7 @@ class ReportAccumulateByRequest: 'average_accuracy_rate': { 'imei_number': IterAvg(), 'purchase_date': IterAvg(), - 'retailer_name': IterAvg(), + 'retailername': IterAvg(), 'sold_to_party': IterAvg(), 'invoice_no': IterAvg() }, @@ -218,7 +218,6 @@ class ReportAccumulateByRequest: self.data[this_month][1][this_day] = self.update_day(self.data[this_month][1][this_day], _report_file) # Update the subtotal of the day def count_transactions_within_day(self, date_string): - # convert this day into timezone.datetime at UTC start_date = datetime.strptime(date_string, "%Y%m%d") start_date_with_timezone = timezone.make_aware(start_date) end_date_with_timezone = start_date_with_timezone + timezone.timedelta(days=1) @@ -303,8 +302,8 @@ class ReportAccumulateByRequest: for key in settings.FIELD: _data[month][1][day]["average_accuracy_rate"][key] = _data[month][1][day]["average_accuracy_rate"][key]() - for accuracy_type in ["feedback_accuracy", key]: - _data[month][1][day][accuracy_type]["imei_number"] = _data[month][1][day]["feedback_accuracy"]["imei_number"]() + for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]: + _data[month][1][day][accuracy_type][key] = _data[month][1][day][accuracy_type][key]() _data[month][1][day]["review_progress"] = _data[month][1][day]["review_progress"].count(1)/(_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) if (_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) >0 else 0 _data[month][1][day].pop("report_files") @@ -318,7 +317,7 @@ class ReportAccumulateByRequest: _data[month][0]["average_processing_time"][key] = _data[month][0]["average_processing_time"][key]() for key in settings.FIELD: _data[month][0]["average_accuracy_rate"][key] = _data[month][0]["average_accuracy_rate"][key]() - for accuracy_type in ["feedback_accuracy", key]: + for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]: _data[month][0][accuracy_type][key] = _data[month][0][accuracy_type][key]() _data[month][0]["review_progress"] = _data[month][0]["review_progress"].count(1)/(_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) if (_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) >0 else 0 return _data @@ -532,7 +531,6 @@ def first_of_list(the_list): def extract_report_detail_list(report_detail_list, lower=False, in_percent=True): data = [] for report_file in report_detail_list: - # FIXME: #79 Fill None with value data.append({ "Subs": report_file.subsidiary, "Request ID": report_file.correspond_request_id, @@ -540,19 +538,19 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True) "Image type": report_file.doc_type, "IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])), "IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])), - "IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])), + "IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])) if report_file.reviewed_result else None, "IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])), - "Invoice_Number_User": report_file.feedback_result.get("invoice_no", None), + "Invoice_Number_User": report_file.feedback_result.get("invoice_no", None) if report_file.feedback_result else None, "Invoice_Number_OCR": report_file.predict_result.get("invoice_no", None), - "Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None), + "Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None) if report_file.reviewed_result else None, "Invoice_Number_Accuracy": first_of_list(report_file.feedback_accuracy.get("invoice_no", [None])), "Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None), "Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []), - "Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None), + "Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None) if report_file.reviewed_result else None, "Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])), "Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None), "Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None), - "Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None), + "Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None) if report_file.reviewed_result else None, "Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])), "OCR Image Accuracy": report_file.acc, "OCR Image Speed (seconds)": report_file.time_cost, @@ -638,7 +636,7 @@ def align_fine_result(ready_predict, fine_result): ready_predict["purchase_date"] = [None] if fine_result["retailername"] and not ready_predict["retailername"]: ready_predict["retailername"] = [None] - if ready_predict["invoice_no"] and not fine_result["invoice_no"]: + if ready_predict.get("invoice_no", None) and not fine_result.get("invoice_no", None): fine_result["invoice_no"] = [None] fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))] return ready_predict, fine_result diff --git a/cope2n-fe/src/components/report-detail/report-overview-table.tsx b/cope2n-fe/src/components/report-detail/report-overview-table.tsx index cc2b3b1..dd39d00 100644 --- a/cope2n-fe/src/components/report-detail/report-overview-table.tsx +++ b/cope2n-fe/src/components/report-detail/report-overview-table.tsx @@ -17,6 +17,7 @@ interface DataType { snImeiAAR: number; // AAR: Average Accuracy Rate purchaseDateAAR: number; retailerNameAAR: number; + invoiceNumberAAR: number; snImeiAPT: number; // APT: Average Processing Time invoiceAPT: number; snImeiTC: number; // TC: transaction count @@ -211,6 +212,20 @@ const columns: TableColumnsType = [ ); }, }, + { + title: 'Invoice number', + dataIndex: 'invoiceNumberAAR', + key: 'invoiceNumberAAR', + width: '130px', + render: (_, record) => { + const isAbnormal = ensureMin(record.invoiceNumberAAR, 0.98); + return ( + + {formatPercent(record.invoiceNumberAAR)} + + ); + }, + }, ], }, { @@ -285,6 +300,7 @@ const ReportOverViewTable: React.FC = ({ snImeiAAR: item.average_accuracy_rate.imei, purchaseDateAAR: item.average_accuracy_rate.purchase_date, retailerNameAAR: item.average_accuracy_rate.retailer_name, + invoiceNumberAAR: item.average_accuracy_rate.invoice_no, snImeiAPT: item.average_processing_time.imei, invoiceAPT: item.average_processing_time.invoice, snImeiTC: item.usage.imei, diff --git a/cope2n-fe/src/components/report-detail/report-table.tsx b/cope2n-fe/src/components/report-detail/report-table.tsx index c59099d..0011f50 100644 --- a/cope2n-fe/src/components/report-detail/report-table.tsx +++ b/cope2n-fe/src/components/report-detail/report-table.tsx @@ -103,6 +103,19 @@ const ReportTable: React.FC = () => { ); }, }, + { + title: 'Invoice number Accuracy', + dataIndex: 'Invoice number Acc', + key: 'Invoice number Acc', + render: (_, record) => { + const isAbnormal = ensureMin(record['Invoice number Acc'], 0.95); + return ( + + {formatPercent(record['Invoice number Acc'])} + + ); + }, + }, { title: 'Retailer Accuracy', diff --git a/document-classification-kv-demo b/document-classification-kv-demo deleted file mode 160000 index 220954c..0000000 --- a/document-classification-kv-demo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 220954c5c6bfed15e93e26b2adacf28ff8b75baf