tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: #79, #80

Browse Source
This commit is contained in:
dx-tan 2024-03-14 10:11:20 +07:00
parent 44900ac555
commit 044c668c54
7 changed files with 55 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cope2n-api/fwd/settings.py
View File

@ -114,7 +114,7 @@ DATABASES = {
'PASSWORD': env.str("DB_PASSWORD", None),
'HOST': env.str("DB_HOST", None),
'PORT': env.str("DB_PORT", None),
'CONN_MAX_AGE': None,
'CONN_MAX_AGE': 0,
}
}

1
cope2n-api/fwd_api/api/accuracy_view.py
View File

@ -428,6 +428,7 @@ class AccuracyViewSet(viewsets.ViewSet):
"End Date": report.end_at,
"No. Requests": report.number_request,
"Status": report.status,
"Invoice number Acc": acc["invoice_no"],
"Purchase Date Acc": acc["purchase_date"],
"Retailer Acc": acc["retailername"],
"IMEI Acc": acc["imei_number"],

21
cope2n-api/fwd_api/celery_worker/process_report_tasks.py
View File

@ -3,7 +3,7 @@ import traceback
from fwd_api.models import SubscriptionRequest, Report, ReportFile
from fwd_api.celery_worker.worker import app
from ..utils import s3 as S3Util
from ..utils.accuracy import (update_temp_accuracy, IterAvg, calculate_and_save_subcription_file,
from ..utils.accuracy import (update_temp_accuracy, IterAvg,
count_transactions, extract_report_detail_list, calculate_a_request,
ReportAccumulateByRequest, create_billing_data)
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3, save_images_to_csv_briefly
@ -78,15 +78,18 @@ def create_accuracy_report(report_id, **kwargs):
accuracy = {"feedback" :{"imei_number": IterAvg(),
"purchase_date": IterAvg(),
"retailername": IterAvg(),
"sold_to_party": IterAvg(),},
"sold_to_party": IterAvg(),
"invoice_no": IterAvg()},
"reviewed" :{"imei_number": IterAvg(),
"purchase_date": IterAvg(),
"retailername": IterAvg(),
"sold_to_party": IterAvg(),},
"sold_to_party": IterAvg(),
"invoice_no": IterAvg()},
"acumulated":{"imei_number": IterAvg(),
"purchase_date": IterAvg(),
"retailername": IterAvg(),
"sold_to_party": IterAvg(),}
"sold_to_party": IterAvg(),
"invoice_no": IterAvg()}
} # {"imei": {"acc": 0.1, count: 1}, ...}
time_cost = {"invoice": IterAvg(),
"imei": IterAvg()}
@ -123,9 +126,9 @@ def create_accuracy_report(report_id, **kwargs):
number_images += request_att["total_images"]
number_bad_images += request_att["bad_images"]
bad_image_list += request_att["bad_image_list"]
update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=settings.FIELD)
update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=settings.FIELD)
update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=settings.FIELD)
time_cost["imei"].add(request_att["time_cost"].get("imei", []))
time_cost["invoice"].add(request_att["time_cost"].get("invoice", []))
@ -158,7 +161,7 @@ def create_accuracy_report(report_id, **kwargs):
"acumulated": {}}
for acc_type in ["feedback", "reviewed", "acumulated"]:
avg_acc = IterAvg()
for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]:
for key in settings.FIELD:
acumulated_acc[acc_type][key] = accuracy[acc_type][key]()
acumulated_acc[acc_type][key + "_count"] = accuracy[acc_type][key].count
avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"])
@ -235,7 +238,7 @@ def create_billing_report(report_id, **kwargs):
"reviewed": {},
"acumulated": {}}
for acc_type in ["feedback", "reviewed", "acumulated"]:
for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]:
for key in settings.FIELD:
acumulated_acc[acc_type][key] = None
acumulated_acc[acc_type][key + "_count"] = None
acumulated_acc[acc_type]["avg"] = None

26
cope2n-api/fwd_api/utils/accuracy.py
View File

@ -38,9 +38,9 @@ class ReportAccumulateByRequest:
'bad_percent': 0
},
'average_accuracy_rate': {
'imei': IterAvg(),
'imei_number': IterAvg(),
'purchase_date': IterAvg(),
'retailer_name': IterAvg(),
'retailername': IterAvg(),
'sold_to_party': IterAvg(),
'invoice_no': IterAvg()
},
@ -86,7 +86,7 @@ class ReportAccumulateByRequest:
'average_accuracy_rate': {
'imei_number': IterAvg(),
'purchase_date': IterAvg(),
'retailer_name': IterAvg(),
'retailername': IterAvg(),
'sold_to_party': IterAvg(),
'invoice_no': IterAvg()
},
@ -218,7 +218,6 @@ class ReportAccumulateByRequest:
self.data[this_month][1][this_day] = self.update_day(self.data[this_month][1][this_day], _report_file) # Update the subtotal of the day
def count_transactions_within_day(self, date_string):
# convert this day into timezone.datetime at UTC
start_date = datetime.strptime(date_string, "%Y%m%d")
start_date_with_timezone = timezone.make_aware(start_date)
end_date_with_timezone = start_date_with_timezone + timezone.timedelta(days=1)
@ -303,8 +302,8 @@ class ReportAccumulateByRequest:
for key in settings.FIELD:
_data[month][1][day]["average_accuracy_rate"][key] = _data[month][1][day]["average_accuracy_rate"][key]()
for accuracy_type in ["feedback_accuracy", key]:
_data[month][1][day][accuracy_type]["imei_number"] = _data[month][1][day]["feedback_accuracy"]["imei_number"]()
for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
_data[month][1][day][accuracy_type][key] = _data[month][1][day][accuracy_type][key]()
_data[month][1][day]["review_progress"] = _data[month][1][day]["review_progress"].count(1)/(_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) if (_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) >0 else 0
_data[month][1][day].pop("report_files")
@ -318,7 +317,7 @@ class ReportAccumulateByRequest:
_data[month][0]["average_processing_time"][key] = _data[month][0]["average_processing_time"][key]()
for key in settings.FIELD:
_data[month][0]["average_accuracy_rate"][key] = _data[month][0]["average_accuracy_rate"][key]()
for accuracy_type in ["feedback_accuracy", key]:
for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
_data[month][0][accuracy_type][key] = _data[month][0][accuracy_type][key]()
_data[month][0]["review_progress"] = _data[month][0]["review_progress"].count(1)/(_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) if (_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) >0 else 0
return _data
@ -532,7 +531,6 @@ def first_of_list(the_list):
def extract_report_detail_list(report_detail_list, lower=False, in_percent=True):
data = []
for report_file in report_detail_list:
# FIXME: #79 Fill None with value
data.append({
"Subs": report_file.subsidiary,
"Request ID": report_file.correspond_request_id,
@ -540,19 +538,19 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True)
"Image type": report_file.doc_type,
"IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])),
"IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])),
"IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])),
"IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])) if report_file.reviewed_result else None,
"IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])),
"Invoice_Number_User": report_file.feedback_result.get("invoice_no", None),
"Invoice_Number_User": report_file.feedback_result.get("invoice_no", None) if report_file.feedback_result else None,
"Invoice_Number_OCR": report_file.predict_result.get("invoice_no", None),
"Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None),
"Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None) if report_file.reviewed_result else None,
"Invoice_Number_Accuracy": first_of_list(report_file.feedback_accuracy.get("invoice_no", [None])),
"Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None),
"Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []),
"Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None),
"Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None) if report_file.reviewed_result else None,
"Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])),
"Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None),
"Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None),
"Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None),
"Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None) if report_file.reviewed_result else None,
"Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])),
"OCR Image Accuracy": report_file.acc,
"OCR Image Speed (seconds)": report_file.time_cost,
@ -638,7 +636,7 @@ def align_fine_result(ready_predict, fine_result):
ready_predict["purchase_date"] = [None]
if fine_result["retailername"] and not ready_predict["retailername"]:
ready_predict["retailername"] = [None]
if ready_predict["invoice_no"] and not fine_result["invoice_no"]:
if ready_predict.get("invoice_no", None) and not fine_result.get("invoice_no", None):
fine_result["invoice_no"] = [None]
fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))]
return ready_predict, fine_result

16
cope2n-fe/src/components/report-detail/report-overview-table.tsx
View File

@ -17,6 +17,7 @@ interface DataType {
snImeiAAR: number; // AAR: Average Accuracy Rate
purchaseDateAAR: number;
retailerNameAAR: number;
invoiceNumberAAR: number;
snImeiAPT: number; // APT: Average Processing Time
invoiceAPT: number;
snImeiTC: number; // TC: transaction count
@ -211,6 +212,20 @@ const columns: TableColumnsType<DataType> = [
);
},
},
{
title: 'Invoice number',
dataIndex: 'invoiceNumberAAR',
key: 'invoiceNumberAAR',
width: '130px',
render: (_, record) => {
const isAbnormal = ensureMin(record.invoiceNumberAAR, 0.98);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
{formatPercent(record.invoiceNumberAAR)}
</span>
);
},
},
],
},
{
@ -285,6 +300,7 @@ const ReportOverViewTable: React.FC<ReportOverViewTableProps> = ({
snImeiAAR: item.average_accuracy_rate.imei,
purchaseDateAAR: item.average_accuracy_rate.purchase_date,
retailerNameAAR: item.average_accuracy_rate.retailer_name,
invoiceNumberAAR: item.average_accuracy_rate.invoice_no,
snImeiAPT: item.average_processing_time.imei,
invoiceAPT: item.average_processing_time.invoice,
snImeiTC: item.usage.imei,

13
cope2n-fe/src/components/report-detail/report-table.tsx
View File

@ -103,6 +103,19 @@ const ReportTable: React.FC = () => {
);
},
},
{
title: 'Invoice number Accuracy',
dataIndex: 'Invoice number Acc',
key: 'Invoice number Acc',
render: (_, record) => {
const isAbnormal = ensureMin(record['Invoice number Acc'], 0.95);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
{formatPercent(record['Invoice number Acc'])}
</span>
);
},
},
{
title: 'Retailer Accuracy',

1
document-classification-kv-demo

@ -1 +0,0 @@
Subproject commit 220954c5c6bfed15e93e26b2adacf28ff8b75baf