diff --git a/cope2n-api/fwd/settings.py b/cope2n-api/fwd/settings.py index 7513fcd..5e5ed7b 100755 --- a/cope2n-api/fwd/settings.py +++ b/cope2n-api/fwd/settings.py @@ -241,6 +241,7 @@ BAD_THRESHOLD = 0.75 NEED_REVIEW = 1.0 SUB_FOR_BILLING = ["all", "seao"] +FIELD = ["imei_number", "purchase_date", "retailername", "sold_to_party", "invoice_no"] CACHES = { 'default': { diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index 346ea8b..859f6ff 100755 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -41,7 +41,8 @@ class ReportAccumulateByRequest: 'imei': IterAvg(), 'purchase_date': IterAvg(), 'retailer_name': IterAvg(), - 'sold_to_party': IterAvg() + 'sold_to_party': IterAvg(), + 'invoice_no': IterAvg(), }, 'average_processing_time': { 'imei': IterAvg(), @@ -57,13 +58,15 @@ class ReportAccumulateByRequest: 'imei_number': IterAvg(), 'purchase_date': IterAvg(), 'retailername': IterAvg(), - 'sold_to_party': IterAvg() + 'sold_to_party': IterAvg(), + 'invoice_no': IterAvg() }, 'reviewed_accuracy': { 'imei_number': IterAvg(), 'purchase_date': IterAvg(), 'retailername': IterAvg(), - 'sold_to_party': IterAvg() + 'sold_to_party': IterAvg(), + 'invoice_no': IterAvg() }, 'num_request': 0, "review_progress": [] @@ -81,10 +84,11 @@ class ReportAccumulateByRequest: 'bad_percent': 0 }, 'average_accuracy_rate': { - 'imei': IterAvg(), + 'imei_number': IterAvg(), 'purchase_date': IterAvg(), 'retailer_name': IterAvg(), - 'sold_to_party': IterAvg() + 'sold_to_party': IterAvg(), + 'invoice_no': IterAvg() }, 'average_processing_time': { 'imei': IterAvg(), @@ -100,13 +104,15 @@ class ReportAccumulateByRequest: 'imei_number': IterAvg(), 'purchase_date': IterAvg(), 'retailername': IterAvg(), - 'sold_to_party': IterAvg() + 'sold_to_party': IterAvg(), + 'invoice_no': IterAvg() }, 'reviewed_accuracy': { 'imei_number': IterAvg(), 'purchase_date': IterAvg(), 'retailername': IterAvg(), - 'sold_to_party': IterAvg() + 'sold_to_party': IterAvg(), + 'invoice_no': IterAvg() }, "report_files": [], "num_request": 0, @@ -127,20 +133,12 @@ class ReportAccumulateByRequest: total["num_imei"] += 1 if doc_type == "imei" else 0 total["num_invoice"] += 1 if doc_type == "invoice" else 0 - if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : - total["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) - total["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) - total["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) - total["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) - elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: - total["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) - total["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) - total["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) - total["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) - - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + for key in settings.FIELD: + if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : + total["average_accuracy_rate"][key].add(report_file.reviewed_accuracy.get(key, [])) + elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: + total["average_accuracy_rate"][key].add(report_file.feedback_accuracy.get(key, [])) total["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: total["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) if not total["average_processing_time"].get(report_file.doc_type, None): @@ -174,20 +172,12 @@ class ReportAccumulateByRequest: day_data["num_invoice"] += 1 if doc_type == "invoice" else 0 day_data["report_files"].append(report_file) - if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0: - day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) - day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) - day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) - day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) - elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: - day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) - day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) - day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) - day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) - - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + for key in settings.FIELD: + if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0: + day_data["average_accuracy_rate"][key].add(report_file.reviewed_accuracy.get(key, [])) + elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: + day_data["average_accuracy_rate"][key].add(report_file.feedback_accuracy.get(key, [])) day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) if not day_data["average_processing_time"].get(report_file.doc_type, None): @@ -264,7 +254,7 @@ class ReportAccumulateByRequest: "reviewed_accuracy": {}} for acc_type in ["feedback_accuracy", "reviewed_accuracy"]: avg_acc = IterAvg() - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + for key in settings.FIELD: acumulated_acc[acc_type][key] = self.data[month][1][day][acc_type][key]() acumulated_acc[acc_type][key+"_count"] = self.data[month][1][day][acc_type][key].count avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"]) @@ -308,21 +298,13 @@ class ReportAccumulateByRequest: for day in _data[month][1].keys(): num_transaction_imei += _data[month][1][day]["usage"].get("imei", 0) num_transaction_invoice += _data[month][1][day]["usage"].get("invoice", 0) - _data[month][1][day]["average_accuracy_rate"]["imei"] = _data[month][1][day]["average_accuracy_rate"]["imei"]() - _data[month][1][day]["average_accuracy_rate"]["purchase_date"] = _data[month][1][day]["average_accuracy_rate"]["purchase_date"]() - _data[month][1][day]["average_accuracy_rate"]["retailer_name"] = _data[month][1][day]["average_accuracy_rate"]["retailer_name"]() - _data[month][1][day]["average_accuracy_rate"]["sold_to_party"] = _data[month][1][day]["average_accuracy_rate"]["sold_to_party"]() for key in _data[month][1][day]["average_processing_time"].keys(): _data[month][1][day]["average_processing_time"][key] = _data[month][1][day]["average_processing_time"][key]() - _data[month][1][day]["feedback_accuracy"]["imei_number"] = _data[month][1][day]["feedback_accuracy"]["imei_number"]() - _data[month][1][day]["feedback_accuracy"]["purchase_date"] = _data[month][1][day]["feedback_accuracy"]["purchase_date"]() - _data[month][1][day]["feedback_accuracy"]["retailername"] = _data[month][1][day]["feedback_accuracy"]["retailername"]() - _data[month][1][day]["feedback_accuracy"]["sold_to_party"] = _data[month][1][day]["feedback_accuracy"]["sold_to_party"]() - _data[month][1][day]["reviewed_accuracy"]["imei_number"] = _data[month][1][day]["reviewed_accuracy"]["imei_number"]() - _data[month][1][day]["reviewed_accuracy"]["purchase_date"] = _data[month][1][day]["reviewed_accuracy"]["purchase_date"]() - _data[month][1][day]["reviewed_accuracy"]["retailername"] = _data[month][1][day]["reviewed_accuracy"]["retailername"]() - _data[month][1][day]["reviewed_accuracy"]["sold_to_party"] = _data[month][1][day]["reviewed_accuracy"]["sold_to_party"]() + for key in settings.FIELD: + _data[month][1][day]["average_accuracy_rate"][key] = _data[month][1][day]["average_accuracy_rate"][key]() + for accuracy_type in ["feedback_accuracy", key]: + _data[month][1][day][accuracy_type]["imei_number"] = _data[month][1][day]["feedback_accuracy"]["imei_number"]() _data[month][1][day]["review_progress"] = _data[month][1][day]["review_progress"].count(1)/(_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) if (_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) >0 else 0 _data[month][1][day].pop("report_files") @@ -332,23 +314,13 @@ class ReportAccumulateByRequest: _data[month][0]["usage"]["imei"] = num_transaction_imei _data[month][0]["usage"]["invoice"] = num_transaction_invoice _data[month][0]["usage"]["total_images"] = num_transaction_invoice + num_transaction_imei - _data[month][0]["average_accuracy_rate"]["imei"] = _data[month][0]["average_accuracy_rate"]["imei"]() - _data[month][0]["average_accuracy_rate"]["purchase_date"] = _data[month][0]["average_accuracy_rate"]["purchase_date"]() - _data[month][0]["average_accuracy_rate"]["retailer_name"] = _data[month][0]["average_accuracy_rate"]["retailer_name"]() - _data[month][0]["average_accuracy_rate"]["sold_to_party"] = _data[month][0]["average_accuracy_rate"]["sold_to_party"]() for key in _data[month][0]["average_processing_time"].keys(): _data[month][0]["average_processing_time"][key] = _data[month][0]["average_processing_time"][key]() - - _data[month][0]["feedback_accuracy"]["imei_number"] = _data[month][0]["feedback_accuracy"]["imei_number"]() - _data[month][0]["feedback_accuracy"]["purchase_date"] = _data[month][0]["feedback_accuracy"]["purchase_date"]() - _data[month][0]["feedback_accuracy"]["retailername"] = _data[month][0]["feedback_accuracy"]["retailername"]() - _data[month][0]["feedback_accuracy"]["sold_to_party"] = _data[month][0]["feedback_accuracy"]["sold_to_party"]() - _data[month][0]["reviewed_accuracy"]["imei_number"] = _data[month][0]["reviewed_accuracy"]["imei_number"]() - _data[month][0]["reviewed_accuracy"]["purchase_date"] = _data[month][0]["reviewed_accuracy"]["purchase_date"]() - _data[month][0]["reviewed_accuracy"]["retailername"] = _data[month][0]["reviewed_accuracy"]["retailername"]() - _data[month][0]["reviewed_accuracy"]["sold_to_party"] = _data[month][0]["reviewed_accuracy"]["sold_to_party"]() + for key in settings.FIELD: + _data[month][0]["average_accuracy_rate"][key] = _data[month][0]["average_accuracy_rate"][key]() + for accuracy_type in ["feedback_accuracy", key]: + _data[month][0][accuracy_type][key] = _data[month][0][accuracy_type][key]() _data[month][0]["review_progress"] = _data[month][0]["review_progress"].count(1)/(_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) if (_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) >0 else 0 - return _data class MonthReportAccumulate: @@ -553,6 +525,7 @@ def first_of_list(the_list): def extract_report_detail_list(report_detail_list, lower=False, in_percent=True): data = [] for report_file in report_detail_list: + # FIXME: #79 Fill None with value data.append({ "Subs": report_file.subsidiary, "Request ID": report_file.correspond_request_id, @@ -560,12 +533,19 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True) "Image type": report_file.doc_type, "IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])), "IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])), + "IMEI Revised": None, "IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])), + "Invoice_Number_User": None, + "Invoice_Number_OCR": None, + "Invoice_Number Revised": None, + "Invoice_Number_Accuracy": None, "Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None), "Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []), + "Invoice_Purchase Date Revised": None, "Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])), "Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None), "Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None), + "Invoice_Purchase Date Revised": None, "Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])), "OCR Image Accuracy": report_file.acc, "OCR Image Speed (seconds)": report_file.time_cost,