From 97521413bec269350b58a5767e2ab34ce8b13127 Mon Sep 17 00:00:00 2001 From: TannedCung Date: Tue, 11 Jun 2024 19:11:56 +0700 Subject: [PATCH] Fix: misalign bw dashboard and report --- .../celery_worker/process_report_tasks.py | 6 +- cope2n-api/fwd_api/utils/accuracy.py | 77 +++++++++++++++++-- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py index 70cb083..1d89693 100755 --- a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py @@ -189,9 +189,9 @@ def create_accuracy_report(report_id, **kwargs): avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"]) acumulated_acc[acc_type]["avg"] = avg_acc() - report.feedback_accuracy = acumulated_acc["feedback"] - report.reviewed_accuracy = acumulated_acc["reviewed"] - report.combined_accuracy = acumulated_acc["acumulated"] + report.feedback_accuracy = _save_data["report"]["feedback_accuracy"] + report.reviewed_accuracy = _save_data["report"]["reviewed_accuracy"] + report.combined_accuracy = _save_data["report"]["average_accuracy_rate"] report.num_reviewed = review_progress.count(1) report.num_not_reviewed = review_progress.count(0) diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index ff49189..71a0ace 100755 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -27,7 +27,7 @@ class ReportAccumulateByRequest: self.sub = sub self.current_time = None self.data = {} # {"month": [total, {"day": day_data}]} - self.total_format = { + self.month_format = { 'subs': "+", 'extraction_date': "Subtotal ()", 'num_imei': 0, @@ -122,10 +122,11 @@ class ReportAccumulateByRequest: "num_request": 0, "review_progress": [] }, + self.report = copy.deepcopy(self.month_format) @staticmethod def update_total(total, report_file): - # Update total = update month + # Update total = update total if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS: total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 @@ -163,6 +164,46 @@ class ReportAccumulateByRequest: total["review_progress"].append(report_file.review_status) return total + @staticmethod + def update_month(month, report_file): + # Update month = update month + if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS: + month["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 + month["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 + month["total_images"] += 1 + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + month["num_imei"] += 1 if doc_type == "imei" else 0 + month["num_invoice"] += 1 if doc_type == "invoice" else 0 + + for key in settings.FIELD: + if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : + month["average_accuracy_rate"][key].add(report_file.reviewed_accuracy.get(key, [])) + elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: + month["average_accuracy_rate"][key].add(report_file.feedback_accuracy.get(key, [])) + month["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) + month["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) + + if not month["average_processing_time"].get(report_file.doc_type, None): + print(f"[WARM]: Weird doctype: {report_file.doc_type}") + month["average_processing_time"][report_file.doc_type] = IterAvg() + month["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 + month["average_processing_time"]["avg"].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 + + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + month["usage"]["imei"] += 1 if doc_type == "imei" else 0 + month["usage"]["invoice"] += 1 if doc_type == "invoice" else 0 + month["usage"]["total_images"] += 1 + month["review_progress"].append(report_file.review_status) + return month + @staticmethod def update_day(day_data, report_file): if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS: @@ -198,7 +239,7 @@ class ReportAccumulateByRequest: this_month = timezone.localtime(request.created_at).strftime("%Y%m") this_day = timezone.localtime(request.created_at).strftime("%Y%m%d") if not self.data.get(this_month, None): - self.data[this_month] = [copy.deepcopy(self.total_format), {}] + self.data[this_month] = [copy.deepcopy(self.month_format), {}] self.data[this_month][0]["extraction_date"] = "Subtotal (" + timezone.localtime(request.created_at).strftime("%Y-%m") + ")" if not self.data[this_month][1].get(this_day, None): print(f"[INFO] Adding a new day: {this_day} for report: {report.id} ...") @@ -222,7 +263,8 @@ class ReportAccumulateByRequest: for t in _report_file.reviewed_accuracy.keys(): _report_file.reviewed_accuracy[t] = [] - self.data[this_month][0] = self.update_total(self.data[this_month][0], _report_file) # Update the subtotal within the month + self.report = self.update_total(self.report, _report_file) + self.data[this_month][0] = self.update_month(self.data[this_month][0], _report_file) # Update the subtotal within the month self.data[this_month][1][this_day] = self.update_day(self.data[this_month][1][this_day], _report_file) # Update the subtotal of the day def count_transactions_within_day(self, date_string): @@ -232,10 +274,11 @@ class ReportAccumulateByRequest: return count_transactions(start_date_with_timezone, end_date_with_timezone, self.sub) def save(self, root_report_id, is_daily_report=False, include_test=False): - report_data = self.get() + report_data, overall_report = self.get() fine_data = [] save_data = {"file": {"overview": f"{root_report_id}/{root_report_id}.xlsx"}, - "data": fine_data} # {"sub_report_id": "S3 location", "data": fine_data} + "data": fine_data, # {"sub_report_id": "S3 location", "data": fine_data} + "report": overall_report} # extract data month_keys = list(report_data.keys()) month_keys.sort(reverse=True) @@ -298,6 +341,26 @@ class ReportAccumulateByRequest: def get(self) -> Any: # FIXME: This looks like a junk _data = copy.deepcopy(self.data) + _report = copy.deepcopy(self.report) + # export report data + for key in _report["average_processing_time"].keys(): + _report["average_processing_time"][key] = _report["average_processing_time"][key]() + + avg_acc = 0 + count_acc = 0 + for key in settings.FIELD: + _report["average_accuracy_rate"][key] = _report["average_accuracy_rate"][key]() + for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]: + avg_acc = (avg_acc*count_acc + _report[accuracy_type][key].avg*_report[accuracy_type][key].count) / (_report[accuracy_type][key].count + count_acc) + count_acc += _report[accuracy_type][key].count + + _report[accuracy_type][key] = _report[accuracy_type][key]() + _report["average_accuracy_rate"]["avg"] = avg_acc + + _report["review_progress"] = _report["review_progress"].count(1)/(_report["review_progress"].count(0)+ _report["review_progress"].count(1)) if (_report["review_progress"].count(0)+ _report["review_progress"].count(1)) >0 else 0 + _report["images_quality"]["successful_percent"] = _report["images_quality"]["successful"]/_report["total_images"] if _report["total_images"] > 0 else 0 + _report["images_quality"]["bad_percent"] = _report["images_quality"]["bad"]/_report["total_images"] if _report["total_images"] > 0 else 0 + # export data for dashboard for month in _data.keys(): _data[month][0]["images_quality"]["successful_percent"] = _data[month][0]["images_quality"]["successful"]/_data[month][0]["total_images"] if _data[month][0]["total_images"] > 0 else 0 _data[month][0]["images_quality"]["bad_percent"] = _data[month][0]["images_quality"]["bad"]/_data[month][0]["total_images"] if _data[month][0]["total_images"] > 0 else 0 @@ -329,7 +392,7 @@ class ReportAccumulateByRequest: for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]: _data[month][0][accuracy_type][key] = _data[month][0][accuracy_type][key]() _data[month][0]["review_progress"] = _data[month][0]["review_progress"].count(1)/(_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) if (_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) >0 else 0 - return _data + return _data, _report class MonthReportAccumulate: def __init__(self):