Fix: misalign bw dashboard and report

cope2n-api/fwd_api/celery_worker/process_report_tasks.py

@@ -189,9 +189,9 @@ def create_accuracy_report(report_id, **kwargs):
                 avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"])
             acumulated_acc[acc_type]["avg"] = avg_acc()
 
-        report.feedback_accuracy = acumulated_acc["feedback"]
+        report.feedback_accuracy = _save_data["report"]["feedback_accuracy"]
-        report.reviewed_accuracy = acumulated_acc["reviewed"]
+        report.reviewed_accuracy = _save_data["report"]["reviewed_accuracy"]
-        report.combined_accuracy = acumulated_acc["acumulated"]
+        report.combined_accuracy = _save_data["report"]["average_accuracy_rate"]
 
         report.num_reviewed = review_progress.count(1)
         report.num_not_reviewed = review_progress.count(0)

cope2n-api/fwd_api/utils/accuracy.py

@@ -27,7 +27,7 @@ class ReportAccumulateByRequest:
         self.sub = sub
         self.current_time = None
         self.data = {} # {"month": [total, {"day": day_data}]}
-        self.total_format = {
+        self.month_format = {
                     'subs': "+",
                     'extraction_date': "Subtotal ()",
                     'num_imei': 0,
@@ -122,10 +122,11 @@ class ReportAccumulateByRequest:
                     "num_request": 0,
                     "review_progress": []
                 },
+        self.report = copy.deepcopy(self.month_format)
     
     @staticmethod
     def update_total(total, report_file):
-        # Update total = update month
+        # Update total = update total
         if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
             total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0
             total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0
@@ -163,6 +164,46 @@ class ReportAccumulateByRequest:
             total["review_progress"].append(report_file.review_status)
         return total
 
+    @staticmethod
+    def update_month(month, report_file):
+        # Update month = update month
+        if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
+            month["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0
+            month["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0
+            month["total_images"] += 1
+            doc_type = "imei"
+            if report_file.doc_type in ["imei", "invoice", "all"]:
+                doc_type = report_file.doc_type
+            else:
+                print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}")
+            month["num_imei"] += 1 if doc_type == "imei" else 0
+            month["num_invoice"] += 1 if doc_type == "invoice" else 0
+
+            for key in settings.FIELD:
+                if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 :
+                    month["average_accuracy_rate"][key].add(report_file.reviewed_accuracy.get(key, []))
+                elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0:
+                    month["average_accuracy_rate"][key].add(report_file.feedback_accuracy.get(key, []))
+                month["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, []))
+                month["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, []))
+
+            if not month["average_processing_time"].get(report_file.doc_type, None):
+                print(f"[WARM]: Weird doctype: {report_file.doc_type}")
+                month["average_processing_time"][report_file.doc_type] = IterAvg()
+            month["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0
+            month["average_processing_time"]["avg"].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 
+
+            doc_type = "imei"
+            if report_file.doc_type in ["imei", "invoice", "all"]:
+                doc_type = report_file.doc_type
+            else:
+                print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}")
+            month["usage"]["imei"] += 1 if doc_type == "imei" else 0
+            month["usage"]["invoice"] += 1 if doc_type == "invoice" else 0
+            month["usage"]["total_images"] += 1
+            month["review_progress"].append(report_file.review_status)
+        return month
+
     @staticmethod
     def update_day(day_data, report_file):
         if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
@@ -198,7 +239,7 @@ class ReportAccumulateByRequest:
         this_month = timezone.localtime(request.created_at).strftime("%Y%m")
         this_day = timezone.localtime(request.created_at).strftime("%Y%m%d")
         if not self.data.get(this_month, None):
-            self.data[this_month] = [copy.deepcopy(self.total_format), {}]
+            self.data[this_month] = [copy.deepcopy(self.month_format), {}]
             self.data[this_month][0]["extraction_date"] = "Subtotal (" + timezone.localtime(request.created_at).strftime("%Y-%m") + ")"
         if not self.data[this_month][1].get(this_day, None):
             print(f"[INFO] Adding a new day: {this_day} for report: {report.id} ...")
@@ -222,7 +263,8 @@ class ReportAccumulateByRequest:
                 for t in _report_file.reviewed_accuracy.keys():
                     _report_file.reviewed_accuracy[t] = []
 
-            self.data[this_month][0] = self.update_total(self.data[this_month][0], _report_file) # Update the subtotal within the month
+            self.report = self.update_total(self.report, _report_file)
+            self.data[this_month][0] = self.update_month(self.data[this_month][0], _report_file) # Update the subtotal within the month
             self.data[this_month][1][this_day] = self.update_day(self.data[this_month][1][this_day], _report_file) # Update the subtotal of the day
 
     def count_transactions_within_day(self, date_string):
@@ -232,10 +274,11 @@ class ReportAccumulateByRequest:
         return count_transactions(start_date_with_timezone, end_date_with_timezone, self.sub)
 
     def save(self, root_report_id, is_daily_report=False, include_test=False):
-        report_data = self.get()
+        report_data, overall_report = self.get()
         fine_data = []
         save_data = {"file": {"overview": f"{root_report_id}/{root_report_id}.xlsx"},
-                    "data": fine_data} # {"sub_report_id": "S3 location", "data": fine_data}
+                    "data": fine_data, # {"sub_report_id": "S3 location", "data": fine_data}
+                    "report": overall_report}
         # extract data
         month_keys = list(report_data.keys())
         month_keys.sort(reverse=True)
@@ -298,6 +341,26 @@ class ReportAccumulateByRequest:
     def get(self) -> Any:
         # FIXME: This looks like a junk
         _data = copy.deepcopy(self.data)
+        _report = copy.deepcopy(self.report)
+        # export report data
+        for key in _report["average_processing_time"].keys():
+            _report["average_processing_time"][key] = _report["average_processing_time"][key]()
+    
+        avg_acc = 0
+        count_acc = 0
+        for key in settings.FIELD:
+            _report["average_accuracy_rate"][key] = _report["average_accuracy_rate"][key]()
+            for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
+                avg_acc = (avg_acc*count_acc + _report[accuracy_type][key].avg*_report[accuracy_type][key].count) / (_report[accuracy_type][key].count + count_acc)
+                count_acc += _report[accuracy_type][key].count
+                
+                _report[accuracy_type][key] = _report[accuracy_type][key]()
+        _report["average_accuracy_rate"]["avg"] = avg_acc
+
+        _report["review_progress"] = _report["review_progress"].count(1)/(_report["review_progress"].count(0)+ _report["review_progress"].count(1)) if (_report["review_progress"].count(0)+ _report["review_progress"].count(1)) >0 else 0
+        _report["images_quality"]["successful_percent"] = _report["images_quality"]["successful"]/_report["total_images"]  if _report["total_images"] > 0 else 0
+        _report["images_quality"]["bad_percent"] = _report["images_quality"]["bad"]/_report["total_images"] if _report["total_images"] > 0 else 0
+        # export data for dashboard
         for month in _data.keys():
             _data[month][0]["images_quality"]["successful_percent"] = _data[month][0]["images_quality"]["successful"]/_data[month][0]["total_images"] if _data[month][0]["total_images"] > 0 else 0
             _data[month][0]["images_quality"]["bad_percent"] = _data[month][0]["images_quality"]["bad"]/_data[month][0]["total_images"] if _data[month][0]["total_images"] > 0 else 0
@@ -329,7 +392,7 @@ class ReportAccumulateByRequest:
                 for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
                     _data[month][0][accuracy_type][key] = _data[month][0][accuracy_type][key]()
             _data[month][0]["review_progress"] = _data[month][0]["review_progress"].count(1)/(_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) if (_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) >0 else 0
-        return _data
+        return _data, _report
     
 class MonthReportAccumulate:
     def __init__(self):