tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: misalign bw dashboard and report

Browse Source
This commit is contained in:
TannedCung 2024-06-11 19:11:56 +07:00
parent 65479e6844
commit 97521413be
2 changed files with 73 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cope2n-api/fwd_api/celery_worker/process_report_tasks.py
View File

@ -189,9 +189,9 @@ def create_accuracy_report(report_id, **kwargs):
avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"])
acumulated_acc[acc_type]["avg"] = avg_acc()
report.feedback_accuracy = acumulated_acc["feedback"]
report.reviewed_accuracy = acumulated_acc["reviewed"]
report.combined_accuracy = acumulated_acc["acumulated"]
report.feedback_accuracy = _save_data["report"]["feedback_accuracy"]
report.reviewed_accuracy = _save_data["report"]["reviewed_accuracy"]
report.combined_accuracy = _save_data["report"]["average_accuracy_rate"]
report.num_reviewed = review_progress.count(1)
report.num_not_reviewed = review_progress.count(0)

77
cope2n-api/fwd_api/utils/accuracy.py
View File

@ -27,7 +27,7 @@ class ReportAccumulateByRequest:
self.sub = sub
self.current_time = None
self.data = {} # {"month": [total, {"day": day_data}]}
self.total_format = {
self.month_format = {
'subs': "+",
'extraction_date': "Subtotal ()",
'num_imei': 0,
@ -122,10 +122,11 @@ class ReportAccumulateByRequest:
"num_request": 0,
"review_progress": []
},
self.report = copy.deepcopy(self.month_format)
@staticmethod
def update_total(total, report_file):
# Update total = update month
# Update total = update total
if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0
total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0
@ -163,6 +164,46 @@ class ReportAccumulateByRequest:
total["review_progress"].append(report_file.review_status)
return total
@staticmethod
def update_month(month, report_file):
# Update month = update month
if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
month["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0
month["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0
month["total_images"] += 1
doc_type = "imei"
if report_file.doc_type in ["imei", "invoice", "all"]:
doc_type = report_file.doc_type
else:
print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}")
month["num_imei"] += 1 if doc_type == "imei" else 0
month["num_invoice"] += 1 if doc_type == "invoice" else 0
for key in settings.FIELD:
if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 :
month["average_accuracy_rate"][key].add(report_file.reviewed_accuracy.get(key, []))
elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0:
month["average_accuracy_rate"][key].add(report_file.feedback_accuracy.get(key, []))
month["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, []))
month["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, []))
if not month["average_processing_time"].get(report_file.doc_type, None):
print(f"[WARM]: Weird doctype: {report_file.doc_type}")
month["average_processing_time"][report_file.doc_type] = IterAvg()
month["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0
month["average_processing_time"]["avg"].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0
doc_type = "imei"
if report_file.doc_type in ["imei", "invoice", "all"]:
doc_type = report_file.doc_type
else:
print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}")
month["usage"]["imei"] += 1 if doc_type == "imei" else 0
month["usage"]["invoice"] += 1 if doc_type == "invoice" else 0
month["usage"]["total_images"] += 1
month["review_progress"].append(report_file.review_status)
return month
@staticmethod
def update_day(day_data, report_file):
if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
@ -198,7 +239,7 @@ class ReportAccumulateByRequest:
this_month = timezone.localtime(request.created_at).strftime("%Y%m")
this_day = timezone.localtime(request.created_at).strftime("%Y%m%d")
if not self.data.get(this_month, None):
self.data[this_month] = [copy.deepcopy(self.total_format), {}]
self.data[this_month] = [copy.deepcopy(self.month_format), {}]
self.data[this_month][0]["extraction_date"] = "Subtotal (" + timezone.localtime(request.created_at).strftime("%Y-%m") + ")"
if not self.data[this_month][1].get(this_day, None):
print(f"[INFO] Adding a new day: {this_day} for report: {report.id} ...")
@ -222,7 +263,8 @@ class ReportAccumulateByRequest:
for t in _report_file.reviewed_accuracy.keys():
_report_file.reviewed_accuracy[t] = []
self.data[this_month][0] = self.update_total(self.data[this_month][0], _report_file) # Update the subtotal within the month
self.report = self.update_total(self.report, _report_file)
self.data[this_month][0] = self.update_month(self.data[this_month][0], _report_file) # Update the subtotal within the month
self.data[this_month][1][this_day] = self.update_day(self.data[this_month][1][this_day], _report_file) # Update the subtotal of the day
def count_transactions_within_day(self, date_string):
@ -232,10 +274,11 @@ class ReportAccumulateByRequest:
return count_transactions(start_date_with_timezone, end_date_with_timezone, self.sub)
def save(self, root_report_id, is_daily_report=False, include_test=False):
report_data = self.get()
report_data, overall_report = self.get()
fine_data = []
save_data = {"file": {"overview": f"{root_report_id}/{root_report_id}.xlsx"},
"data": fine_data} # {"sub_report_id": "S3 location", "data": fine_data}
"data": fine_data, # {"sub_report_id": "S3 location", "data": fine_data}
"report": overall_report}
# extract data
month_keys = list(report_data.keys())
month_keys.sort(reverse=True)
@ -298,6 +341,26 @@ class ReportAccumulateByRequest:
def get(self) -> Any:
# FIXME: This looks like a junk
_data = copy.deepcopy(self.data)
_report = copy.deepcopy(self.report)
# export report data
for key in _report["average_processing_time"].keys():
_report["average_processing_time"][key] = _report["average_processing_time"][key]()
avg_acc = 0
count_acc = 0
for key in settings.FIELD:
_report["average_accuracy_rate"][key] = _report["average_accuracy_rate"][key]()
for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
avg_acc = (avg_acc*count_acc + _report[accuracy_type][key].avg*_report[accuracy_type][key].count) / (_report[accuracy_type][key].count + count_acc)
count_acc += _report[accuracy_type][key].count
_report[accuracy_type][key] = _report[accuracy_type][key]()
_report["average_accuracy_rate"]["avg"] = avg_acc
_report["review_progress"] = _report["review_progress"].count(1)/(_report["review_progress"].count(0)+ _report["review_progress"].count(1)) if (_report["review_progress"].count(0)+ _report["review_progress"].count(1)) >0 else 0
_report["images_quality"]["successful_percent"] = _report["images_quality"]["successful"]/_report["total_images"] if _report["total_images"] > 0 else 0
_report["images_quality"]["bad_percent"] = _report["images_quality"]["bad"]/_report["total_images"] if _report["total_images"] > 0 else 0
# export data for dashboard
for month in _data.keys():
_data[month][0]["images_quality"]["successful_percent"] = _data[month][0]["images_quality"]["successful"]/_data[month][0]["total_images"] if _data[month][0]["total_images"] > 0 else 0
_data[month][0]["images_quality"]["bad_percent"] = _data[month][0]["images_quality"]["bad"]/_data[month][0]["total_images"] if _data[month][0]["total_images"] > 0 else 0
@ -329,7 +392,7 @@ class ReportAccumulateByRequest:
for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
_data[month][0][accuracy_type][key] = _data[month][0][accuracy_type][key]()
_data[month][0]["review_progress"] = _data[month][0]["review_progress"].count(1)/(_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) if (_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) >0 else 0
return _data
return _data, _report
class MonthReportAccumulate:
def __init__(self):