tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: #67

Browse Source
This commit is contained in:
dx-tan 2024-03-06 15:09:29 +07:00
parent 3581c95a8e
commit adc8c8bbbd
6 changed files with 89 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cope2n-api/fwd_api/api/accuracy_view.py
View File

@ -20,6 +20,7 @@ from ..utils.file import download_from_S3, dict2xlsx, save_report_to_S3, build_S
from ..utils.redis import RedisUtils from ..utils.redis import RedisUtils
from ..utils.process import string_to_boolean from ..utils.process import string_to_boolean
from ..utils.cache import get_cache, set_cache from ..utils.cache import get_cache, set_cache
from fwd_api.constant.common import FileCategory
from ..request.ReportCreationSerializer import ReportCreationSerializer from ..request.ReportCreationSerializer import ReportCreationSerializer
from ..utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long from ..utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long
from ..utils.report import aggregate_overview from ..utils.report import aggregate_overview
@ -110,7 +111,7 @@ class AccuracyViewSet(viewsets.ViewSet):
subsidiary = request.data.get("subsidiary", "all") subsidiary = request.data.get("subsidiary", "all")
subsidiary = map_subsidiary_long_to_short(subsidiary) subsidiary = map_subsidiary_long_to_short(subsidiary)
base_query = Q() base_query = Q(status=200)
if start_date_str or end_date_str: if start_date_str or end_date_str:
try: try:
start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%d') # We care only about day precision only start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%d') # We care only about day precision only
@ -670,7 +671,7 @@ class AccuracyViewSet(viewsets.ViewSet):
data = [] data = []
files = [] files = []
subscription_request_files = SubscriptionRequestFile.objects.filter(request=subscription_request.id) subscription_request_files = SubscriptionRequestFile.objects.filter(request=subscription_request.id, file_category=FileCategory.Origin.value)
for subscription_request_file in subscription_request_files: for subscription_request_file in subscription_request_files:
sub = subscription_request.subscription sub = subscription_request.subscription

4
cope2n-api/fwd_api/celery_worker/process_report_tasks.py
View File

@ -261,6 +261,8 @@ def make_a_report_2(report_id, query_set):
report.average_OCR_time = {"invoice": time_cost["invoice"](), "imei": time_cost["imei"](), report.average_OCR_time = {"invoice": time_cost["invoice"](), "imei": time_cost["imei"](),
"invoice_count": time_cost["invoice"].count, "imei_count": time_cost["imei"].count} "invoice_count": time_cost["invoice"].count, "imei_count": time_cost["imei"].count}
report.average_OCR_time["invoice"] = 0 if report.average_OCR_time["invoice"] is None else report.average_OCR_time["invoice"]
report.average_OCR_time["imei"] = 0 if report.average_OCR_time["imei"] is None else report.average_OCR_time["imei"]
report.average_OCR_time["avg"] = (report.average_OCR_time["invoice"]*report.average_OCR_time["invoice_count"] + report.average_OCR_time["imei"]*report.average_OCR_time["imei_count"])/(report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) if (report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) > 0 else None report.average_OCR_time["avg"] = (report.average_OCR_time["invoice"]*report.average_OCR_time["invoice_count"] + report.average_OCR_time["imei"]*report.average_OCR_time["imei_count"])/(report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) if (report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) > 0 else None
report.number_imei_transaction = transaction_att.get("imei", 0) report.number_imei_transaction = transaction_att.get("imei", 0)
@ -311,7 +313,7 @@ def make_a_report_2(report_id, query_set):
for key in keys: for key in keys:
if report_fine_data[i][key]: if report_fine_data[i][key]:
for x_key in report_fine_data[i][key].keys(): for x_key in report_fine_data[i][key].keys():
report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 if report_fine_data[i][key][x_key] is not None else None
data_workbook = dict2xlsx(report_fine_data, _type='report') data_workbook = dict2xlsx(report_fine_data, _type='report')
overview_filename = query_set["subsidiary"] + "_" + query_set["report_overview_duration"] + ".xlsx" overview_filename = query_set["subsidiary"] + "_" + query_set["report_overview_duration"] + ".xlsx"
local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT) local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT)

142
cope2n-api/fwd_api/utils/accuracy.py
View File

@ -115,84 +115,86 @@ class ReportAccumulateByRequest:
@staticmethod @staticmethod
def update_total(total, report_file): def update_total(total, report_file):
total["total_images"] += 1 if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0
total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0
doc_type = "imei" total["total_images"] += 1
if report_file.doc_type in ["imei", "invoice", "all"]: doc_type = "imei"
doc_type = report_file.doc_type if report_file.doc_type in ["imei", "invoice", "all"]:
else: doc_type = report_file.doc_type
print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") else:
total["num_imei"] += 1 if doc_type == "imei" else 0 print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}")
total["num_invoice"] += 1 if doc_type == "invoice" else 0 total["num_imei"] += 1 if doc_type == "imei" else 0
total["num_invoice"] += 1 if doc_type == "invoice" else 0
if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 :
total["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) total["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", []))
total["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) total["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", []))
total["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) total["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", []))
total["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) total["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", []))
elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0:
total["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) total["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", []))
total["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) total["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", []))
total["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) total["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", []))
total["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) total["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", []))
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
total["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) total["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, []))
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
total["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) total["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, []))
if not total["average_processing_time"].get(report_file.doc_type, None): if not total["average_processing_time"].get(report_file.doc_type, None):
print(f"[WARM]: Weird doctype: {report_file.doc_type}") print(f"[WARM]: Weird doctype: {report_file.doc_type}")
total["average_processing_time"][report_file.doc_type] = IterAvg() total["average_processing_time"][report_file.doc_type] = IterAvg()
total["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 total["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0
doc_type = "imei" doc_type = "imei"
if report_file.doc_type in ["imei", "invoice", "all"]: if report_file.doc_type in ["imei", "invoice", "all"]:
doc_type = report_file.doc_type doc_type = report_file.doc_type
else: else:
print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}")
total["usage"]["imei"] += 1 if doc_type == "imei" else 0 total["usage"]["imei"] += 1 if doc_type == "imei" else 0
total["usage"]["invoice"] += 1 if doc_type == "invoice" else 0 total["usage"]["invoice"] += 1 if doc_type == "invoice" else 0
total["usage"]["total_images"] += 1 total["usage"]["total_images"] += 1
total["review_progress"].append(report_file.review_status) total["review_progress"].append(report_file.review_status)
return total return total
@staticmethod @staticmethod
def update_day(day_data, report_file): def update_day(day_data, report_file):
day_data["total_images"] += 1 if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS:
day_data["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 day_data["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0
day_data["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 day_data["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0
doc_type = "imei" day_data["total_images"] += 1
if report_file.doc_type in ["imei", "invoice", "all"]: doc_type = "imei"
doc_type = report_file.doc_type if report_file.doc_type in ["imei", "invoice", "all"]:
else: doc_type = report_file.doc_type
print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") else:
day_data["num_imei"] += 1 if doc_type == "imei" else 0 print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}")
day_data["num_invoice"] += 1 if doc_type == "invoice" else 0 day_data["num_imei"] += 1 if doc_type == "imei" else 0
day_data["report_files"].append(report_file) day_data["num_invoice"] += 1 if doc_type == "invoice" else 0
day_data["report_files"].append(report_file)
if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0:
day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0:
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", []))
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", []))
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", []))
elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", []))
day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0:
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", []))
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", []))
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", []))
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", []))
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, []))
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, []))
if not day_data["average_processing_time"].get(report_file.doc_type, None): if not day_data["average_processing_time"].get(report_file.doc_type, None):
print(f"[WARM]: Weird doctype: {report_file.doc_type}") print(f"[WARM]: Weird doctype: {report_file.doc_type}")
day_data["average_processing_time"][report_file.doc_type] = IterAvg() day_data["average_processing_time"][report_file.doc_type] = IterAvg()
day_data["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 day_data["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0
day_data["review_progress"].append(report_file.review_status) day_data["review_progress"].append(report_file.review_status)
return day_data return day_data
def add(self, request, report_files): def add(self, request, report_files):
@ -255,6 +257,8 @@ class ReportAccumulateByRequest:
_average_OCR_time = {"invoice": self.data[month][1][day]["average_processing_time"]["invoice"](), "imei": self.data[month][1][day]["average_processing_time"]["imei"](), _average_OCR_time = {"invoice": self.data[month][1][day]["average_processing_time"]["invoice"](), "imei": self.data[month][1][day]["average_processing_time"]["imei"](),
"invoice_count": self.data[month][1][day]["average_processing_time"]["invoice"].count, "imei_count": self.data[month][1][day]["average_processing_time"]["imei"].count} "invoice_count": self.data[month][1][day]["average_processing_time"]["invoice"].count, "imei_count": self.data[month][1][day]["average_processing_time"]["imei"].count}
_average_OCR_time["invoice"] = 0 if _average_OCR_time["invoice"] is None else _average_OCR_time["invoice"]
_average_OCR_time["imei"] = 0 if _average_OCR_time["imei"] is None else _average_OCR_time["imei"]
_average_OCR_time["avg"] = (_average_OCR_time["invoice"]*_average_OCR_time["invoice_count"] + _average_OCR_time["imei"]*_average_OCR_time["imei_count"])/(_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) if (_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) > 0 else None _average_OCR_time["avg"] = (_average_OCR_time["invoice"]*_average_OCR_time["invoice_count"] + _average_OCR_time["imei"]*_average_OCR_time["imei_count"])/(_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) if (_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) > 0 else None
acumulated_acc = {"feedback_accuracy": {}, acumulated_acc = {"feedback_accuracy": {},
"reviewed_accuracy": {}} "reviewed_accuracy": {}}
@ -527,6 +531,8 @@ class IterAvg:
self.avg = (self.avg*(self.count-count) + avg*count)/(self.count) self.avg = (self.avg*(self.count-count) + avg*count)/(self.count)
def __call__(self): def __call__(self):
if self.count == 0:
return None
return self.avg return self.avg
def validate_feedback_file(feedback, predict): def validate_feedback_file(feedback, predict):

1
cope2n-api/fwd_api/utils/cache.py
View File

@ -10,7 +10,6 @@ def set_cache(key, value):
this_cache.save() this_cache.save()
return this_cache return this_cache
def get_cache(key): def get_cache(key):
value = {} value = {}
cache = Caching.objects.filter(key=key) cache = Caching.objects.filter(key=key)

2
cope2n-fe/src/utils/metric-format.ts
View File

@ -1,5 +1,5 @@
export const formatPercent = (value: number, floatingPoint: number = 1) => { export const formatPercent = (value: number, floatingPoint: number = 1) => {
if (value === 0) { if (value === null || value === undefined) {
return '-'; return '-';
} }
if (value < 100.0) { if (value < 100.0) {

16
docker-compose-dev.yml
View File

@ -89,12 +89,12 @@ services:
depends_on: depends_on:
db-sbt: db-sbt:
condition: service_started condition: service_started
command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && # command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input &&
python manage.py makemigrations && # python manage.py makemigrations &&
python manage.py migrate && # python manage.py migrate &&
python manage.py compilemessages && # python manage.py compilemessages &&
gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod # gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
# command: bash -c "tail -f > /dev/null" command: bash -c "tail -f > /dev/null"
minio: minio:
image: minio/minio image: minio/minio
@ -179,8 +179,8 @@ services:
- ./cope2n-api:/app - ./cope2n-api:/app
working_dir: /app working_dir: /app
command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" # command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
# command: bash -c "tail -f > /dev/null" command: bash -c "tail -f > /dev/null"
# Back-end persistent # Back-end persistent
db-sbt: db-sbt: