Fix: double record, acc issues
This commit is contained in:
parent
46a89c9b26
commit
005d742750
@ -224,6 +224,8 @@ OVERVIEW_REFRESH_INTERVAL = 2
|
||||
OVERVIEW_REPORT_ROOT = "overview"
|
||||
OVERVIEW_REPORT_DURATION = ["30d", "7d"]
|
||||
|
||||
ACC_EXCLUDE_RESEASONS = ["Invalid Input", "Handwritten information", "handwritten"]
|
||||
|
||||
SUBS = {
|
||||
"SEAU": "AU",
|
||||
"SESP": "SG",
|
||||
|
||||
@ -382,7 +382,7 @@ class AccuracyViewSet(viewsets.ViewSet):
|
||||
for key in acc_keys:
|
||||
fb = report.feedback_accuracy.get(key, 0) if report.feedback_accuracy else 0
|
||||
rv = report.reviewed_accuracy.get(key, 0) if report.reviewed_accuracy else 0
|
||||
acc[key] = max([fb, rv])
|
||||
acc[key] = report.combined_accuracy.get(key, 0) if report.combined_accuracy else max([fb, rv])
|
||||
data.append({
|
||||
"ID": report.id,
|
||||
"Created Date": report.created_at,
|
||||
|
||||
@ -14,7 +14,7 @@ from ..utils import file as FileUtils
|
||||
from ..utils import process as ProcessUtil
|
||||
from ..utils import s3 as S3Util
|
||||
from ..utils.accuracy import validate_feedback_file
|
||||
from fwd_api.constant.common import ProcessType
|
||||
from fwd_api.constant.common import FileCategory
|
||||
import csv
|
||||
import json
|
||||
import copy
|
||||
@ -114,6 +114,9 @@ def process_csv_feedback(csv_file_path, feedback_id):
|
||||
for k, v in sub_rq.ai_inference_profile.items():
|
||||
time_cost[k.split("_")[0]].append(v["inference"][1][0] - v["inference"][0] + (v["postprocess"][1]-v["postprocess"][0]))
|
||||
for i, image in enumerate(images):
|
||||
if image.file_category != FileCategory.Origin.value:
|
||||
# skip break files, which are not responsible for storing data
|
||||
continue
|
||||
_predict_result = copy.deepcopy(predict_result_to_ready(sub_rq.predict_result))
|
||||
_feedback_result = copy.deepcopy(sub_rq.feedback_result)
|
||||
_reviewed_result = copy.deepcopy(sub_rq.reviewed_result)
|
||||
@ -128,12 +131,10 @@ def process_csv_feedback(csv_file_path, feedback_id):
|
||||
_predict_result["imei_number"] = []
|
||||
if _feedback_result:
|
||||
_feedback_result["imei_number"] = []
|
||||
else:
|
||||
None
|
||||
|
||||
if _reviewed_result:
|
||||
_reviewed_result["imei_number"] = []
|
||||
else:
|
||||
None
|
||||
|
||||
else:
|
||||
try:
|
||||
_predict_result = {"retailername": None, "sold_to_party": None, "purchase_date": [], "imei_number": [_predict_result["imei_number"][image.index_in_request]]}
|
||||
|
||||
@ -4,7 +4,7 @@ from fwd_api.models import SubscriptionRequest, Report, ReportFile
|
||||
from fwd_api.celery_worker.worker import app
|
||||
from ..utils import s3 as S3Util
|
||||
from ..utils.accuracy import update_temp_accuracy, IterAvg, calculate_and_save_subcription_file, count_transactions, extract_report_detail_list, calculate_a_request, ReportAccumulateByRequest
|
||||
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3
|
||||
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3, save_images_to_csv_briefly
|
||||
from ..utils import time_stuff
|
||||
from ..utils.redis import RedisUtils
|
||||
from django.utils import timezone
|
||||
@ -187,8 +187,6 @@ def make_a_report_2(report_id, query_set):
|
||||
base_query &= Q(is_reviewed=True)
|
||||
elif query_set["is_reviewed"] == "not reviewed":
|
||||
base_query &= Q(is_reviewed=False)
|
||||
# elif query_set["is_reviewed"] == "all":
|
||||
# pass
|
||||
|
||||
errors = []
|
||||
# Create a placeholder to fill
|
||||
@ -197,12 +195,17 @@ def make_a_report_2(report_id, query_set):
|
||||
"retailername": IterAvg(),
|
||||
"sold_to_party": IterAvg(),},
|
||||
"reviewed" :{"imei_number": IterAvg(),
|
||||
"purchase_date": IterAvg(),
|
||||
"retailername": IterAvg(),
|
||||
"sold_to_party": IterAvg(),},
|
||||
"acumulated":{"imei_number": IterAvg(),
|
||||
"purchase_date": IterAvg(),
|
||||
"retailername": IterAvg(),
|
||||
"sold_to_party": IterAvg(),}
|
||||
} # {"imei": {"acc": 0.1, count: 1}, ...}
|
||||
time_cost = {"invoice": IterAvg(),
|
||||
"imei": IterAvg()}
|
||||
bad_image_list = []
|
||||
number_images = 0
|
||||
number_bad_images = 0
|
||||
# TODO: Multithreading
|
||||
@ -232,8 +235,10 @@ def make_a_report_2(report_id, query_set):
|
||||
request.save()
|
||||
number_images += request_att["total_images"]
|
||||
number_bad_images += request_att["bad_images"]
|
||||
bad_image_list += request_att["bad_image_list"]
|
||||
update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
|
||||
update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
|
||||
update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
|
||||
|
||||
time_cost["imei"].add(request_att["time_cost"].get("imei", []))
|
||||
time_cost["invoice"].add(request_att["time_cost"].get("invoice", []))
|
||||
@ -259,8 +264,9 @@ def make_a_report_2(report_id, query_set):
|
||||
report.number_invoice_transaction = transaction_att.get("invoice", 0)
|
||||
|
||||
acumulated_acc = {"feedback": {},
|
||||
"reviewed": {}}
|
||||
for acc_type in ["feedback", "reviewed"]:
|
||||
"reviewed": {},
|
||||
"acumulated": {}}
|
||||
for acc_type in ["feedback", "reviewed", "acumulated"]:
|
||||
avg_acc = IterAvg()
|
||||
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
|
||||
acumulated_acc[acc_type][key] = accuracy[acc_type][key]()
|
||||
@ -270,10 +276,13 @@ def make_a_report_2(report_id, query_set):
|
||||
|
||||
report.feedback_accuracy = acumulated_acc["feedback"]
|
||||
report.reviewed_accuracy = acumulated_acc["reviewed"]
|
||||
report.combined_accuracy = acumulated_acc["acumulated"]
|
||||
|
||||
report.errors = "|".join(errors)
|
||||
report.status = "Ready"
|
||||
report.save()
|
||||
# Save a list of bad images to csv file for debugging
|
||||
save_images_to_csv_briefly(report.report_id, bad_image_list)
|
||||
# Saving a xlsx file
|
||||
data = extract_report_detail_list(report_files, lower=True)
|
||||
data_workbook = dict2xlsx(data, _type='report_detail')
|
||||
|
||||
@ -0,0 +1,166 @@
|
||||
# myapp/management/commands/mycustomcommand.py
|
||||
from django.core.management.base import BaseCommand
|
||||
from tqdm import tqdm
|
||||
from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest
|
||||
from fwd_api.utils.accuracy import predict_result_to_ready
|
||||
import traceback
|
||||
import copy
|
||||
import csv
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = 'Refactor database for image level'
|
||||
|
||||
def add_arguments(self, parser):
|
||||
# Add your command-line arguments here
|
||||
parser.add_argument('test', type=str, help='Value for the argument')
|
||||
|
||||
def process_request(self, request, predict_result, user_feedback, reviewed_result):
|
||||
if len(request.request_id.split(".")[0].split("_")) < 2:
|
||||
return
|
||||
|
||||
request_feedback = copy.deepcopy(request.feedback_result)
|
||||
request_review = copy.deepcopy(request.reviewed_result)
|
||||
|
||||
if not request_feedback:
|
||||
request_feedback = {
|
||||
"request_id": request.request_id,
|
||||
"imei_number": [],
|
||||
"retailername": "",
|
||||
"purchase_date": "",
|
||||
"sold_to_party": ""
|
||||
}
|
||||
|
||||
if not request_review:
|
||||
request_review = {
|
||||
"request_id": request.request_id,
|
||||
"imei_number": [],
|
||||
"retailername": "",
|
||||
"purchase_date": "",
|
||||
"sold_to_party": ""
|
||||
}
|
||||
|
||||
images = SubscriptionRequestFile.objects.filter(request=request)
|
||||
is_match = False
|
||||
try:
|
||||
for i, image in enumerate(images):
|
||||
if not request.predict_result:
|
||||
raise KeyError(f"Key predict_result not found in {request.request_id}")
|
||||
if request.predict_result.get("status", 200) != 200:
|
||||
raise AttributeError(f"Failed request: {request.request_id}")
|
||||
|
||||
for field in ['retailername', 'purchase_date', 'imei_number']:
|
||||
# if image.feedback_result[field] is not None:
|
||||
# print(f"image.feedback_result[field] is not None is not None - field: {field}")
|
||||
# else:
|
||||
# print("image.feedback_result[field] is None")
|
||||
|
||||
# if image.feedback_result[field] == user_feedback:
|
||||
# print("image.feedback_result[field] == user_feedback")
|
||||
# else:
|
||||
# print(f"NOT image.feedback_result[field] == user_feedback - field: {field} - image.feedback_result[field]:{image.feedback_result[field]} - user_feedback:{user_feedback}")
|
||||
|
||||
# if (field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback):
|
||||
# print("(field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback)")
|
||||
# else:
|
||||
# print(f"NOT (field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback) - field: {field}")
|
||||
|
||||
# if image.feedback_result[field] is not None and ((field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback) or image.feedback_result[field] == user_feedback):
|
||||
is_match = True
|
||||
|
||||
if field == 'imei_number':
|
||||
if not reviewed_result == request_review:
|
||||
request_review["imei_number"].append(reviewed_result)
|
||||
if not user_feedback == request_feedback:
|
||||
request_feedback["imei_number"].append(user_feedback)
|
||||
else:
|
||||
if not reviewed_result == request_review:
|
||||
request_review[field] = reviewed_result
|
||||
if not user_feedback == request_feedback:
|
||||
request_feedback[field] = user_feedback
|
||||
|
||||
_predict_result = copy.deepcopy(predict_result_to_ready(request.predict_result))
|
||||
_feedback_result = copy.deepcopy(request.feedback_result)
|
||||
_reviewed_result = copy.deepcopy(request.reviewed_result)
|
||||
|
||||
if not _feedback_result:
|
||||
_feedback_result = {
|
||||
"imei_number": [],
|
||||
"retailername": "",
|
||||
"purchase_date": "",
|
||||
"sold_to_party": ""
|
||||
}
|
||||
if not _reviewed_result:
|
||||
_reviewed_result = {
|
||||
"imei_number": [],
|
||||
"retailername": "",
|
||||
"purchase_date": "",
|
||||
"sold_to_party": ""
|
||||
}
|
||||
|
||||
if image.doc_type == "invoice":
|
||||
_predict_result[field] = predict_result
|
||||
_predict_result["imei_number"] = []
|
||||
if _feedback_result:
|
||||
_feedback_result[field] = user_feedback
|
||||
_feedback_result["imei_number"] = []
|
||||
else:
|
||||
None
|
||||
if _reviewed_result:
|
||||
_reviewed_result[field] = reviewed_result
|
||||
_reviewed_result["imei_number"] = []
|
||||
else:
|
||||
None
|
||||
else:
|
||||
_predict_result = {
|
||||
"retailername": None,
|
||||
"sold_to_party": None,
|
||||
"purchase_date": [],
|
||||
"imei_number": [predict_result]
|
||||
}
|
||||
_feedback_result = {
|
||||
"retailername": None,
|
||||
"sold_to_party": None,
|
||||
"purchase_date": None,
|
||||
"imei_number": [user_feedback]
|
||||
} if _feedback_result else None
|
||||
_reviewed_result = {
|
||||
"retailername": None,
|
||||
"sold_to_party": None,
|
||||
"purchase_date": None,
|
||||
"imei_number": [reviewed_result]
|
||||
} if _reviewed_result else None
|
||||
image.predict_result = _predict_result
|
||||
image.feedback_result = _feedback_result
|
||||
image.reviewed_result = _reviewed_result
|
||||
image.save()
|
||||
# request.feedback_result = request_feedback
|
||||
request.reviewed_result = request_review
|
||||
request.feedback_result["request_id"] = request.request_id
|
||||
request.reviewed_result["request_id"] = request.request_id
|
||||
request.is_reviewed = True
|
||||
request.save()
|
||||
|
||||
except Exception as e:
|
||||
self.stdout.write(self.style.ERROR(f"Request: {request.request_id} failed with {e}"))
|
||||
print(traceback.format_exc())
|
||||
|
||||
if not is_match:
|
||||
print("FAIL =====>", "image.feedback_result: ", image.feedback_result, "| predict_result: ", predict_result, " | user_feedback: ", user_feedback, "| reviewed_result: ", reviewed_result)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
test = options['test']
|
||||
#open csv file
|
||||
with open(test, 'r') as csvfile:
|
||||
reader = csv.reader(csvfile)
|
||||
index = 0
|
||||
for row in reader:
|
||||
if index != 0:
|
||||
request = SubscriptionRequest.objects.filter(request_id=row[0]).first()
|
||||
if not request:
|
||||
# print("Not found ====>", row)
|
||||
continue
|
||||
else:
|
||||
self.process_request(request, row[4], row[3], row[5])
|
||||
index += 1
|
||||
|
||||
self.stdout.write(self.style.SUCCESS('Sample Django management command executed successfully!'))
|
||||
@ -0,0 +1,18 @@
|
||||
# Generated by Django 4.1.3 on 2024-02-18 05:19
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('fwd_api', '0181_reportfile_subsidiary'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='report',
|
||||
name='combined_accuracy',
|
||||
field=models.JSONField(null=True),
|
||||
),
|
||||
]
|
||||
@ -39,3 +39,4 @@ class Report(models.Model):
|
||||
|
||||
feedback_accuracy = models.JSONField(null=True)
|
||||
reviewed_accuracy = models.JSONField(null=True)
|
||||
combined_accuracy = models.JSONField(null=True)
|
||||
@ -5,7 +5,7 @@ import copy
|
||||
from typing import Any
|
||||
from .ocr_utils.ocr_metrics import eval_ocr_metric
|
||||
from .ocr_utils.sbt_report import post_processing_str
|
||||
import uuid
|
||||
from fwd_api.constant.common import FileCategory
|
||||
from fwd_api.models import SubscriptionRequest, SubscriptionRequestFile, ReportFile
|
||||
from ..celery_worker.client_connector import c_connector
|
||||
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3
|
||||
@ -173,20 +173,20 @@ class ReportAccumulateByRequest:
|
||||
day_data["report_files"].append(report_file)
|
||||
|
||||
if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0:
|
||||
day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", 0))
|
||||
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", 0))
|
||||
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", 0))
|
||||
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", 0))
|
||||
day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", []))
|
||||
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", []))
|
||||
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", []))
|
||||
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", []))
|
||||
elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0:
|
||||
day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", 0))
|
||||
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", 0))
|
||||
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", 0))
|
||||
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", 0))
|
||||
day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", []))
|
||||
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", []))
|
||||
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", []))
|
||||
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", []))
|
||||
|
||||
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
|
||||
day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, 0))
|
||||
day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, []))
|
||||
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
|
||||
day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, 0))
|
||||
day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, []))
|
||||
|
||||
if not day_data["average_processing_time"].get(report_file.doc_type, None):
|
||||
print(f"[WARM]: Weird doctype: {report_file.doc_type}")
|
||||
@ -196,14 +196,14 @@ class ReportAccumulateByRequest:
|
||||
return day_data
|
||||
|
||||
def add(self, request, report_files):
|
||||
this_month = request.created_at.strftime("%Y%m")
|
||||
this_day = request.created_at.strftime("%Y%m%d")
|
||||
this_month = timezone.localtime(request.created_at).strftime("%Y%m")
|
||||
this_day = timezone.localtime(request.created_at).strftime("%Y%m%d")
|
||||
if not self.data.get(this_month, None):
|
||||
self.data[this_month] = [copy.deepcopy(self.total_format), {}]
|
||||
self.data[this_month][0]["extraction_date"] = "Subtotal (" + request.created_at.strftime("%Y-%m") + ")"
|
||||
self.data[this_month][0]["extraction_date"] = "Subtotal (" + timezone.localtime(request.created_at).strftime("%Y-%m") + ")"
|
||||
if not self.data[this_month][1].get(this_day, None):
|
||||
self.data[this_month][1][this_day] = copy.deepcopy(self.day_format)[0]
|
||||
self.data[this_month][1][this_day]['extraction_date'] = request.created_at.strftime("%Y-%m-%d")
|
||||
self.data[this_month][1][this_day]['extraction_date'] = timezone.localtime(request.created_at).strftime("%Y-%m-%d")
|
||||
usage = self.count_transactions_within_day(this_day)
|
||||
self.data[this_month][1][this_day]["usage"]["imei"] = usage.get("imei", 0)
|
||||
self.data[this_month][1][this_day]["usage"]["invoice"] = usage.get("invoice", 0)
|
||||
@ -625,14 +625,14 @@ def align_fine_result(ready_predict, fine_result):
|
||||
# print(f"[DEBUG]: fine_result: {fine_result}")
|
||||
# print(f"[DEBUG]: ready_predict: {ready_predict}")
|
||||
if fine_result:
|
||||
if isinstance(ready_predict["purchase_date"], str):
|
||||
ready_predict["purchase_date"] = [ready_predict["purchase_date"]]
|
||||
# ready_predict.save()
|
||||
if fine_result["purchase_date"] and len(ready_predict["purchase_date"]) == 0:
|
||||
ready_predict["purchase_date"] = [None]
|
||||
if fine_result["retailername"] and not ready_predict["retailername"]:
|
||||
ready_predict["retailername"] = [None]
|
||||
# if ready_predict["retailername"] and not fine_result["retailername"]:
|
||||
# fine_result["retailername"] = [None]
|
||||
fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))]
|
||||
# fine_result["retailername"] = None if len(ready_predict["purchase_date"]))]
|
||||
# else:
|
||||
# fine_result = {}
|
||||
# for key in ready_predict.keys():
|
||||
@ -668,6 +668,12 @@ def calculate_accuracy(key_name, inference, target):
|
||||
target[key_name] = []
|
||||
else:
|
||||
target[key_name] = [target[key_name]]
|
||||
# Realign lenght for mis predicted/feedback/reivew result
|
||||
if len(target[key_name]) == 0 and len(inference[key_name]) > 0:
|
||||
target[key_name] = [None for _ in range(len(inference[key_name]))]
|
||||
elif len(inference[key_name]) == 0 and len(target[key_name]) > 0:
|
||||
target[key_name] = [None for _ in range(len(inference[key_name]))]
|
||||
|
||||
for i, v in enumerate(inference[key_name]):
|
||||
# TODO: target[key_name][i] is None, ""
|
||||
x = post_processing_str(key_name, inference[key_name][i], is_gt=False)
|
||||
@ -777,6 +783,11 @@ def calculate_a_request(report, request):
|
||||
"sold_to_party": [],
|
||||
},
|
||||
"reviewed": {"imei_number": [],
|
||||
"purchase_date": [],
|
||||
"retailername": [],
|
||||
"sold_to_party": [],
|
||||
},
|
||||
"acumulated":{"imei_number": [],
|
||||
"purchase_date": [],
|
||||
"retailername": [],
|
||||
"sold_to_party": [],
|
||||
@ -784,10 +795,15 @@ def calculate_a_request(report, request):
|
||||
"err": [],
|
||||
"time_cost": {},
|
||||
"total_images": 0,
|
||||
"bad_images": 0}
|
||||
images = SubscriptionRequestFile.objects.filter(request=request)
|
||||
"bad_images": 0,
|
||||
"bad_image_list": [],
|
||||
}
|
||||
images = SubscriptionRequestFile.objects.filter(request=request, file_category=FileCategory.Origin.value)
|
||||
report_files = []
|
||||
for image in images:
|
||||
if image.reason in settings.ACC_EXCLUDE_RESEASONS:
|
||||
continue
|
||||
|
||||
status, att = calculate_subcription_file(image)
|
||||
if status != 200:
|
||||
continue
|
||||
@ -805,6 +821,8 @@ def calculate_a_request(report, request):
|
||||
_sub = map_subsidiary_short_to_long(request.redemption_id[:2])
|
||||
else:
|
||||
print(f"[WARM]: empty redemption_id, check request: {request.request_id}")
|
||||
if att["is_bad_image"]:
|
||||
request_att["bad_image_list"].append(image.file_name)
|
||||
new_report_file = ReportFile(report=report,
|
||||
subsidiary=_sub,
|
||||
correspond_request_id=request.request_id,
|
||||
@ -838,11 +856,16 @@ def calculate_a_request(report, request):
|
||||
request_att["acc"]["reviewed"]["retailername"] += att["acc"]["reviewed"]["retailername"]
|
||||
request_att["acc"]["reviewed"]["sold_to_party"] += att["acc"]["reviewed"]["sold_to_party"]
|
||||
|
||||
request_att["acc"]["acumulated"]["imei_number"] += att["acc"]["reviewed"]["imei_number"] if att["acc"]["reviewed"]["imei_number"] else att["acc"]["feedback"]["imei_number"]
|
||||
request_att["acc"]["acumulated"]["purchase_date"] += att["acc"]["reviewed"]["purchase_date"] if att["acc"]["reviewed"]["purchase_date"] else att["acc"]["feedback"]["purchase_date"]
|
||||
request_att["acc"]["acumulated"]["retailername"] += att["acc"]["reviewed"]["retailername"] if att["acc"]["reviewed"]["retailername"] else att["acc"]["feedback"]["retailername"]
|
||||
request_att["acc"]["acumulated"]["sold_to_party"] += att["acc"]["reviewed"]["sold_to_party"] if att["acc"]["reviewed"]["sold_to_party"] else att["acc"]["feedback"]["sold_to_party"]
|
||||
|
||||
request_att["bad_images"] += int(att["is_bad_image"])
|
||||
request_att["total_images"] += 1
|
||||
request_att["err"] += att["err"]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(f"[ERROR]: failed to calculate request: {request.request_id} - request_file: {image.file_name} because of {e}")
|
||||
continue
|
||||
|
||||
return request_att, report_files
|
||||
@ -870,11 +893,20 @@ def calculate_subcription_file(subcription_request_file):
|
||||
att["acc"]["reviewed"][key_name], _ = calculate_accuracy(key_name, inference_result, reviewed_result)
|
||||
except Exception as e:
|
||||
att["err"].append(str(e))
|
||||
# print(f"[DEBUG]: predict_result: {subcription_request_file.predict_result}")
|
||||
# print(f"[DEBUG]: e: {e} -key_name: {key_name}")
|
||||
subcription_request_file.feedback_accuracy = att["acc"]["feedback"]
|
||||
subcription_request_file.reviewed_accuracy = att["acc"]["reviewed"]
|
||||
subcription_request_file.save()
|
||||
avg_reviewed = calculate_avg_accuracy(att["acc"], "reviewed", ["retailername", "sold_to_party", "purchase_date", "imei_number"])
|
||||
avg_feedback = calculate_avg_accuracy(att["acc"], "feedback", ["retailername", "sold_to_party", "purchase_date", "imei_number"])
|
||||
if avg_feedback is not None or avg_reviewed is not None:
|
||||
avg_acc = max([x for x in [avg_feedback, avg_reviewed] if x is not None])
|
||||
avg_acc = 0
|
||||
if avg_feedback is not None:
|
||||
avg_acc = avg_feedback
|
||||
if avg_reviewed is not None:
|
||||
avg_acc = avg_reviewed
|
||||
|
||||
if avg_acc < BAD_THRESHOLD:
|
||||
att["is_bad_image"] = True
|
||||
# exclude bad images
|
||||
@ -947,6 +979,12 @@ def calculate_attributions(request): # for one request, return in order
|
||||
|
||||
return acc, data, time_cost, image_quality_num, error
|
||||
|
||||
def mean_list(l):
|
||||
l = [x for x in l if x is not None]
|
||||
if len(l) == 0:
|
||||
return 0
|
||||
return sum(l)/len(l)
|
||||
|
||||
def shadow_report(report_id, query):
|
||||
c_connector.make_a_report_2(
|
||||
(report_id, query))
|
||||
@ -250,6 +250,37 @@ def save_file_with_path(file_name: str, file: TemporaryUploadedFile, quality, fo
|
||||
raise ServiceUnavailableException()
|
||||
return file_path
|
||||
|
||||
def save_images_to_csv_briefly(id, image_filenames):
|
||||
# columns = ["request_id", "file_name", "predict_result", "feedback_result", "reviewed_result", "feedback_accuracy", "reviewed_accuracy"]
|
||||
columns = ["request_id", "file_name", "predict_result", "feedback_result", "reviewed_result", "feedback_accuracy", "reviewed_accuracy"]
|
||||
|
||||
# get the SubcriptionRequestFile list
|
||||
images = SubscriptionRequestFile.objects.filter(file_name__in=image_filenames)
|
||||
# Create a CSV writer object
|
||||
folder_path = os.path.join(settings.MEDIA_ROOT, "report", id)
|
||||
file_path = os.path.join(folder_path, "bad_images.csv")
|
||||
os.makedirs(folder_path, exist_ok = True)
|
||||
|
||||
csv_file = open(file_path, "w", newline="")
|
||||
csv_writer = csv.DictWriter(csv_file, fieldnames=columns)
|
||||
csv_writer.writeheader()
|
||||
|
||||
# Write data to the CSV file
|
||||
for subscription_request_file in images:
|
||||
row = {
|
||||
"request_id": subscription_request_file.request.request_id,
|
||||
"file_name" : subscription_request_file.file_name,
|
||||
"predict_result": subscription_request_file.predict_result,
|
||||
"feedback_result": subscription_request_file.feedback_result,
|
||||
"reviewed_result": subscription_request_file.reviewed_result,
|
||||
# "feedback_accuracy": subscription_request_file.feedback_accuracy,
|
||||
# "reviewed_accuracy": subscription_request_file.reviewed_accuracy,
|
||||
}
|
||||
csv_writer.writerow(row)
|
||||
# Close the CSV file
|
||||
csv_file.close()
|
||||
# save to S3
|
||||
save_report_to_S3(id, file_path)
|
||||
|
||||
def resize_and_save_file(file_name: str, rq: SubscriptionRequest, file: TemporaryUploadedFile, quality: int):
|
||||
try:
|
||||
|
||||
@ -15,8 +15,8 @@ login_token = None
|
||||
# Define the login credentials
|
||||
login_credentials = {
|
||||
'username': 'sbt',
|
||||
'password': '7Eg4AbWIXDnufgn'
|
||||
# 'password': 'abc'
|
||||
# 'password': '7Eg4AbWIXDnufgn'
|
||||
'password': 'abc'
|
||||
}
|
||||
|
||||
# Define the command to call the update API
|
||||
|
||||
Loading…
Reference in New Issue
Block a user