tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #87 from SDSRV-IDP/trungpt/invoice_no

Browse Source 
Add Invoice_no
This commit is contained in:
Đỗ Xuân Tân 2024-03-13 15:39:25 +07:00 committed by GitHub Enterprise
commit 44900ac555
10 changed files with 80 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
cope2n-ai-fi/Dockerfile
View File

@ -24,10 +24,10 @@ RUN python -m pip install 'git+https://github.com/facebookresearch/detectron2.gi
# Install SDSV packages # Install SDSV packages
COPY . /workspace/cope2n-ai-fi COPY . /workspace/cope2n-ai-fi
RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsv_dewarp && pip3 install -v -e . --no-cache-dir RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr/sdsvocr/externals/sdsv_dewarp && pip3 install -v -e . --no-cache-dir
RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsvtd && pip3 install -v -e . --no-cache-dir RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr/sdsvocr/externals/sdsvtd && pip3 install -v -e . --no-cache-dir
RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsvtr && pip3 install -v -e . --no-cache-dir RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr/sdsvocr/externals/sdsvtr && pip3 install -v -e . --no-cache-dir
RUN cd /workspace/cope2n-ai-fi/modules/sdsvkvu/sdsvkvu/externals/sdsvocr && pip3 install -v -e . --no-cache-dir
# COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsv_dewarp /tmp/sdsv_dewarp # COPY ./modules/sdsvkvu/sdsvkvu/externals/sdsvocr/externals/sdsv_dewarp /tmp/sdsv_dewarp
# RUN cd /tmp/sdsv_dewarp && pip install -v -e . --no-cache-dir # RUN cd /tmp/sdsv_dewarp && pip install -v -e . --no-cache-dir
@ -64,4 +64,5 @@ WORKDIR /workspace
ENV PYTHONPATH="." ENV PYTHONPATH="."
ENV TZ="Asia/Ho_Chi_Minh" ENV TZ="Asia/Ho_Chi_Minh"
CMD [ "sh", "run.sh"] CMD [ "sh", "run.sh"]
# ENTRYPOINT [ "sleep", "infinity" ]

3
cope2n-ai-fi/api/sdsap_sbt/prediction_sbt.py
View File

@ -38,7 +38,7 @@ def sbt_predict(image_url, engine) -> None:
os.makedirs(save_dir, exist_ok = True) os.makedirs(save_dir, exist_ok = True)
tmp_image_path = os.path.join(save_dir, f"{uuid.uuid4()}.jpg") tmp_image_path = os.path.join(save_dir, f"{uuid.uuid4()}.jpg")
cv2.imwrite(tmp_image_path, img) cv2.imwrite(tmp_image_path, img)
outputs = process_img(img_path=tmp_image_path, outputs = process_img(img=tmp_image_path,
save_dir=save_dir, save_dir=save_dir,
engine=engine, engine=engine,
export_all=False, # False export_all=False, # False
@ -71,7 +71,6 @@ def predict(page_numb, image_url):
""" """
sbt_result = sbt_predict(image_url, engine=sbt_engine) sbt_result = sbt_predict(image_url, engine=sbt_engine)
print(sbt_result)
output_dict = { output_dict = {
"document_type": "invoice", "document_type": "invoice",
"document_class": " ", "document_class": " ",

3
cope2n-ai-fi/common/utils_kvu/split_docs.py
View File

@ -107,6 +107,7 @@ def merge_sbt_output(loutputs):
merged_output = [] merged_output = []
combined_output = {"retailername": None, combined_output = {"retailername": None,
"sold_to_party": None, "sold_to_party": None,
"invoice_no": None,
"purchase_date": [], "purchase_date": [],
"imei_number": []} # place holder for the output "imei_number": []} # place holder for the output
for output in loutputs: for output in loutputs:
@ -123,7 +124,7 @@ def merge_sbt_output(loutputs):
combined_output[field["label"]].append(field["value"]) combined_output[field["label"]].append(field["value"])
if output['doc_type'] == "invoice": if output['doc_type'] == "invoice":
for field in fields: for field in fields:
if field["label"] in ["retailername", "sold_to_party", "purchase_date"] : if field["label"] in ["retailername", "sold_to_party", "purchase_date", "invoice_no"] :
if isinstance(combined_output[field["label"]], list): if isinstance(combined_output[field["label"]], list):
if field["value"] is not None: if field["value"] is not None:
if isinstance(field["value"], list): if isinstance(field["value"], list):

2
cope2n-ai-fi/requirements.txt
View File

@ -9,6 +9,6 @@ pymupdf
easydict easydict
imagesize==1.4.1 imagesize==1.4.1
pdf2image==1.16.3 pdf2image==1.17.0
redis==5.0.1 redis==5.0.1
celery==5.3.6 celery==5.3.6

8
cope2n-api/fwd_api/api/accuracy_view.py
View File

@ -409,7 +409,7 @@ class AccuracyViewSet(viewsets.ViewSet):
data = [] data = []
for report in page: for report in page:
acc_keys = ["purchase_date", "retailername", "imei_number", "avg"] acc_keys = ["purchase_date", "retailername", "invoice_no", "imei_number", "avg"]
acc = {} acc = {}
for key in acc_keys: for key in acc_keys:
fb = report.feedback_accuracy.get(key, 0) if report.feedback_accuracy else 0 fb = report.feedback_accuracy.get(key, 0) if report.feedback_accuracy else 0
@ -649,7 +649,7 @@ class AccuracyViewSet(viewsets.ViewSet):
'properties': { 'properties': {
'reviewed_result': { 'reviewed_result': {
'type': 'string', 'type': 'string',
'default': '''{"request_id": "Sample request_id", "imei_number": ["sample_imei1", "sample_imei2"], "retailername": "Sample Retailer", "purchase_date": "01/01/1970", "sold_to_party": "Sample party"}''', 'default': '''{"request_id": "Sample request_id", "imei_number": ["sample_imei1", "sample_imei2"], "retailername": "Sample Retailer", "purchase_date": "01/01/1970", "sold_to_party": "Sample party", "invoice_no": "Sample Invoice no"}''',
}, },
}, },
}, },
@ -673,6 +673,7 @@ class AccuracyViewSet(viewsets.ViewSet):
"request_id": subscription_request.request_id, "request_id": subscription_request.request_id,
"retailername": None, "retailername": None,
"sold_to_party": None, "sold_to_party": None,
"invoice_no": None,
"purchase_date": None, "purchase_date": None,
"imei_number": [] "imei_number": []
} }
@ -774,7 +775,7 @@ class AccuracyViewSet(viewsets.ViewSet):
raise InvalidException(excArgs=f'reviewed_result') raise InvalidException(excArgs=f'reviewed_result')
reviewed_result = data["reviewed_result"] reviewed_result = data["reviewed_result"]
for field in ['retailername', 'sold_to_party', 'purchase_date', 'imei_number']: for field in ['retailername', 'sold_to_party', 'invoice_no', 'purchase_date', 'imei_number']:
if not field in reviewed_result.keys(): if not field in reviewed_result.keys():
raise RequiredFieldException(excArgs=f'reviewed_result.{field}') raise RequiredFieldException(excArgs=f'reviewed_result.{field}')
reviewed_result['request_id'] = request_id reviewed_result['request_id'] = request_id
@ -787,6 +788,7 @@ class AccuracyViewSet(viewsets.ViewSet):
subscription_request_file.reviewed_result = { subscription_request_file.reviewed_result = {
"retailername": None, "retailername": None,
"sold_to_party": None, "sold_to_party": None,
"invoice_no": None,
"purchase_date": [], "purchase_date": [],
"imei_number": []} "imei_number": []}
if len(reviewed_result["imei_number"]) - 1 >= subscription_request_file.index_in_request: if len(reviewed_result["imei_number"]) - 1 >= subscription_request_file.index_in_request:

5
cope2n-api/fwd_api/api/ctel_view.py
View File

@ -320,6 +320,9 @@ class CtelViewSet(viewsets.ViewSet):
'retailername': { 'retailername': {
'type': 'string', 'type': 'string',
}, },
'invoice_no': {
'type': 'string',
},
'sold_to_party': { 'sold_to_party': {
'type': 'string', 'type': 'string',
}, },
@ -336,7 +339,7 @@ class CtelViewSet(viewsets.ViewSet):
} }
}, },
}, },
'required': ['request_id', 'retailername', 'sold_to_party', 'purchase_date', 'imei_number'] 'required': ['request_id', 'retailername', 'invoice_no', 'sold_to_party', 'purchase_date', 'imei_number']
} }
}, responses=None, tags=['OCR']) }, responses=None, tags=['OCR'])
@action(detail=False, url_path="images/feedback", methods=["POST"]) @action(detail=False, url_path="images/feedback", methods=["POST"])

11
cope2n-api/fwd_api/celery_worker/internal_task.py
View File

@ -90,11 +90,13 @@ def process_csv_feedback(csv_file_path, feedback_id):
imei2 = row.get('imeiNumber2') imei2 = row.get('imeiNumber2')
purchase_date = row.get('Purchase Date') purchase_date = row.get('Purchase Date')
retailer = row.get('retailer') retailer = row.get('retailer')
invoice_no = row.get('invoice_no')
sold_to_party = row.get('Sold to party') sold_to_party = row.get('Sold to party')
server_time = float(row.get('timetakenmilli')) server_time = float(row.get('timetakenmilli'))
fb['request_id'] = request_id fb['request_id'] = request_id
fb['retailername'] = retailer fb['retailername'] = retailer
fb['sold_to_party'] = sold_to_party fb['sold_to_party'] = sold_to_party
fb["invoice_no"] = invoice_no
fb['purchase_date'] = purchase_date fb['purchase_date'] = purchase_date
fb['imei_number'] = [imei1, imei2] fb['imei_number'] = [imei1, imei2]
sub_rq.feedback_result = fb sub_rq.feedback_result = fb
@ -119,7 +121,6 @@ def process_csv_feedback(csv_file_path, feedback_id):
continue continue
_predict_result = copy.deepcopy(predict_result_to_ready(sub_rq.predict_result)) _predict_result = copy.deepcopy(predict_result_to_ready(sub_rq.predict_result))
_feedback_result = copy.deepcopy(sub_rq.feedback_result) _feedback_result = copy.deepcopy(sub_rq.feedback_result)
# _reviewed_result = copy.deepcopy(sub_rq.reviewed_result)
try: try:
image.processing_time = time_cost.get(image.doc_type, [0 for _ in range(image.index_in_request)])[image.index_in_request] image.processing_time = time_cost.get(image.doc_type, [0 for _ in range(image.index_in_request)])[image.index_in_request]
except Exception as e: except Exception as e:
@ -132,14 +133,10 @@ def process_csv_feedback(csv_file_path, feedback_id):
if _feedback_result: if _feedback_result:
_feedback_result["imei_number"] = [] _feedback_result["imei_number"] = []
# if _reviewed_result:
# _reviewed_result["imei_number"] = []
else: else:
try: try:
_predict_result = {"retailername": None, "sold_to_party": None, "purchase_date": [], "imei_number": [_predict_result["imei_number"][image.index_in_request]]} _predict_result = {"retailername": None, "sold_to_party": None, "invoice_no": None, "purchase_date": [], "imei_number": [_predict_result["imei_number"][image.index_in_request]]}
_feedback_result = {"retailername": None, "sold_to_party": None, "purchase_date": None, "imei_number": [_feedback_result["imei_number"][image.index_in_request]]} if _feedback_result else None _feedback_result = {"retailername": None, "sold_to_party": None, "invoice_no": None, "purchase_date": None, "imei_number": [_feedback_result["imei_number"][image.index_in_request]]} if _feedback_result else None
# _reviewed_result = {"retailername": None, "sold_to_party": None, "purchase_date": None, "imei_number": [_reviewed_result["imei_number"][image.index_in_request]]} if _reviewed_result else None
except Exception as e: except Exception as e:
print (f"[ERROR]: {request_id} - {e}") print (f"[ERROR]: {request_id} - {e}")
image.predict_result = _predict_result image.predict_result = _predict_result

145
cope2n-api/fwd_api/celery_worker/process_report_tasks.py
View File

@ -35,135 +35,6 @@ def mean_list(l):
return 0 return 0
return sum(l)/len(l) return sum(l)/len(l)
@app.task(name='make_a_report')
def make_a_report(report_id, query_set):
# TODO: to be deprecated
try:
start_date = timezone.datetime.strptime(query_set["start_date_str"], '%Y-%m-%dT%H:%M:%S%z')
end_date = timezone.datetime.strptime(query_set["end_date_str"], '%Y-%m-%dT%H:%M:%S%z')
base_query = Q(created_at__range=(start_date, end_date))
if query_set["request_id"]:
base_query &= Q(request_id=query_set["request_id"])
if query_set["redemption_id"]:
base_query &= Q(redemption_id=query_set["redemption_id"])
base_query &= Q(is_test_request=False)
if isinstance(query_set["include_test"], str):
query_set["include_test"] = True if query_set["include_test"].lower() in ["true", "yes", "1"] else False
if query_set["include_test"]:
# base_query = ~base_query
base_query.children = base_query.children[:-1]
elif isinstance(query_set["include_test"], bool):
if query_set["include_test"]:
base_query = ~base_query
if isinstance(query_set["subsidiary"], str):
if query_set["subsidiary"] and query_set["subsidiary"].lower().replace(" ", "")!="all":
base_query &= Q(redemption_id__startswith=query_set["subsidiary"])
if isinstance(query_set["is_reviewed"], str):
if query_set["is_reviewed"] == "reviewed":
base_query &= Q(is_reviewed=True)
elif query_set["is_reviewed"] == "not reviewed":
base_query &= Q(is_reviewed=False)
# elif query_set["is_reviewed"] == "all":
# pass
errors = []
# Create a placeholder to fill
accuracy = {"feedback" :{"imei_number": IterAvg(),
"purchase_date": IterAvg(),
"retailername": IterAvg(),
"sold_to_party": IterAvg(),},
"reviewed" :{"imei_number": IterAvg(),
"purchase_date": IterAvg(),
"retailername": IterAvg(),
"sold_to_party": IterAvg(),}
} # {"imei": {"acc": 0.1, count: 1}, ...}
time_cost = {"invoice": IterAvg(),
"imei": IterAvg()}
number_images = 0
number_bad_images = 0
# TODO: Multithreading
# Calculate accuracy, processing time, ....Then save.
subscription_requests = SubscriptionRequest.objects.filter(base_query).order_by('created_at')
report: Report = \
Report.objects.filter(report_id=report_id).first()
# TODO: number of transaction by doc type
num_request = 0
for request in subscription_requests:
if request.status != 200 or not (request.reviewed_result or request.feedback_result):
# Failed requests or lack of reviewed_result/feedback_result
continue
request_att = calculate_and_save_subcription_file(report, request)
request.feedback_accuracy = {"imei_number" : mean_list(request_att["acc"]["feedback"].get("imei_number", [None])),
"purchase_date" : mean_list(request_att["acc"]["feedback"].get("purchase_date", [None])),
"retailername" : mean_list(request_att["acc"]["feedback"].get("retailername", [None])),
"sold_to_party" : mean_list(request_att["acc"]["feedback"].get("sold_to_party", [None]))}
request.reviewed_accuracy = {"imei_number" : mean_list(request_att["acc"]["reviewed"].get("imei_number", [None])),
"purchase_date" : mean_list(request_att["acc"]["reviewed"].get("purchase_date", [None])),
"retailername" : mean_list(request_att["acc"]["reviewed"].get("retailername", [None])),
"sold_to_party" : mean_list(request_att["acc"]["reviewed"].get("sold_to_party", [None]))}
request.save()
number_images += request_att["total_images"]
number_bad_images += request_att["bad_images"]
update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
time_cost["imei"].add(request_att["time_cost"].get("imei", []))
time_cost["invoice"].add(request_att["time_cost"].get("invoice", []))
errors += request_att["err"]
num_request += 1
transaction_att = count_transactions(start_date, end_date, report.subsidiary)
# Do saving process
report.number_request = num_request
report.number_images = number_images
report.number_imei = time_cost["imei"].count
report.number_invoice = time_cost["invoice"].count
report.number_bad_images = number_bad_images
# FIXME: refactor this data stream for endurability
report.average_OCR_time = {"invoice": time_cost["invoice"](), "imei": time_cost["imei"](),
"invoice_count": time_cost["invoice"].count, "imei_count": time_cost["imei"].count}
report.average_OCR_time["invoice"] = 0 if report.average_OCR_time["invoice"] is None else report.average_OCR_time["invoice"]
report.average_OCR_time["imei"] = 0 if report.average_OCR_time["imei"] is None else report.average_OCR_time["imei"]
report.average_OCR_time["avg"] = (report.average_OCR_time["invoice"]*report.average_OCR_time["invoice_count"] + report.average_OCR_time["imei"]*report.average_OCR_time["imei_count"])/(report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) if (report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) > 0 else None
report.number_imei_transaction = transaction_att.get("imei", 0)
report.number_invoice_transaction = transaction_att.get("invoice", 0)
acumulated_acc = {"feedback": {},
"reviewed": {}}
for acc_type in ["feedback", "reviewed"]:
avg_acc = IterAvg()
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
acumulated_acc[acc_type][key] = accuracy[acc_type][key]()
acumulated_acc[acc_type][key+"_count"] = accuracy[acc_type][key].count
avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"])
acumulated_acc[acc_type]["avg"] = avg_acc()
report.feedback_accuracy = acumulated_acc["feedback"]
report.reviewed_accuracy = acumulated_acc["reviewed"]
report.errors = "|".join(errors)
report.status = "Ready"
report.save()
# Saving a xlsx file
report_files = ReportFile.objects.filter(report=report)
data = extract_report_detail_list(report_files, lower=True)
data_workbook = dict2xlsx(data, _type='report_detail')
local_workbook = save_workbook_file(report.report_id + ".xlsx", report, data_workbook)
s3_key=save_report_to_S3(report.report_id, local_workbook)
except IndexError as e:
print(e)
traceback.print_exc()
print("NotFound request by report id, %d", report_id)
except Exception as e:
print("[ERROR]: an error occured while processing report: ", report_id)
traceback.print_exc()
return 400
@app.task(name='make_a_report_2') @app.task(name='make_a_report_2')
def make_a_report_2(report_id, query_set): def make_a_report_2(report_id, query_set):
report_type = query_set.pop("report_type", "accuracy") report_type = query_set.pop("report_type", "accuracy")
@ -241,18 +112,20 @@ def create_accuracy_report(report_id, **kwargs):
request.feedback_accuracy = {"imei_number": mean_list(request_att["acc"]["feedback"].get("imei_number", [None])), request.feedback_accuracy = {"imei_number": mean_list(request_att["acc"]["feedback"].get("imei_number", [None])),
"purchase_date": mean_list(request_att["acc"]["feedback"].get("purchase_date", [None])), "purchase_date": mean_list(request_att["acc"]["feedback"].get("purchase_date", [None])),
"retailername": mean_list(request_att["acc"]["feedback"].get("retailername", [None])), "retailername": mean_list(request_att["acc"]["feedback"].get("retailername", [None])),
"sold_to_party": mean_list(request_att["acc"]["feedback"].get("sold_to_party", [None]))} "sold_to_party": mean_list(request_att["acc"]["feedback"].get("sold_to_party", [None])),
"invoice_no": mean_list(request_att["acc"]["feedback"].get("invoice_no", [None]))}
request.reviewed_accuracy = {"imei_number": mean_list(request_att["acc"]["reviewed"].get("imei_number", [None])), request.reviewed_accuracy = {"imei_number": mean_list(request_att["acc"]["reviewed"].get("imei_number", [None])),
"purchase_date": mean_list(request_att["acc"]["reviewed"].get("purchase_date", [None])), "purchase_date": mean_list(request_att["acc"]["reviewed"].get("purchase_date", [None])),
"retailername": mean_list(request_att["acc"]["reviewed"].get("retailername", [None])), "retailername": mean_list(request_att["acc"]["reviewed"].get("retailername", [None])),
"sold_to_party": mean_list(request_att["acc"]["reviewed"].get("sold_to_party", [None]))} "sold_to_party": mean_list(request_att["acc"]["reviewed"].get("sold_to_party", [None])),
"invoice_no": mean_list(request_att["acc"]["reviewed"].get("invoice_no", [None]))}
request.save() request.save()
number_images += request_att["total_images"] number_images += request_att["total_images"]
number_bad_images += request_att["bad_images"] number_bad_images += request_att["bad_images"]
bad_image_list += request_att["bad_image_list"] bad_image_list += request_att["bad_image_list"]
update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"]) update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"]) update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"]) update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
time_cost["imei"].add(request_att["time_cost"].get("imei", [])) time_cost["imei"].add(request_att["time_cost"].get("imei", []))
time_cost["invoice"].add(request_att["time_cost"].get("invoice", [])) time_cost["invoice"].add(request_att["time_cost"].get("invoice", []))
@ -285,7 +158,7 @@ def create_accuracy_report(report_id, **kwargs):
"acumulated": {}} "acumulated": {}}
for acc_type in ["feedback", "reviewed", "acumulated"]: for acc_type in ["feedback", "reviewed", "acumulated"]:
avg_acc = IterAvg() avg_acc = IterAvg()
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]:
acumulated_acc[acc_type][key] = accuracy[acc_type][key]() acumulated_acc[acc_type][key] = accuracy[acc_type][key]()
acumulated_acc[acc_type][key + "_count"] = accuracy[acc_type][key].count acumulated_acc[acc_type][key + "_count"] = accuracy[acc_type][key].count
avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"]) avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"])
@ -362,7 +235,7 @@ def create_billing_report(report_id, **kwargs):
"reviewed": {}, "reviewed": {},
"acumulated": {}} "acumulated": {}}
for acc_type in ["feedback", "reviewed", "acumulated"]: for acc_type in ["feedback", "reviewed", "acumulated"]:
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]:
acumulated_acc[acc_type][key] = None acumulated_acc[acc_type][key] = None
acumulated_acc[acc_type][key + "_count"] = None acumulated_acc[acc_type][key + "_count"] = None
acumulated_acc[acc_type]["avg"] = None acumulated_acc[acc_type]["avg"] = None

21
cope2n-api/fwd_api/celery_worker/process_result_tasks.py
View File

@ -29,8 +29,8 @@ def aggregate_result(results):
des_result["content"]["total_pages"] = 0 des_result["content"]["total_pages"] = 0
des_result["content"]["ocr_num_pages"] = 0 des_result["content"]["ocr_num_pages"] = 0
des_result["content"]["document"][0]["end_page"] = 0 des_result["content"]["document"][0]["end_page"] = 0
des_result["content"]["document"][0]["content"][3]["value"] = [None for _ in range(doc_types.count("imei"))] des_result["content"]["document"][0]["content"][4]["value"] = [None for _ in range(doc_types.count("imei"))]
des_result["content"]["document"][0]["content"][2]["value"] = [] des_result["content"]["document"][0]["content"][3]["value"] = []
imei_count = 0 imei_count = 0
for doc_type, result in sorted_results: for doc_type, result in sorted_results:
@ -38,12 +38,14 @@ def aggregate_result(results):
des_result["content"]["ocr_num_pages"] += 1 des_result["content"]["ocr_num_pages"] += 1
des_result["content"]["document"][0]["end_page"] += 1 des_result["content"]["document"][0]["end_page"] += 1
if doc_type == "imei": if doc_type == "imei":
des_result["content"]["document"][0]["content"][3]["value"][imei_count] = result["content"]["document"][0]["content"][3]["value"][0] if len(result["content"]["document"][0]["content"][4]["value"]):
des_result["content"]["document"][0]["content"][4]["value"][imei_count] = result["content"]["document"][0]["content"][4]["value"][0]
imei_count += 1 imei_count += 1
elif doc_type == "invoice": elif doc_type == "invoice":
des_result["content"]["document"][0]["content"][0]["value"] = result["content"]["document"][0]["content"][0]["value"] des_result["content"]["document"][0]["content"][0]["value"] = result["content"]["document"][0]["content"][0]["value"]
des_result["content"]["document"][0]["content"][1]["value"] = result["content"]["document"][0]["content"][1]["value"] des_result["content"]["document"][0]["content"][1]["value"] = result["content"]["document"][0]["content"][1]["value"]
des_result["content"]["document"][0]["content"][2]["value"] += result["content"]["document"][0]["content"][2]["value"] des_result["content"]["document"][0]["content"][2]["value"] = result["content"]["document"][0]["content"][2]["value"]
des_result["content"]["document"][0]["content"][3]["value"] += result["content"]["document"][0]["content"][3]["value"]
elif doc_type == "all": elif doc_type == "all":
des_result.update(result) des_result.update(result)
else: else:
@ -151,7 +153,6 @@ def process_invoice_sbt_result(rq_id, result, metadata):
index_in_request = metadata.pop("index_to_image_type", 0) index_in_request = metadata.pop("index_to_image_type", 0)
result["metadata"] = metadata result["metadata"] = metadata
_update_subscription_rq_file(request_id=rq, index_in_request=index_in_request, doc_type=image_type, result=result) _update_subscription_rq_file(request_id=rq, index_in_request=index_in_request, doc_type=image_type, result=result)
status = result.get("status", 200) status = result.get("status", 200)
redis_client.set_cache(rq_id, page_index, result) redis_client.set_cache(rq_id, page_index, result)
done = rq.pages == redis_client.get_size(rq_id) done = rq.pages == redis_client.get_size(rq_id)
@ -194,9 +195,9 @@ def process_invoice_sbt_result(rq_id, result, metadata):
def _update_subscription_rq_file(request_id, index_in_request, doc_type, result): def _update_subscription_rq_file(request_id, index_in_request, doc_type, result):
image = SubscriptionRequestFile.objects.filter(request=request_id, index_in_request=index_in_request, doc_type=doc_type).first() image = SubscriptionRequestFile.objects.filter(request=request_id, index_in_request=index_in_request, doc_type=doc_type).first()
retailer_name = None retailer_name = None
sold_to_party = None sold_to_party = None
invoice_no = None
purchase_date = [] purchase_date = []
imei_number = [] imei_number = []
predicted_res = __get_actual_predict_result(result=result) predicted_res = __get_actual_predict_result(result=result)
@ -208,19 +209,23 @@ def _update_subscription_rq_file(request_id, index_in_request, doc_type, result)
sold_to_party = elem['value'] sold_to_party = elem['value']
elif elem["label"] == "purchase_date": elif elem["label"] == "purchase_date":
purchase_date = elem['value'] purchase_date = elem['value']
elif elem["label"] == "invoice_no":
invoice_no = elem['value']
else: else:
imei_number = elem['value'] imei_number = elem['value']
if doc_type=='invoice': if doc_type=='invoice':
_predict_result = { _predict_result = {
"retailername": retailer_name, "retailername": retailer_name,
"sold_to_party": sold_to_party, "sold_to_party": sold_to_party,
"invoice_no": invoice_no,
"purchase_date": purchase_date, "purchase_date": purchase_date,
"imei_number": [] "imei_number": []
} }
else: else:
_predict_result = { _predict_result = {
"retailername": None, "retailername": None,
"sold_to_party": None, "sold_to_party": None,
"invoice_no": None,
"purchase_date": [], "purchase_date": [],
"imei_number": imei_number "imei_number": imei_number
} }

197
cope2n-api/fwd_api/utils/accuracy.py
View File

@ -17,7 +17,7 @@ from fwd import settings
from ..models import SubscriptionRequest, Report, ReportFile from ..models import SubscriptionRequest, Report, ReportFile
import json import json
valid_keys = ["retailername", "sold_to_party", "purchase_date", "imei_number"] valid_keys = ["retailername", "sold_to_party", "invoice_no", "purchase_date", "imei_number"]
class ReportAccumulateByRequest: class ReportAccumulateByRequest:
def __init__(self, sub): def __init__(self, sub):
@ -42,7 +42,7 @@ class ReportAccumulateByRequest:
'purchase_date': IterAvg(), 'purchase_date': IterAvg(),
'retailer_name': IterAvg(), 'retailer_name': IterAvg(),
'sold_to_party': IterAvg(), 'sold_to_party': IterAvg(),
'invoice_no': IterAvg(), 'invoice_no': IterAvg()
}, },
'average_processing_time': { 'average_processing_time': {
'imei': IterAvg(), 'imei': IterAvg(),
@ -339,7 +339,8 @@ class MonthReportAccumulate:
'average_accuracy_rate': { 'average_accuracy_rate': {
'imei': IterAvg(), 'imei': IterAvg(),
'purchase_date': IterAvg(), 'purchase_date': IterAvg(),
'retailer_name': IterAvg() 'retailer_name': IterAvg(),
'invoice_no': IterAvg()
}, },
'average_processing_time': { 'average_processing_time': {
'imei': IterAvg(), 'imei': IterAvg(),
@ -366,7 +367,8 @@ class MonthReportAccumulate:
'average_accuracy_rate': { 'average_accuracy_rate': {
'imei': 0, 'imei': 0,
'purchase_date': 0, 'purchase_date': 0,
'retailer_name': 0 'retailer_name': 0,
'invoice_no': 0
}, },
'average_processing_time': { 'average_processing_time': {
'imei': 0, 'imei': 0,
@ -388,10 +390,12 @@ class MonthReportAccumulate:
self.total["average_accuracy_rate"]["imei"].add_avg(report.reviewed_accuracy.get("imei_number", 0), report.reviewed_accuracy.get("imei_number_count", 0)) self.total["average_accuracy_rate"]["imei"].add_avg(report.reviewed_accuracy.get("imei_number", 0), report.reviewed_accuracy.get("imei_number_count", 0))
self.total["average_accuracy_rate"]["purchase_date"].add_avg(report.reviewed_accuracy.get("purchase_date", 0), report.reviewed_accuracy.get("purchase_date_count", 0)) self.total["average_accuracy_rate"]["purchase_date"].add_avg(report.reviewed_accuracy.get("purchase_date", 0), report.reviewed_accuracy.get("purchase_date_count", 0))
self.total["average_accuracy_rate"]["retailer_name"].add_avg(report.reviewed_accuracy.get("retailername", 0), report.reviewed_accuracy.get("retailername_count", 0)) self.total["average_accuracy_rate"]["retailer_name"].add_avg(report.reviewed_accuracy.get("retailername", 0), report.reviewed_accuracy.get("retailername_count", 0))
self.total["average_accuracy_rate"]["invoice_no"].add_avg(report.reviewed_accuracy.get("invoice_no", 0), report.reviewed_accuracy.get("invoice_no_count", 0))
elif sum([ report.feedback_accuracy[x] for x in report.feedback_accuracy.keys() if "_count" not in x]) > 0: elif sum([ report.feedback_accuracy[x] for x in report.feedback_accuracy.keys() if "_count" not in x]) > 0:
self.total["average_accuracy_rate"]["imei"].add_avg(report.feedback_accuracy.get("imei_number", 0), report.feedback_accuracy.get("imei_number_count", 0)) self.total["average_accuracy_rate"]["imei"].add_avg(report.feedback_accuracy.get("imei_number", 0), report.feedback_accuracy.get("imei_number_count", 0))
self.total["average_accuracy_rate"]["purchase_date"].add_avg(report.feedback_accuracy.get("purchase_date", 0), report.feedback_accuracy.get("purchase_date_count", 0)) self.total["average_accuracy_rate"]["purchase_date"].add_avg(report.feedback_accuracy.get("purchase_date", 0), report.feedback_accuracy.get("purchase_date_count", 0))
self.total["average_accuracy_rate"]["retailer_name"].add_avg(report.feedback_accuracy.get("retailername", 0), report.feedback_accuracy.get("retailername_count", 0)) self.total["average_accuracy_rate"]["retailer_name"].add_avg(report.feedback_accuracy.get("retailername", 0), report.feedback_accuracy.get("retailername_count", 0))
self.total["average_accuracy_rate"]["invoice_no"].add_avg(report.feedback_accuracy.get("invoice_no", 0), report.feedback_accuracy.get("invoice_no_count", 0))
self.total["average_processing_time"]["imei"].add_avg(report.average_OCR_time.get("imei", 0), report.average_OCR_time.get("imei_count", 0)) if report.average_OCR_time else 0 self.total["average_processing_time"]["imei"].add_avg(report.average_OCR_time.get("imei", 0), report.average_OCR_time.get("imei_count", 0)) if report.average_OCR_time else 0
self.total["average_processing_time"]["invoice"].add_avg(report.average_OCR_time.get("invoice", 0), report.average_OCR_time.get("invoice_count", 0)) if report.average_OCR_time else 0 self.total["average_processing_time"]["invoice"].add_avg(report.average_OCR_time.get("invoice", 0), report.average_OCR_time.get("invoice_count", 0)) if report.average_OCR_time else 0
@ -425,10 +429,13 @@ class MonthReportAccumulate:
new_data["average_accuracy_rate"]["imei"] = report.reviewed_accuracy.get("imei_number", None) new_data["average_accuracy_rate"]["imei"] = report.reviewed_accuracy.get("imei_number", None)
new_data["average_accuracy_rate"]["purchase_date"] = report.reviewed_accuracy.get("purchase_date", None) new_data["average_accuracy_rate"]["purchase_date"] = report.reviewed_accuracy.get("purchase_date", None)
new_data["average_accuracy_rate"]["retailer_name"] = report.reviewed_accuracy.get("retailername", None) new_data["average_accuracy_rate"]["retailer_name"] = report.reviewed_accuracy.get("retailername", None)
new_data["average_accuracy_rate"]["invoice_no"] = report.reviewed_accuracy.get("invoice_no", None)
elif sum([ report.feedback_accuracy[x] for x in report.feedback_accuracy.keys() if "_count" not in x]): elif sum([ report.feedback_accuracy[x] for x in report.feedback_accuracy.keys() if "_count" not in x]):
new_data["average_accuracy_rate"]["imei"] = report.feedback_accuracy.get("imei_number", None) new_data["average_accuracy_rate"]["imei"] = report.feedback_accuracy.get("imei_number", None)
new_data["average_accuracy_rate"]["purchase_date"] = report.feedback_accuracy.get("purchase_date", None) new_data["average_accuracy_rate"]["purchase_date"] = report.feedback_accuracy.get("purchase_date", None)
new_data["average_accuracy_rate"]["retailer_name"] = report.feedback_accuracy.get("retailername", None) new_data["average_accuracy_rate"]["retailer_name"] = report.feedback_accuracy.get("retailername", None)
new_data["average_accuracy_rate"]["invoice_no"] = report.feedback_accuracy.get("invoice_no", None)
new_data["average_processing_time"]["imei"] = report.average_OCR_time.get("imei", 0) if report.average_OCR_time else 0 new_data["average_processing_time"]["imei"] = report.average_OCR_time.get("imei", 0) if report.average_OCR_time else 0
new_data["average_processing_time"]["invoice"] = report.average_OCR_time.get("invoice", 0) if report.average_OCR_time else 0 new_data["average_processing_time"]["invoice"] = report.average_OCR_time.get("invoice", 0) if report.average_OCR_time else 0
new_data["usage"]["imei"] = report.number_imei_transaction new_data["usage"]["imei"] = report.number_imei_transaction
@ -516,7 +523,7 @@ def validate_feedback_file(feedback, predict):
if num_imei_feedback != num_imei_predict: if num_imei_feedback != num_imei_predict:
return False return False
return True return True
def first_of_list(the_list): def first_of_list(the_list):
if not the_list: if not the_list:
return None return None
@ -533,19 +540,19 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True)
"Image type": report_file.doc_type, "Image type": report_file.doc_type,
"IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])), "IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])),
"IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])), "IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])),
"IMEI Revised": None, "IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])),
"IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])), "IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])),
"Invoice_Number_User": None, "Invoice_Number_User": report_file.feedback_result.get("invoice_no", None),
"Invoice_Number_OCR": None, "Invoice_Number_OCR": report_file.predict_result.get("invoice_no", None),
"Invoice_Number Revised": None, "Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None),
"Invoice_Number_Accuracy": None, "Invoice_Number_Accuracy": first_of_list(report_file.feedback_accuracy.get("invoice_no", [None])),
"Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None), "Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None),
"Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []), "Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []),
"Invoice_Purchase Date Revised": None, "Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None),
"Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])), "Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])),
"Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None), "Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None),
"Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None), "Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None),
"Invoice_Purchase Date Revised": None, "Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None),
"Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])), "Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])),
"OCR Image Accuracy": report_file.acc, "OCR Image Accuracy": report_file.acc,
"OCR Image Speed (seconds)": report_file.time_cost, "OCR Image Speed (seconds)": report_file.time_cost,
@ -555,6 +562,7 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True)
"IMEI_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("imei_number", [None])), "IMEI_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("imei_number", [None])),
"Purchase Date_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("purchase_date", [None])), "Purchase Date_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("purchase_date", [None])),
"Retailer_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("retailername", [None])), "Retailer_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("retailername", [None])),
"Invoice_Number_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("invoice_no", [None]))
}) })
if lower: if lower:
for i, dat in enumerate(data): for i, dat in enumerate(data):
@ -610,14 +618,16 @@ def convert_datetime_format(date_string: str, is_gt=False) -> str:
def predict_result_to_ready(result): def predict_result_to_ready(result):
dict_result = {"retailername": "", dict_result = {"retailername": "",
"sold_to_party": "", "sold_to_party": "",
"invoice_no": "",
"purchase_date": [], "purchase_date": [],
"imei_number": [],} "imei_number": [],}
if not result: if not result:
return dict_result return dict_result
dict_result["retailername"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}])[0].get("value", None) dict_result["retailername"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}])[0].get("value", None)
dict_result["sold_to_party"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}])[1].get("value", None) dict_result["sold_to_party"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}])[1].get("value", None)
dict_result["purchase_date"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}])[2].get("value", []) dict_result["invoice_no"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}])[2].get("value", None)
dict_result["imei_number"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}, {}])[3].get("value", []) dict_result["purchase_date"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}])[3].get("value", [])
dict_result["imei_number"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}, {}])[4].get("value", [])
return dict_result return dict_result
def align_fine_result(ready_predict, fine_result): def align_fine_result(ready_predict, fine_result):
@ -628,15 +638,9 @@ def align_fine_result(ready_predict, fine_result):
ready_predict["purchase_date"] = [None] ready_predict["purchase_date"] = [None]
if fine_result["retailername"] and not ready_predict["retailername"]: if fine_result["retailername"] and not ready_predict["retailername"]:
ready_predict["retailername"] = [None] ready_predict["retailername"] = [None]
# if ready_predict["retailername"] and not fine_result["retailername"]: if ready_predict["invoice_no"] and not fine_result["invoice_no"]:
# fine_result["retailername"] = [None] fine_result["invoice_no"] = [None]
fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))] fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))]
# fine_result["retailername"] = None if len(ready_predict["purchase_date"]))]
# else:
# fine_result = {}
# for key in ready_predict.keys():
# fine_result[key] = []
# fine_result["purchase_date"] = [None for _ in range(len(ready_predict["purchase_date"]))]
return ready_predict, fine_result return ready_predict, fine_result
def update_temp_accuracy(accuracy, acc, keys): def update_temp_accuracy(accuracy, acc, keys):
@ -704,76 +708,6 @@ def calculate_avg_accuracy(acc, type, keys=[]):
return sum(acc_list)/len(acc_list) if len(acc_list) > 0 else None return sum(acc_list)/len(acc_list) if len(acc_list) > 0 else None
# Deprecated
def calculate_and_save_subcription_file(report, request):
request_att = {"acc": {"feedback": {"imei_number": [],
"purchase_date": [],
"retailername": [],
"sold_to_party": [],
},
"reviewed": {"imei_number": [],
"purchase_date": [],
"retailername": [],
"sold_to_party": [],
}},
"err": [],
"time_cost": {},
"total_images": 0,
"bad_images": 0}
images = SubscriptionRequestFile.objects.filter(request=request)
for image in images:
status, att = calculate_subcription_file(image)
if status != 200:
continue
image.feedback_accuracy = att["acc"]["feedback"]
image.reviewed_accuracy = att["acc"]["reviewed"]
image.is_bad_image_quality = att["is_bad_image"]
image.save()
new_report_file = ReportFile(report=report,
correspond_request_id=request.request_id,
correspond_redemption_id=request.redemption_id,
doc_type=image.doc_type,
predict_result=image.predict_result,
feedback_result=image.feedback_result,
reviewed_result=image.reviewed_result,
feedback_accuracy=att["acc"]["feedback"],
reviewed_accuracy=att["acc"]["reviewed"],
acc=att["avg_acc"],
time_cost=image.processing_time,
is_bad_image=att["is_bad_image"],
bad_image_reason=image.reason,
counter_measures=image.counter_measures,
error="|".join(att["err"])
)
new_report_file.save()
if request_att["time_cost"].get(image.doc_type, None):
request_att["time_cost"][image.doc_type].append(image.processing_time)
else:
request_att["time_cost"][image.doc_type] = [image.processing_time]
try:
request_att["acc"]["feedback"]["imei_number"] += att["acc"]["feedback"]["imei_number"]
request_att["acc"]["feedback"]["purchase_date"] += att["acc"]["feedback"]["purchase_date"]
request_att["acc"]["feedback"]["retailername"] += att["acc"]["feedback"]["retailername"]
request_att["acc"]["feedback"]["sold_to_party"] += att["acc"]["feedback"]["sold_to_party"]
request_att["acc"]["reviewed"]["imei_number"] += att["acc"]["reviewed"]["imei_number"]
request_att["acc"]["reviewed"]["purchase_date"] += att["acc"]["reviewed"]["purchase_date"]
request_att["acc"]["reviewed"]["retailername"] += att["acc"]["reviewed"]["retailername"]
request_att["acc"]["reviewed"]["sold_to_party"] += att["acc"]["reviewed"]["sold_to_party"]
request_att["bad_images"] += int(att["is_bad_image"])
request_att["total_images"] += 1
request_att["err"] += att["err"]
except Exception as e:
print(e)
continue
return request_att
# def result_maximize_list_values(result, acc):
# for k in acc.keys():
# if isinstance(acc[k], list) and len(acc[k]) > 0:
def acc_maximize_list_values(acc): def acc_maximize_list_values(acc):
pos = {} pos = {}
for k in acc.keys(): for k in acc.keys():
@ -838,16 +772,19 @@ def calculate_a_request(report, request):
"purchase_date": [], "purchase_date": [],
"retailername": [], "retailername": [],
"sold_to_party": [], "sold_to_party": [],
"invoice_no": [],
}, },
"reviewed": {"imei_number": [], "reviewed": {"imei_number": [],
"purchase_date": [], "purchase_date": [],
"retailername": [], "retailername": [],
"sold_to_party": [], "sold_to_party": [],
"invoice_no": [],
}, },
"acumulated":{"imei_number": [], "acumulated":{"imei_number": [],
"purchase_date": [], "purchase_date": [],
"retailername": [], "retailername": [],
"sold_to_party": [], "sold_to_party": [],
"invoice_no": [],
}}, }},
"err": [], "err": [],
"time_cost": {"imei": [], "time_cost": {"imei": [],
@ -936,16 +873,19 @@ def calculate_a_request(report, request):
request_att["acc"]["feedback"]["purchase_date"] += _att["acc"]["feedback"]["purchase_date"] request_att["acc"]["feedback"]["purchase_date"] += _att["acc"]["feedback"]["purchase_date"]
request_att["acc"]["feedback"]["retailername"] += _att["acc"]["feedback"]["retailername"] request_att["acc"]["feedback"]["retailername"] += _att["acc"]["feedback"]["retailername"]
request_att["acc"]["feedback"]["sold_to_party"] += _att["acc"]["feedback"]["sold_to_party"] request_att["acc"]["feedback"]["sold_to_party"] += _att["acc"]["feedback"]["sold_to_party"]
request_att["acc"]["feedback"]["invoice_no"] += _att["acc"]["feedback"]["invoice_no"]
request_att["acc"]["reviewed"]["imei_number"] += _att["acc"]["reviewed"]["imei_number"] request_att["acc"]["reviewed"]["imei_number"] += _att["acc"]["reviewed"]["imei_number"]
request_att["acc"]["reviewed"]["purchase_date"] += _att["acc"]["reviewed"]["purchase_date"] request_att["acc"]["reviewed"]["purchase_date"] += _att["acc"]["reviewed"]["purchase_date"]
request_att["acc"]["reviewed"]["retailername"] += _att["acc"]["reviewed"]["retailername"] request_att["acc"]["reviewed"]["retailername"] += _att["acc"]["reviewed"]["retailername"]
request_att["acc"]["reviewed"]["sold_to_party"] += _att["acc"]["reviewed"]["sold_to_party"] request_att["acc"]["reviewed"]["sold_to_party"] += _att["acc"]["reviewed"]["sold_to_party"]
request_att["acc"]["reviewed"]["invoice_no"] += _att["acc"]["reviewed"]["invoice_no"]
request_att["acc"]["acumulated"]["imei_number"] += _att["acc"]["reviewed"]["imei_number"] if _att["acc"]["reviewed"]["imei_number"] else _att["acc"]["feedback"]["imei_number"] request_att["acc"]["acumulated"]["imei_number"] += _att["acc"]["reviewed"]["imei_number"] if _att["acc"]["reviewed"]["imei_number"] else _att["acc"]["feedback"]["imei_number"]
request_att["acc"]["acumulated"]["purchase_date"] += _att["acc"]["reviewed"]["purchase_date"] if _att["acc"]["reviewed"]["purchase_date"] else _att["acc"]["feedback"]["purchase_date"] request_att["acc"]["acumulated"]["purchase_date"] += _att["acc"]["reviewed"]["purchase_date"] if _att["acc"]["reviewed"]["purchase_date"] else _att["acc"]["feedback"]["purchase_date"]
request_att["acc"]["acumulated"]["retailername"] += _att["acc"]["reviewed"]["retailername"] if _att["acc"]["reviewed"]["retailername"] else _att["acc"]["feedback"]["retailername"] request_att["acc"]["acumulated"]["retailername"] += _att["acc"]["reviewed"]["retailername"] if _att["acc"]["reviewed"]["retailername"] else _att["acc"]["feedback"]["retailername"]
request_att["acc"]["acumulated"]["sold_to_party"] += _att["acc"]["reviewed"]["sold_to_party"] if _att["acc"]["reviewed"]["sold_to_party"] else _att["acc"]["feedback"]["sold_to_party"] request_att["acc"]["acumulated"]["sold_to_party"] += _att["acc"]["reviewed"]["sold_to_party"] if _att["acc"]["reviewed"]["sold_to_party"] else _att["acc"]["feedback"]["sold_to_party"]
request_att["acc"]["acumulated"]["invoice_no"] += _att["acc"]["reviewed"]["invoice_no"] if _att["acc"]["reviewed"]["invoice_no"] else _att["acc"]["feedback"]["invoice_no"]
if image.reason not in settings.ACC_EXCLUDE_RESEASONS: if image.reason not in settings.ACC_EXCLUDE_RESEASONS:
request_att["bad_images"] += int(_att["is_bad_image"]) request_att["bad_images"] += int(_att["is_bad_image"])
@ -973,10 +913,6 @@ def calculate_subcription_file(subcription_request_file):
inference_result = copy.deepcopy(subcription_request_file.predict_result) inference_result = copy.deepcopy(subcription_request_file.predict_result)
inference_result, feedback_result = align_fine_result(inference_result, copy.deepcopy(subcription_request_file.feedback_result)) inference_result, feedback_result = align_fine_result(inference_result, copy.deepcopy(subcription_request_file.feedback_result))
inference_result, reviewed_result = align_fine_result(inference_result, copy.deepcopy(subcription_request_file.reviewed_result)) inference_result, reviewed_result = align_fine_result(inference_result, copy.deepcopy(subcription_request_file.reviewed_result))
# print(f"[DEBUG]: predict_result: {subcription_request_file.predict_result}")
# print(f"[DEBUG]: inference_result: {inference_result}")
# print(f"[DEBUG]: feedback_result: {feedback_result}")
# print(f"[DEBUG]: reviewed_result: {reviewed_result}")
for key_name in valid_keys: for key_name in valid_keys:
try: try:
@ -988,8 +924,8 @@ def calculate_subcription_file(subcription_request_file):
# print(f"[DEBUG]: e: {e} -key_name: {key_name}") # print(f"[DEBUG]: e: {e} -key_name: {key_name}")
subcription_request_file.feedback_accuracy = att["acc"]["feedback"] subcription_request_file.feedback_accuracy = att["acc"]["feedback"]
subcription_request_file.reviewed_accuracy = att["acc"]["reviewed"] subcription_request_file.reviewed_accuracy = att["acc"]["reviewed"]
avg_reviewed = calculate_avg_accuracy(att["acc"], "reviewed", ["retailername", "sold_to_party", "purchase_date", "imei_number"]) avg_reviewed = calculate_avg_accuracy(att["acc"], "reviewed", ["retailername", "sold_to_party", "invoice_no", "purchase_date", "imei_number"])
avg_feedback = calculate_avg_accuracy(att["acc"], "feedback", ["retailername", "sold_to_party", "purchase_date", "imei_number"]) avg_feedback = calculate_avg_accuracy(att["acc"], "feedback", ["retailername", "sold_to_party", "invoice_no", "purchase_date", "imei_number"])
if avg_feedback is not None or avg_reviewed is not None: if avg_feedback is not None or avg_reviewed is not None:
avg_acc = 0 avg_acc = 0
if avg_feedback is not None: if avg_feedback is not None:
@ -1009,68 +945,6 @@ def calculate_subcription_file(subcription_request_file):
att["is_bad_image"] = True att["is_bad_image"] = True
return 200, att return 200, att
def calculate_attributions(request): # for one request, return in order
# Deprecated
acc = {"feedback": {},
"reviewed": {}} # {"feedback": {"retailername": [0.1], "sold_to_party":[0.9], "purchase_date":[0.6], "imei_number":[0.8]},
# "reviewed": {"retailername": [0.1], "sold_to_party":[0.9], "purchase_date":[0.6], "imei_number":[0.8]}}
data = {"feedback": {},
"reviewed": {}} # {"feedback": {"retailername": [[ocr, feedback], ...], "sold_to_party":[[ocr, feedback], ...], "purchase_date":[[ocr, feedback], ...], "imei_number":[[ocr, feedback], ...]}}
# {"reviewed": {"retailername": [[ocr, reviewed], ...], "sold_to_party":[[ocr, reviewed], ...], "purchase_date":[[ocr, reviewed], ...], "imei_number":[[ocr, reviewed], ...]}}
time_cost = {} # {"imei": [0.1], "invoice": [0.1]}
image_quality_num = [0, 0] # [good, bad]
image_quality_num[0] = len(request.doc_type.split(","))
error = ""
inference_result = predict_result_to_ready(request.predict_result)
reviewed_result = align_fine_result(inference_result, request.reviewed_result)
feedback_result = align_fine_result(inference_result, request.feedback_result)
# accuracy calculation
for key_name in valid_keys:
if isinstance(inference_result[key_name], list):
if len(inference_result[key_name]) != len(reviewed_result.get(key_name, [])):
error = f"Request {request.request_id} failed with different {key_name} in predict and reviewed_result"
break
if len(inference_result[key_name]) != len(feedback_result.get(key_name, [])):
error = f"Request {request.request_id} failed with different {key_name} in predict and feedback_result"
break
# calculate accuracy for feedback result
acc["feedback"][key_name], data["feedback"][key_name] = calculate_accuracy(key_name, inference_result, feedback_result)
acc["reviewed"][key_name], data["reviewed"][key_name] = calculate_accuracy(key_name, inference_result, reviewed_result)
else:
inference_result[key_name] = [inference_result[key_name]]
feedback_result[key_name] = [feedback_result[key_name]]
reviewed_result[key_name] = [reviewed_result[key_name]]
acc["feedback"][key_name], data["feedback"][key_name] = calculate_accuracy(key_name, inference_result, feedback_result)
acc["reviewed"][key_name], data["reviewed"][key_name] = calculate_accuracy(key_name, inference_result, reviewed_result)
acc["feedback"]["purchase_date"] = [max(acc["feedback"]["purchase_date"])] if len(acc["feedback"]["purchase_date"]) > 0 else []
acc["reviewed"]["purchase_date"] = [max(acc["reviewed"]["purchase_date"])] if len(acc["reviewed"]["purchase_date"]) > 0 else []
# Count for bad and total images
avg_invoice_feedback = calculate_avg_accuracy(acc, "feedback", ["retailername", "sold_to_party", "purchase_date"])
avg_invoice_reviewed = calculate_avg_accuracy(acc, "reviewed", ["retailername", "sold_to_party", "purchase_date"])
if avg_invoice_feedback is not None or avg_invoice_reviewed is not None:
if max([x for x in [avg_invoice_feedback, avg_invoice_reviewed] if x is not None]) < settings.BAD_THRESHOLD:
image_quality_num[1] += 1
for i, _ in enumerate(acc["feedback"]["imei_number"]):
if acc["feedback"]["imei_number"][i] is not None and acc["reviewed"]["imei_number"][i] is not None:
if max([x for x in [acc["feedback"]["imei_number"][i], acc["reviewed"]["imei_number"][i]] if x is not None]) < settings.BAD_THRESHOLD:
image_quality_num[1] += 1
# time cost and quality calculation
# TODO: to be deprecated, doc_type would be in file level in the future
try:
for doc_type, doc_profile in request.ai_inference_profile.items():
doc_type = doc_type.split("_")[0]
inference_time = doc_profile["inference"][1][0] - doc_profile["inference"][0]
postprocess_time = doc_profile["postprocess"][1] - doc_profile["postprocess"][0]
time_cost[doc_type].append(inference_time + postprocess_time)
except Exception as e:
error = f"Request id {request.request_id} failed with error: {e}"
return acc, data, time_cost, image_quality_num, error
def mean_list(l): def mean_list(l):
l = [x for x in l if x is not None] l = [x for x in l if x is not None]
if len(l) == 0: if len(l) == 0:
@ -1078,5 +952,4 @@ def mean_list(l):
return sum(l)/len(l) return sum(l)/len(l)
def shadow_report(report_id, query): def shadow_report(report_id, query):
c_connector.make_a_report_2( c_connector.make_a_report_2((report_id, query))
(report_id, query))