tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://code.sdsdev.co.kr/SDSRV-IDP/sbt-idp into vietanh99-update-xlsx

Browse Source
This commit is contained in:
daovietanh99 2024-02-19 16:59:57 +07:00
commit 787cb2ff6c
21 changed files with 1197 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -39,3 +39,5 @@ cope2n-ai-fi/Dockerfile_old_work
cope2n-api/public/SBT_report_20240122.csv
Jan.csv
*.csv
cope2n-api/reviewed/date.xlsx
cope2n-api/reviewed/retailer.xlsx

2
cope2n-ai-fi/modules/sdsvkvu

@ -1 +1 @@
Subproject commit b6d4fab46f7f8689dd6b050cfbff2faa6a6f3fec
Subproject commit d01de312ab86db554ffa2f1b01396ef8d56b78ed

2
cope2n-api/fwd/settings.py
View File

@ -224,6 +224,8 @@ OVERVIEW_REFRESH_INTERVAL = 2
OVERVIEW_REPORT_ROOT = "overview"
OVERVIEW_REPORT_DURATION = ["30d", "7d"]
ACC_EXCLUDE_RESEASONS = ["Invalid Input", "Handwritten information", "handwritten"]
SUBS = {
"SEAU": "AU",
"SESP": "SG",

2
cope2n-api/fwd_api/api/accuracy_view.py
View File

@ -382,7 +382,7 @@ class AccuracyViewSet(viewsets.ViewSet):
for key in acc_keys:
fb = report.feedback_accuracy.get(key, 0) if report.feedback_accuracy else 0
rv = report.reviewed_accuracy.get(key, 0) if report.reviewed_accuracy else 0
acc[key] = max([fb, rv])
acc[key] = report.combined_accuracy.get(key, 0) if report.combined_accuracy else max([fb, rv])
data.append({
"ID": report.id,
"Created Date": report.created_at,

11
cope2n-api/fwd_api/celery_worker/internal_task.py
View File

@ -14,7 +14,7 @@ from ..utils import file as FileUtils
from ..utils import process as ProcessUtil
from ..utils import s3 as S3Util
from ..utils.accuracy import validate_feedback_file
from fwd_api.constant.common import ProcessType
from fwd_api.constant.common import FileCategory
import csv
import json
import copy
@ -114,6 +114,9 @@ def process_csv_feedback(csv_file_path, feedback_id):
for k, v in sub_rq.ai_inference_profile.items():
time_cost[k.split("_")[0]].append(v["inference"][1][0] - v["inference"][0] + (v["postprocess"][1]-v["postprocess"][0]))
for i, image in enumerate(images):
if image.file_category != FileCategory.Origin.value:
# skip break files, which are not responsible for storing data
continue
_predict_result = copy.deepcopy(predict_result_to_ready(sub_rq.predict_result))
_feedback_result = copy.deepcopy(sub_rq.feedback_result)
_reviewed_result = copy.deepcopy(sub_rq.reviewed_result)
@ -128,12 +131,10 @@ def process_csv_feedback(csv_file_path, feedback_id):
_predict_result["imei_number"] = []
if _feedback_result:
_feedback_result["imei_number"] = []
else:
None
if _reviewed_result:
_reviewed_result["imei_number"] = []
else:
None
else:
try:
_predict_result = {"retailername": None, "sold_to_party": None, "purchase_date": [], "imei_number": [_predict_result["imei_number"][image.index_in_request]]}

19
cope2n-api/fwd_api/celery_worker/process_report_tasks.py
View File

@ -4,7 +4,7 @@ from fwd_api.models import SubscriptionRequest, Report, ReportFile
from fwd_api.celery_worker.worker import app
from ..utils import s3 as S3Util
from ..utils.accuracy import update_temp_accuracy, IterAvg, calculate_and_save_subcription_file, count_transactions, extract_report_detail_list, calculate_a_request, ReportAccumulateByRequest
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3, save_images_to_csv_briefly
from ..utils import time_stuff
from ..utils.redis import RedisUtils
from django.utils import timezone
@ -187,8 +187,6 @@ def make_a_report_2(report_id, query_set):
base_query &= Q(is_reviewed=True)
elif query_set["is_reviewed"] == "not reviewed":
base_query &= Q(is_reviewed=False)
# elif query_set["is_reviewed"] == "all":
# pass
errors = []
# Create a placeholder to fill
@ -197,12 +195,17 @@ def make_a_report_2(report_id, query_set):
"retailername": IterAvg(),
"sold_to_party": IterAvg(),},
"reviewed" :{"imei_number": IterAvg(),
"purchase_date": IterAvg(),
"retailername": IterAvg(),
"sold_to_party": IterAvg(),},
"acumulated":{"imei_number": IterAvg(),
"purchase_date": IterAvg(),
"retailername": IterAvg(),
"sold_to_party": IterAvg(),}
} # {"imei": {"acc": 0.1, count: 1}, ...}
time_cost = {"invoice": IterAvg(),
"imei": IterAvg()}
bad_image_list = []
number_images = 0
number_bad_images = 0
# TODO: Multithreading
@ -232,8 +235,10 @@ def make_a_report_2(report_id, query_set):
request.save()
number_images += request_att["total_images"]
number_bad_images += request_att["bad_images"]
bad_image_list += request_att["bad_image_list"]
update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=["imei_number", "purchase_date", "retailername", "sold_to_party"])
time_cost["imei"].add(request_att["time_cost"].get("imei", []))
time_cost["invoice"].add(request_att["time_cost"].get("invoice", []))
@ -259,8 +264,9 @@ def make_a_report_2(report_id, query_set):
report.number_invoice_transaction = transaction_att.get("invoice", 0)
acumulated_acc = {"feedback": {},
"reviewed": {}}
for acc_type in ["feedback", "reviewed"]:
"reviewed": {},
"acumulated": {}}
for acc_type in ["feedback", "reviewed", "acumulated"]:
avg_acc = IterAvg()
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
acumulated_acc[acc_type][key] = accuracy[acc_type][key]()
@ -270,10 +276,13 @@ def make_a_report_2(report_id, query_set):
report.feedback_accuracy = acumulated_acc["feedback"]
report.reviewed_accuracy = acumulated_acc["reviewed"]
report.combined_accuracy = acumulated_acc["acumulated"]
report.errors = "|".join(errors)
report.status = "Ready"
report.save()
# Save a list of bad images to csv file for debugging
save_images_to_csv_briefly(report.report_id, bad_image_list)
# Saving a xlsx file
data = extract_report_detail_list(report_files, lower=True)
data_workbook = dict2xlsx(data, _type='report_detail')

166
cope2n-api/fwd_api/management/commands/migrate-csv-new-col-index.py Normal file
View File

@ -0,0 +1,166 @@
# myapp/management/commands/mycustomcommand.py
from django.core.management.base import BaseCommand
from tqdm import tqdm
from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest
from fwd_api.utils.accuracy import predict_result_to_ready
import traceback
import copy
import csv
class Command(BaseCommand):
help = 'Refactor database for image level'
def add_arguments(self, parser):
# Add your command-line arguments here
parser.add_argument('test', type=str, help='Value for the argument')
def process_request(self, request, predict_result, user_feedback, reviewed_result):
if len(request.request_id.split(".")[0].split("_")) < 2:
return
request_feedback = copy.deepcopy(request.feedback_result)
request_review = copy.deepcopy(request.reviewed_result)
if not request_feedback:
request_feedback = {
"request_id": request.request_id,
"imei_number": [],
"retailername": "",
"purchase_date": "",
"sold_to_party": ""
}
if not request_review:
request_review = {
"request_id": request.request_id,
"imei_number": [],
"retailername": "",
"purchase_date": "",
"sold_to_party": ""
}
images = SubscriptionRequestFile.objects.filter(request=request)
is_match = False
try:
for i, image in enumerate(images):
if not request.predict_result:
raise KeyError(f"Key predict_result not found in {request.request_id}")
if request.predict_result.get("status", 200) != 200:
raise AttributeError(f"Failed request: {request.request_id}")
for field in ['retailername', 'purchase_date', 'imei_number']:
# if image.feedback_result[field] is not None:
# print(f"image.feedback_result[field] is not None is not None - field: {field}")
# else:
# print("image.feedback_result[field] is None")
# if image.feedback_result[field] == user_feedback:
# print("image.feedback_result[field] == user_feedback")
# else:
# print(f"NOT image.feedback_result[field] == user_feedback - field: {field} - image.feedback_result[field]:{image.feedback_result[field]} - user_feedback:{user_feedback}")
# if (field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback):
# print("(field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback)")
# else:
# print(f"NOT (field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback) - field: {field}")
# if image.feedback_result[field] is not None and ((field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback) or image.feedback_result[field] == user_feedback):
is_match = True
if field == 'imei_number':
if not reviewed_result == request_review:
request_review["imei_number"].append(reviewed_result)
if not user_feedback == request_feedback:
request_feedback["imei_number"].append(user_feedback)
else:
if not reviewed_result == request_review:
request_review[field] = reviewed_result
if not user_feedback == request_feedback:
request_feedback[field] = user_feedback
_predict_result = copy.deepcopy(predict_result_to_ready(request.predict_result))
_feedback_result = copy.deepcopy(request.feedback_result)
_reviewed_result = copy.deepcopy(request.reviewed_result)
if not _feedback_result:
_feedback_result = {
"imei_number": [],
"retailername": "",
"purchase_date": "",
"sold_to_party": ""
}
if not _reviewed_result:
_reviewed_result = {
"imei_number": [],
"retailername": "",
"purchase_date": "",
"sold_to_party": ""
}
if image.doc_type == "invoice":
_predict_result[field] = predict_result
_predict_result["imei_number"] = []
if _feedback_result:
_feedback_result[field] = user_feedback
_feedback_result["imei_number"] = []
else:
None
if _reviewed_result:
_reviewed_result[field] = reviewed_result
_reviewed_result["imei_number"] = []
else:
None
else:
_predict_result = {
"retailername": None,
"sold_to_party": None,
"purchase_date": [],
"imei_number": [predict_result]
}
_feedback_result = {
"retailername": None,
"sold_to_party": None,
"purchase_date": None,
"imei_number": [user_feedback]
} if _feedback_result else None
_reviewed_result = {
"retailername": None,
"sold_to_party": None,
"purchase_date": None,
"imei_number": [reviewed_result]
} if _reviewed_result else None
image.predict_result = _predict_result
image.feedback_result = _feedback_result
image.reviewed_result = _reviewed_result
image.save()
# request.feedback_result = request_feedback
request.reviewed_result = request_review
request.feedback_result["request_id"] = request.request_id
request.reviewed_result["request_id"] = request.request_id
request.is_reviewed = True
request.save()
except Exception as e:
self.stdout.write(self.style.ERROR(f"Request: {request.request_id} failed with {e}"))
print(traceback.format_exc())
if not is_match:
print("FAIL =====>", "image.feedback_result: ", image.feedback_result, "| predict_result: ", predict_result, " | user_feedback: ", user_feedback, "| reviewed_result: ", reviewed_result)
def handle(self, *args, **options):
test = options['test']
#open csv file
with open(test, 'r') as csvfile:
reader = csv.reader(csvfile)
index = 0
for row in reader:
if index != 0:
request = SubscriptionRequest.objects.filter(request_id=row[0]).first()
if not request:
# print("Not found ====>", row)
continue
else:
self.process_request(request, row[4], row[3], row[5])
index += 1
self.stdout.write(self.style.SUCCESS('Sample Django management command executed successfully!'))

18
cope2n-api/fwd_api/migrations/0182_report_combined_accuracy.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.3 on 2024-02-18 05:19
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0181_reportfile_subsidiary'),
]
operations = [
migrations.AddField(
model_name='report',
name='combined_accuracy',
field=models.JSONField(null=True),
),
]

3
cope2n-api/fwd_api/models/Report.py
View File

@ -38,4 +38,5 @@ class Report(models.Model):
average_client_time = models.JSONField(null=True) # {"invoice": 0.1, "imei": 0.1}
feedback_accuracy = models.JSONField(null=True)
reviewed_accuracy = models.JSONField(null=True)
reviewed_accuracy = models.JSONField(null=True)
combined_accuracy = models.JSONField(null=True)

2
cope2n-api/fwd_api/models/SubscriptionRequestFile.py
View File

@ -31,4 +31,4 @@ class SubscriptionRequestFile(models.Model):
reviewed_result = models.JSONField(null=True)
feedback_accuracy = models.JSONField(null=True)
reviewed_accuracy = models.JSONField(null=True)
reviewed_accuracy = models.JSONField(null=True)

86
cope2n-api/fwd_api/utils/accuracy.py
View File

@ -5,7 +5,7 @@ import copy
from typing import Any
from .ocr_utils.ocr_metrics import eval_ocr_metric
from .ocr_utils.sbt_report import post_processing_str
import uuid
from fwd_api.constant.common import FileCategory
from fwd_api.models import SubscriptionRequest, SubscriptionRequestFile, ReportFile
from ..celery_worker.client_connector import c_connector
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3
@ -172,21 +172,21 @@ class ReportAccumulateByRequest:
day_data["num_invoice"] += 1 if doc_type == "invoice" else 0
day_data["report_files"].append(report_file)
if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 :
day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", 0))
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", 0))
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", 0))
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", 0))
if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0:
day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", []))
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", []))
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", []))
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", []))
elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0:
day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", 0))
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", 0))
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", 0))
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", 0))
day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", []))
day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", []))
day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", []))
day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", []))
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, 0))
day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, []))
for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]:
day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, 0))
day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, []))
if not day_data["average_processing_time"].get(report_file.doc_type, None):
print(f"[WARM]: Weird doctype: {report_file.doc_type}")
@ -196,14 +196,14 @@ class ReportAccumulateByRequest:
return day_data
def add(self, request, report_files):
this_month = request.created_at.strftime("%Y%m")
this_day = request.created_at.strftime("%Y%m%d")
this_month = timezone.localtime(request.created_at).strftime("%Y%m")
this_day = timezone.localtime(request.created_at).strftime("%Y%m%d")
if not self.data.get(this_month, None):
self.data[this_month] = [copy.deepcopy(self.total_format), {}]
self.data[this_month][0]["extraction_date"] = "Subtotal (" + request.created_at.strftime("%Y-%m") + ")"
self.data[this_month][0]["extraction_date"] = "Subtotal (" + timezone.localtime(request.created_at).strftime("%Y-%m") + ")"
if not self.data[this_month][1].get(this_day, None):
self.data[this_month][1][this_day] = copy.deepcopy(self.day_format)[0]
self.data[this_month][1][this_day]['extraction_date'] = request.created_at.strftime("%Y-%m-%d")
self.data[this_month][1][this_day]['extraction_date'] = timezone.localtime(request.created_at).strftime("%Y-%m-%d")
usage = self.count_transactions_within_day(this_day)
self.data[this_month][1][this_day]["usage"]["imei"] = usage.get("imei", 0)
self.data[this_month][1][this_day]["usage"]["invoice"] = usage.get("invoice", 0)
@ -625,14 +625,14 @@ def align_fine_result(ready_predict, fine_result):
# print(f"[DEBUG]: fine_result: {fine_result}")
# print(f"[DEBUG]: ready_predict: {ready_predict}")
if fine_result:
if isinstance(ready_predict["purchase_date"], str):
ready_predict["purchase_date"] = [ready_predict["purchase_date"]]
# ready_predict.save()
if fine_result["purchase_date"] and len(ready_predict["purchase_date"]) == 0:
ready_predict["purchase_date"] = [None]
if fine_result["retailername"] and not ready_predict["retailername"]:
ready_predict["retailername"] = [None]
fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))]
# if ready_predict["retailername"] and not fine_result["retailername"]:
# fine_result["retailername"] = [None]
fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))]
# fine_result["retailername"] = None if len(ready_predict["purchase_date"]))]
# else:
# fine_result = {}
# for key in ready_predict.keys():
@ -668,6 +668,12 @@ def calculate_accuracy(key_name, inference, target):
target[key_name] = []
else:
target[key_name] = [target[key_name]]
# Realign lenght for mis predicted/feedback/reivew result
if len(target[key_name]) == 0 and len(inference[key_name]) > 0:
target[key_name] = [None for _ in range(len(inference[key_name]))]
elif len(inference[key_name]) == 0 and len(target[key_name]) > 0:
target[key_name] = [None for _ in range(len(inference[key_name]))]
for i, v in enumerate(inference[key_name]):
# TODO: target[key_name][i] is None, ""
x = post_processing_str(key_name, inference[key_name][i], is_gt=False)
@ -777,6 +783,11 @@ def calculate_a_request(report, request):
"sold_to_party": [],
},
"reviewed": {"imei_number": [],
"purchase_date": [],
"retailername": [],
"sold_to_party": [],
},
"acumulated":{"imei_number": [],
"purchase_date": [],
"retailername": [],
"sold_to_party": [],
@ -784,10 +795,15 @@ def calculate_a_request(report, request):
"err": [],
"time_cost": {},
"total_images": 0,
"bad_images": 0}
images = SubscriptionRequestFile.objects.filter(request=request)
"bad_images": 0,
"bad_image_list": [],
}
images = SubscriptionRequestFile.objects.filter(request=request, file_category=FileCategory.Origin.value)
report_files = []
for image in images:
if image.reason in settings.ACC_EXCLUDE_RESEASONS:
continue
status, att = calculate_subcription_file(image)
if status != 200:
continue
@ -805,6 +821,8 @@ def calculate_a_request(report, request):
_sub = map_subsidiary_short_to_long(request.redemption_id[:2])
else:
print(f"[WARM]: empty redemption_id, check request: {request.request_id}")
if att["is_bad_image"]:
request_att["bad_image_list"].append(image.file_name)
new_report_file = ReportFile(report=report,
subsidiary=_sub,
correspond_request_id=request.request_id,
@ -838,11 +856,16 @@ def calculate_a_request(report, request):
request_att["acc"]["reviewed"]["retailername"] += att["acc"]["reviewed"]["retailername"]
request_att["acc"]["reviewed"]["sold_to_party"] += att["acc"]["reviewed"]["sold_to_party"]
request_att["acc"]["acumulated"]["imei_number"] += att["acc"]["reviewed"]["imei_number"] if att["acc"]["reviewed"]["imei_number"] else att["acc"]["feedback"]["imei_number"]
request_att["acc"]["acumulated"]["purchase_date"] += att["acc"]["reviewed"]["purchase_date"] if att["acc"]["reviewed"]["purchase_date"] else att["acc"]["feedback"]["purchase_date"]
request_att["acc"]["acumulated"]["retailername"] += att["acc"]["reviewed"]["retailername"] if att["acc"]["reviewed"]["retailername"] else att["acc"]["feedback"]["retailername"]
request_att["acc"]["acumulated"]["sold_to_party"] += att["acc"]["reviewed"]["sold_to_party"] if att["acc"]["reviewed"]["sold_to_party"] else att["acc"]["feedback"]["sold_to_party"]
request_att["bad_images"] += int(att["is_bad_image"])
request_att["total_images"] += 1
request_att["err"] += att["err"]
except Exception as e:
print(e)
print(f"[ERROR]: failed to calculate request: {request.request_id} - request_file: {image.file_name} because of {e}")
continue
return request_att, report_files
@ -870,11 +893,20 @@ def calculate_subcription_file(subcription_request_file):
att["acc"]["reviewed"][key_name], _ = calculate_accuracy(key_name, inference_result, reviewed_result)
except Exception as e:
att["err"].append(str(e))
# print(f"[DEBUG]: predict_result: {subcription_request_file.predict_result}")
# print(f"[DEBUG]: e: {e} -key_name: {key_name}")
subcription_request_file.feedback_accuracy = att["acc"]["feedback"]
subcription_request_file.reviewed_accuracy = att["acc"]["reviewed"]
subcription_request_file.save()
avg_reviewed = calculate_avg_accuracy(att["acc"], "reviewed", ["retailername", "sold_to_party", "purchase_date", "imei_number"])
avg_feedback = calculate_avg_accuracy(att["acc"], "feedback", ["retailername", "sold_to_party", "purchase_date", "imei_number"])
if avg_feedback is not None or avg_reviewed is not None:
avg_acc = max([x for x in [avg_feedback, avg_reviewed] if x is not None])
avg_acc = 0
if avg_feedback is not None:
avg_acc = avg_feedback
if avg_reviewed is not None:
avg_acc = avg_reviewed
if avg_acc < BAD_THRESHOLD:
att["is_bad_image"] = True
# exclude bad images
@ -947,6 +979,12 @@ def calculate_attributions(request): # for one request, return in order
return acc, data, time_cost, image_quality_num, error
def mean_list(l):
l = [x for x in l if x is not None]
if len(l) == 0:
return 0
return sum(l)/len(l)
def shadow_report(report_id, query):
c_connector.make_a_report_2(
(report_id, query))

31
cope2n-api/fwd_api/utils/file.py
View File

@ -250,6 +250,37 @@ def save_file_with_path(file_name: str, file: TemporaryUploadedFile, quality, fo
raise ServiceUnavailableException()
return file_path
def save_images_to_csv_briefly(id, image_filenames):
# columns = ["request_id", "file_name", "predict_result", "feedback_result", "reviewed_result", "feedback_accuracy", "reviewed_accuracy"]
columns = ["request_id", "file_name", "predict_result", "feedback_result", "reviewed_result", "feedback_accuracy", "reviewed_accuracy"]
# get the SubcriptionRequestFile list
images = SubscriptionRequestFile.objects.filter(file_name__in=image_filenames)
# Create a CSV writer object
folder_path = os.path.join(settings.MEDIA_ROOT, "report", id)
file_path = os.path.join(folder_path, "bad_images.csv")
os.makedirs(folder_path, exist_ok = True)
csv_file = open(file_path, "w", newline="")
csv_writer = csv.DictWriter(csv_file, fieldnames=columns)
csv_writer.writeheader()
# Write data to the CSV file
for subscription_request_file in images:
row = {
"request_id": subscription_request_file.request.request_id,
"file_name" : subscription_request_file.file_name,
"predict_result": subscription_request_file.predict_result,
"feedback_result": subscription_request_file.feedback_result,
"reviewed_result": subscription_request_file.reviewed_result,
# "feedback_accuracy": subscription_request_file.feedback_accuracy,
# "reviewed_accuracy": subscription_request_file.reviewed_accuracy,
}
csv_writer.writerow(row)
# Close the CSV file
csv_file.close()
# save to S3
save_report_to_S3(id, file_path)
def resize_and_save_file(file_name: str, rq: SubscriptionRequest, file: TemporaryUploadedFile, quality: int):
try:

4
cope2n-api/scripts/script.py
View File

@ -15,8 +15,8 @@ login_token = None
# Define the login credentials
login_credentials = {
'username': 'sbt',
'password': '7Eg4AbWIXDnufgn'
# 'password': 'abc'
# 'password': '7Eg4AbWIXDnufgn'
'password': 'abc'
}
# Define the command to call the update API

698
cope2n-fe/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

41
cope2n-fe/package.json
View File

@ -27,26 +27,27 @@
},
"dependencies": {
"@ant-design/colors": "^6.0.0",
"@ant-design/icons": "^4.8.0",
"@ant-design/plots": "^1.2.3",
"@ant-design/pro-layout": "^7.10.3",
"@babel/core": "^7.13.10",
"@tanstack/react-query": "^4.20.4",
"antd": "^5.4.0",
"axios": "^1.2.2",
"chart.js": "^4.4.1",
"history": "^5.3.0",
"lodash-es": "^4.17.21",
"mousetrap": "^1.6.5",
"process": "^0.11.10",
"react": "^18.2.0",
"react-chartjs-2": "^5.2.0",
"react-dom": "^18.2.0",
"react-json-view-lite": "^1.2.1",
"react-office-viewer": "^1.0.4",
"react-router-dom": "^6.6.1",
"styled-components": "^5.3.6",
"uuid": "^9.0.0"
"@ant-design/icons": "^4.8.0",
"@ant-design/plots": "^1.2.3",
"@ant-design/pro-layout": "^7.10.3",
"@babel/core": "^7.13.10",
"@cyntler/react-doc-viewer": "^1.14.1",
"@tanstack/react-query": "^4.20.4",
"antd": "^5.4.0",
"axios": "^1.2.2",
"chart.js": "^4.4.1",
"history": "^5.3.0",
"lodash-es": "^4.17.21",
"mousetrap": "^1.6.5",
"process": "^0.11.10",
"react": "^18.2.0",
"react-chartjs-2": "^5.2.0",
"react-dom": "^18.2.0",
"react-json-view-lite": "^1.2.1",
"react-office-viewer": "^1.0.4",
"react-router-dom": "^6.6.1",
"styled-components": "^5.3.6",
"uuid": "^9.0.0"
},
"devDependencies": {
"@babel/plugin-syntax-jsx": "^7.12.13",

BIN
cope2n-fe/public/dummy.pdf Normal file
View File

Binary file not shown.

4
cope2n-fe/src/components/left-menu/index.tsx
View File

@ -1,4 +1,4 @@
import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined } from '@ant-design/icons';
import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined, FileSearchOutlined } from '@ant-design/icons';
import { t } from '@lingui/macro';
import { Menu, MenuProps } from 'antd';
import React from 'react';
@ -34,8 +34,8 @@ function LeftMenu() {
const generalSubItems = [
getItem(t`Dashboard`, '/dashboard', <AppstoreOutlined />),
// getItem(t`Reviews`, '/reviews', <FileSearchOutlined />),
getItem(t`Reports`, '/reports', <BarChartOutlined />),
// getItem(t`Review`, '/reviews', <FileSearchOutlined />),
getItem(t`Inference`, '/inference', <RotateRightOutlined />),
// getItem(t`Users`, '/users', <UsergroupAddOutlined />),
];

11
cope2n-fe/src/components/report-detail/report-overview-table.tsx
View File

@ -69,7 +69,8 @@ const columns: TableColumnsType<DataType> = [
width: '130px',
className: 'hide-border-right',
render: (_, record) => {
return <span>{record.snImeiTC + record.invoiceTC}</span>;
const value = record.snImeiTC + record.invoiceTC;
return <span>{value ? value : '-'}</span>;
},
},
{
@ -81,12 +82,20 @@ const columns: TableColumnsType<DataType> = [
key: 'snImeiTC',
width: '50px',
className: 'show-border-left',
render: (_, record) => {
const value = record.snImeiTC;
return <span>{value ? value : '-'}</span>;
},
},
{
title: 'Invoice',
dataIndex: 'invoiceTC',
key: 'invoiceTC',
width: '50px',
render: (_, record) => {
const value = record.invoiceTC;
return <span>{value ? value : '-'}</span>;
},
},
],
},

184
cope2n-fe/src/pages/reviews/index.tsx
View File

@ -1,5 +1,181 @@
function Reviews() {
return <div>Reviews</div>;
}
import { t } from '@lingui/macro';
import { Button, message, Upload, Input, Table } from 'antd';
import { SbtPageHeader } from 'components/page-header';
import { useState } from 'react';
import { Layout } from 'antd';
import FileViewer from '@cyntler/react-doc-viewer';
const { Sider, Content } = Layout;
export default Reviews;
const siderStyle: React.CSSProperties = {
backgroundColor: '#fafafa',
padding: 10,
width: 200,
};
const fileList = [
{
name: "invoice.pdf",
url: "/dummpy.pdf",
type: "invoice",
isBadQuality: false,
},
{
name: "invoice.pdf",
url: "/dummpy.pdf",
type: "imei",
isBadQuality: true,
}
]
const dataSource = [
{
key: '1',
value: 'Mike',
},
{
key: '2',
value: 'Mike',
},
{
key: '3',
value: 'Mike',
},
];
const columns = [
{
title: 'Key',
dataIndex: 'key',
key: 'key',
},
{
title: 'Predicted',
dataIndex: 'value',
key: 'value',
},
{
title: 'Submitted',
dataIndex: 'value',
key: 'value',
},
{
title: 'Revised',
dataIndex: 'value',
key: 'value',
},
];
const FileCard = ({ file, isSelected, onClick }) => {
return (
<div style={{
border: '1px solid #ccc',
width: '200px',
backgroundColor: isSelected ? '#d4ecff' : '#fff',
padding: '4px 8px',
marginRight: '4px',
marginTop: '4px',
}} onClick={onClick}>
<div>
<span style={{
fontSize: '12px',
color: '#333',
fontWeight: 'bold',
padding: '4px 8px',
}}>{file.type.toUpperCase()}</span>
<span style={{
fontSize: '12px',
color: '#aaa',
fontWeight: 'bold',
padding: '4px 8px',
}}>{file.name}</span>
</div>
</div>
);
};
const InferencePage = () => {
const [selectedFileId, setSelectedFileId] = useState(0);
const selectFileByIndex = (index) => {
setSelectedFileId(index);
};
return (
<>
{/* <SbtPageHeader
title={t`Result Review`}
/> */}
<Layout style={{
overflow: 'hidden',
width: '100%',
maxWidth: '100%',
minHeight: 'calc(100vh - 100px)',
maxHeight: 'calc(100vh - 100px)',
display: 'flex',
padding: '8px',
}}>
<Content style={{
textAlign: 'center',
color: '#fff',
backgroundColor: '#efefef',
height: '100%',
display: 'flex',
flexDirection: 'column',
flexGrow: 1,
}}>
<div style={{
border: "1px solid #ccc",
flexGrow: 1,
height: '500px',
}}>
<FileViewer documents={
[
{ uri: "/dummy.pdf" }
]
} config={{
header: {
disableHeader: true,
disableFileName: true,
retainURLParams: true,
},
csvDelimiter: ",", // "," as default,
pdfVerticalScrollByDefault: true, // false as default
}} />
</div>
<div
style={{
width: "100%",
display: "flex",
flexDirection: "row",
height: "100px",
flexGrow: 0,
}}>
{fileList.map((file, index) => (
<FileCard key={index} file={file} isSelected={index === selectedFileId} onClick={
() => {
setSelectedFileId(index);
}
} />
))}
</div>
</Content>
<Sider width="400px" style={siderStyle}>
<h2 style={{ margin: "0 0 10px 0" }}>Overview</h2>
<Input size='small' addonBefore="Request ID" style={{ marginBottom: "4px" }} readOnly />
<Input size='small' addonBefore="Redemption" style={{ marginBottom: "4px" }} readOnly />
<Input size='small' addonBefore="Uploaded date" style={{ marginBottom: "4px" }} readOnly />
<Input size='small' addonBefore="Request time" style={{ marginBottom: "4px" }} readOnly />
<Input size='small' addonBefore="Processing time" style={{ marginBottom: "4px" }} readOnly />
<div style={{ marginBottom: "8px", marginTop: "8px", display: "flex" }}>
<Button type="primary" size='middle'>Confirm result</Button>
</div>
<Table dataSource={dataSource} columns={columns} />
</Sider>
</Layout>
</>
);
};
export default InferencePage;

2
deploy_images.sh
View File

@ -6,7 +6,7 @@ tag=$1
echo "[INFO] Tag received from Python: $tag"
# echo "[INFO] Updating everything the remote..."
echo "[INFO] Updating everything the remote..."
git submodule update --recursive --remote
echo "[INFO] Pushing AI image with tag: $tag..."

4
docker-compose-dev.yml
View File

@ -174,8 +174,8 @@ services:
- ./cope2n-api:/app
working_dir: /app
# command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
command: bash -c "tail -f > /dev/null"
command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
# command: bash -c "tail -f > /dev/null"
# Back-end persistent
db-sbt: