Merge pull request #88 from SDSRV-IDP/dev/20240319

Fix: #79, #80

cope2n-api/fwd/settings.py

@@ -114,7 +114,7 @@ DATABASES = {
         'PASSWORD': env.str("DB_PASSWORD", None),
         'HOST': env.str("DB_HOST", None),
         'PORT': env.str("DB_PORT", None),
-        'CONN_MAX_AGE': None,
+        'CONN_MAX_AGE': 0,
     }
 }
 

cope2n-api/fwd_api/api/accuracy_view.py

@@ -428,6 +428,7 @@ class AccuracyViewSet(viewsets.ViewSet):
                     "End Date": report.end_at,
                     "No. Requests": report.number_request,
                     "Status": report.status,
+                    "Invoice number Acc": acc["invoice_no"],
                     "Purchase Date Acc": acc["purchase_date"],
                     "Retailer Acc": acc["retailername"],
                     "IMEI Acc": acc["imei_number"],

cope2n-api/fwd_api/celery_worker/process_report_tasks.py

@@ -3,7 +3,7 @@ import traceback
 from fwd_api.models import SubscriptionRequest, Report, ReportFile
 from fwd_api.celery_worker.worker import app
 from ..utils import s3 as S3Util
-from ..utils.accuracy import (update_temp_accuracy, IterAvg, calculate_and_save_subcription_file,
+from ..utils.accuracy import (update_temp_accuracy, IterAvg,
                               count_transactions, extract_report_detail_list, calculate_a_request,
                               ReportAccumulateByRequest, create_billing_data)
 from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3, save_images_to_csv_briefly
@@ -78,15 +78,18 @@ def create_accuracy_report(report_id, **kwargs):
         accuracy = {"feedback" :{"imei_number":  IterAvg(),
                                 "purchase_date":  IterAvg(),
                                 "retailername":  IterAvg(),
-                                "sold_to_party":  IterAvg(),},
+                                "sold_to_party":  IterAvg(),
+                                "invoice_no": IterAvg()},
                     "reviewed" :{"imei_number":  IterAvg(),
                                 "purchase_date":  IterAvg(),
                                 "retailername":  IterAvg(),
-                                "sold_to_party":  IterAvg(),},
+                                "sold_to_party":  IterAvg(),
+                                "invoice_no": IterAvg()},
                     "acumulated":{"imei_number":  IterAvg(),
                                 "purchase_date":  IterAvg(),
                                 "retailername":  IterAvg(),
-                                "sold_to_party":  IterAvg(),}
+                                "sold_to_party":  IterAvg(),
+                                "invoice_no": IterAvg()}
                     }  # {"imei": {"acc": 0.1, count: 1}, ...}
         time_cost = {"invoice":  IterAvg(),
                      "imei":  IterAvg()}
@@ -123,9 +126,9 @@ def create_accuracy_report(report_id, **kwargs):
             number_images += request_att["total_images"]
             number_bad_images += request_att["bad_images"]
             bad_image_list += request_att["bad_image_list"]
-            update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
+            update_temp_accuracy(accuracy["feedback"], request_att["acc"]["feedback"], keys=settings.FIELD)
-            update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
+            update_temp_accuracy(accuracy["reviewed"], request_att["acc"]["reviewed"], keys=settings.FIELD)
-            update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"])
+            update_temp_accuracy(accuracy["acumulated"], request_att["acc"]["acumulated"], keys=settings.FIELD)
 
             time_cost["imei"].add(request_att["time_cost"].get("imei", []))
             time_cost["invoice"].add(request_att["time_cost"].get("invoice", []))
@@ -158,7 +161,7 @@ def create_accuracy_report(report_id, **kwargs):
                           "acumulated": {}}
         for acc_type in ["feedback", "reviewed", "acumulated"]:
             avg_acc = IterAvg()
-            for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]:
+            for key in settings.FIELD:
                 acumulated_acc[acc_type][key] = accuracy[acc_type][key]()
                 acumulated_acc[acc_type][key + "_count"] = accuracy[acc_type][key].count
                 avg_acc.add_avg(acumulated_acc[acc_type][key], acumulated_acc[acc_type][key+"_count"])
@@ -235,7 +238,7 @@ def create_billing_report(report_id, **kwargs):
                           "reviewed": {},
                           "acumulated": {}}
         for acc_type in ["feedback", "reviewed", "acumulated"]:
-            for key in ["imei_number", "purchase_date", "invoice_no", "retailername", "sold_to_party"]:
+            for key in settings.FIELD:
                 acumulated_acc[acc_type][key] = None
                 acumulated_acc[acc_type][key + "_count"] = None
             acumulated_acc[acc_type]["avg"] = None

cope2n-api/fwd_api/utils/accuracy.py

@@ -38,9 +38,9 @@ class ReportAccumulateByRequest:
                         'bad_percent': 0
                     },
                     'average_accuracy_rate': {
-                        'imei': IterAvg(),
+                        'imei_number': IterAvg(),
                         'purchase_date': IterAvg(),
-                        'retailer_name': IterAvg(),
+                        'retailername': IterAvg(),
                         'sold_to_party': IterAvg(),
                         'invoice_no': IterAvg()
                     },
@@ -86,7 +86,7 @@ class ReportAccumulateByRequest:
                     'average_accuracy_rate': {
                         'imei_number': IterAvg(),
                         'purchase_date': IterAvg(),
-                        'retailer_name': IterAvg(),
+                        'retailername': IterAvg(),
                         'sold_to_party': IterAvg(),
                         'invoice_no': IterAvg()
                     },
@@ -218,7 +218,6 @@ class ReportAccumulateByRequest:
             self.data[this_month][1][this_day] = self.update_day(self.data[this_month][1][this_day], _report_file) # Update the subtotal of the day
 
     def count_transactions_within_day(self, date_string):
-        # convert this day into timezone.datetime at UTC
         start_date = datetime.strptime(date_string, "%Y%m%d")
         start_date_with_timezone = timezone.make_aware(start_date)
         end_date_with_timezone = start_date_with_timezone + timezone.timedelta(days=1)
@@ -303,8 +302,8 @@ class ReportAccumulateByRequest:
             
                 for key in settings.FIELD:
                     _data[month][1][day]["average_accuracy_rate"][key] = _data[month][1][day]["average_accuracy_rate"][key]()
-                    for accuracy_type in ["feedback_accuracy", key]:
+                    for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
-                        _data[month][1][day][accuracy_type]["imei_number"] = _data[month][1][day]["feedback_accuracy"]["imei_number"]()
+                        _data[month][1][day][accuracy_type][key] = _data[month][1][day][accuracy_type][key]()
                 _data[month][1][day]["review_progress"] = _data[month][1][day]["review_progress"].count(1)/(_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) if (_data[month][1][day]["review_progress"].count(0)+ _data[month][1][day]["review_progress"].count(1)) >0 else 0
                 _data[month][1][day].pop("report_files")
 
@@ -318,7 +317,7 @@ class ReportAccumulateByRequest:
                 _data[month][0]["average_processing_time"][key] = _data[month][0]["average_processing_time"][key]()
             for key in settings.FIELD:
                 _data[month][0]["average_accuracy_rate"][key] = _data[month][0]["average_accuracy_rate"][key]()
-                for accuracy_type in ["feedback_accuracy", key]:
+                for accuracy_type in ["feedback_accuracy", "reviewed_accuracy"]:
                     _data[month][0][accuracy_type][key] = _data[month][0][accuracy_type][key]()
             _data[month][0]["review_progress"] = _data[month][0]["review_progress"].count(1)/(_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) if (_data[month][0]["review_progress"].count(0)+ _data[month][0]["review_progress"].count(1)) >0 else 0
         return _data
@@ -532,7 +531,6 @@ def first_of_list(the_list):
 def extract_report_detail_list(report_detail_list, lower=False, in_percent=True):
     data = []
     for report_file in report_detail_list:
-        # FIXME: #79 Fill None with value
         data.append({
             "Subs": report_file.subsidiary,
             "Request ID": report_file.correspond_request_id,
@@ -540,19 +538,19 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True)
             "Image type": report_file.doc_type,
             "IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])),
             "IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])),
-            "IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])),
+            "IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])) if report_file.reviewed_result else None,
             "IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])),
-            "Invoice_Number_User": report_file.feedback_result.get("invoice_no", None),
+            "Invoice_Number_User": report_file.feedback_result.get("invoice_no", None) if report_file.feedback_result else None,
             "Invoice_Number_OCR": report_file.predict_result.get("invoice_no", None),
-            "Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None),
+            "Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None) if report_file.reviewed_result else None,
             "Invoice_Number_Accuracy": first_of_list(report_file.feedback_accuracy.get("invoice_no", [None])),
             "Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None),
             "Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []),
-            "Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None),
+            "Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None) if report_file.reviewed_result else None,
             "Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])),
             "Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None),
             "Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None),
-            "Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None),
+            "Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None) if report_file.reviewed_result else None,
             "Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])),
             "OCR Image Accuracy": report_file.acc,
             "OCR Image Speed (seconds)": report_file.time_cost,
@@ -638,7 +636,7 @@ def align_fine_result(ready_predict, fine_result):
             ready_predict["purchase_date"] = [None]
         if fine_result["retailername"] and not ready_predict["retailername"]:
             ready_predict["retailername"] = [None]
-        if ready_predict["invoice_no"] and not fine_result["invoice_no"]:
+        if ready_predict.get("invoice_no", None) and not fine_result.get("invoice_no", None):
             fine_result["invoice_no"] = [None]
         fine_result["purchase_date"] = [fine_result["purchase_date"] for _ in range(len(ready_predict["purchase_date"]))]
     return ready_predict, fine_result

cope2n-fe/src/components/report-detail/report-overview-table.tsx

@@ -17,6 +17,7 @@ interface DataType {
   snImeiAAR: number; // AAR: Average Accuracy Rate
   purchaseDateAAR: number;
   retailerNameAAR: number;
+  invoiceNumberAAR: number;
   snImeiAPT: number; // APT: Average Processing Time
   invoiceAPT: number;
   snImeiTC: number; // TC: transaction count
@@ -211,6 +212,20 @@ const columns: TableColumnsType<DataType> = [
           );
         },
       },
+      {
+        title: 'Invoice number',
+        dataIndex: 'invoiceNumberAAR',
+        key: 'invoiceNumberAAR',
+        width: '130px',
+        render: (_, record) => {
+          const isAbnormal = ensureMin(record.invoiceNumberAAR, 0.98);
+          return (
+            <span style={{ color: isAbnormal ? 'red' : '' }}>
+              {formatPercent(record.invoiceNumberAAR)}
+            </span>
+          );
+        },
+      },
     ],
   },
   {
@@ -285,6 +300,7 @@ const ReportOverViewTable: React.FC<ReportOverViewTableProps> = ({
         snImeiAAR: item.average_accuracy_rate.imei,
         purchaseDateAAR: item.average_accuracy_rate.purchase_date,
         retailerNameAAR: item.average_accuracy_rate.retailer_name,
+        invoiceNumberAAR: item.average_accuracy_rate.invoice_no,
         snImeiAPT: item.average_processing_time.imei,
         invoiceAPT: item.average_processing_time.invoice,
         snImeiTC: item.usage.imei,

cope2n-fe/src/components/report-detail/report-table.tsx

@@ -103,6 +103,19 @@ const ReportTable: React.FC = () => {
         );
       },
     },
+    {
+      title: 'Invoice number Accuracy',
+      dataIndex: 'Invoice number Acc',
+      key: 'Invoice number Acc',
+      render: (_, record) => {
+        const isAbnormal = ensureMin(record['Invoice number Acc'], 0.95);
+        return (
+          <span style={{ color: isAbnormal ? 'red' : '' }}>
+            {formatPercent(record['Invoice number Acc'])}
+          </span>
+        );
+      },
+    },
 
     {
       title: 'Retailer Accuracy',

document-classification-kv-demo

@@ -1 +0,0 @@
-Subproject commit 220954c5c6bfed15e93e26b2adacf28ff8b75baf