From 4e48909e88f3b3a7a3122d58b734fdd201c2a12c Mon Sep 17 00:00:00 2001 From: dx-tan Date: Tue, 26 Mar 2024 16:26:10 +0700 Subject: [PATCH 1/3] Fix: #101 --- .gitignore | 3 +- cope2n-ai-fi/modules/sdsvkvu | 2 +- cope2n-api/fwd_api/api/accuracy_view.py | 13 ++- .../fwd_api/celery_worker/internal_task.py | 1 - .../celery_worker/process_report_tasks.py | 48 ++++++---- ...eportfile_correspond_request_created_at.py | 19 ++++ .../0187_report_s3_dashboard_file_name.py | 18 ++++ cope2n-api/fwd_api/models/Report.py | 1 + cope2n-api/fwd_api/utils/file.py | 4 +- cope2n-fe/.env.development | 2 +- .../report-detail/report-overview-table.tsx | 44 ++++++--- .../components/report-detail/report-table.tsx | 32 ++++--- cope2n-fe/src/locales/en/messages.json | 1 - cope2n-fe/src/locales/vi/messages.json | 1 - cope2n-fe/src/models/report.ts | 1 + .../src/pages/reports/report_detail/index.tsx | 4 +- cope2n-fe/src/request/report.ts | 4 +- docker-compose-dev.yml | 91 ++++++++++--------- test_samples/test_26/result.json | 2 +- test_samples/test_27/result.json | 2 +- tests/test_set_26_jpg_invoice.py | 2 +- tests/test_set_27_jpeg_invoice.py | 2 +- 22 files changed, 192 insertions(+), 105 deletions(-) create mode 100644 cope2n-api/fwd_api/migrations/0186_reportfile_correspond_request_created_at.py create mode 100644 cope2n-api/fwd_api/migrations/0187_report_s3_dashboard_file_name.py diff --git a/.gitignore b/.gitignore index 8d7c64e..8b3fae4 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,5 @@ cope2n-api/reviewed/date.xlsx cope2n-api/reviewed/retailer.xlsx /scripts/* scripts/crawl_database.py -redundant_models/ \ No newline at end of file +redundant_models/ +junk.json \ No newline at end of file diff --git a/cope2n-ai-fi/modules/sdsvkvu b/cope2n-ai-fi/modules/sdsvkvu index 2012803..46a612a 160000 --- a/cope2n-ai-fi/modules/sdsvkvu +++ b/cope2n-ai-fi/modules/sdsvkvu @@ -1 +1 @@ -Subproject commit 20128037dbfca217fa3d3ca4551cc7f8ae8a190e +Subproject commit 46a612a003c411406988b83b3dd6299d2a458366 diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index 97a7f5f..fdf35cf 100755 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -277,6 +277,7 @@ class AccuracyViewSet(viewsets.ViewSet): start_at=start_date, end_at=end_date, status="Processing", + report_type=report_type, ) new_report.save() # Background job to calculate accuracy @@ -437,6 +438,7 @@ class AccuracyViewSet(viewsets.ViewSet): "Avg. OCR Processing Time": processing_time, "report_id": report.report_id, "Subsidiary": map_subsidiary_short_to_long(report.subsidiary), + "Report Type": report.report_type, }) response = { @@ -628,11 +630,16 @@ class AccuracyViewSet(viewsets.ViewSet): report = Report.objects.filter(report_id=report_id).first() # download from s3 to local target_timezone = pytz.timezone(settings.TIME_ZONE) - tmp_file = "/tmp/" + report.subsidiary + "_" + report.start_at.astimezone(target_timezone).strftime("%Y%m%d") + "_" + report.end_at.astimezone(target_timezone).strftime("%Y%m%d") + "_created_on_" + report.created_at.astimezone(target_timezone).strftime("%Y%m%d") + ".xlsx" - os.makedirs("/tmp", exist_ok=True) if not report.S3_file_name: raise NotFoundException(excArgs="S3 file name") - download_from_S3(report.S3_file_name, tmp_file) + if not report.S3_dashboard_file_name: + raise NotFoundException(excArgs="S3 dashboard file name") + + file_name = report.S3_file_name if request.query_params["report_expression"] == "detail" else report.S3_dashboard_file_name + tmp_file = "/tmp/" + request.query_params["report_expression"] + "_" + report.subsidiary + "_" + report.start_at.astimezone(target_timezone).strftime("%Y%m%d") + "_" + report.end_at.astimezone(target_timezone).strftime("%Y%m%d") + "_created_on_" + report.created_at.astimezone(target_timezone).strftime("%Y%m%d") + ".xlsx" + os.makedirs("/tmp", exist_ok=True) + + download_from_S3(file_name, tmp_file) file = open(tmp_file, 'rb') response = FileResponse(file, status=200) diff --git a/cope2n-api/fwd_api/celery_worker/internal_task.py b/cope2n-api/fwd_api/celery_worker/internal_task.py index ac13d84..d44c6d1 100755 --- a/cope2n-api/fwd_api/celery_worker/internal_task.py +++ b/cope2n-api/fwd_api/celery_worker/internal_task.py @@ -267,7 +267,6 @@ def upload_report_to_s3(local_file_path, s3_key, report_id, delay): if report_id: report = Report.objects.filter(report_id=report_id)[0] report.S3_uploaded = True - report.S3_file_name = s3_key report.save() except Exception as e: logger.error(f"Unable to set S3: {e}") diff --git a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py index a8f1d7e..820be0d 100755 --- a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py @@ -14,6 +14,7 @@ from django.utils import timezone from django.db.models import Q import json import copy +import os from celery.utils.log import get_task_logger from fwd import settings @@ -177,36 +178,45 @@ def create_accuracy_report(report_id, **kwargs): report.errors = "|".join(errors) report.status = "Ready" + + detail_file_name = "detail_" + report.report_id + ".xlsx" + dashboard_file_name = "overview_" + report.report_id + ".xlsx" + report.S3_file_name = os.path.join("report", report.report_id, detail_file_name) + report.S3_dashboard_file_name = os.path.join("report", report.report_id, dashboard_file_name) report.save() + # Save a list of bad images to csv file for debugging save_images_to_csv_briefly(report.report_id, bad_image_list) # Saving a xlsx file data = extract_report_detail_list(report_files, lower=True) data_workbook = dict2xlsx(data, _type='report_detail') - local_workbook = save_workbook_file(report.report_id + ".xlsx", report, data_workbook) + local_workbook = save_workbook_file(detail_file_name, report, data_workbook) s3_key = save_report_to_S3(report.report_id, local_workbook, 5) + # Save overview dashboard + # multiple accuracy by 100 + save_data = copy.deepcopy(_save_data) + review_key = "review_progress" + for i, dat in enumerate(report_fine_data): + report_fine_data[i][review_key] = report_fine_data[i][review_key]*100 + keys = [x for x in list(dat.keys()) if "accuracy" in x.lower()] + keys_percent = "images_quality" + for x_key in report_fine_data[i][keys_percent].keys(): + if "percent" not in x_key: + continue + report_fine_data[i][keys_percent][x_key] = report_fine_data[i][keys_percent][x_key]*100 + for key in keys: + if report_fine_data[i][key]: + for x_key in report_fine_data[i][key].keys(): + report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 if report_fine_data[i][key][x_key] is not None else None + data_workbook = dict2xlsx(report_fine_data, _type='report') if kwargs["is_daily_report"]: - # Save overview dashboard - # multiple accuracy by 100 - save_data = copy.deepcopy(_save_data) - review_key = "review_progress" - for i, dat in enumerate(report_fine_data): - report_fine_data[i][review_key] = report_fine_data[i][review_key]*100 - keys = [x for x in list(dat.keys()) if "accuracy" in x.lower()] - keys_percent = "images_quality" - for x_key in report_fine_data[i][keys_percent].keys(): - if "percent" not in x_key: - continue - report_fine_data[i][keys_percent][x_key] = report_fine_data[i][keys_percent][x_key]*100 - for key in keys: - if report_fine_data[i][key]: - for x_key in report_fine_data[i][key].keys(): - report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 if report_fine_data[i][key][x_key] is not None else None - data_workbook = dict2xlsx(report_fine_data, _type='report') overview_filename = kwargs["subsidiary"] + "_" + kwargs["report_overview_duration"] + ".xlsx" local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT) - s3_key = save_report_to_S3(report.report_id, local_workbook) set_cache(overview_filename.replace(".xlsx", ""), save_data) + else: + overview_filename = dashboard_file_name + local_workbook = save_workbook_file(overview_filename, report, data_workbook) + s3_key = save_report_to_S3(report.report_id, local_workbook) except IndexError as e: print(e) diff --git a/cope2n-api/fwd_api/migrations/0186_reportfile_correspond_request_created_at.py b/cope2n-api/fwd_api/migrations/0186_reportfile_correspond_request_created_at.py new file mode 100644 index 0000000..8a23004 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0186_reportfile_correspond_request_created_at.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1.3 on 2024-03-21 08:58 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0185_report_report_type'), + ] + + operations = [ + migrations.AddField( + model_name='reportfile', + name='correspond_request_created_at', + field=models.DateTimeField(db_index=True, default=django.utils.timezone.now), + ), + ] diff --git a/cope2n-api/fwd_api/migrations/0187_report_s3_dashboard_file_name.py b/cope2n-api/fwd_api/migrations/0187_report_s3_dashboard_file_name.py new file mode 100644 index 0000000..e127f37 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0187_report_s3_dashboard_file_name.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2024-03-26 07:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0186_reportfile_correspond_request_created_at'), + ] + + operations = [ + migrations.AddField( + model_name='report', + name='S3_dashboard_file_name', + field=models.TextField(default=None, null=True), + ), + ] diff --git a/cope2n-api/fwd_api/models/Report.py b/cope2n-api/fwd_api/models/Report.py index ebfb393..d52a455 100755 --- a/cope2n-api/fwd_api/models/Report.py +++ b/cope2n-api/fwd_api/models/Report.py @@ -24,6 +24,7 @@ class Report(models.Model): # Data S3_uploaded = models.BooleanField(default=False) S3_file_name = models.TextField(default=None, null=True) + S3_dashboard_file_name = models.TextField(default=None, null=True) number_request = models.IntegerField(default=0) number_images = models.IntegerField(default=0) number_bad_images = models.IntegerField(default=0) diff --git a/cope2n-api/fwd_api/utils/file.py b/cope2n-api/fwd_api/utils/file.py index 0435b5f..b3ea680 100755 --- a/cope2n-api/fwd_api/utils/file.py +++ b/cope2n-api/fwd_api/utils/file.py @@ -498,9 +498,9 @@ def dump_excel_report(input: json): 'J': 'images_quality.successful_percent', 'K': 'images_quality.bad', 'L': 'images_quality.bad_percent', - 'M': 'average_accuracy_rate.imei', + 'M': 'average_accuracy_rate.imei_number', 'N': 'average_accuracy_rate.purchase_date', - 'O': 'average_accuracy_rate.retailer_name', + 'O': 'average_accuracy_rate.retailername', 'P': 'average_accuracy_rate.invoice_number', 'Q': 'average_processing_time.imei', 'R': 'average_processing_time.invoice', diff --git a/cope2n-fe/.env.development b/cope2n-fe/.env.development index a27afc6..b3f7364 100644 --- a/cope2n-fe/.env.development +++ b/cope2n-fe/.env.development @@ -1,3 +1,3 @@ VITE_PORT=8080 -VITE_PROXY=http://42.96.42.13:9881 +VITE_PROXY=http://42.96.42.13:9000 VITE_KUBEFLOW_HOST=https://107.120.133.22:8085 \ No newline at end of file diff --git a/cope2n-fe/src/components/report-detail/report-overview-table.tsx b/cope2n-fe/src/components/report-detail/report-overview-table.tsx index b96a140..b572872 100644 --- a/cope2n-fe/src/components/report-detail/report-overview-table.tsx +++ b/cope2n-fe/src/components/report-detail/report-overview-table.tsx @@ -24,7 +24,8 @@ interface DataType { purchaseDateAAR: number; retailerNameAAR: number; invoiceNumberAAR: number; - avgAPT: number; // APT: Average Processing Time + snImeiAPT: number; // APT: Average Processing Time + invoiceAPT: number; snImeiTC: number; // TC: transaction count invoiceTC: number; reviewProgress: number; @@ -255,16 +256,34 @@ const columns: TableColumnsType = [ }, { title: 'Average Processing Time Per Image (Seconds)', - dataIndex: 'avgAPT', - key: 'avgAPT', - render: (_, record) => { - const isAbnormal = ensureMax(record.avgAPT, 2); - return ( - - {formatNumber(record?.avgAPT)} - - ); - }, + children: [ + { + title: 'SN/IMEI', + dataIndex: 'snImeiAPT', + key: 'snImeiAPT', + render: (_, record) => { + const isAbnormal = ensureMax(record.snImeiAPT, 2); + return ( + + {formatNumber(record?.snImeiAPT)} + + ); + }, + }, + { + title: 'Invoice', + dataIndex: 'invoiceAPT', + key: 'invoiceAPT', + render: (_, record) => { + const isAbnormal = ensureMax(record.invoiceAPT, 2); + return ( + + {formatNumber(record?.invoiceAPT)} + + ); + }, + }, + ], }, { title: 'Review Progress', @@ -310,7 +329,8 @@ const ReportOverViewTable: React.FC = ({ purchaseDateAAR: item.average_accuracy_rate.purchase_date, retailerNameAAR: item.average_accuracy_rate.retailername, invoiceNumberAAR: item.average_accuracy_rate.invoice_no, - avgAPT: item.average_processing_time.avg, + snImeiAPT: item.average_processing_time.imei, + invoiceAPT: item.average_processing_time.invoice, snImeiTC: item.usage.imei, invoiceTC: item.usage.invoice, reviewProgress: item.review_progress, diff --git a/cope2n-fe/src/components/report-detail/report-table.tsx b/cope2n-fe/src/components/report-detail/report-table.tsx index 1b8c509..ef4a5b0 100644 --- a/cope2n-fe/src/components/report-detail/report-table.tsx +++ b/cope2n-fe/src/components/report-detail/report-table.tsx @@ -1,3 +1,4 @@ +import { VerticalAlignBottomOutlined } from '@ant-design/icons'; import type { TableColumnsType } from 'antd'; import { Button, Table } from 'antd'; import { ReportDetail } from 'models'; @@ -25,8 +26,8 @@ const ReportTable: React.FC = () => { page_size: 10, })); - const handleDownloadReport = async (report_id: string) => { - const { file, filename } = await downloadReport(report_id); + const handleDownloadReport = async (report_id: string, report_expression: string) => { + const { file, filename } = await downloadReport(report_id, report_expression); const anchorElement = document.createElement('a'); anchorElement.href = URL.createObjectURL(file); anchorElement.download = filename; @@ -111,7 +112,7 @@ const ReportTable: React.FC = () => { render: (_, record) => { const isAbnormal = ensureMin(record['Purchase Date Acc'], 0.95); return ( - + {formatPercent(record['Purchase Date Acc'])} ); @@ -124,7 +125,7 @@ const ReportTable: React.FC = () => { render: (_, record) => { const isAbnormal = ensureMin(record['Invoice number Acc'], 0.95); return ( - + {formatPercent(record['Invoice number Acc'])} ); @@ -138,7 +139,7 @@ const ReportTable: React.FC = () => { render: (_, record) => { const isAbnormal = ensureMin(record['Retailer Acc'], 0.95); return ( - + {formatPercent(record['Retailer Acc'])} ); @@ -151,7 +152,7 @@ const ReportTable: React.FC = () => { render: (_, record) => { const isAbnormal = ensureMin(record['IMEI Acc'], 0.95); return ( - + {formatPercent(record['IMEI Acc'])} ); @@ -164,7 +165,7 @@ const ReportTable: React.FC = () => { render: (_, record) => { const isAbnormal = ensureMin(record['Avg. Accuracy'], 0.95); return ( - + {formatPercent(record['Avg. Accuracy'])} ); @@ -177,7 +178,7 @@ const ReportTable: React.FC = () => { render: (_, record) => { const isAbnormal = ensureMax(record['Avg. OCR Processing Time'], 2); return ( - + {formatNumber(record['Avg. OCR Processing Time'], 1)} ); @@ -187,20 +188,27 @@ const ReportTable: React.FC = () => { title: 'Actions', dataIndex: 'actions', key: 'actions', - width: 240, + width: 340, render: (_, record) => { return ( -
+
- +
); diff --git a/cope2n-fe/src/locales/en/messages.json b/cope2n-fe/src/locales/en/messages.json index eaee229..7718262 100644 --- a/cope2n-fe/src/locales/en/messages.json +++ b/cope2n-fe/src/locales/en/messages.json @@ -51,7 +51,6 @@ "This field must not have more than {MAX_USERNAME_LENGTH} characters": "This field must not have more than {MAX_USERNAME_LENGTH} characters", "Too blurry text": "Too blurry text", "Too small text": "Too small text", - "Upload files to process. The requests here will not be used in accuracy or payment calculations.": "Upload files to process. The requests here will not be used in accuracy or payment calculations.", "User log in successfully": "User log in successfully", "Username": "Username", "Username must not have more than {MAX_USERNAME_LENGTH} characters": "Username must not have more than {MAX_USERNAME_LENGTH} characters", diff --git a/cope2n-fe/src/locales/vi/messages.json b/cope2n-fe/src/locales/vi/messages.json index 07a56ed..3ca8f0b 100644 --- a/cope2n-fe/src/locales/vi/messages.json +++ b/cope2n-fe/src/locales/vi/messages.json @@ -51,7 +51,6 @@ "This field must not have more than {MAX_USERNAME_LENGTH} characters": "Độ dài chuỗi không được vượt quá {MAX_USERNAME_LENGTH} kí tự", "Too blurry text": "", "Too small text": "", - "Upload files to process. The requests here will not be used in accuracy or payment calculations.": "", "User log in successfully": "Đăng nhập thành công", "Username": "Tên tài khoản", "Username must not have more than {MAX_USERNAME_LENGTH} characters": "Tên tài khoản không được chứa nhiều hơn {MAX_USERNAME_LENGTH} kí tự", diff --git a/cope2n-fe/src/models/report.ts b/cope2n-fe/src/models/report.ts index 58b8af2..a5351fe 100644 --- a/cope2n-fe/src/models/report.ts +++ b/cope2n-fe/src/models/report.ts @@ -91,6 +91,7 @@ export interface ReportDetail { 'Avg Accuracy': any; 'Avg. Client Request Time': number; 'Avg. OCR Processing Time': number; + 'Report Type': string; report_id: string; } diff --git a/cope2n-fe/src/pages/reports/report_detail/index.tsx b/cope2n-fe/src/pages/reports/report_detail/index.tsx index 272eb60..a16d055 100644 --- a/cope2n-fe/src/pages/reports/report_detail/index.tsx +++ b/cope2n-fe/src/pages/reports/report_detail/index.tsx @@ -47,7 +47,7 @@ const ReportDetail = () => { }); const report_data = data as ReportDetailList; const handleDownloadReport = async () => { - const {file, filename} = await downloadReport(id); + const {file, filename} = await downloadReport(id, "detail"); const anchorElement = document.createElement('a'); anchorElement.href = URL.createObjectURL(file); anchorElement.download = filename; @@ -63,7 +63,7 @@ const ReportDetail = () => { // Download and show report useEffect(() => { try { - downloadReport(id, (fileDetails) => { + downloadReport(id, "detail", (fileDetails) => { if (!fileDetails?.file) { setError("The report has not been ready to preview."); } diff --git a/cope2n-fe/src/request/report.ts b/cope2n-fe/src/request/report.ts index 1b58042..357e0c0 100644 --- a/cope2n-fe/src/request/report.ts +++ b/cope2n-fe/src/request/report.ts @@ -85,9 +85,9 @@ export async function getOverViewReport(params?: DashboardOverviewParams) { } } -export async function downloadReport(report_id: string, downloadFinishedCallback?: (fileDetails: any) => void) { +export async function downloadReport(report_id: string, report_expression: string, downloadFinishedCallback?: (fileDetails: any) => void) { try { - const response = await API.get(`/ctel/get_report_file/${report_id}/`, { + const response = await API.get(`/ctel/get_report_file/${report_id}/?report_expression=${report_expression}`, { responseType: 'blob', // Important }); let filename = "report.xlsx"; diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index ba812e6..9b67208 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -63,8 +63,8 @@ services: - AUTH_TOKEN_LIFE_TIME=${AUTH_TOKEN_LIFE_TIME} - IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME} - INTERNAL_SDS_KEY=${INTERNAL_SDS_KEY} - - ADMIN_USER_NAME=${FI_USER_NAME} - - ADMIN_PASSWORD=${FI_PASSWORD} + - ADMIN_USER_NAME=${ADMIN_USER_NAME} + - ADMIN_PASSWORD=${ADMIN_PASSWORD} - STANDARD_USER_NAME=${STANDARD_USER_NAME} - STANDARD_PASSWORD=${STANDARD_PASSWORD} - S3_ENDPOINT=${S3_ENDPOINT} @@ -89,6 +89,11 @@ services: depends_on: db-sbt: condition: service_started + # command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && + # python manage.py makemigrations && + # python manage.py migrate && + # python manage.py compilemessages && + # gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod command: "sleep infinity" minio: @@ -174,8 +179,8 @@ services: - ./cope2n-api:/app working_dir: /app - command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" - # command: bash -c "tail -f > /dev/null" + # command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" + command: bash -c "tail -f > /dev/null" # Back-end persistent db-sbt: @@ -204,46 +209,46 @@ services: - RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER} - RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS} - # # Front-end services - # fe-sbt: - # restart: always - # build: - # context: cope2n-fe - # shm_size: 10gb - # dockerfile: Dockerfile - # image: sidp/cope2n-fe-fi-sbt:latest - # shm_size: 10gb - # privileged: true - # ports: - # - 9881:80 - # depends_on: - # be-ctel-sbt: - # condition: service_started - # be-celery-sbt: - # condition: service_started - # environment: - # - VITE_PROXY=http://be-ctel-sbt:${BASE_PORT} - # - VITE_API_BASE_URL=http://fe-sbt:80 - # volumes: - # - BE_static:/backend-static - # networks: - # - ctel-sbt + # Front-end services + fe-sbt: + restart: always + build: + context: cope2n-fe + shm_size: 10gb + dockerfile: Dockerfile + image: sidp/cope2n-fe-fi-sbt:latest + shm_size: 10gb + privileged: true + ports: + - 9881:80 + depends_on: + be-ctel-sbt: + condition: service_started + be-celery-sbt: + condition: service_started + environment: + - VITE_PROXY=http://be-ctel-sbt:${BASE_PORT} + - VITE_API_BASE_URL=http://fe-sbt:80 + volumes: + - BE_static:/backend-static + networks: + - ctel-sbt - # dashboard_refresh: - # build: - # context: api-cronjob - # dockerfile: Dockerfile - # image: sidp/api-caller-sbt:latest - # environment: - # - PROXY=http://be-ctel-sbt:9000 - # - ADMIN_USER_NAME=${ADMIN_USER_NAME} - # - ADMIN_PASSWORD=${ADMIN_PASSWORD} - # depends_on: - # be-ctel-sbt: - # condition: service_healthy - # restart: always - # networks: - # - ctel-sbt + dashboard_refresh: + build: + context: api-cronjob + dockerfile: Dockerfile + image: sidp/api-caller-sbt:latest + environment: + - PROXY=http://be-ctel-sbt:9000 + - ADMIN_USER_NAME=${ADMIN_USER_NAME} + - ADMIN_PASSWORD=${ADMIN_PASSWORD} + depends_on: + be-ctel-sbt: + condition: service_healthy + restart: always + networks: + - ctel-sbt volumes: db_data: diff --git a/test_samples/test_26/result.json b/test_samples/test_26/result.json index f5e379a..415cd37 100644 --- a/test_samples/test_26/result.json +++ b/test_samples/test_26/result.json @@ -24,6 +24,6 @@ } ], "doc_type": "sbt_document", - "end_page": 1, + "end_page": 2, "start_page": 1 } \ No newline at end of file diff --git a/test_samples/test_27/result.json b/test_samples/test_27/result.json index 5e8f437..f94b4f1 100644 --- a/test_samples/test_27/result.json +++ b/test_samples/test_27/result.json @@ -24,6 +24,6 @@ } ], "doc_type": "sbt_document", - "end_page": 1, + "end_page": 2, "start_page": 1 } \ No newline at end of file diff --git a/tests/test_set_26_jpg_invoice.py b/tests/test_set_26_jpg_invoice.py index 863a295..185b7c3 100644 --- a/tests/test_set_26_jpg_invoice.py +++ b/tests/test_set_26_jpg_invoice.py @@ -12,7 +12,7 @@ token = login(HOST, USERNAME, PASSWORD) def test_invoice_number(): - invoice_files = [] + invoice_files = ["test_samples/test_26/invoice_no_jpg.jpg",] imei_files = [ "test_samples/test_26/invoice_no_jpg.jpg", ] diff --git a/tests/test_set_27_jpeg_invoice.py b/tests/test_set_27_jpeg_invoice.py index 619210b..5a0a67d 100644 --- a/tests/test_set_27_jpeg_invoice.py +++ b/tests/test_set_27_jpeg_invoice.py @@ -12,7 +12,7 @@ token = login(HOST, USERNAME, PASSWORD) def test_invoice_number(): - invoice_files = [] + invoice_files = ["test_samples/test_27/invoice_no_jpeg.jpeg",] imei_files = [ "test_samples/test_27/invoice_no_jpeg.jpeg", ] From f7dadf2cb699ea9ab7aae8a65c7171c9f3907e64 Mon Sep 17 00:00:00 2001 From: dx-tan Date: Fri, 29 Mar 2024 17:47:10 +0700 Subject: [PATCH 2/3] Fix: #101 --- cope2n-api/fwd_api/api/accuracy_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index fdf35cf..9fb9268 100755 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -632,7 +632,7 @@ class AccuracyViewSet(viewsets.ViewSet): target_timezone = pytz.timezone(settings.TIME_ZONE) if not report.S3_file_name: raise NotFoundException(excArgs="S3 file name") - if not report.S3_dashboard_file_name: + if not report.S3_dashboard_file_name and request.query_params["report_expression"] != "detail": raise NotFoundException(excArgs="S3 dashboard file name") file_name = report.S3_file_name if request.query_params["report_expression"] == "detail" else report.S3_dashboard_file_name From 580f099200414b4ab74419929bfb63843d72d6fa Mon Sep 17 00:00:00 2001 From: dx-tan Date: Fri, 29 Mar 2024 17:47:28 +0700 Subject: [PATCH 3/3] Fix: #103 --- .../celery_worker/process_report_tasks.py | 7 +++++-- .../fwd_api/management/commands/migrate-csv.py | 15 ++++++++++----- cope2n-api/fwd_api/utils/accuracy.py | 6 +++--- .../components/report-detail/report-table.tsx | 12 +++++++++++- docker-compose-dev.yml | 16 ++++++++-------- 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py index 820be0d..506762d 100755 --- a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py @@ -260,10 +260,13 @@ def create_billing_report(report_id, **kwargs): report.average_OCR_time = {"invoice": None, "imei": None, "invoice_count": None, "imei_count": None, "avg": None} report.status = "Ready" + detail_file_name = "detail_" + report.report_id + ".xlsx" + dashboard_file_name = "overview_" + report.report_id + ".xlsx" + report.S3_file_name = os.path.join("report", report.report_id, detail_file_name) + report.S3_dashboard_file_name = os.path.join("report", report.report_id, dashboard_file_name) report.save() data_workbook = dict2xlsx(billing_data, _type='billing_report') - local_workbook = save_workbook_file( - report.report_id + ".xlsx", report, data_workbook) + local_workbook = save_workbook_file(detail_file_name, report, data_workbook) s3_key = save_report_to_S3(report.report_id, local_workbook) except IndexError as e: print(e) diff --git a/cope2n-api/fwd_api/management/commands/migrate-csv.py b/cope2n-api/fwd_api/management/commands/migrate-csv.py index b4ad435..440992e 100644 --- a/cope2n-api/fwd_api/management/commands/migrate-csv.py +++ b/cope2n-api/fwd_api/management/commands/migrate-csv.py @@ -34,7 +34,8 @@ class Command(BaseCommand): "imei_number": [], "retailername": "", "purchase_date": "", - "sold_to_party": "" + "sold_to_party": "", + "invoice_no": "" } images = SubscriptionRequestFile.objects.filter(request=request, file_category=FileCategory.Origin.value) @@ -46,7 +47,7 @@ class Command(BaseCommand): if request.predict_result.get("status", 200) != 200: raise AttributeError(f"Failed request: {request.request_id}") - for field in ['retailername', 'purchase_date', 'imei_number']: + for field in ['retailername', 'purchase_date', 'imei_number', 'invoice_no']: if image.feedback_result[field] is not None and ((field == 'imei_number' and len(image.feedback_result[field]) > 0 and image.feedback_result[field][0] == user_feedback) or image.feedback_result[field] == user_feedback): is_match = True @@ -65,10 +66,11 @@ class Command(BaseCommand): "imei_number": [], "retailername": "", "purchase_date": "", - "sold_to_party": "" + "sold_to_party": "", + "invoice_no": "" } - if image.doc_type == "invoice" and field in ['retailername', 'purchase_date']: + if image.doc_type == "invoice" and field in ['retailername', 'purchase_date', 'invoice_no']: if _reviewed_result: _reviewed_result[field] = reviewed_result _reviewed_result["imei_number"] = [] @@ -79,11 +81,14 @@ class Command(BaseCommand): "retailername": None, "sold_to_party": None, "purchase_date": None, - "imei_number": [reviewed_result] + "imei_number": [reviewed_result], + "invoice_no": None } if _reviewed_result else None image.reviewed_result = _reviewed_result if reason: image.reason = reason + else: + image.reason = "" if counter: image.counter_measures = counter image.save() diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index d334b2a..833a4fd 100755 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -584,7 +584,7 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True) def format_datetime_for_report_detail(ocr_extraction_date): naived_time = timezone.make_naive(ocr_extraction_date) ocr_extraction_date = timezone.make_aware(value=naived_time, timezone=timezone.get_current_timezone()) - format_to_date = '%Y/%m/%d' + format_to_date = '%Y-%m-%d' ocr_extraction_date = ocr_extraction_date.strftime(format_to_date) return ocr_extraction_date @@ -755,8 +755,8 @@ def create_billing_data(subscription_requests): _sub = map_subsidiary_short_to_long(request.redemption_id[:2]) redemption_id = request.redemption_id - format_to_time = '%Y/%m/%d %H:%M' - format_to_date = '%Y/%m/%d' + format_to_time = '%Y-%m-%d %H:%M' + format_to_date = '%Y-%m-%d' format_to_month = '%B %Y' naive_created_time = timezone.make_naive(request.created_at) rq_created_at = timezone.make_aware(value=naive_created_time, timezone=timezone.get_current_timezone()) diff --git a/cope2n-fe/src/components/report-detail/report-table.tsx b/cope2n-fe/src/components/report-detail/report-table.tsx index ef4a5b0..17ac154 100644 --- a/cope2n-fe/src/components/report-detail/report-table.tsx +++ b/cope2n-fe/src/components/report-detail/report-table.tsx @@ -59,6 +59,16 @@ const ReportTable: React.FC = () => { }, width: 110, }, + { + title: 'Report Type', + dataIndex: 'Report Type', + key: 'Report Type', + render: (_, record) => { + const reportType = record["Report Type"]; + const uppercasedReportType = reportType.charAt(0).toUpperCase() + reportType.slice(1); + return <>{uppercasedReportType}; + }, + }, { title: 'Start Date', dataIndex: 'Start Date', @@ -172,7 +182,7 @@ const ReportTable: React.FC = () => { }, }, { - title: 'Average Processing Time (ms)', + title: 'Average Processing Time (seconds)', dataIndex: 'Avg. OCR Processing Time', key: 'Avg. OCR Processing Time', render: (_, record) => { diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 9b67208..3d31685 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -89,12 +89,12 @@ services: depends_on: db-sbt: condition: service_started - # command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && - # python manage.py makemigrations && - # python manage.py migrate && - # python manage.py compilemessages && - # gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod - command: "sleep infinity" + command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && + python manage.py makemigrations && + python manage.py migrate && + python manage.py compilemessages && + gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod + # command: "sleep infinity" minio: image: minio/minio @@ -179,8 +179,8 @@ services: - ./cope2n-api:/app working_dir: /app - # command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" - command: bash -c "tail -f > /dev/null" + command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" + # command: bash -c "tail -f > /dev/null" # Back-end persistent db-sbt: