tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: #101

Browse Source
This commit is contained in:
dx-tan 2024-03-26 16:26:10 +07:00
parent d0a088d068
commit 4e48909e88
22 changed files with 192 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -44,3 +44,4 @@ cope2n-api/reviewed/retailer.xlsx
/scripts/*
scripts/crawl_database.py
redundant_models/
junk.json

2
cope2n-ai-fi/modules/sdsvkvu

@ -1 +1 @@
Subproject commit 20128037dbfca217fa3d3ca4551cc7f8ae8a190e
Subproject commit 46a612a003c411406988b83b3dd6299d2a458366

13
cope2n-api/fwd_api/api/accuracy_view.py
View File

@ -277,6 +277,7 @@ class AccuracyViewSet(viewsets.ViewSet):
start_at=start_date,
end_at=end_date,
status="Processing",
report_type=report_type,
)
new_report.save()
# Background job to calculate accuracy
@ -437,6 +438,7 @@ class AccuracyViewSet(viewsets.ViewSet):
"Avg. OCR Processing Time": processing_time,
"report_id": report.report_id,
"Subsidiary": map_subsidiary_short_to_long(report.subsidiary),
"Report Type": report.report_type,
})
response = {
@ -628,11 +630,16 @@ class AccuracyViewSet(viewsets.ViewSet):
report = Report.objects.filter(report_id=report_id).first()
# download from s3 to local
target_timezone = pytz.timezone(settings.TIME_ZONE)
tmp_file = "/tmp/" + report.subsidiary + "_" + report.start_at.astimezone(target_timezone).strftime("%Y%m%d") + "_" + report.end_at.astimezone(target_timezone).strftime("%Y%m%d") + "_created_on_" + report.created_at.astimezone(target_timezone).strftime("%Y%m%d") + ".xlsx"
os.makedirs("/tmp", exist_ok=True)
if not report.S3_file_name:
raise NotFoundException(excArgs="S3 file name")
download_from_S3(report.S3_file_name, tmp_file)
if not report.S3_dashboard_file_name:
raise NotFoundException(excArgs="S3 dashboard file name")
file_name = report.S3_file_name if request.query_params["report_expression"] == "detail" else report.S3_dashboard_file_name
tmp_file = "/tmp/" + request.query_params["report_expression"] + "_" + report.subsidiary + "_" + report.start_at.astimezone(target_timezone).strftime("%Y%m%d") + "_" + report.end_at.astimezone(target_timezone).strftime("%Y%m%d") + "_created_on_" + report.created_at.astimezone(target_timezone).strftime("%Y%m%d") + ".xlsx"
os.makedirs("/tmp", exist_ok=True)
download_from_S3(file_name, tmp_file)
file = open(tmp_file, 'rb')
response = FileResponse(file, status=200)

1
cope2n-api/fwd_api/celery_worker/internal_task.py
View File

@ -267,7 +267,6 @@ def upload_report_to_s3(local_file_path, s3_key, report_id, delay):
if report_id:
report = Report.objects.filter(report_id=report_id)[0]
report.S3_uploaded = True
report.S3_file_name = s3_key
report.save()
except Exception as e:
logger.error(f"Unable to set S3: {e}")

48
cope2n-api/fwd_api/celery_worker/process_report_tasks.py
View File

@ -14,6 +14,7 @@ from django.utils import timezone
from django.db.models import Q
import json
import copy
import os
from celery.utils.log import get_task_logger
from fwd import settings
@ -177,36 +178,45 @@ def create_accuracy_report(report_id, **kwargs):
report.errors = "|".join(errors)
report.status = "Ready"
detail_file_name = "detail_" + report.report_id + ".xlsx"
dashboard_file_name = "overview_" + report.report_id + ".xlsx"
report.S3_file_name = os.path.join("report", report.report_id, detail_file_name)
report.S3_dashboard_file_name = os.path.join("report", report.report_id, dashboard_file_name)
report.save()
# Save a list of bad images to csv file for debugging
save_images_to_csv_briefly(report.report_id, bad_image_list)
# Saving a xlsx file
data = extract_report_detail_list(report_files, lower=True)
data_workbook = dict2xlsx(data, _type='report_detail')
local_workbook = save_workbook_file(report.report_id + ".xlsx", report, data_workbook)
local_workbook = save_workbook_file(detail_file_name, report, data_workbook)
s3_key = save_report_to_S3(report.report_id, local_workbook, 5)
# Save overview dashboard
# multiple accuracy by 100
save_data = copy.deepcopy(_save_data)
review_key = "review_progress"
for i, dat in enumerate(report_fine_data):
report_fine_data[i][review_key] = report_fine_data[i][review_key]*100
keys = [x for x in list(dat.keys()) if "accuracy" in x.lower()]
keys_percent = "images_quality"
for x_key in report_fine_data[i][keys_percent].keys():
if "percent" not in x_key:
continue
report_fine_data[i][keys_percent][x_key] = report_fine_data[i][keys_percent][x_key]*100
for key in keys:
if report_fine_data[i][key]:
for x_key in report_fine_data[i][key].keys():
report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 if report_fine_data[i][key][x_key] is not None else None
data_workbook = dict2xlsx(report_fine_data, _type='report')
if kwargs["is_daily_report"]:
# Save overview dashboard
# multiple accuracy by 100
save_data = copy.deepcopy(_save_data)
review_key = "review_progress"
for i, dat in enumerate(report_fine_data):
report_fine_data[i][review_key] = report_fine_data[i][review_key]*100
keys = [x for x in list(dat.keys()) if "accuracy" in x.lower()]
keys_percent = "images_quality"
for x_key in report_fine_data[i][keys_percent].keys():
if "percent" not in x_key:
continue
report_fine_data[i][keys_percent][x_key] = report_fine_data[i][keys_percent][x_key]*100
for key in keys:
if report_fine_data[i][key]:
for x_key in report_fine_data[i][key].keys():
report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 if report_fine_data[i][key][x_key] is not None else None
data_workbook = dict2xlsx(report_fine_data, _type='report')
overview_filename = kwargs["subsidiary"] + "_" + kwargs["report_overview_duration"] + ".xlsx"
local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT)
s3_key = save_report_to_S3(report.report_id, local_workbook)
set_cache(overview_filename.replace(".xlsx", ""), save_data)
else:
overview_filename = dashboard_file_name
local_workbook = save_workbook_file(overview_filename, report, data_workbook)
s3_key = save_report_to_S3(report.report_id, local_workbook)
except IndexError as e:
print(e)

19
cope2n-api/fwd_api/migrations/0186_reportfile_correspond_request_created_at.py Normal file
View File

@ -0,0 +1,19 @@
# Generated by Django 4.1.3 on 2024-03-21 08:58
from django.db import migrations, models
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0185_report_report_type'),
]
operations = [
migrations.AddField(
model_name='reportfile',
name='correspond_request_created_at',
field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
),
]

18
cope2n-api/fwd_api/migrations/0187_report_s3_dashboard_file_name.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.3 on 2024-03-26 07:28
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0186_reportfile_correspond_request_created_at'),
]
operations = [
migrations.AddField(
model_name='report',
name='S3_dashboard_file_name',
field=models.TextField(default=None, null=True),
),
]

1
cope2n-api/fwd_api/models/Report.py
View File

@ -24,6 +24,7 @@ class Report(models.Model):
# Data
S3_uploaded = models.BooleanField(default=False)
S3_file_name = models.TextField(default=None, null=True)
S3_dashboard_file_name = models.TextField(default=None, null=True)
number_request = models.IntegerField(default=0)
number_images = models.IntegerField(default=0)
number_bad_images = models.IntegerField(default=0)

4
cope2n-api/fwd_api/utils/file.py
View File

@ -498,9 +498,9 @@ def dump_excel_report(input: json):
'J': 'images_quality.successful_percent',
'K': 'images_quality.bad',
'L': 'images_quality.bad_percent',
'M': 'average_accuracy_rate.imei',
'M': 'average_accuracy_rate.imei_number',
'N': 'average_accuracy_rate.purchase_date',
'O': 'average_accuracy_rate.retailer_name',
'O': 'average_accuracy_rate.retailername',
'P': 'average_accuracy_rate.invoice_number',
'Q': 'average_processing_time.imei',
'R': 'average_processing_time.invoice',

2
cope2n-fe/.env.development
View File

@ -1,3 +1,3 @@
VITE_PORT=8080
VITE_PROXY=http://42.96.42.13:9881
VITE_PROXY=http://42.96.42.13:9000
VITE_KUBEFLOW_HOST=https://107.120.133.22:8085

44
cope2n-fe/src/components/report-detail/report-overview-table.tsx
View File

@ -24,7 +24,8 @@ interface DataType {
purchaseDateAAR: number;
retailerNameAAR: number;
invoiceNumberAAR: number;
avgAPT: number; // APT: Average Processing Time
snImeiAPT: number; // APT: Average Processing Time
invoiceAPT: number;
snImeiTC: number; // TC: transaction count
invoiceTC: number;
reviewProgress: number;
@ -255,16 +256,34 @@ const columns: TableColumnsType<DataType> = [
},
{
title: 'Average Processing Time Per Image (Seconds)',
dataIndex: 'avgAPT',
key: 'avgAPT',
render: (_, record) => {
const isAbnormal = ensureMax(record.avgAPT, 2);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
{formatNumber(record?.avgAPT)}
</span>
);
},
children: [
{
title: 'SN/IMEI',
dataIndex: 'snImeiAPT',
key: 'snImeiAPT',
render: (_, record) => {
const isAbnormal = ensureMax(record.snImeiAPT, 2);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
{formatNumber(record?.snImeiAPT)}
</span>
);
},
},
{
title: 'Invoice',
dataIndex: 'invoiceAPT',
key: 'invoiceAPT',
render: (_, record) => {
const isAbnormal = ensureMax(record.invoiceAPT, 2);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
{formatNumber(record?.invoiceAPT)}
</span>
);
},
},
],
},
{
title: 'Review Progress',
@ -310,7 +329,8 @@ const ReportOverViewTable: React.FC<ReportOverViewTableProps> = ({
purchaseDateAAR: item.average_accuracy_rate.purchase_date,
retailerNameAAR: item.average_accuracy_rate.retailername,
invoiceNumberAAR: item.average_accuracy_rate.invoice_no,
avgAPT: item.average_processing_time.avg,
snImeiAPT: item.average_processing_time.imei,
invoiceAPT: item.average_processing_time.invoice,
snImeiTC: item.usage.imei,
invoiceTC: item.usage.invoice,
reviewProgress: item.review_progress,

32
cope2n-fe/src/components/report-detail/report-table.tsx
View File

@ -1,3 +1,4 @@
import { VerticalAlignBottomOutlined } from '@ant-design/icons';
import type { TableColumnsType } from 'antd';
import { Button, Table } from 'antd';
import { ReportDetail } from 'models';
@ -25,8 +26,8 @@ const ReportTable: React.FC = () => {
page_size: 10,
}));
const handleDownloadReport = async (report_id: string) => {
const { file, filename } = await downloadReport(report_id);
const handleDownloadReport = async (report_id: string, report_expression: string) => {
const { file, filename } = await downloadReport(report_id, report_expression);
const anchorElement = document.createElement('a');
anchorElement.href = URL.createObjectURL(file);
anchorElement.download = filename;
@ -111,7 +112,7 @@ const ReportTable: React.FC = () => {
render: (_, record) => {
const isAbnormal = ensureMin(record['Purchase Date Acc'], 0.95);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
<span style={{ color: isAbnormal ? 'red' : '' }} key='Purchase Date Acc'>
{formatPercent(record['Purchase Date Acc'])}
</span>
);
@ -124,7 +125,7 @@ const ReportTable: React.FC = () => {
render: (_, record) => {
const isAbnormal = ensureMin(record['Invoice number Acc'], 0.95);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
<span style={{ color: isAbnormal ? 'red' : '' }} key='Invoice number Acc'>
{formatPercent(record['Invoice number Acc'])}
</span>
);
@ -138,7 +139,7 @@ const ReportTable: React.FC = () => {
render: (_, record) => {
const isAbnormal = ensureMin(record['Retailer Acc'], 0.95);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
<span style={{ color: isAbnormal ? 'red' : '' }} key='Retailer Acc'>
{formatPercent(record['Retailer Acc'])}
</span>
);
@ -151,7 +152,7 @@ const ReportTable: React.FC = () => {
render: (_, record) => {
const isAbnormal = ensureMin(record['IMEI Acc'], 0.95);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
<span style={{ color: isAbnormal ? 'red' : '' }} key='IMEI Acc'>
{formatPercent(record['IMEI Acc'])}
</span>
);
@ -164,7 +165,7 @@ const ReportTable: React.FC = () => {
render: (_, record) => {
const isAbnormal = ensureMin(record['Avg. Accuracy'], 0.95);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
<span style={{ color: isAbnormal ? 'red' : '' }} key='Avg. Accuracy'>
{formatPercent(record['Avg. Accuracy'])}
</span>
);
@ -177,7 +178,7 @@ const ReportTable: React.FC = () => {
render: (_, record) => {
const isAbnormal = ensureMax(record['Avg. OCR Processing Time'], 2);
return (
<span style={{ color: isAbnormal ? 'red' : '' }}>
<span style={{ color: isAbnormal ? 'red' : '' }} key='Avg. OCR Processing Time'>
{formatNumber(record['Avg. OCR Processing Time'], 1)}
</span>
);
@ -187,20 +188,27 @@ const ReportTable: React.FC = () => {
title: 'Actions',
dataIndex: 'actions',
key: 'actions',
width: 240,
width: 340,
render: (_, record) => {
return (
<div style={{ flexDirection: 'row' }}>
<div style={{ flexDirection: 'row' }} key={"actions"}>
<Button
onClick={() => {
navigate(`/reports/${record.report_id}/detail`);
}}
style={{ marginRight: 10 }}
key={0}
>
Details
</Button>
<Button onClick={() => handleDownloadReport(record.report_id)}>
Download
<Button onClick={() => handleDownloadReport(record.report_id, "detail")}
style={{ marginRight: 2 }} key={1}>
Detail<VerticalAlignBottomOutlined />
</Button>
<Button onClick={() => handleDownloadReport(record.report_id, "dashboard")}
style={{ marginRight: 2 }}
disabled={record["Report Type"]==="billing"} key={2}>
Dashboard<VerticalAlignBottomOutlined />
</Button>
</div>
);

1
cope2n-fe/src/locales/en/messages.json
View File

@ -51,7 +51,6 @@
"This field must not have more than {MAX_USERNAME_LENGTH} characters": "This field must not have more than {MAX_USERNAME_LENGTH} characters",
"Too blurry text": "Too blurry text",
"Too small text": "Too small text",
"Upload files to process. The requests here will not be used in accuracy or payment calculations.": "Upload files to process. The requests here will not be used in accuracy or payment calculations.",
"User log in successfully": "User log in successfully",
"Username": "Username",
"Username must not have more than {MAX_USERNAME_LENGTH} characters": "Username must not have more than {MAX_USERNAME_LENGTH} characters",

1
cope2n-fe/src/locales/vi/messages.json
View File

@ -51,7 +51,6 @@
"This field must not have more than {MAX_USERNAME_LENGTH} characters": "Độ dài chuỗi không được vượt quá {MAX_USERNAME_LENGTH} kí tự",
"Too blurry text": "",
"Too small text": "",
"Upload files to process. The requests here will not be used in accuracy or payment calculations.": "",
"User log in successfully": "Đăng nhập thành công",
"Username": "Tên tài khoản",
"Username must not have more than {MAX_USERNAME_LENGTH} characters": "Tên tài khoản không được chứa nhiều hơn {MAX_USERNAME_LENGTH} kí tự",

1
cope2n-fe/src/models/report.ts
View File

@ -91,6 +91,7 @@ export interface ReportDetail {
'Avg Accuracy': any;
'Avg. Client Request Time': number;
'Avg. OCR Processing Time': number;
'Report Type': string;
report_id: string;
}

4
cope2n-fe/src/pages/reports/report_detail/index.tsx
View File

@ -47,7 +47,7 @@ const ReportDetail = () => {
});
const report_data = data as ReportDetailList;
const handleDownloadReport = async () => {
const {file, filename} = await downloadReport(id);
const {file, filename} = await downloadReport(id, "detail");
const anchorElement = document.createElement('a');
anchorElement.href = URL.createObjectURL(file);
anchorElement.download = filename;
@ -63,7 +63,7 @@ const ReportDetail = () => {
// Download and show report
useEffect(() => {
try {
downloadReport(id, (fileDetails) => {
downloadReport(id, "detail", (fileDetails) => {
if (!fileDetails?.file) {
setError("The report has not been ready to preview.");
}

4
cope2n-fe/src/request/report.ts
View File

@ -85,9 +85,9 @@ export async function getOverViewReport(params?: DashboardOverviewParams) {
}
}
export async function downloadReport(report_id: string, downloadFinishedCallback?: (fileDetails: any) => void) {
export async function downloadReport(report_id: string, report_expression: string, downloadFinishedCallback?: (fileDetails: any) => void) {
try {
const response = await API.get(`/ctel/get_report_file/${report_id}/`, {
const response = await API.get(`/ctel/get_report_file/${report_id}/?report_expression=${report_expression}`, {
responseType: 'blob', // Important
});
let filename = "report.xlsx";

91
docker-compose-dev.yml
View File

@ -63,8 +63,8 @@ services:
- AUTH_TOKEN_LIFE_TIME=${AUTH_TOKEN_LIFE_TIME}
- IMAGE_TOKEN_LIFE_TIME=${IMAGE_TOKEN_LIFE_TIME}
- INTERNAL_SDS_KEY=${INTERNAL_SDS_KEY}
- ADMIN_USER_NAME=${FI_USER_NAME}
- ADMIN_PASSWORD=${FI_PASSWORD}
- ADMIN_USER_NAME=${ADMIN_USER_NAME}
- ADMIN_PASSWORD=${ADMIN_PASSWORD}
- STANDARD_USER_NAME=${STANDARD_USER_NAME}
- STANDARD_PASSWORD=${STANDARD_PASSWORD}
- S3_ENDPOINT=${S3_ENDPOINT}
@ -89,6 +89,11 @@ services:
depends_on:
db-sbt:
condition: service_started
# command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input &&
# python manage.py makemigrations &&
# python manage.py migrate &&
# python manage.py compilemessages &&
# gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
command: "sleep infinity"
minio:
@ -174,8 +179,8 @@ services:
- ./cope2n-api:/app
working_dir: /app
command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
# command: bash -c "tail -f > /dev/null"
# command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
command: bash -c "tail -f > /dev/null"
# Back-end persistent
db-sbt:
@ -204,46 +209,46 @@ services:
- RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER}
- RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS}
# # Front-end services
# fe-sbt:
# restart: always
# build:
# context: cope2n-fe
# shm_size: 10gb
# dockerfile: Dockerfile
# image: sidp/cope2n-fe-fi-sbt:latest
# shm_size: 10gb
# privileged: true
# ports:
# - 9881:80
# depends_on:
# be-ctel-sbt:
# condition: service_started
# be-celery-sbt:
# condition: service_started
# environment:
# - VITE_PROXY=http://be-ctel-sbt:${BASE_PORT}
# - VITE_API_BASE_URL=http://fe-sbt:80
# volumes:
# - BE_static:/backend-static
# networks:
# - ctel-sbt
# Front-end services
fe-sbt:
restart: always
build:
context: cope2n-fe
shm_size: 10gb
dockerfile: Dockerfile
image: sidp/cope2n-fe-fi-sbt:latest
shm_size: 10gb
privileged: true
ports:
- 9881:80
depends_on:
be-ctel-sbt:
condition: service_started
be-celery-sbt:
condition: service_started
environment:
- VITE_PROXY=http://be-ctel-sbt:${BASE_PORT}
- VITE_API_BASE_URL=http://fe-sbt:80
volumes:
- BE_static:/backend-static
networks:
- ctel-sbt
# dashboard_refresh:
# build:
# context: api-cronjob
# dockerfile: Dockerfile
# image: sidp/api-caller-sbt:latest
# environment:
# - PROXY=http://be-ctel-sbt:9000
# - ADMIN_USER_NAME=${ADMIN_USER_NAME}
# - ADMIN_PASSWORD=${ADMIN_PASSWORD}
# depends_on:
# be-ctel-sbt:
# condition: service_healthy
# restart: always
# networks:
# - ctel-sbt
dashboard_refresh:
build:
context: api-cronjob
dockerfile: Dockerfile
image: sidp/api-caller-sbt:latest
environment:
- PROXY=http://be-ctel-sbt:9000
- ADMIN_USER_NAME=${ADMIN_USER_NAME}
- ADMIN_PASSWORD=${ADMIN_PASSWORD}
depends_on:
be-ctel-sbt:
condition: service_healthy
restart: always
networks:
- ctel-sbt
volumes:
db_data:

2
test_samples/test_26/result.json
View File

@ -24,6 +24,6 @@
}
],
"doc_type": "sbt_document",
"end_page": 1,
"end_page": 2,
"start_page": 1
}

2
test_samples/test_27/result.json
View File

@ -24,6 +24,6 @@
}
],
"doc_type": "sbt_document",
"end_page": 1,
"end_page": 2,
"start_page": 1
}

2
tests/test_set_26_jpg_invoice.py
View File

@ -12,7 +12,7 @@ token = login(HOST, USERNAME, PASSWORD)
def test_invoice_number():
invoice_files = []
invoice_files = ["test_samples/test_26/invoice_no_jpg.jpg",]
imei_files = [
"test_samples/test_26/invoice_no_jpg.jpg",
]

2
tests/test_set_27_jpeg_invoice.py
View File

@ -12,7 +12,7 @@ token = login(HOST, USERNAME, PASSWORD)
def test_invoice_number():
invoice_files = []
invoice_files = ["test_samples/test_27/invoice_no_jpeg.jpeg",]
imei_files = [
"test_samples/test_27/invoice_no_jpeg.jpeg",
]