tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #163 from SDSRV-IDP/dev/search_content

Browse Source 
Dev/search content
This commit is contained in:
Nguyễn Duy Hiển 2024-12-05 14:46:24 +07:00 committed by GitHub Enterprise
commit 590a08c119
19 changed files with 346 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cope2n-api/Dockerfile
View File

@ -14,7 +14,7 @@ RUN pip install --upgrade pip
RUN pip install uvicorn gunicorn Celery
# For intergration with sdskvu
RUN pip install pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
RUN pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
RUN pip install -U openmim==0.3.7 --no-cache-dir
RUN mim install mmcv-full==1.7.2
# RUN pip install fastdeploy-gpu-python==1.0.7 -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html --no-cache-dir

1
cope2n-api/fwd/settings.py
View File

@ -245,6 +245,7 @@ FIELDS_BY_SUB = {
BAD_THRESHOLD = 0.75
NEED_REVIEW = 1.0
DOC_TYPES = ["imei", "invoice"]
SUB_FOR_BILLING = ["all", "seao"]
FIELD = ["imei_number", "purchase_date", "retailername", "sold_to_party", "invoice_no"]

61
cope2n-api/fwd_api/api/accuracy_view.py
View File

@ -78,6 +78,7 @@ class AccuracyViewSet(viewsets.ViewSet):
description='Which subsidiary to be included',
type=OpenApiTypes.STR,
enum=list(settings.SUBS.keys()),
required=True
),
OpenApiParameter(
name='request_id',
@ -112,6 +113,42 @@ class AccuracyViewSet(viewsets.ViewSet):
type=OpenApiTypes.FLOAT,
required=False
),
OpenApiParameter(
name='predict_result',
location=OpenApiParameter.QUERY,
description='Filter by predict result',
type=OpenApiTypes.STR,
required=False
),
OpenApiParameter(
name='feedback_result',
location=OpenApiParameter.QUERY,
description='Filter by feedback result',
type=OpenApiTypes.STR,
required=False
),
OpenApiParameter(
name='reviewed_result',
location=OpenApiParameter.QUERY,
description='Filter by reviewed result',
type=OpenApiTypes.STR,
required=False
),
OpenApiParameter(
name='doc_type',
location=OpenApiParameter.QUERY,
description='Filter by document type',
type=OpenApiTypes.STR,
enum=list(settings.DOC_TYPES),
required=False
),
OpenApiParameter(
name='bad_reason',
location=OpenApiParameter.QUERY,
description='Filter by bad reason',
type=OpenApiTypes.STR,
required=False
),
],
responses=None, tags=['Accuracy']
)
@ -129,7 +166,11 @@ class AccuracyViewSet(viewsets.ViewSet):
include_test = request.GET.get('includes_test', False)
subsidiary = request.GET.get("subsidiary", "all")
max_accuracy = float(request.GET.get("max_accuracy", 100))
# subsidiary = map_subsidiary_long_to_short(subsidiary)
predict_result = request.GET.get('predict_result', None)
feedback_result = request.GET.get('feedback_result', None)
reviewed_result = request.GET.get('reviewed_result', None)
doc_type = request.GET.get('doc_type', None)
bad_reason = request.GET.get('bad_reason', None)
base_query = Q(status=200)
if start_date_str or end_date_str:
@ -193,10 +234,28 @@ class AccuracyViewSet(viewsets.ViewSet):
base_query &= Q(
redemption_id__startswith=map_subsidiary_long_to_short(subsidiary))
if predict_result:
base_query &= Q(predict_result__icontains=predict_result)
if feedback_result:
base_query &= Q(feedback_result__icontains=feedback_result)
if reviewed_result:
base_query &= Q(reviewed_result__icontains=reviewed_result)
if doc_type:
if doc_type.lower() == 'invoice':
base_query &= Q(doc_type__regex=r'^invoice(,\s*invoice)*$')
elif doc_type.lower() == 'imei':
base_query &= Q(doc_type__regex=r'^imei(,\s*imei)*$')
if isinstance(max_accuracy, float):
base_query &= Q(raw_accuracy__lt=(
max_accuracy/100)) | Q(raw_accuracy__isnull=True)
if bad_reason:
bad_reason_subquery = SubscriptionRequestFile.objects.filter(
reason__icontains=bad_reason
).values_list('request_id', flat=True)
base_query &= Q(id__in=bad_reason_subquery)
subscription_requests = SubscriptionRequest.objects.filter(
base_query).order_by('created_at')

31
cope2n-api/fwd_api/migrations/0195_alter_subscriptionrequest_request_id_and_more.py Normal file
View File

@ -0,0 +1,31 @@
# Generated by Django 4.1.3 on 2024-11-05 02:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0194_alter_semiautocorrection_feedback_accuracy_and_more'),
]
operations = [
migrations.AlterField(
model_name='subscriptionrequest',
name='request_id',
field=models.CharField(db_index=True, max_length=200),
),
migrations.AlterField(
model_name='userprofile',
name='id',
field=models.AutoField(db_index=True, primary_key=True, serialize=False),
),
migrations.AddIndex(
model_name='subscriptionrequestfile',
index=models.Index(fields=['request', 'index_in_request', 'doc_type'], name='fwd_api_sub_request_890e13_idx'),
),
migrations.AddIndex(
model_name='subscriptionrequestfile',
index=models.Index(fields=['request', 'file_name'], name='fwd_api_sub_request_1a42cd_idx'),
),
]

26
cope2n-api/fwd_api/migrations/0196_subscriptionrequestfile_idx_gin_predict_result_and_more.py Normal file
View File

@ -0,0 +1,26 @@
# Generated by Django 4.1.3 on 2024-11-28 07:10
import django.contrib.postgres.indexes
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0195_alter_subscriptionrequest_request_id_and_more'),
]
operations = [
migrations.AddIndex(
model_name='subscriptionrequestfile',
index=django.contrib.postgres.indexes.GinIndex(fields=['predict_result'], name='idx_gin_predict_result'),
),
migrations.AddIndex(
model_name='subscriptionrequestfile',
index=django.contrib.postgres.indexes.GinIndex(fields=['feedback_result'], name='idx_gin_feedback_result'),
),
migrations.AddIndex(
model_name='subscriptionrequestfile',
index=django.contrib.postgres.indexes.GinIndex(fields=['reviewed_result'], name='idx_gin_reviewed_result'),
),
]

26
cope2n-api/fwd_api/migrations/0197_subscriptionrequest_idx_gin_rq_predict_result_and_more.py Normal file
View File

@ -0,0 +1,26 @@
# Generated by Django 4.1.3 on 2024-11-28 09:04
import django.contrib.postgres.indexes
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0196_subscriptionrequestfile_idx_gin_predict_result_and_more'),
]
operations = [
migrations.AddIndex(
model_name='subscriptionrequest',
index=django.contrib.postgres.indexes.GinIndex(fields=['predict_result'], name='idx_gin_rq_predict_result'),
),
migrations.AddIndex(
model_name='subscriptionrequest',
index=django.contrib.postgres.indexes.GinIndex(fields=['feedback_result'], name='idx_gin_rq_feedback_result'),
),
migrations.AddIndex(
model_name='subscriptionrequest',
index=django.contrib.postgres.indexes.GinIndex(fields=['reviewed_result'], name='idx_gin_rq_reviewed_result'),
),
]

18
cope2n-api/fwd_api/migrations/0198_alter_subscriptionrequest_doc_type.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.3 on 2024-11-28 09:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0197_subscriptionrequest_idx_gin_rq_predict_result_and_more'),
]
operations = [
migrations.AlterField(
model_name='subscriptionrequest',
name='doc_type',
field=models.CharField(db_index=True, max_length=100),
),
]

18
cope2n-api/fwd_api/migrations/0199_alter_subscriptionrequestfile_reason.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.3 on 2024-12-04 10:18
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0198_alter_subscriptionrequest_doc_type'),
]
operations = [
migrations.AlterField(
model_name='subscriptionrequestfile',
name='reason',
field=models.TextField(blank=True, db_index=True),
),
]

10
cope2n-api/fwd_api/models/SubscriptionRequest.py
View File

@ -1,5 +1,6 @@
from django.db import models
from django.utils import timezone
from django.contrib.postgres.indexes import GinIndex
from fwd_api.models.Subscription import Subscription
@ -7,7 +8,7 @@ class SubscriptionRequest(models.Model):
id = models.AutoField(primary_key=True)
pages: int = models.IntegerField()
pages_left: int = models.IntegerField(default=1)
doc_type: str = models.CharField(max_length=100)
doc_type: str = models.CharField(max_length=100, db_index=True)
request_id = models.CharField(max_length=200, db_index=True) # Change to request_id
redemption_id = models.CharField(max_length=200, null=True)
process_type = models.CharField(max_length=200) # driver/id/invoice
@ -39,3 +40,10 @@ class SubscriptionRequest(models.Model):
is_reviewed = models.BooleanField(default=False)
is_required = models.BooleanField(default=True)
subsidiary = models.CharField(default="", null=True, max_length=200)
class Meta:
indexes = [
GinIndex(fields=['predict_result'], name='idx_gin_rq_predict_result'),
GinIndex(fields=['feedback_result'], name='idx_gin_rq_feedback_result'),
GinIndex(fields=['reviewed_result'], name='idx_gin_rq_reviewed_result'),
]

5
cope2n-api/fwd_api/models/SubscriptionRequestFile.py
View File

@ -1,5 +1,5 @@
from django.db import models
from django.contrib.postgres.indexes import GinIndex
from fwd_api.constant.common import FileCategory
from fwd_api.models import SubscriptionRequest
from fwd_api.models.fields.EncryptedCharField import EncryptedCharField
@ -40,5 +40,8 @@ class SubscriptionRequestFile(models.Model):
indexes = [
models.Index(fields=['request', 'index_in_request', 'doc_type']), # For updating results
models.Index(fields=['request', 'file_name']), # for getting image files by AI
GinIndex(fields=['predict_result'], name='idx_gin_predict_result'),
GinIndex(fields=['feedback_result'], name='idx_gin_feedback_result'),
GinIndex(fields=['reviewed_result'], name='idx_gin_reviewed_result'),
]

2
cope2n-fe/package.json
View File

@ -83,7 +83,7 @@
"prettier-plugin-organize-imports": "^3.2.1",
"rollup-plugin-visualizer": "^5.9.0",
"sass": "^1.57.1",
"typescript": "5.6.3",
"typescript": "^5.6.3",
"vite": "^4.0.3",
"vite-plugin-svgr": "^2.4.0",
"vite-tsconfig-paths": "^4.0.3"

2
cope2n-fe/src/components/left-menu/index.tsx
View File

@ -1,4 +1,4 @@
import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined, FileSearchOutlined } from '@ant-design/icons';
import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined } from '@ant-design/icons';
import { t } from '@lingui/macro';
import { Menu, MenuProps } from 'antd';
import React from 'react';

1
cope2n-fe/src/components/spin/index.tsx
View File

@ -4,7 +4,6 @@ export function GlobalSpin() {
return (
<Spin
size='large'
tip='Loading ...'
style={{
position: 'fixed',
top: '50%',

6
cope2n-fe/src/locales/en/messages.json
View File

@ -70,5 +70,9 @@
"You are only allowed to upload {0} file.": "You are only allowed to upload {0} file.",
"You have unsaved changes!": "You have unsaved changes!",
"Your current password has expired. Please change your password to continue.": "Your current password has expired. Please change your password to continue.",
"max_accuracy": "Max accuracy"
"max_accuracy": "Max accuracy",
"docType": "Only type",
"Feedback Result": "Feedback Result",
"Predict Result": "Predict Result",
"Reviewed Result": "Reviewed Result"
}

6
cope2n-fe/src/locales/vi/messages.json
View File

@ -70,5 +70,9 @@
"You are only allowed to upload {0} file.": "Bạn chỉ được phép tải lên {0}.",
"You have unsaved changes!": "Bạn có những thay đổi chưa được lưu!",
"Your current password has expired. Please change your password to continue.": "",
"max_accuracy": "Độ chính xác tối đa"
"max_accuracy": "Độ chính xác tối đa",
"DocType": "Kiểu tài liệu",
"Feedback Result": "",
"Predict Result": "",
"Reviewed Result": ""
}

32
cope2n-fe/src/pages/reviews2/api.ts
View File

@ -2,10 +2,14 @@ import { baseURL } from 'request/api';
import { RecentRequest } from './const';
export const fetchAllRequests = async (
filterDateRange,
filterSubsidiaries,
filterReviewState,
filterIncludeTests,
filterDateRange: any[],
filterSubsidiaries: string,
filterReviewState: string,
filterIncludeTests: string,
filterDoctype: string,
filterFeedbackResult: string,
filterPredictResult: string,
filterReviewedResult: string,
page = 1,
page_size = 20,
max_accuracy = 100,
@ -25,6 +29,20 @@ export const fetchAllRequests = async (
if (filterIncludeTests) {
filterStr += `includes_test=${filterIncludeTests}&`;
}
// add 4 more field
if (filterDoctype) {
filterStr += `doc_type=${filterDoctype}&`;
}
if (filterFeedbackResult) {
filterStr += `feedback_result=${filterFeedbackResult}&`;
}
if (filterPredictResult) {
filterStr += `predict_result=${filterPredictResult}&`;
}
if (filterReviewedResult) {
filterStr += `reviewed_result=${filterReviewedResult}&`;
}
//
if (startDate && endDate) {
filterStr += `start_date=${startDate}&end_date=${endDate}&`;
}
@ -54,7 +72,7 @@ export const confirmRequest = async (requestID: any) => {
body: JSON.stringify({ request_file_results: [] }),
})
};
export const updateRevisedDataByFile = async (
@ -77,7 +95,7 @@ export const updateRevisedDataByFile = async (
)
};
export const fetchRequest = async (id) => {
export const fetchRequest = async (id: string) => {
const token = localStorage.getItem('sbt-token') || '';
const response = await fetch(`${baseURL}/ctel/request/${id}/`, {
method: 'GET',
@ -87,7 +105,7 @@ export const fetchRequest = async (id) => {
});
return await (
await response.json()
).subscription_requests[0];
).subscription_requests?.[0] || null;
};
export const addRecentRequest = (

6
cope2n-fe/src/pages/reviews2/const.ts
View File

@ -48,6 +48,12 @@ export const SUBSIDIARIES = [
{ value: 'SEIN', label: 'SEIN' },
];
export const DOCTYPE = [
{ value: null, label: '--' },
{ value: 'imei', label: 'imei' },
{ value: 'invoice', label: 'invoice' },
];
export const SOURCE_KEYS = [
'retailername',
'sold_to_party',

133
cope2n-fe/src/pages/reviews2/index.tsx
View File

@ -41,6 +41,7 @@ import {
} from './api';
import {
counter_measure_map,
DOCTYPE,
FEEDBACK_ACCURACY,
FEEDBACK_RESULT,
PREDICTED_RESULT,
@ -71,6 +72,10 @@ const ReviewPage = () => {
const [filterAccuracy, setFilterAccuracy] = useState(100);
const [filterReviewState, setFilterReviewState] = useState('all');
const [filterIncludeTests, setFilterIncludesTests] = useState('true');
const [filterDoctype, setFilterDoctype] = useState(null);
const [filterFeedbackResult, setFilterFeedbackResult] = useState('');
const [filterPredictResult, setFilterPredictResult] = useState('');
const [filterReviewedResult, setFilterReviewedResult] = useState('');
// const [requests, setRequests] = useState([]);
const [currentRequest, setCurrentRequest] = useState(null);
const [currentRequestIndex, setCurrentRequestIndex] = useState(1);
@ -109,6 +114,10 @@ const ReviewPage = () => {
filterSubsidiaries,
filterReviewState,
filterIncludeTests,
filterDoctype,
filterFeedbackResult,
filterPredictResult,
filterReviewedResult,
1,
1,
filterAccuracy,
@ -163,7 +172,7 @@ const ReviewPage = () => {
const setAndLoadSelectedFile = async (requestData, index) => {
setSelectedFileId(index);
if (!requestData['Files'][index]) {
if (!requestData?.['Files'][index]) {
setSelectedFileData('FAILED_TO_LOAD_FILE');
setImageLoading(false);
return;
@ -218,6 +227,10 @@ const ReviewPage = () => {
filterSubsidiaries,
filterReviewState,
filterIncludeTests,
filterDoctype,
filterFeedbackResult,
filterPredictResult,
filterReviewedResult,
requestIndex,
1,
filterAccuracy,
@ -273,6 +286,10 @@ const ReviewPage = () => {
filterSubsidiaries,
filterReviewState,
filterIncludeTests,
filterDoctype,
filterFeedbackResult,
filterPredictResult,
filterReviewedResult,
1,
1,
filterAccuracy,
@ -399,18 +416,18 @@ const ReviewPage = () => {
style={
fullscreen
? {
position: 'fixed',
top: 0,
left: 0,
width: '100%',
height: '100%',
backgroundColor: '#fff',
zIndex: 1000,
}
position: 'fixed',
top: 0,
left: 0,
width: '100%',
height: '100%',
backgroundColor: '#fff',
zIndex: 1000,
}
: {
height: '100%',
position: 'relative',
}
height: '100%',
position: 'relative',
}
}
>
<div
@ -561,17 +578,17 @@ const ReviewPage = () => {
overflow: 'auto',
}}
>
<Viewer
{selectedFileData && <Viewer
plugins={[defaultLayoutPluginInstance]}
fileUrl={selectedFileData}
onDocumentLoad={() => setImageLoading(false)}
/>
/>}
</div>
) : (
<div
style={{
flexGrow: 1,
overflow: 'auto',
overflowY: 'auto',
display: 'flex',
justifyContent: 'center',
alignItems: 'center',
@ -739,11 +756,10 @@ const ReviewPage = () => {
>
<Input
size='middle'
value={`Sub: ${filterSubsidiaries}, Date:${
filterDateRange[0]
? filterDateRange[0] + ' to ' + filterDateRange[1]
: 'All'
}, Reviewed: ${filterReviewState}, Tests: ${filterIncludeTests}`}
value={`Sub: ${filterSubsidiaries}, Date:${filterDateRange[0]
? filterDateRange[0] + ' to ' + filterDateRange[1]
: 'All'
}, Reviewed: ${filterReviewState}, Tests: ${filterIncludeTests}`}
readOnly
/>
<Button
@ -764,14 +780,14 @@ const ReviewPage = () => {
let shouldRevised = false;
try {
if (
selectedFileDataSource[data]?.[FEEDBACK_ACCURACY].length > 0
selectedFileDataSource[data]?.[FEEDBACK_ACCURACY]?.length > 0
) {
shouldRevised =
selectedFileDataSource[data][FEEDBACK_ACCURACY][0] < 1;
}
} catch (error) {}
} catch (error) { }
return (
<div style={{ margin: '0 0 8px' }} className='file-input-group'>
<div style={{ margin: '0 0 8px' }} className='file-input-group' key={data}>
<div
style={{
display: 'flex',
@ -858,7 +874,7 @@ const ReviewPage = () => {
options={SOLUTION_BAD_QUALITY}
onChange={setSolution}
value={solution}
// defaultValue={solution}
// defaultValue={solution}
/>
{solution === 'other' && (
<Input
@ -875,7 +891,7 @@ const ReviewPage = () => {
<div style={{ display: 'flex', justifyContent: 'flex-end' }}>
<Button
type='primary'
color='success'
// color='success'
size='middle'
style={{
height: '36px',
@ -1025,6 +1041,73 @@ const ReviewPage = () => {
/>
</Form.Item>
</div>
{/* add 4 more filter fields */}
<div
style={{
marginTop: 10,
display: 'flex',
justifyContent: 'space-between',
marginLeft: 0,
padding: 0,
}}
>
<Form.Item
name='doc_type'
label={t`Only type`}
rules={[
{
required: true,
message: 'Please select a document type',
},
]}
>
<Select
placeholder='Select a document type'
style={{ width: 200 }}
options={DOCTYPE}
value={filterDoctype}
defaultValue={filterDoctype}
onChange={setFilterDoctype}
/>
</Form.Item>
<Form.Item
name='feedback_result'
label={t`Feedback includes`}
>
<Input
defaultValue={filterFeedbackResult}
onChange={(e) => setFilterFeedbackResult(e.target.value)}
/>
</Form.Item>
</div>
<div
style={{
marginTop: 10,
display: 'flex',
justifyContent: 'space-between',
marginLeft: 0,
padding: 0,
}}
>
<Form.Item
name='predict_result'
label={t`Predict includes`}
>
<Input
defaultValue={filterPredictResult}
onChange={(e) => setFilterPredictResult(e.target.value)}
/>
</Form.Item>
<Form.Item
name='reviewed_result'
label={t`Review inculdes`}
>
<Input
defaultValue={filterReviewedResult}
onChange={(e) => setFilterReviewedResult(e.target.value)}
/>
</Form.Item>
</div>
</Form>
</Modal>
<Modal

2
cope2n-fe/src/request/api.ts
View File

@ -11,7 +11,7 @@ const environment = process.env.NODE_ENV;
const AXIOS_TIMEOUT_MS = 30 * 60 * 1000; // This config sastified long-live upload file request
const EXPIRED_PASSWORD_SIGNAL = 'expired_password';
export const baseURL = environment === 'development' ? 'http://107.120.133.27:9881/api' : '/api';
export const baseURL = environment === 'development' ? 'http://107.120.133.27:19001/api' : '/api';
// export const baseURL = '/api';