diff --git a/cope2n-api/Dockerfile b/cope2n-api/Dockerfile index 20eeeb2..f01b570 100755 --- a/cope2n-api/Dockerfile +++ b/cope2n-api/Dockerfile @@ -14,7 +14,7 @@ RUN pip install --upgrade pip RUN pip install uvicorn gunicorn Celery # For intergration with sdskvu -RUN pip install pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116 +RUN pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116 RUN pip install -U openmim==0.3.7 --no-cache-dir RUN mim install mmcv-full==1.7.2 # RUN pip install fastdeploy-gpu-python==1.0.7 -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html --no-cache-dir diff --git a/cope2n-api/fwd/settings.py b/cope2n-api/fwd/settings.py index 86d4832..8f05d50 100755 --- a/cope2n-api/fwd/settings.py +++ b/cope2n-api/fwd/settings.py @@ -245,6 +245,7 @@ FIELDS_BY_SUB = { BAD_THRESHOLD = 0.75 NEED_REVIEW = 1.0 +DOC_TYPES = ["imei", "invoice"] SUB_FOR_BILLING = ["all", "seao"] FIELD = ["imei_number", "purchase_date", "retailername", "sold_to_party", "invoice_no"] diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index e4295f0..9968dae 100755 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -78,6 +78,7 @@ class AccuracyViewSet(viewsets.ViewSet): description='Which subsidiary to be included', type=OpenApiTypes.STR, enum=list(settings.SUBS.keys()), + required=True ), OpenApiParameter( name='request_id', @@ -112,6 +113,42 @@ class AccuracyViewSet(viewsets.ViewSet): type=OpenApiTypes.FLOAT, required=False ), + OpenApiParameter( + name='predict_result', + location=OpenApiParameter.QUERY, + description='Filter by predict result', + type=OpenApiTypes.STR, + required=False + ), + OpenApiParameter( + name='feedback_result', + location=OpenApiParameter.QUERY, + description='Filter by feedback result', + type=OpenApiTypes.STR, + required=False + ), + OpenApiParameter( + name='reviewed_result', + location=OpenApiParameter.QUERY, + description='Filter by reviewed result', + type=OpenApiTypes.STR, + required=False + ), + OpenApiParameter( + name='doc_type', + location=OpenApiParameter.QUERY, + description='Filter by document type', + type=OpenApiTypes.STR, + enum=list(settings.DOC_TYPES), + required=False + ), + OpenApiParameter( + name='bad_reason', + location=OpenApiParameter.QUERY, + description='Filter by bad reason', + type=OpenApiTypes.STR, + required=False + ), ], responses=None, tags=['Accuracy'] ) @@ -129,7 +166,11 @@ class AccuracyViewSet(viewsets.ViewSet): include_test = request.GET.get('includes_test', False) subsidiary = request.GET.get("subsidiary", "all") max_accuracy = float(request.GET.get("max_accuracy", 100)) - # subsidiary = map_subsidiary_long_to_short(subsidiary) + predict_result = request.GET.get('predict_result', None) + feedback_result = request.GET.get('feedback_result', None) + reviewed_result = request.GET.get('reviewed_result', None) + doc_type = request.GET.get('doc_type', None) + bad_reason = request.GET.get('bad_reason', None) base_query = Q(status=200) if start_date_str or end_date_str: @@ -193,10 +234,28 @@ class AccuracyViewSet(viewsets.ViewSet): base_query &= Q( redemption_id__startswith=map_subsidiary_long_to_short(subsidiary)) + if predict_result: + base_query &= Q(predict_result__icontains=predict_result) + if feedback_result: + base_query &= Q(feedback_result__icontains=feedback_result) + if reviewed_result: + base_query &= Q(reviewed_result__icontains=reviewed_result) + if doc_type: + if doc_type.lower() == 'invoice': + base_query &= Q(doc_type__regex=r'^invoice(,\s*invoice)*$') + elif doc_type.lower() == 'imei': + base_query &= Q(doc_type__regex=r'^imei(,\s*imei)*$') + if isinstance(max_accuracy, float): base_query &= Q(raw_accuracy__lt=( max_accuracy/100)) | Q(raw_accuracy__isnull=True) + if bad_reason: + bad_reason_subquery = SubscriptionRequestFile.objects.filter( + reason__icontains=bad_reason + ).values_list('request_id', flat=True) + base_query &= Q(id__in=bad_reason_subquery) + subscription_requests = SubscriptionRequest.objects.filter( base_query).order_by('created_at') diff --git a/cope2n-api/fwd_api/migrations/0195_alter_subscriptionrequest_request_id_and_more.py b/cope2n-api/fwd_api/migrations/0195_alter_subscriptionrequest_request_id_and_more.py new file mode 100644 index 0000000..175920c --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0195_alter_subscriptionrequest_request_id_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.3 on 2024-11-05 02:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0194_alter_semiautocorrection_feedback_accuracy_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='subscriptionrequest', + name='request_id', + field=models.CharField(db_index=True, max_length=200), + ), + migrations.AlterField( + model_name='userprofile', + name='id', + field=models.AutoField(db_index=True, primary_key=True, serialize=False), + ), + migrations.AddIndex( + model_name='subscriptionrequestfile', + index=models.Index(fields=['request', 'index_in_request', 'doc_type'], name='fwd_api_sub_request_890e13_idx'), + ), + migrations.AddIndex( + model_name='subscriptionrequestfile', + index=models.Index(fields=['request', 'file_name'], name='fwd_api_sub_request_1a42cd_idx'), + ), + ] \ No newline at end of file diff --git a/cope2n-api/fwd_api/migrations/0196_subscriptionrequestfile_idx_gin_predict_result_and_more.py b/cope2n-api/fwd_api/migrations/0196_subscriptionrequestfile_idx_gin_predict_result_and_more.py new file mode 100644 index 0000000..561ddaf --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0196_subscriptionrequestfile_idx_gin_predict_result_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 4.1.3 on 2024-11-28 07:10 + +import django.contrib.postgres.indexes +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0195_alter_subscriptionrequest_request_id_and_more'), + ] + + operations = [ + migrations.AddIndex( + model_name='subscriptionrequestfile', + index=django.contrib.postgres.indexes.GinIndex(fields=['predict_result'], name='idx_gin_predict_result'), + ), + migrations.AddIndex( + model_name='subscriptionrequestfile', + index=django.contrib.postgres.indexes.GinIndex(fields=['feedback_result'], name='idx_gin_feedback_result'), + ), + migrations.AddIndex( + model_name='subscriptionrequestfile', + index=django.contrib.postgres.indexes.GinIndex(fields=['reviewed_result'], name='idx_gin_reviewed_result'), + ), + ] diff --git a/cope2n-api/fwd_api/migrations/0197_subscriptionrequest_idx_gin_rq_predict_result_and_more.py b/cope2n-api/fwd_api/migrations/0197_subscriptionrequest_idx_gin_rq_predict_result_and_more.py new file mode 100644 index 0000000..3a8f521 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0197_subscriptionrequest_idx_gin_rq_predict_result_and_more.py @@ -0,0 +1,26 @@ +# Generated by Django 4.1.3 on 2024-11-28 09:04 + +import django.contrib.postgres.indexes +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0196_subscriptionrequestfile_idx_gin_predict_result_and_more'), + ] + + operations = [ + migrations.AddIndex( + model_name='subscriptionrequest', + index=django.contrib.postgres.indexes.GinIndex(fields=['predict_result'], name='idx_gin_rq_predict_result'), + ), + migrations.AddIndex( + model_name='subscriptionrequest', + index=django.contrib.postgres.indexes.GinIndex(fields=['feedback_result'], name='idx_gin_rq_feedback_result'), + ), + migrations.AddIndex( + model_name='subscriptionrequest', + index=django.contrib.postgres.indexes.GinIndex(fields=['reviewed_result'], name='idx_gin_rq_reviewed_result'), + ), + ] diff --git a/cope2n-api/fwd_api/migrations/0198_alter_subscriptionrequest_doc_type.py b/cope2n-api/fwd_api/migrations/0198_alter_subscriptionrequest_doc_type.py new file mode 100644 index 0000000..c553826 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0198_alter_subscriptionrequest_doc_type.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2024-11-28 09:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0197_subscriptionrequest_idx_gin_rq_predict_result_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='subscriptionrequest', + name='doc_type', + field=models.CharField(db_index=True, max_length=100), + ), + ] diff --git a/cope2n-api/fwd_api/migrations/0199_alter_subscriptionrequestfile_reason.py b/cope2n-api/fwd_api/migrations/0199_alter_subscriptionrequestfile_reason.py new file mode 100644 index 0000000..8a245fb --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0199_alter_subscriptionrequestfile_reason.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2024-12-04 10:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0198_alter_subscriptionrequest_doc_type'), + ] + + operations = [ + migrations.AlterField( + model_name='subscriptionrequestfile', + name='reason', + field=models.TextField(blank=True, db_index=True), + ), + ] diff --git a/cope2n-api/fwd_api/models/SubscriptionRequest.py b/cope2n-api/fwd_api/models/SubscriptionRequest.py index 0b89fdf..24fc53b 100755 --- a/cope2n-api/fwd_api/models/SubscriptionRequest.py +++ b/cope2n-api/fwd_api/models/SubscriptionRequest.py @@ -1,5 +1,6 @@ from django.db import models from django.utils import timezone +from django.contrib.postgres.indexes import GinIndex from fwd_api.models.Subscription import Subscription @@ -7,7 +8,7 @@ class SubscriptionRequest(models.Model): id = models.AutoField(primary_key=True) pages: int = models.IntegerField() pages_left: int = models.IntegerField(default=1) - doc_type: str = models.CharField(max_length=100) + doc_type: str = models.CharField(max_length=100, db_index=True) request_id = models.CharField(max_length=200, db_index=True) # Change to request_id redemption_id = models.CharField(max_length=200, null=True) process_type = models.CharField(max_length=200) # driver/id/invoice @@ -39,3 +40,10 @@ class SubscriptionRequest(models.Model): is_reviewed = models.BooleanField(default=False) is_required = models.BooleanField(default=True) subsidiary = models.CharField(default="", null=True, max_length=200) + + class Meta: + indexes = [ + GinIndex(fields=['predict_result'], name='idx_gin_rq_predict_result'), + GinIndex(fields=['feedback_result'], name='idx_gin_rq_feedback_result'), + GinIndex(fields=['reviewed_result'], name='idx_gin_rq_reviewed_result'), + ] \ No newline at end of file diff --git a/cope2n-api/fwd_api/models/SubscriptionRequestFile.py b/cope2n-api/fwd_api/models/SubscriptionRequestFile.py index e9ffc22..1dce06d 100755 --- a/cope2n-api/fwd_api/models/SubscriptionRequestFile.py +++ b/cope2n-api/fwd_api/models/SubscriptionRequestFile.py @@ -1,5 +1,5 @@ from django.db import models - +from django.contrib.postgres.indexes import GinIndex from fwd_api.constant.common import FileCategory from fwd_api.models import SubscriptionRequest from fwd_api.models.fields.EncryptedCharField import EncryptedCharField @@ -40,5 +40,8 @@ class SubscriptionRequestFile(models.Model): indexes = [ models.Index(fields=['request', 'index_in_request', 'doc_type']), # For updating results models.Index(fields=['request', 'file_name']), # for getting image files by AI + GinIndex(fields=['predict_result'], name='idx_gin_predict_result'), + GinIndex(fields=['feedback_result'], name='idx_gin_feedback_result'), + GinIndex(fields=['reviewed_result'], name='idx_gin_reviewed_result'), ] diff --git a/cope2n-fe/package.json b/cope2n-fe/package.json index a05845f..fb1b005 100644 --- a/cope2n-fe/package.json +++ b/cope2n-fe/package.json @@ -83,7 +83,7 @@ "prettier-plugin-organize-imports": "^3.2.1", "rollup-plugin-visualizer": "^5.9.0", "sass": "^1.57.1", - "typescript": "5.6.3", + "typescript": "^5.6.3", "vite": "^4.0.3", "vite-plugin-svgr": "^2.4.0", "vite-tsconfig-paths": "^4.0.3" diff --git a/cope2n-fe/src/components/left-menu/index.tsx b/cope2n-fe/src/components/left-menu/index.tsx index d0390dc..3bee952 100644 --- a/cope2n-fe/src/components/left-menu/index.tsx +++ b/cope2n-fe/src/components/left-menu/index.tsx @@ -1,4 +1,4 @@ -import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined, FileSearchOutlined } from '@ant-design/icons'; +import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined } from '@ant-design/icons'; import { t } from '@lingui/macro'; import { Menu, MenuProps } from 'antd'; import React from 'react'; diff --git a/cope2n-fe/src/components/spin/index.tsx b/cope2n-fe/src/components/spin/index.tsx index 3c1b782..a11eba0 100644 --- a/cope2n-fe/src/components/spin/index.tsx +++ b/cope2n-fe/src/components/spin/index.tsx @@ -4,7 +4,6 @@ export function GlobalSpin() { return ( { body: JSON.stringify({ request_file_results: [] }), }) - + }; export const updateRevisedDataByFile = async ( @@ -77,7 +95,7 @@ export const updateRevisedDataByFile = async ( ) }; -export const fetchRequest = async (id) => { +export const fetchRequest = async (id: string) => { const token = localStorage.getItem('sbt-token') || ''; const response = await fetch(`${baseURL}/ctel/request/${id}/`, { method: 'GET', @@ -87,7 +105,7 @@ export const fetchRequest = async (id) => { }); return await ( await response.json() - ).subscription_requests[0]; + ).subscription_requests?.[0] || null; }; export const addRecentRequest = ( diff --git a/cope2n-fe/src/pages/reviews2/const.ts b/cope2n-fe/src/pages/reviews2/const.ts index 86592c7..7c883dc 100644 --- a/cope2n-fe/src/pages/reviews2/const.ts +++ b/cope2n-fe/src/pages/reviews2/const.ts @@ -48,6 +48,12 @@ export const SUBSIDIARIES = [ { value: 'SEIN', label: 'SEIN' }, ]; +export const DOCTYPE = [ + { value: null, label: '--' }, + { value: 'imei', label: 'imei' }, + { value: 'invoice', label: 'invoice' }, +]; + export const SOURCE_KEYS = [ 'retailername', 'sold_to_party', diff --git a/cope2n-fe/src/pages/reviews2/index.tsx b/cope2n-fe/src/pages/reviews2/index.tsx index 2f73edb..a6d73ab 100644 --- a/cope2n-fe/src/pages/reviews2/index.tsx +++ b/cope2n-fe/src/pages/reviews2/index.tsx @@ -41,6 +41,7 @@ import { } from './api'; import { counter_measure_map, + DOCTYPE, FEEDBACK_ACCURACY, FEEDBACK_RESULT, PREDICTED_RESULT, @@ -71,6 +72,10 @@ const ReviewPage = () => { const [filterAccuracy, setFilterAccuracy] = useState(100); const [filterReviewState, setFilterReviewState] = useState('all'); const [filterIncludeTests, setFilterIncludesTests] = useState('true'); + const [filterDoctype, setFilterDoctype] = useState(null); + const [filterFeedbackResult, setFilterFeedbackResult] = useState(''); + const [filterPredictResult, setFilterPredictResult] = useState(''); + const [filterReviewedResult, setFilterReviewedResult] = useState(''); // const [requests, setRequests] = useState([]); const [currentRequest, setCurrentRequest] = useState(null); const [currentRequestIndex, setCurrentRequestIndex] = useState(1); @@ -109,6 +114,10 @@ const ReviewPage = () => { filterSubsidiaries, filterReviewState, filterIncludeTests, + filterDoctype, + filterFeedbackResult, + filterPredictResult, + filterReviewedResult, 1, 1, filterAccuracy, @@ -163,7 +172,7 @@ const ReviewPage = () => { const setAndLoadSelectedFile = async (requestData, index) => { setSelectedFileId(index); - if (!requestData['Files'][index]) { + if (!requestData?.['Files'][index]) { setSelectedFileData('FAILED_TO_LOAD_FILE'); setImageLoading(false); return; @@ -218,6 +227,10 @@ const ReviewPage = () => { filterSubsidiaries, filterReviewState, filterIncludeTests, + filterDoctype, + filterFeedbackResult, + filterPredictResult, + filterReviewedResult, requestIndex, 1, filterAccuracy, @@ -273,6 +286,10 @@ const ReviewPage = () => { filterSubsidiaries, filterReviewState, filterIncludeTests, + filterDoctype, + filterFeedbackResult, + filterPredictResult, + filterReviewedResult, 1, 1, filterAccuracy, @@ -399,18 +416,18 @@ const ReviewPage = () => { style={ fullscreen ? { - position: 'fixed', - top: 0, - left: 0, - width: '100%', - height: '100%', - backgroundColor: '#fff', - zIndex: 1000, - } + position: 'fixed', + top: 0, + left: 0, + width: '100%', + height: '100%', + backgroundColor: '#fff', + zIndex: 1000, + } : { - height: '100%', - position: 'relative', - } + height: '100%', + position: 'relative', + } } >
{ overflow: 'auto', }} > - setImageLoading(false)} - /> + />}
) : (
{ >
+ {/* add 4 more filter fields */} +
+ + setFilterFeedbackResult(e.target.value)} + /> + +
+
+ + setFilterPredictResult(e.target.value)} + /> + + + setFilterReviewedResult(e.target.value)} + /> + +