Merge pull request #163 from SDSRV-IDP/dev/search_content

Dev/search content

cope2n-api/Dockerfile

@@ -14,7 +14,7 @@ RUN pip install --upgrade pip
 RUN pip install uvicorn gunicorn Celery
 
 # For intergration with sdskvu
-RUN pip install pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
+RUN pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
 RUN pip install -U openmim==0.3.7 --no-cache-dir
 RUN mim install mmcv-full==1.7.2
 # RUN pip install fastdeploy-gpu-python==1.0.7 -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html --no-cache-dir

cope2n-api/fwd/settings.py

@@ -245,6 +245,7 @@ FIELDS_BY_SUB = {
 BAD_THRESHOLD = 0.75
 NEED_REVIEW = 1.0
 
+DOC_TYPES = ["imei", "invoice"]
 SUB_FOR_BILLING = ["all", "seao"]
 FIELD = ["imei_number", "purchase_date", "retailername", "sold_to_party", "invoice_no"]
 

cope2n-api/fwd_api/api/accuracy_view.py

@@ -78,6 +78,7 @@ class AccuracyViewSet(viewsets.ViewSet):
                 description='Which subsidiary to be included',
                 type=OpenApiTypes.STR,
                 enum=list(settings.SUBS.keys()),
+                required=True
             ),
             OpenApiParameter(
                 name='request_id',
@@ -112,6 +113,42 @@ class AccuracyViewSet(viewsets.ViewSet):
                 type=OpenApiTypes.FLOAT,
                 required=False
             ),
+            OpenApiParameter(
+                name='predict_result',
+                location=OpenApiParameter.QUERY,
+                description='Filter by predict result',
+                type=OpenApiTypes.STR,
+                required=False
+            ),
+            OpenApiParameter(
+                name='feedback_result',
+                location=OpenApiParameter.QUERY,
+                description='Filter by feedback result',
+                type=OpenApiTypes.STR,
+                required=False
+            ),
+            OpenApiParameter(
+                name='reviewed_result',
+                location=OpenApiParameter.QUERY,
+                description='Filter by reviewed result',
+                type=OpenApiTypes.STR,
+                required=False
+            ),
+            OpenApiParameter(
+                name='doc_type',
+                location=OpenApiParameter.QUERY,
+                description='Filter by document type',
+                type=OpenApiTypes.STR,
+                enum=list(settings.DOC_TYPES),
+                required=False
+            ),
+            OpenApiParameter(
+                name='bad_reason',
+                location=OpenApiParameter.QUERY,
+                description='Filter by bad reason',
+                type=OpenApiTypes.STR,
+                required=False
+            ),
         ],
         responses=None, tags=['Accuracy']
     )
@@ -129,7 +166,11 @@ class AccuracyViewSet(viewsets.ViewSet):
             include_test = request.GET.get('includes_test', False)
             subsidiary = request.GET.get("subsidiary", "all")
             max_accuracy = float(request.GET.get("max_accuracy", 100))
-            # subsidiary = map_subsidiary_long_to_short(subsidiary)
+            predict_result = request.GET.get('predict_result', None)
+            feedback_result = request.GET.get('feedback_result', None)
+            reviewed_result = request.GET.get('reviewed_result', None)
+            doc_type = request.GET.get('doc_type', None)
+            bad_reason = request.GET.get('bad_reason', None)
 
             base_query = Q(status=200)
             if start_date_str or end_date_str:
@@ -193,10 +234,28 @@ class AccuracyViewSet(viewsets.ViewSet):
                         base_query &= Q(
                             redemption_id__startswith=map_subsidiary_long_to_short(subsidiary))
 
+            if predict_result:
+                base_query &= Q(predict_result__icontains=predict_result)
+            if feedback_result:
+                base_query &= Q(feedback_result__icontains=feedback_result)
+            if reviewed_result:
+                base_query &= Q(reviewed_result__icontains=reviewed_result)
+            if doc_type:
+                if doc_type.lower() == 'invoice':
+                    base_query &= Q(doc_type__regex=r'^invoice(,\s*invoice)*$')
+                elif doc_type.lower() == 'imei':
+                    base_query &= Q(doc_type__regex=r'^imei(,\s*imei)*$')
+
             if isinstance(max_accuracy, float):
                 base_query &= Q(raw_accuracy__lt=(
                     max_accuracy/100)) | Q(raw_accuracy__isnull=True)
 
+            if bad_reason:
+                bad_reason_subquery = SubscriptionRequestFile.objects.filter(
+                    reason__icontains=bad_reason
+                ).values_list('request_id', flat=True)
+                base_query &= Q(id__in=bad_reason_subquery)
+
             subscription_requests = SubscriptionRequest.objects.filter(
                 base_query).order_by('created_at')
 

cope2n-api/fwd_api/migrations/0195_alter_subscriptionrequest_request_id_and_more.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.1.3 on 2024-11-05 02:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0194_alter_semiautocorrection_feedback_accuracy_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='subscriptionrequest',
+            name='request_id',
+            field=models.CharField(db_index=True, max_length=200),
+        ),
+        migrations.AlterField(
+            model_name='userprofile',
+            name='id',
+            field=models.AutoField(db_index=True, primary_key=True, serialize=False),
+        ),
+        migrations.AddIndex(
+            model_name='subscriptionrequestfile',
+            index=models.Index(fields=['request', 'index_in_request', 'doc_type'], name='fwd_api_sub_request_890e13_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='subscriptionrequestfile',
+            index=models.Index(fields=['request', 'file_name'], name='fwd_api_sub_request_1a42cd_idx'),
+        ),
+    ]

cope2n-api/fwd_api/migrations/0196_subscriptionrequestfile_idx_gin_predict_result_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.3 on 2024-11-28 07:10
+
+import django.contrib.postgres.indexes
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0195_alter_subscriptionrequest_request_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='subscriptionrequestfile',
+            index=django.contrib.postgres.indexes.GinIndex(fields=['predict_result'], name='idx_gin_predict_result'),
+        ),
+        migrations.AddIndex(
+            model_name='subscriptionrequestfile',
+            index=django.contrib.postgres.indexes.GinIndex(fields=['feedback_result'], name='idx_gin_feedback_result'),
+        ),
+        migrations.AddIndex(
+            model_name='subscriptionrequestfile',
+            index=django.contrib.postgres.indexes.GinIndex(fields=['reviewed_result'], name='idx_gin_reviewed_result'),
+        ),
+    ]

cope2n-api/fwd_api/migrations/0197_subscriptionrequest_idx_gin_rq_predict_result_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.3 on 2024-11-28 09:04
+
+import django.contrib.postgres.indexes
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0196_subscriptionrequestfile_idx_gin_predict_result_and_more'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='subscriptionrequest',
+            index=django.contrib.postgres.indexes.GinIndex(fields=['predict_result'], name='idx_gin_rq_predict_result'),
+        ),
+        migrations.AddIndex(
+            model_name='subscriptionrequest',
+            index=django.contrib.postgres.indexes.GinIndex(fields=['feedback_result'], name='idx_gin_rq_feedback_result'),
+        ),
+        migrations.AddIndex(
+            model_name='subscriptionrequest',
+            index=django.contrib.postgres.indexes.GinIndex(fields=['reviewed_result'], name='idx_gin_rq_reviewed_result'),
+        ),
+    ]

cope2n-api/fwd_api/migrations/0198_alter_subscriptionrequest_doc_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.3 on 2024-11-28 09:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0197_subscriptionrequest_idx_gin_rq_predict_result_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='subscriptionrequest',
+            name='doc_type',
+            field=models.CharField(db_index=True, max_length=100),
+        ),
+    ]

cope2n-api/fwd_api/migrations/0199_alter_subscriptionrequestfile_reason.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.3 on 2024-12-04 10:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0198_alter_subscriptionrequest_doc_type'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='subscriptionrequestfile',
+            name='reason',
+            field=models.TextField(blank=True, db_index=True),
+        ),
+    ]

cope2n-api/fwd_api/models/SubscriptionRequest.py

@@ -1,5 +1,6 @@
 from django.db import models
 from django.utils import timezone
+from django.contrib.postgres.indexes import GinIndex
 
 from fwd_api.models.Subscription import Subscription
 
@@ -7,7 +8,7 @@ class SubscriptionRequest(models.Model):
     id = models.AutoField(primary_key=True)
     pages: int = models.IntegerField()
     pages_left: int = models.IntegerField(default=1)
-    doc_type: str = models.CharField(max_length=100)
+    doc_type: str = models.CharField(max_length=100, db_index=True)
     request_id = models.CharField(max_length=200, db_index=True)  # Change to request_id
     redemption_id = models.CharField(max_length=200, null=True)
     process_type = models.CharField(max_length=200)  # driver/id/invoice
@@ -39,3 +40,10 @@ class SubscriptionRequest(models.Model):
     is_reviewed = models.BooleanField(default=False)
     is_required = models.BooleanField(default=True)
     subsidiary = models.CharField(default="", null=True, max_length=200)
+
+    class Meta:
+        indexes = [
+            GinIndex(fields=['predict_result'], name='idx_gin_rq_predict_result'),
+            GinIndex(fields=['feedback_result'], name='idx_gin_rq_feedback_result'),
+            GinIndex(fields=['reviewed_result'], name='idx_gin_rq_reviewed_result'),
+        ]

cope2n-api/fwd_api/models/SubscriptionRequestFile.py

@@ -1,5 +1,5 @@
 from django.db import models
-
+from django.contrib.postgres.indexes import GinIndex
 from fwd_api.constant.common import FileCategory
 from fwd_api.models import SubscriptionRequest
 from fwd_api.models.fields.EncryptedCharField import EncryptedCharField
@@ -40,5 +40,8 @@ class SubscriptionRequestFile(models.Model):
         indexes = [
             models.Index(fields=['request', 'index_in_request', 'doc_type']),  # For updating results
             models.Index(fields=['request', 'file_name']), # for getting image files by AI
+            GinIndex(fields=['predict_result'], name='idx_gin_predict_result'),
+            GinIndex(fields=['feedback_result'], name='idx_gin_feedback_result'),
+            GinIndex(fields=['reviewed_result'], name='idx_gin_reviewed_result'),
         ]
 

cope2n-fe/package.json

@@ -83,7 +83,7 @@
     "prettier-plugin-organize-imports": "^3.2.1",
     "rollup-plugin-visualizer": "^5.9.0",
     "sass": "^1.57.1",
-    "typescript": "5.6.3",
+    "typescript": "^5.6.3",
     "vite": "^4.0.3",
     "vite-plugin-svgr": "^2.4.0",
     "vite-tsconfig-paths": "^4.0.3"

cope2n-fe/src/components/left-menu/index.tsx

@@ -1,4 +1,4 @@
-import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined, FileSearchOutlined } from '@ant-design/icons';
+import { AppstoreOutlined, BarChartOutlined, RotateRightOutlined } from '@ant-design/icons';
 import { t } from '@lingui/macro';
 import { Menu, MenuProps } from 'antd';
 import React from 'react';

cope2n-fe/src/components/spin/index.tsx

@@ -4,7 +4,6 @@ export function GlobalSpin() {
   return (
     <Spin
       size='large'
-      tip='Loading ...'
       style={{
         position: 'fixed',
         top: '50%',

cope2n-fe/src/locales/en/messages.json

@@ -70,5 +70,9 @@
   "You are only allowed to upload {0} file.": "You are only allowed to upload {0} file.",
   "You have unsaved changes!": "You have unsaved changes!",
   "Your current password has expired. Please change your password to continue.": "Your current password has expired. Please change your password to continue.",
-  "max_accuracy": "Max accuracy"
+  "max_accuracy": "Max accuracy",
+  "docType": "Only type",
+  "Feedback Result": "Feedback Result",
+  "Predict Result": "Predict Result",
+  "Reviewed Result": "Reviewed Result"
 }

cope2n-fe/src/locales/vi/messages.json

@@ -70,5 +70,9 @@
   "You are only allowed to upload {0} file.": "Bạn chỉ được phép tải lên {0}.",
   "You have unsaved changes!": "Bạn có những thay đổi chưa được lưu!",
   "Your current password has expired. Please change your password to continue.": "",
-  "max_accuracy": "Độ chính xác tối đa"
+  "max_accuracy": "Độ chính xác tối đa",
+  "DocType": "Kiểu tài liệu",
+  "Feedback Result": "",
+  "Predict Result": "",
+  "Reviewed Result": ""
 }

cope2n-fe/src/pages/reviews2/api.ts

@@ -2,10 +2,14 @@ import { baseURL } from 'request/api';
 import { RecentRequest } from './const';
 
 export const fetchAllRequests = async (
-  filterDateRange,
+  filterDateRange: any[],
-  filterSubsidiaries,
+  filterSubsidiaries: string,
-  filterReviewState,
+  filterReviewState: string,
-  filterIncludeTests,
+  filterIncludeTests: string,
+  filterDoctype: string,
+  filterFeedbackResult: string,
+  filterPredictResult: string,
+  filterReviewedResult: string,
   page = 1,
   page_size = 20,
   max_accuracy = 100,
@@ -25,6 +29,20 @@ export const fetchAllRequests = async (
   if (filterIncludeTests) {
     filterStr += `includes_test=${filterIncludeTests}&`;
   }
+  // add 4 more field
+  if (filterDoctype) {
+    filterStr += `doc_type=${filterDoctype}&`;
+  }
+  if (filterFeedbackResult) {
+    filterStr += `feedback_result=${filterFeedbackResult}&`;
+  }
+  if (filterPredictResult) {
+    filterStr += `predict_result=${filterPredictResult}&`;
+  }
+  if (filterReviewedResult) {
+    filterStr += `reviewed_result=${filterReviewedResult}&`;
+  }
+  //
   if (startDate && endDate) {
     filterStr += `start_date=${startDate}&end_date=${endDate}&`;
   }
@@ -54,7 +72,7 @@ export const confirmRequest = async (requestID: any) => {
     body: JSON.stringify({ request_file_results: [] }),
   })
 
-    
+
 };
 
 export const updateRevisedDataByFile = async (
@@ -77,7 +95,7 @@ export const updateRevisedDataByFile = async (
   )
 };
 
-export const fetchRequest = async (id) => {
+export const fetchRequest = async (id: string) => {
   const token = localStorage.getItem('sbt-token') || '';
   const response = await fetch(`${baseURL}/ctel/request/${id}/`, {
     method: 'GET',
@@ -87,7 +105,7 @@ export const fetchRequest = async (id) => {
   });
   return await (
     await response.json()
-  ).subscription_requests[0];
+  ).subscription_requests?.[0] || null;
 };
 
 export const addRecentRequest = (

cope2n-fe/src/pages/reviews2/const.ts

@@ -48,6 +48,12 @@ export const SUBSIDIARIES = [
   { value: 'SEIN', label: 'SEIN' },
 ];
 
+export const DOCTYPE = [
+  { value: null, label: '--' },
+  { value: 'imei', label: 'imei' },
+  { value: 'invoice', label: 'invoice' },
+];
+
 export const SOURCE_KEYS = [
   'retailername',
   'sold_to_party',

cope2n-fe/src/pages/reviews2/index.tsx

@@ -41,6 +41,7 @@ import {
 } from './api';
 import {
   counter_measure_map,
+  DOCTYPE,
   FEEDBACK_ACCURACY,
   FEEDBACK_RESULT,
   PREDICTED_RESULT,
@@ -71,6 +72,10 @@ const ReviewPage = () => {
   const [filterAccuracy, setFilterAccuracy] = useState(100);
   const [filterReviewState, setFilterReviewState] = useState('all');
   const [filterIncludeTests, setFilterIncludesTests] = useState('true');
+  const [filterDoctype, setFilterDoctype] = useState(null);
+  const [filterFeedbackResult, setFilterFeedbackResult] = useState('');
+  const [filterPredictResult, setFilterPredictResult] = useState('');
+  const [filterReviewedResult, setFilterReviewedResult] = useState('');
   // const [requests, setRequests] = useState([]);
   const [currentRequest, setCurrentRequest] = useState(null);
   const [currentRequestIndex, setCurrentRequestIndex] = useState(1);
@@ -109,6 +114,10 @@ const ReviewPage = () => {
         filterSubsidiaries,
         filterReviewState,
         filterIncludeTests,
+        filterDoctype,
+        filterFeedbackResult,
+        filterPredictResult,
+        filterReviewedResult,
         1,
         1,
         filterAccuracy,
@@ -163,7 +172,7 @@ const ReviewPage = () => {
 
   const setAndLoadSelectedFile = async (requestData, index) => {
     setSelectedFileId(index);
-    if (!requestData['Files'][index]) {
+    if (!requestData?.['Files'][index]) {
       setSelectedFileData('FAILED_TO_LOAD_FILE');
       setImageLoading(false);
       return;
@@ -218,6 +227,10 @@ const ReviewPage = () => {
       filterSubsidiaries,
       filterReviewState,
       filterIncludeTests,
+      filterDoctype,
+      filterFeedbackResult,
+      filterPredictResult,
+      filterReviewedResult,
       requestIndex,
       1,
       filterAccuracy,
@@ -273,6 +286,10 @@ const ReviewPage = () => {
       filterSubsidiaries,
       filterReviewState,
       filterIncludeTests,
+      filterDoctype,
+      filterFeedbackResult,
+      filterPredictResult,
+      filterReviewedResult,
       1,
       1,
       filterAccuracy,
@@ -399,18 +416,18 @@ const ReviewPage = () => {
       style={
         fullscreen
           ? {
-              position: 'fixed',
+            position: 'fixed',
-              top: 0,
+            top: 0,
-              left: 0,
+            left: 0,
-              width: '100%',
+            width: '100%',
-              height: '100%',
+            height: '100%',
-              backgroundColor: '#fff',
+            backgroundColor: '#fff',
-              zIndex: 1000,
+            zIndex: 1000,
-            }
+          }
           : {
-              height: '100%',
+            height: '100%',
-              position: 'relative',
+            position: 'relative',
-            }
+          }
       }
     >
       <div
@@ -561,17 +578,17 @@ const ReviewPage = () => {
                 overflow: 'auto',
               }}
             >
-              <Viewer
+              {selectedFileData && <Viewer
                 plugins={[defaultLayoutPluginInstance]}
                 fileUrl={selectedFileData}
                 onDocumentLoad={() => setImageLoading(false)}
-              />
+              />}
             </div>
           ) : (
             <div
               style={{
                 flexGrow: 1,
-                overflow: 'auto',
+                overflowY: 'auto',
                 display: 'flex',
                 justifyContent: 'center',
                 alignItems: 'center',
@@ -739,11 +756,10 @@ const ReviewPage = () => {
           >
             <Input
               size='middle'
-              value={`Sub: ${filterSubsidiaries}, Date:${
+              value={`Sub: ${filterSubsidiaries}, Date:${filterDateRange[0]
-                filterDateRange[0]
+                ? filterDateRange[0] + ' to ' + filterDateRange[1]
-                  ? filterDateRange[0] + ' to ' + filterDateRange[1]
+                : 'All'
-                  : 'All'
+                }, Reviewed: ${filterReviewState}, Tests: ${filterIncludeTests}`}
-              }, Reviewed: ${filterReviewState}, Tests: ${filterIncludeTests}`}
               readOnly
             />
             <Button
@@ -764,14 +780,14 @@ const ReviewPage = () => {
               let shouldRevised = false;
               try {
                 if (
-                  selectedFileDataSource[data]?.[FEEDBACK_ACCURACY].length > 0
+                  selectedFileDataSource[data]?.[FEEDBACK_ACCURACY]?.length > 0
                 ) {
                   shouldRevised =
                     selectedFileDataSource[data][FEEDBACK_ACCURACY][0] < 1;
                 }
-              } catch (error) {}
+              } catch (error) { }
               return (
-                <div style={{ margin: '0 0 8px' }} className='file-input-group'>
+                <div style={{ margin: '0 0 8px' }} className='file-input-group' key={data}>
                   <div
                     style={{
                       display: 'flex',
@@ -858,7 +874,7 @@ const ReviewPage = () => {
                 options={SOLUTION_BAD_QUALITY}
                 onChange={setSolution}
                 value={solution}
-                // defaultValue={solution}
+              // defaultValue={solution}
               />
               {solution === 'other' && (
                 <Input
@@ -875,7 +891,7 @@ const ReviewPage = () => {
           <div style={{ display: 'flex', justifyContent: 'flex-end' }}>
             <Button
               type='primary'
-              color='success'
+              // color='success'
               size='middle'
               style={{
                 height: '36px',
@@ -1025,6 +1041,73 @@ const ReviewPage = () => {
               />
             </Form.Item>
           </div>
+          {/* add 4 more filter fields */}
+          <div
+            style={{
+              marginTop: 10,
+              display: 'flex',
+              justifyContent: 'space-between',
+              marginLeft: 0,
+              padding: 0,
+            }}
+          >
+            <Form.Item
+              name='doc_type'
+              label={t`Only type`}
+              rules={[
+                {
+                  required: true,
+                  message: 'Please select a document type',
+                },
+              ]}
+            >
+              <Select
+                placeholder='Select a document type'
+                style={{ width: 200 }}
+                options={DOCTYPE}
+                value={filterDoctype}
+                defaultValue={filterDoctype}
+                onChange={setFilterDoctype}
+              />
+            </Form.Item>
+            <Form.Item
+              name='feedback_result'
+              label={t`Feedback includes`}
+            >
+              <Input
+                defaultValue={filterFeedbackResult}
+                onChange={(e) => setFilterFeedbackResult(e.target.value)}
+              />
+            </Form.Item>
+          </div>
+          <div
+            style={{
+              marginTop: 10,
+              display: 'flex',
+              justifyContent: 'space-between',
+              marginLeft: 0,
+              padding: 0,
+            }}
+          >
+            <Form.Item
+              name='predict_result'
+              label={t`Predict includes`}
+            >
+              <Input
+                defaultValue={filterPredictResult}
+                onChange={(e) => setFilterPredictResult(e.target.value)}
+              />
+            </Form.Item>
+            <Form.Item
+              name='reviewed_result'
+              label={t`Review inculdes`}
+            >
+              <Input
+                defaultValue={filterReviewedResult}
+                onChange={(e) => setFilterReviewedResult(e.target.value)}
+              />
+            </Form.Item>
+          </div>
         </Form>
       </Modal>
       <Modal

cope2n-fe/src/request/api.ts

@@ -11,7 +11,7 @@ const environment = process.env.NODE_ENV;
 const AXIOS_TIMEOUT_MS = 30 * 60 * 1000; // This config sastified long-live upload file request
 const EXPIRED_PASSWORD_SIGNAL = 'expired_password';
 
-export const baseURL = environment === 'development' ? 'http://107.120.133.27:9881/api' : '/api';
+export const baseURL = environment === 'development' ? 'http://107.120.133.27:19001/api' : '/api';
 // export const baseURL = '/api';