diff --git a/.gitignore b/.gitignore index b5cd87e..8d7c64e 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ Jan.csv cope2n-api/reviewed/date.xlsx cope2n-api/reviewed/retailer.xlsx /scripts/* +scripts/crawl_database.py +redundant_models/ \ No newline at end of file diff --git a/api-cronjob/run.py b/api-cronjob/run.py index 3a2f225..90ae97a 100644 --- a/api-cronjob/run.py +++ b/api-cronjob/run.py @@ -4,7 +4,7 @@ import requests from datetime import datetime # Get the proxy URL from the environment variable -interval = 60*60*1 # 1 minute +interval = 60*60*3 # 1 minute update_cost = int(60*2) proxy_url = os.getenv('PROXY', "localhost") user = os.getenv('ADMIN_USER_NAME', "") diff --git a/cope2n-ai-fi/Dockerfile b/cope2n-ai-fi/Dockerfile index ed5d957..156497d 100755 --- a/cope2n-ai-fi/Dockerfile +++ b/cope2n-ai-fi/Dockerfile @@ -15,7 +15,7 @@ RUN pip install mmcv==1.6.0 -f https://download.openmmlab.com/mmcv/dst/cu116/tor RUN ln -s /opt/conda/lib/python3.10/site-packages/torch/lib/libcudnn.so.8 /usr/lib/libcudnn.so && \ ln -s /opt/conda/lib/libcublas.so /usr/lib/libcublas.so -# RUN python -m pip install paddlepaddle-gpu==2.4.2.post116 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html --no-cache-dir +RUN python -m pip install paddlepaddle-gpu==2.4.2.post116 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html --no-cache-dir RUN pip install fastdeploy-gpu-python==1.0.7 -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html --no-cache-dir diff --git a/cope2n-ai-fi/configs/sdsap_sbt/configs.py b/cope2n-ai-fi/configs/sdsap_sbt/configs.py index 402c836..1defd15 100755 --- a/cope2n-ai-fi/configs/sdsap_sbt/configs.py +++ b/cope2n-ai-fi/configs/sdsap_sbt/configs.py @@ -29,7 +29,7 @@ kvu_model = { "option": "sbt_v2", "model": { "pretrained_model_path": "/workspace/cope2n-ai-fi/weights/layoutxlm-base", - "config": "/workspace/cope2n-ai-fi/weights/models/sdsvkvu/key_value_understanding_for_sbt-20231219-143837/base.yaml", - "checkpoint": "/workspace/cope2n-ai-fi/weights/models/sdsvkvu/key_value_understanding_for_sbt-20231219-143837/checkpoints/best_model.pth" + "config": "/workspace/cope2n-ai-fi/weights/models/sdsvkvu/key_value_understanding_for_sbt-20240315-175030/base.yaml", + "checkpoint": "/workspace/cope2n-ai-fi/weights/models/sdsvkvu/key_value_understanding_for_sbt-20240315-175030/checkpoints/best_model.pth" } } \ No newline at end of file diff --git a/cope2n-ai-fi/modules/sdsvkvu b/cope2n-ai-fi/modules/sdsvkvu index 671d791..2012803 160000 --- a/cope2n-ai-fi/modules/sdsvkvu +++ b/cope2n-ai-fi/modules/sdsvkvu @@ -1 +1 @@ -Subproject commit 671d7917c657ad185a06772e0b707b45fe59788a +Subproject commit 20128037dbfca217fa3d3ca4551cc7f8ae8a190e diff --git a/cope2n-api/Dockerfile b/cope2n-api/Dockerfile index c841ccb..bc603be 100755 --- a/cope2n-api/Dockerfile +++ b/cope2n-api/Dockerfile @@ -17,6 +17,7 @@ RUN pip install uvicorn gunicorn Celery RUN pip install pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116 RUN pip install -U openmim==0.3.7 --no-cache-dir RUN mim install mmcv-full==1.7.2 +RUN pip install fastdeploy-gpu-python==1.0.7 -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html --no-cache-dir # End intergration with sdskvu USER ${UID} diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index 6c751bf..97a7f5f 100755 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -16,7 +16,7 @@ import json from ..exception.exceptions import InvalidException, RequiredFieldException, NotFoundException from ..models import SubscriptionRequest, Report, ReportFile, SubscriptionRequestFile from ..utils.accuracy import shadow_report, MonthReportAccumulate, first_of_list, extract_report_detail_list, IterAvg -from ..utils.file import download_from_S3, dict2xlsx, save_report_to_S3, build_S3_url +from ..utils.file import download_from_S3, dict2xlsx, save_report_to_S3, build_S3_url, validate_review from ..utils.redis import RedisUtils from ..utils.process import string_to_boolean from ..utils.cache import get_cache, set_cache @@ -695,8 +695,10 @@ class AccuracyViewSet(viewsets.ViewSet): if not reviewed_result: reviewed_result = copy.deepcopy(sample_result) + reviewed_result["imei_number"] = [None for _ in range(subscription_request.doc_type.split(",").count("imei"))] if not feedback_result: feedback_result = copy.deepcopy(sample_result) + feedback_result["imei_number"] = [None for _ in range(subscription_request.doc_type.split(",").count("imei"))] if not predicted_result: predicted_result = copy.deepcopy(sample_result) @@ -726,8 +728,10 @@ class AccuracyViewSet(viewsets.ViewSet): if not reviewed_result: reviewed_result = copy.deepcopy(sample_result) + reviewed_result["imei_number"] = [None for _ in range(subscription_request.doc_type.split(",").count("imei"))] if not feedback_result: feedback_result = copy.deepcopy(sample_result) + feedback_result["imei_number"] = [None for _ in range(subscription_request.doc_type.split(",").count("imei"))] if not predicted_result: predicted_result = copy.deepcopy(sample_result) @@ -760,25 +764,22 @@ class AccuracyViewSet(viewsets.ViewSet): elif request.method == 'POST': data = request.data - base_query = Q(request_id=request_id) - subscription_request = SubscriptionRequest.objects.filter(base_query) if subscription_request.count() == 0: raise NotFoundException(excArgs=request_id) subscription_request = subscription_request.first() - subscription_request_files = SubscriptionRequestFile.objects.filter(request=subscription_request.id) if "reviewed_result" not in data: raise InvalidException(excArgs=f'reviewed_result') reviewed_result = data["reviewed_result"] - for field in ['retailername', 'sold_to_party', 'invoice_no', 'purchase_date', 'imei_number']: - if not field in reviewed_result.keys(): - raise RequiredFieldException(excArgs=f'reviewed_result.{field}') + if not subscription_request.predict_result: + raise InvalidException(excArgs=f'request_id') + validate_review(reviewed_result, len(subscription_request.predict_result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}, {}])[3].get("value", []))) reviewed_result['request_id'] = request_id for subscription_request_file in subscription_request_files: @@ -849,5 +850,7 @@ class AccuracyViewSet(viewsets.ViewSet): is_available = True if not is_available: raise NotFoundException(excArgs=request_id + "/" + request_image_id) + return JsonResponse({'message': 'success.'}, status=200) + else: return JsonResponse({'error': 'Invalid request method.'}, status=405) diff --git a/cope2n-api/fwd_api/management/commands/migrate-database-fill-invoice-no.py b/cope2n-api/fwd_api/management/commands/migrate-database-fill-invoice-no.py index 6f7077b..ace89de 100644 --- a/cope2n-api/fwd_api/management/commands/migrate-database-fill-invoice-no.py +++ b/cope2n-api/fwd_api/management/commands/migrate-database-fill-invoice-no.py @@ -36,7 +36,7 @@ class Command(BaseCommand): if value["label"] == "invoice_no": invoice_no_exited = True if not invoice_no_exited: - request.predict_result["content"]["document"][0]["content"].insert(2, {"label": "invoice_no","value": None}) + request.predict_result["content"]["document"][0]["content"].insert(4, {"label": "invoice_no","value": None}) if isinstance(request.feedback_result, dict): request.feedback_result[KEY] = None if isinstance(request.reviewed_result, dict): diff --git a/cope2n-api/fwd_api/management/commands/migrate-database-mark-requests.py b/cope2n-api/fwd_api/management/commands/migrate-database-mark-requests.py new file mode 100644 index 0000000..d1be5f1 --- /dev/null +++ b/cope2n-api/fwd_api/management/commands/migrate-database-mark-requests.py @@ -0,0 +1,79 @@ +# myapp/management/commands/mycustomcommand.py +from django.core.management.base import BaseCommand +from tqdm import tqdm +from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest +from fwd_api.exception.exceptions import InvalidException +from fwd_api.utils.accuracy import predict_result_to_ready +import traceback +import copy +from django.utils import timezone + +TEST_IMEI_VALUES = ["357822611219904", "RFAW2022FED", "5AWH14MT400396N", "0HU33NIW200044K", "0GJG4DBW200318X", "358975990917032", "350731691693549", "R52W70BHDWX", "R52W70BHDWX", "350073345090297", "0PBL3NHW500023N"] +TEST_RETAILER_VALUES = ["Best Denki", "Arrow Communication"] +KEY = "invoice_no" + +class Command(BaseCommand): + help = 'Refactor database for image level' + + def add_arguments(self, parser): + # Add your command-line arguments here + parser.add_argument('start', type=str, help='start date, sample: 2023-01-02T00:00:00+0700') + parser.add_argument('end', type=str, help='end date, sample: 2024-01-25T00:00:00+0800') + + def process_request(self, request, test_requests, supicious_requests): + if len(request.request_id.split(".")[0].split("_")) < 2: + return + + if not request.predict_result: + self.stdout.write(self.style.WARNING(f"Key predict_result not found in {request.request_id}")) + return + if request.predict_result.get("status", 200) != 200: + self.stdout.write(self.style.WARNING(f"Key predict_result not found in {request.request_id}")) + return + + imeis = [] + retailername = None + + for content in request.predict_result["content"]["document"][0]["content"]: + if content["label"] == "retailername": + retailername = content["value"] + elif content["label"] == "imei_number": + imeis = content["value"] + if retailername in TEST_RETAILER_VALUES and any(imei in TEST_IMEI_VALUES for imei in imeis): + request.is_test_request= True + request.save() + test_requests.append(request.request_id) + + elif any(imei in TEST_IMEI_VALUES for imei in imeis): + request.is_test_request= True + request.save() + test_requests.append(request.request_id) + + elif retailername in TEST_RETAILER_VALUES: + supicious_requests.append(request.request_id) + + def handle(self, *args, **options): + start = options['start'] + end = options['end'] + + test_requests = [] + supicious_requests = [] + + if start or end: + try: + start_date = timezone.datetime.strptime(start, '%Y-%m-%dT%H:%M:%S%z') # We care only about day precision only + end_date = timezone.datetime.strptime(end, '%Y-%m-%dT%H:%M:%S%z') + except Exception as e: + print(f"[INFO]: start: {start}") + print(f"[INFO]: end: {end}") + raise InvalidException(excArgs="Date format") + subcription_iter = SubscriptionRequest.objects.filter(created_at__range=(start_date, end_date)) + else: + subcription_iter = SubscriptionRequest.objects.all() + + for request in tqdm(subcription_iter.iterator()): + self.process_request(request, test_requests, supicious_requests) + self.stdout.write(self.style.SUCCESS('Sample Django management command executed successfully!')) + self.stdout.write(self.style.SUCCESS(f'Test request: \n {test_requests} \n =============================')) + self.stdout.write(self.style.SUCCESS(f'Supicious request: \n {supicious_requests} \n =============================')) + diff --git a/cope2n-api/fwd_api/management/commands/migrate-database-test-request-24-Feb.py b/cope2n-api/fwd_api/management/commands/migrate-database-test-request-24-Feb.py new file mode 100644 index 0000000..7a8cda3 --- /dev/null +++ b/cope2n-api/fwd_api/management/commands/migrate-database-test-request-24-Feb.py @@ -0,0 +1,49 @@ +# myapp/management/commands/mycustomcommand.py +from django.core.management.base import BaseCommand +from tqdm import tqdm +from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest +from fwd_api.exception.exceptions import InvalidException +from fwd_api.utils.accuracy import predict_result_to_ready +import traceback +import copy +from django.utils import timezone + +class Command(BaseCommand): + help = 'Refactor database for image level' + + def add_arguments(self, parser): + # Add your command-line arguments here + parser.add_argument('start', type=str, help='start date, sample: 2023-01-02T00:00:00+0700') + parser.add_argument('end', type=str, help='end date, sample: 2024-01-25T00:00:00+0800') + + def process_request(self, request): + # if not request.predict_result: + # self.stdout.write(self.style.WARNING(f"Key predict_result not found in {request.request_id}")) + # return + # if request.predict_result.get("status", 200) != 200: + # self.stdout.write(self.style.WARNING(f"Key predict_result not found in {request.request_id}")) + # return + + if not request.feedback_result or not request.redemption_id: + request.is_test_request= True + request.save() + + def handle(self, *args, **options): + start = options['start'] + end = options['end'] + + if start or end: + try: + start_date = timezone.datetime.strptime(start, '%Y-%m-%dT%H:%M:%S%z') # We care only about day precision only + end_date = timezone.datetime.strptime(end, '%Y-%m-%dT%H:%M:%S%z') + except Exception as e: + print(f"[INFO]: start: {start}") + print(f"[INFO]: end: {end}") + raise InvalidException(excArgs="Date format") + subcription_iter = SubscriptionRequest.objects.filter(created_at__range=(start_date, end_date)) + else: + subcription_iter = SubscriptionRequest.objects.all() + + for request in tqdm(subcription_iter.iterator()): + self.process_request(request) + self.stdout.write(self.style.SUCCESS('Sample Django management command executed successfully!')) \ No newline at end of file diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index 4aaaa21..6b31911 100755 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -536,7 +536,7 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True) "Request ID": report_file.correspond_request_id, "Redemption Number": report_file.correspond_redemption_id, "Image type": report_file.doc_type, - "IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])), + "IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])) if report_file.feedback_result else None, "IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])), "IMEI Revised": first_of_list(report_file.reviewed_result.get("imei_number", [None])) if report_file.reviewed_result else None, "IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])), @@ -544,11 +544,11 @@ def extract_report_detail_list(report_detail_list, lower=False, in_percent=True) "Invoice_Number_OCR": report_file.predict_result.get("invoice_no", None), "Invoice_Number Revised": report_file.reviewed_result.get("invoice_no", None) if report_file.reviewed_result else None, "Invoice_Number_Accuracy": first_of_list(report_file.feedback_accuracy.get("invoice_no", [None])), - "Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None), + "Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None) if report_file.feedback_result else None, "Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []), "Invoice_Purchase Date Revised": report_file.reviewed_result.get("purchase_date", None) if report_file.reviewed_result else None, "Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])), - "Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None), + "Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None) if report_file.feedback_result else None, "Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None), "Invoice_Retailer Revised": report_file.reviewed_result.get("retailername", None) if report_file.reviewed_result else None, "Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])), @@ -623,9 +623,9 @@ def predict_result_to_ready(result): return dict_result dict_result["retailername"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}])[0].get("value", None) dict_result["sold_to_party"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}])[1].get("value", None) - dict_result["invoice_no"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}])[2].get("value", None) - dict_result["purchase_date"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}])[3].get("value", []) - dict_result["imei_number"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}, {}])[4].get("value", []) + dict_result["purchase_date"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}])[2].get("value", []) + dict_result["imei_number"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}, {}])[3].get("value", []) + dict_result["invoice_no"] = result.get("content", {}).get("document", [{}])[0].get("content", [{}, {}, {}, {}, {}])[4].get("value", None) return dict_result def align_fine_result(ready_predict, fine_result): diff --git a/cope2n-api/fwd_api/utils/file.py b/cope2n-api/fwd_api/utils/file.py index fea47c0..fa2a1cd 100755 --- a/cope2n-api/fwd_api/utils/file.py +++ b/cope2n-api/fwd_api/utils/file.py @@ -80,6 +80,13 @@ def validate_feedback_file(csv_file_path): if missing_columns: raise RequiredColumnException(excArgs=str(missing_columns)) +def validate_review(review, num_imei): + for field in settings.FIELD: + if not field in review.keys(): + raise RequiredFieldException(excArgs=f'reviewed_result.{field}') + if not isinstance(review["imei_number"], list) or len(review["imei_number"]) != num_imei: + raise InvalidException(excArgs=f'imei_number') + def validate_list_file(files, max_file_num=settings.MAX_UPLOAD_FILES_IN_A_REQUEST, min_file_num=1, file_field="files"): total_file_size = 0 if len(files) < min_file_num: @@ -140,7 +147,7 @@ def get_folder_path(rq: SubscriptionRequest): logger = get_task_logger(__name__) request_id = str(rq.request_id) - logger.info(f"[DEBUG]: rq.process_type: {rq.process_type}") + logger.debug(f"rq.process_type: {rq.process_type}") p_type = ProcessUtil.map_process_type_to_folder_name(int(rq.process_type)) sub_id = str(rq.subscription.id) user_id = str(rq.subscription.user.id) diff --git a/cope2n-api/fwd_api/utils/sdsvkvu b/cope2n-api/fwd_api/utils/sdsvkvu index 671d791..2012803 160000 --- a/cope2n-api/fwd_api/utils/sdsvkvu +++ b/cope2n-api/fwd_api/utils/sdsvkvu @@ -1 +1 @@ -Subproject commit 671d7917c657ad185a06772e0b707b45fe59788a +Subproject commit 20128037dbfca217fa3d3ca4551cc7f8ae8a190e diff --git a/cope2n-fe/src/components/report-detail/report-overview-table.tsx b/cope2n-fe/src/components/report-detail/report-overview-table.tsx index dd39d00..5a8f55c 100644 --- a/cope2n-fe/src/components/report-detail/report-overview-table.tsx +++ b/cope2n-fe/src/components/report-detail/report-overview-table.tsx @@ -297,9 +297,9 @@ const ReportOverViewTable: React.FC = ({ successfulPercentage: item.images_quality.successful_percent, badNumber: item.images_quality.bad, badPercentage: item.images_quality.bad_percent, - snImeiAAR: item.average_accuracy_rate.imei, + snImeiAAR: item.average_accuracy_rate.imei_number, purchaseDateAAR: item.average_accuracy_rate.purchase_date, - retailerNameAAR: item.average_accuracy_rate.retailer_name, + retailerNameAAR: item.average_accuracy_rate.retailername, invoiceNumberAAR: item.average_accuracy_rate.invoice_no, snImeiAPT: item.average_processing_time.imei, invoiceAPT: item.average_processing_time.invoice, diff --git a/scripts/crawl_database.py b/scripts/crawl_database.py index 7cfacd5..1d5143b 100644 --- a/scripts/crawl_database.py +++ b/scripts/crawl_database.py @@ -8,7 +8,7 @@ from dotenv import load_dotenv load_dotenv("../.env_prod") -OUTPUT_NAME = "5Jan" +OUTPUT_NAME = "possible_test" # Database connection details db_host = os.environ.get('DB_HOST', "") @@ -26,69 +26,188 @@ secret_key = os.environ.get('S3_SECRET_KEY', "") # Request IDs for filtering request_ids = [ -'SAP_20240104082259_85c7f4dd262946d183dbec826fc6709e', -'SAP_20240104082709_c05319c56fd3422dbf133aee33fc3e10', -'SAP_20240104091512_23ae1a81f1314be0a27ebeae0e8fa0d7', -'SAP_20240104091512_23ae1a81f1314be0a27ebeae0e8fa0d7', -'SAP_20240104091816_025c90b9789246ed811772003622fa0d', -'SAP_20240104092541_5c71e535f07c4cc8803b45336ec70f77', -'SAP_20240104100259_5a667d33cb914e7ba5a4447b9e17d649', -'SAP_20240104101145_a7010bac159f47bc95d5866e6c5f5bdf', -'SAP_20240104105702_95252229252b4e238add117919ce882a', -'SAP_20240104112108_34b2cca84a42473ca77bc316e787fe2e', -'SAP_20240104114038_dd57ecf7982c4a5eaf1409f5ef050fab', -'SAP_20240104115942_1b77f411791940a4a85c838c2e9931ad', -'SAP_20240104120746_d63319f4cde343d894f9b89706756a9d', -'SAP_20240104123607_48d25c04fec6411dbf013c6a19054e77', -'SAP_20240104130957_ece21bad331b4f2cad0887693331aa3a', -'SAP_20240104131228_edebee4000ae4bd382feaea5d6c82031', -'SAP_20240104132641_97909efd013f45e89d83d36a5ea35c52', -'SAP_20240104133527_ad55f6ee667643ba8ae65e9ef1c32418', -'SAP_20240104134014_2d2cdbc1b06a44868ce1b32cdb53864f', -'SAP_20240104134425_9b37555ef8094153838e6048f7c63c9b', -'SAP_20240104134457_55a1cf1e371146d995c8849cc0ba7c7b', -'SAP_20240104134609_3f7d308e467d43dbb59a7bcc02e3a7d2', -'SAP_20240104134709_c708daf83f7e4aa69ab9696afe1a9081', -'SAP_20240104135007_44b7a30c5e9c41a0b8065ac4e7000223', -'SAP_20240104141547_7203ddb915274e99a08ae6e54ec49cbd', -'SAP_20240104141559_62fd19a6179248ecb4ff15b33338b294', -'SAP_20240104142352_68699cbe140f4264b858981a3ac67e40', -'SAP_20240104143937_801931cc1f344a4ca8384dfe13d1accc', -'SAP_20240104144730_3180a8919e604e26a188ce051465c392', -'SAP_20240104144933_3380f64019634769befed49e9a671bc6', -'SAP_20240104151239_76ae2f1d02444f7fabbc104eb77fe45f', -'SAP_20240104151243_61775c88685d434d98bb9fc7a9889b8e', -'SAP_20240104151243_61775c88685d434d98bb9fc7a9889b8e', -'SAP_20240104151243_61775c88685d434d98bb9fc7a9889b8e', -'SAP_20240104151638_a08a61448a58459a8f2209f64e54c213', -'SAP_20240104152030_479259e84c5b449499df2cb1023e91ac', -'SAP_20240104160108_a03634c80583454494b77efcdecbcc71', -'SAP_20240104160108_a03634c80583454494b77efcdecbcc71', -'SAP_20240104160311_e7cb02a11bbd4ea1906b3758e97f33ab', -'SAP_20240104161305_89c5518563224ab89345439dffd504a5', -'SAP_20240104161305_89c5518563224ab89345439dffd504a5', -'SAP_20240104164022_0b94af24db9d4ebe9af2086a4bd3cd7e', -'SAP_20240104170837_58165ec9f88d4e4aa3095ba3dda201d7', -'SAP_20240104171740_10279cfebbf344f184bbb429cb9a15ad', -'SAP_20240104175202_247892a4dc7f40f28eafac9c2ad85971', -'SAP_20240104180517_8ce7a1981dc743e08e09284fd904d536', -'SAP_20240104182034_406bac0ab0684727b9efb1bb9b422026', -'SAP_20240104182426_92a48bb4b85a4c3abb48e0d7cf727777', -'SAP_20240104183506_aa1fa7d6774a4509a142a6f4a7b5af29', -'SAP_20240104185716_f9d464e42c314370910913b37133e6c3', -'SAP_20240104190220_573244d03bb8408dbca422ff60eb527a', -'SAP_20240104191236_deedcc588b7b4928a950f7dc2ce4230c', -'SAP_20240104191236_deedcc588b7b4928a950f7dc2ce4230c', -'SAP_20240104192614_990bf10c38e144a7bf489548d356720e', -'SAP_20240104192614_990bf10c38e144a7bf489548d356720e', -'SAP_20240104212143_f8c1b4a6e6e443fcb5e882c7a5b917f3', -'SAP_20240104212924_ee1998a60d6848af9576292ac383037f', -'SAP_20240104214418_f8e1abf808c8499097ecddf014d401c7', -'SAP_20240104214619_8d27c05a9ce74b738b20195cb816bfbf', -'SAP_20240104215037_477863cdc0aa4d5fa1f05bbb0ae673ed', -'SAP_20240104221543_37605982df624324ad2594e268054361', -'SAP_20240104225026_acacd06ea6de4a738bc47683dc53f378', -'SAP_20240104235743_b48aa3e744ed428795171d84066adefe', +'SAP_20240105150247_cd45453e440c41068c8aa5ce804c6ac5', +'SAP_20240111145530_219e783311404e12bb8b14a26eec804b', +'SAP_20240111143618_a2bfed95f0624f65a668309be72251d4', +'SAP_20240206142539_9445f405c2a144f2adb43cb70fc66030', +'SAP_20240209133540_2e4edf0837e142dca7062d370d359116', +'SAP_20240209170013_0cfbd55689bc4bb18029883570987395', +'SAP_20240130145752_399dfa7936e34471b54571f7a425ced0', +'SAP_20240203101503_fa876d1a323b4199a5e3af7d56d0d0f1', +'SAP_20240206201610_411c4a42c6bc4a0bb3c23cf00a44bbf4', +'SAP_20240130151625_4c06ab2728304c9f873e3f6a8cb1cc6a', +'SAP_20240130153633_c911feda8307419dbe21420c2599bf12', +'SAP_20240203114112_13dcee9f08b244f6ab8440bca2993d8b', +'SAP_20240207213859_80b1fe039eca440696cb222ba7d58244', +'SAP_20240128114117_0a352b8f90034800bd460ec0b676bd71', +'SAP_20240130182943_385335a6c7d046b08b9ba19c9733c45f', +'SAP_20240130185043_4bc097bccaa243929b0e0f3e679607ff', +'SAP_20240130191730_1b0169695d724f4883021cf831e3fb80', +'SAP_20240127095627_ca5483e31c04418f9d8e27c75bf3017b', +'SAP_20240127080133_ba41c7301cb742c097576b7a033f714c', +'SAP_20240127080912_79b186e7075d4adbb2f36729909eafa7', +'SAP_20240127081511_5a1593167da84e6abc7699a3ad5a8917', +'SAP_20240202095744_c22254156de143f9b9288d12301015ae', +'SAP_20240127143437_b5f16f906fb3401f8d56233c6d357291', +'SAP_20240130203642_163aa3f319a340a4a4b118b99e708a0f', +'SAP_20240130221931_bf4245a4efb04f31b9b001c240d73bb5', +'SAP_20240130222513_b6b1a0b81b9c4be0a9f859c2d6758589', +'SAP_20240130223545_8e43ce0bb37b49a3bfc012d76c2861fd', +'SAP_20240130232330_a44faa31a2314e2ead7a0aabcc8f22c6', +'SAP_20240130233355_77f4d20c4b554b39bc1045c19213ec3b', +'SAP_20240130204736_0fbefc13ea8542289c2fa67d4f36f270', +'SAP_20240129194345_ff0578a42fd34ed0969d1b83abccb160', +'SAP_20240131094443_77ce08812b5045c497d9b2a05a23da7b', +'SAP_20240130210829_93c0bf0277cf41ac96ae9eadb31e61d4', +'SAP_20240131114352_5b01f9fe496f48bea8b1511ad29c8958', +'SAP_20240201113203_f3c964e2a44841ee88dab86b8bdd6c59', +'SAP_20240131135943_f5b89290a0e743bfa192a29a4dfba3d0', +'SAP_20240131141657_af0db06da4194bb596018135d0aad6ba', +'SAP_20240131153605_3f6f2431b0e04c979d2b102cba3a92d0', +'SAP_20240131162709_01b2db786d824f3abf3d51efe1f14906', +'SAP_20240131172119_f123f0f43c184fe5ab6ac94e87762d40', +'SAP_20240131175029_7c6b587a59624b62aad72325d64a27f5', +'SAP_20240206203220_6c815ef66d214635bc14e0affa425e22', +'SAP_20240130131509_edaadcc6e82b474ba6888662d64ae5f5', +'SAP_20240130183515_a7dc68c51cd14694a6a0190d5896af3a', +'SAP_20240131225503_9a91f1e995904b5cacf795e91d7de1f3', +'SAP_20240131080747_80c2b0701bc44ad2acce0e278069db9e', +'SAP_20240201085824_0b22f093f45c47abab7298c8567a0187', +'SAP_20240203155516_1f84eac8680841169fb11a202896181f', +'SAP_20240131130549_05ed8fb82d384e7e97daa66af70d11f5', +'SAP_20240201130640_0b9d4ac0c28b45f783110b39f5ec7d3f', +'SAP_20240131225955_7221e7854d1646ed914fdada7a1a88af', +'SAP_20240201000050_20963ca447cd4a0db15f2c29f74c7e39', +'SAP_20240201110455_f5179cbb464e42a0b8e6e119813f6469', +'SAP_20240201113407_6225cb07f56b4bd8bf4ec1e1b62df7db', +'SAP_20240206150912_82c8cfedd5ae4b55ab8bb4c1ec8b6303', +'SAP_20240201153138_115d71aba51340b09a057f7ca2403b0a', +'SAP_20240201173238_8858ef3098c94c619890dfaa2d2f1b0c', +'SAP_20240201214111_670687c1bc984e4caf5f06de8e3aaf36', +'SAP_20240201214838_d62c8bcfbafc410f98629bf28060ae34', +'SAP_20240207142000_0090e42e0bec4643acf2c4bdd311f6d0', +'SAP_20240201012315_b3c3a77e558b4b47b48a0c6756fdd1dc', +'SAP_20240201115010_21f021971f0a4dff861b343d7d1645f2', +'SAP_20240201163135_5495e7cf28fb4a1e838c55f587869706', +'SAP_20240201164329_05c40987df7f4451ab62e58dd1d05d4b', +'SAP_20240130193934_379de93d3501496e94bd02ac994a2d31', +'SAP_20240201215219_7c4c7cbc3636440f888c3d85cd1e17a7', +'SAP_20240202104415_e4389d6576c9405e8b2cc5d8392502be', +'SAP_20240202113205_fbae574e17f24cd2a3dafd51bd9279e2', +'SAP_20240206120308_7d1924951da347c4a4c336a6dc662b7d', +'SAP_20240203201702_677762cdb5bc484bbef3bd945355f2f3', +'SAP_20240201000252_d7d23365a3334f58bfe02d865b261532', +'SAP_20240201000514_97ef0eeeb538463ba1abcf1bfa66110c', +'SAP_20240201005704_21be390b7e6c43129fdb9cf7022f8c85', +'SAP_20240203000154_8ff7effa54264bd5ad551a6917b03cf0', +'SAP_20240203133112_12719d7ba12f42cca731877e3517d19f', +'SAP_20240201114044_3da4f899c4e241b59eea8ecd7ddeea55', +'SAP_20240201114519_cbc54d32f65f47eabe6d6ac4a6cbef3a', +'SAP_20240201121617_d294849ddb1d4a5b8acb26b35d1d7c56', +'SAP_20240201125936_70fcf9ea03ef4ef19b3e822e3e5b4ed0', +'SAP_20240204133921_28de5f72a5294ca88202d01874ddb898', +'SAP_20240204134555_6252a7f3d3584d1b9fef50f9f882bb78', +'SAP_20240204135801_696e5acff18f4fd99067b3d9cc13acc8', +'SAP_20240204184026_0095125a337a45399c6627443fb2809c', +'SAP_20240204202837_8ecf60184d714fa7b74352b301897d8e', +'SAP_20240204203505_edd7ac14d5f54529978ac4e1cac98275', +'SAP_20240204214531_59af28a5eb7a4a219e0327b5b5ac4198', +'SAP_20240202203759_efcbd8e5c4e14e4db1885f1a2b455698', +'SAP_20240207155502_8a8281d4e0e94e42921550248c1c9253', +'SAP_20240205104502_5d4d1435fa964d918db245bac7f29825', +'SAP_20240205122344_051e356d6e47434d81b117b3b716df7e', +'SAP_20240203220754_948aecccf29c4ee4b38bb31d1866f7c8', +'SAP_20240201231307_71dc1557e46248f2921f544c160a50bc', +'SAP_20240205142554_192372a311544fd2b0d8e12330d3e131', +'SAP_20240201231409_293633e97745457e80f7e22984b967ef', +'SAP_20240205152002_06515566f15041988718e1cc2279376f', +'SAP_20240202103100_3f2bf5c5397b4adeafe66369dfd406e1', +'SAP_20240204140541_04c3647483ca45dbbb2d3698aa2a2f9e', +'SAP_20240202114015_dab36ad3f6d94fc09360ced2e7303098', +'SAP_20240203214357_f1809ee448394419b45dc577b599d9f8', +'SAP_20240209131241_7042d1b45b3447999e713e92d08f275e', +'SAP_20240205105355_61830207b0874ba9a2e4bdb8b5d2f23f', +'SAP_20240130150814_bcbd1cb941244fd888e89de91910b0b9', +'SAP_20240205150922_92b3cd37ed414363846ef97eca6b767e', +'SAP_20240205104327_add42791a9e04769814f90a612ebe325', +'SAP_20240206142303_be619241683949da8130171c6ab944c7', +'SAP_20240205200647_2a4afa2cc24b4378a209553746920985', +'SAP_20240206005021_9635b064af604ff6b4f5780b9beb1c3f', +'SAP_20240208190424_97a6ec6e60204ed2a08af99057ed46bb', +'SAP_20240206010454_f70bbe6ec0d345dc9ecfd0a34db9cf0c', +'SAP_20240206142856_9293e55ecd28499580dc6267f909a282', +'SAP_20240207144719_b870eed54f3f42eeb3ef136aa91fec19', +'SAP_20240208104533_f8576e640af04ec8a139e469ca9745a5', +'SAP_20240206154445_b0464949fd7341ab91dc248e49f7ccd9', +'SAP_20240208224819_e5b10a142d4841dcb7f67827ccd2b9f6', +'SAP_20240115152747_d4d73d7dc03e4dbbbc8034dce7b45c1d', +'SAP_20240203133442_3b2b1af9cc2a469cae7fb84190e49e0f', +'SAP_20240207201452_142d0db8532248dcb863e6323375fa2e', +'SAP_20240207203540_50e31f5ba9094d88a707cf8e906c51cf', +'SAP_20240125114600_231b3c239d0f408fbcbc97a975b6a286', +'SAP_20240125134355_b575051032ce42a2bd9a4f7a0825a9ba', +'SAP_20240125145604_de6d55acae44488f94afe7ba49868c35', +'SAP_20240208222636_7a65cfc778fa4ff6be8a75042eac3549', +'SAP_20240208231626_8a39b7c36c6845a0ac264a08da4f596f', +'SAP_20240204135652_eacb604d6da1447ab6e0195419d4cbfb', +'SAP_20240207155340_6b30a07dd3ba41baa3d1902dcc1f37a6', +'SAP_20240205183046_711fb8da646c45bba4ce7789a7cfcd49', +'SAP_20240201112759_b6ec8e1c0b45495a90c22d5a9305ceb5', +'SAP_20240204184514_3e4b9b6e10cb4ce3af14f25e4873f009', +'SAP_20240205213153_e98fa3147643494b9e2de32e4c3382e3', +'SAP_20240205173852_f2b3b98fb28b43cdac2cab9bb35170a1', +'SAP_20240129153233_367de58c8b504d0bbc5b5d71a48442e1', +'SAP_20240206202807_bbfdd9dae29a42d59f5b69ca1d6c091f', +'SAP_20240203194418_ade51ce92c2a463fbec3ae75cbae6b0e', +'SAP_20240203214100_b245fb7d115a4842a77534084f0544c7', +'SAP_20240207142500_71cbd6cfdd984baca36e953904a09726', +'SAP_20240207143331_d583e47af987422ba07b76dcdda7cb57', +'SAP_20240125114657_582106f68a9a4c0a8a89beffa1a4ad2a', +'SAP_20240130073108_c2b438564c534798babe7fc7dc4dabc0', +'SAP_20240205122103_f62bce47060f4077875a2e891a31c3ac', +'SAP_20240131150338_c602f037b83d4862a92dd8f582246a08', +'SAP_20240207135750_c25b4a95ce434ad2ae195817c42bc332', +'SAP_20240128114401_c8ef9be4fe604f19bbed477e84b3bb0d', +'SAP_20240203183625_8a282ed896ea48d6b565c12edbb4149e', +'SAP_20240130093313_a273212ddc8741fea13c121cbfe56b41', +'SAP_20240130094904_13ccfd9311b34bc89e8f6f17e84a9823', +'SAP_20240129113741_a9ff76f80b77491790d8dcbb80e7404b', +'SAP_20240131212309_64f9623dfd0440ba862457a5ce43baa6', +'SAP_20240207141147_a5d526e312a1474caddbe81f2c9653e7', +'SAP_20240130131627_d22774c04918466b8eca12db9df9a4b0', +'SAP_20240130131732_cc8747a1754b4dadacd85d0f221d482c', +'SAP_20240201102500_b10ffe3e4d384eb48727e88b9a04cb69', +'SAP_20240201103923_cabc4189b8ed4e79b1dd5c0af1b719a0', +'SAP_20240130210435_5a330b99a3b3470e81fde0be7a87e21c', +'SAP_20240130220948_f580aad038c6449eace07e381b69d923', +'SAP_20240130082854_02c13ded93ba434c9ca7028805f7e870', +'SAP_20240208083548_ccfd1889cd6e4a6abbc85616ee50fd43', +'SAP_20240208141202_35ba5fb5480e4180b97109395fb82210', +'SAP_20240208165449_250444df8c91401ab7c6487969e10a87', +'SAP_20240204184848_4ed9674a9cff4b7d9798392655e7ad0c', +'SAP_20240208211623_52693b1ca6c34cd4a86096eeae6492c6', +'SAP_20240207140158_389feecc33af4280b1eaa6d2f1f1f3af', +'SAP_20240207140304_49f05af86b42464790ae752231ae74c9', +'SAP_20240129203245_4d7fbc485cbf448783ca7e5931c28544', +'SAP_20240207150318_9779a58bed9847f980d1d18e8bf0d1af', +'SAP_20240207151949_cb25425b6fd04e6f8f24d53d24d1d5e4', +'SAP_20240129234649_79af35670f8c4db8be1c40a1007a7df3', +'SAP_20240207135511_bfd85fcdb55947848cc566dc87f390cf', +'SAP_20240207143245_0e5ceb9737174404b0f48c9a012b485e', +'SAP_20240129194208_443d0c7e86404e33a10bb5a26a8b67a6', +'SAP_20240130083633_4ee6ab512a6f40af8771ec35d5a93293', +'SAP_20240131081559_a8344a059cdc4363ad641fa76185d6a6', +'SAP_20240207141507_01e78eea090649daaf1827053191acf6', +'SAP_20240207142012_3558b44105b44fe6a5e805803a9d5aad', +'SAP_20240130154409_78a36295728a4df28983b1b55432515d', +'SAP_20240207193406_ea2d379e900649aba8d066749537ac2d', +'SAP_20240207212515_c04427ed27af413298ef131956996a9c', +'SAP_20240205125127_cbe5503d09774b12904b0f9e0665a51e', +'SAP_20240209154036_3bac79b974974afc8f77189d7748935d', +'SAP_20240204135604_80a5109a5f5b46618ff1df1bee8dbe11', +'SAP_20240208211217_0db522e3a44145f9aa0ea6e855fbf38a', +'SAP_20240208105747_3117b91ff2f84b7ca6ebd80a800b42ea', +'SAP_20240204140735_371a26579ebc455f930a2ca638a3f7fd', ] # Connect to the PostgreSQL database diff --git a/test_samples/test_15/result.json b/test_samples/test_15/result.json index 8e54bf5..49a832a 100644 --- a/test_samples/test_15/result.json +++ b/test_samples/test_15/result.json @@ -17,6 +17,10 @@ "value": [ "357822611219904" ] + }, + { + "label": "invoice_no", + "value": null } ], "doc_type": "sbt_document", diff --git a/test_samples/test_17/result.json b/test_samples/test_17/result.json index 97f584e..8749e3d 100644 --- a/test_samples/test_17/result.json +++ b/test_samples/test_17/result.json @@ -17,6 +17,10 @@ "value": [ "RFAW2022FED" ] + }, + { + "label": "invoice_no", + "value": null } ], "doc_type": "sbt_document", diff --git a/test_samples/test_21.json b/test_samples/test_21.json index 9c9caeb..9e37991 100644 --- a/test_samples/test_21.json +++ b/test_samples/test_21.json @@ -19,6 +19,10 @@ "value": [ "357822611219904" ] + }, + { + "label": "invoice_no", + "value": null } ], "doc_type": "sbt_document", diff --git a/test_samples/test_22.json b/test_samples/test_22.json index 599c50a..ed417b5 100644 --- a/test_samples/test_22.json +++ b/test_samples/test_22.json @@ -20,6 +20,10 @@ "357822611219904", "RFAW2022FED" ] + }, + { + "label": "invoice_no", + "value": null } ], "doc_type": "sbt_document", diff --git a/test_samples/test_23.json b/test_samples/test_23.json index 9735e3b..d7b1146 100644 --- a/test_samples/test_23.json +++ b/test_samples/test_23.json @@ -21,6 +21,10 @@ "RFAW2022FED", "5AWH14MT400396N" ] + }, + { + "label": "invoice_no", + "value": null } ], "doc_type": "sbt_document", diff --git a/test_samples/test_24.json b/test_samples/test_24.json index 84f155c..fc4ad9b 100644 --- a/test_samples/test_24.json +++ b/test_samples/test_24.json @@ -22,6 +22,10 @@ "5AWH14MT400396N", "0HU33NIW200044K" ] + }, + { + "label": "invoice_no", + "value": null } ], "doc_type": "sbt_document", diff --git a/test_samples/test_25.json b/test_samples/test_25.json index 92a2439..06b2e7a 100644 --- a/test_samples/test_25.json +++ b/test_samples/test_25.json @@ -23,6 +23,10 @@ "0HU33NIW200044K", "0GJG4DBW200318X" ] + }, + { + "label": "invoice_no", + "value": null } ], "doc_type": "sbt_document", diff --git a/tests/test_set_07_01_invoice_06_imei.py b/tests/test_set_07_01_invoice_06_imei.py index 27b906c..6642199 100644 --- a/tests/test_set_07_01_invoice_06_imei.py +++ b/tests/test_set_07_01_invoice_06_imei.py @@ -76,6 +76,10 @@ def test_1_invoice_5_imei(): "0HU33NIW200044K", "0GJG4DBW200318X" ] + }, + { + "label": "invoice_no", + "value": None } ], "doc_type": "sbt_document",