From 82fa5ced60bf568c287a8d82a1ea80b6ccef84df Mon Sep 17 00:00:00 2001 From: dx-tan Date: Fri, 29 Mar 2024 18:34:02 +0700 Subject: [PATCH 1/2] Add: sub to API --- cope2n-api/fwd_api/api/ctel_view.py | 4 ++++ cope2n-api/fwd_api/celery_worker/internal_task.py | 1 + cope2n-api/fwd_api/models/SubscriptionRequest.py | 1 + cope2n-api/fwd_api/models/SubscriptionRequestFile.py | 1 + cope2n-api/fwd_api/utils/process.py | 11 +++++++++++ scripts/crawl_database_by_time.py | 8 +++++--- 6 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cope2n-api/fwd_api/api/ctel_view.py b/cope2n-api/fwd_api/api/ctel_view.py index 593613c..e46e026 100755 --- a/cope2n-api/fwd_api/api/ctel_view.py +++ b/cope2n-api/fwd_api/api/ctel_view.py @@ -198,6 +198,9 @@ class CtelViewSet(viewsets.ViewSet): 'redemption_ID': { 'type': 'string' }, + 'subsidiary': { + 'type': 'string' + }, 'is_test_request': { 'type': 'boolean', }, @@ -245,6 +248,7 @@ class CtelViewSet(viewsets.ViewSet): provider_code=provider_code, subscription=sub, redemption_id=validated_data["redemption_ID"], + subsidiary=validated_data["subsidiary"], is_test_request=is_test_request) new_request.save() diff --git a/cope2n-api/fwd_api/celery_worker/internal_task.py b/cope2n-api/fwd_api/celery_worker/internal_task.py index d44c6d1..ec53777 100755 --- a/cope2n-api/fwd_api/celery_worker/internal_task.py +++ b/cope2n-api/fwd_api/celery_worker/internal_task.py @@ -222,6 +222,7 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files): file_meta["index_in_request"] = i file_meta["preprocessing_time"] = preprocessing_time file_meta["index_to_image_type"] = b_url["index_to_image_type"] + file_meta["subsidiary"] = new_request.subsidiary to_queue.append((fractorized_request_id, sub_id, [b_url], user_id, p_type, file_meta)) # Send to next queue diff --git a/cope2n-api/fwd_api/models/SubscriptionRequest.py b/cope2n-api/fwd_api/models/SubscriptionRequest.py index 9ca9ac2..871b2f0 100755 --- a/cope2n-api/fwd_api/models/SubscriptionRequest.py +++ b/cope2n-api/fwd_api/models/SubscriptionRequest.py @@ -36,3 +36,4 @@ class SubscriptionRequest(models.Model): total_memory = models.FloatField(default=-1) gpu_stats = models.CharField(max_length=100, null=True) is_reviewed = models.BooleanField(default=False) + subsidiary = models.CharField(default="", null=True, max_length=200) diff --git a/cope2n-api/fwd_api/models/SubscriptionRequestFile.py b/cope2n-api/fwd_api/models/SubscriptionRequestFile.py index 4055b48..cf873b4 100755 --- a/cope2n-api/fwd_api/models/SubscriptionRequestFile.py +++ b/cope2n-api/fwd_api/models/SubscriptionRequestFile.py @@ -25,6 +25,7 @@ class SubscriptionRequestFile(models.Model): processing_time = models.FloatField(default=-1) # in milisecond reason = models.TextField(blank=True) counter_measures = models.TextField(blank=True) + # subsidiary = models.CharField(default="", null=True, max_length=200) # save already in SubscriptionRequest predict_result = models.JSONField(null=True) feedback_result = models.JSONField(null=True) diff --git a/cope2n-api/fwd_api/utils/process.py b/cope2n-api/fwd_api/utils/process.py index c89b42f..9e02381 100644 --- a/cope2n-api/fwd_api/utils/process.py +++ b/cope2n-api/fwd_api/utils/process.py @@ -18,6 +18,7 @@ from fwd_api.exception.exceptions import NumberOfBoxLimitReachedException, \ ServiceUnavailableException, DuplicateEntityException, LimitReachedException, BadGatewayException from fwd_api.utils import date as DateUtil from fwd_api.utils import file as FileUtils +from fwd_api.utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long from ..constant.common import ProcessType, TEMPLATE_BOX_TYPE, EntityStatus from ..exception.exceptions import InvalidException, NotFoundException, \ PermissionDeniedException, RequiredFieldException, InvalidException, InvalidDecompressedSizeException @@ -144,6 +145,16 @@ def sbt_validate_ocr_request_and_get(request, subscription): validated_data['is_test_request'] = string_to_boolean(request.data.get('is_test_request', "false")) # print(f"[DEBUG]: is_test_request: ", validated_data['is_test_request']) + subsidiary = request.data.get("subsidiary", None) + valid_subs = list(settings.SUBS.keys())[:-2] # remove "ALL" and "SEAO" + # TODO: subsidiary will be a required field in the future + if not subsidiary: + validated_data['subsidiary'] = None + else: + if not subsidiary or subsidiary not in valid_subs: + raise InvalidException(excArgs="subsidiary") + validated_data['subsidiary'] = map_subsidiary_long_to_short(subsidiary) + return validated_data def string_to_boolean(value): diff --git a/scripts/crawl_database_by_time.py b/scripts/crawl_database_by_time.py index 4628592..2addcc8 100644 --- a/scripts/crawl_database_by_time.py +++ b/scripts/crawl_database_by_time.py @@ -13,9 +13,9 @@ load_dotenv("../.env_prod") tz = pytz.timezone('Asia/Singapore') -OUTPUT_NAME = "Feb29" -START_DATE = datetime(2024, 2, 29) -END_DATE = datetime(2024, 3, 1) +OUTPUT_NAME = "0303-0327" +START_DATE = datetime(2024, 3, 3) +END_DATE = datetime(2024, 3, 27) START_DATE = timezone.make_aware(START_DATE, tz) END_DATE = timezone.make_aware(END_DATE, tz) @@ -83,6 +83,8 @@ for request_id in tqdm(request_ids): folder_key = f"{s3_folder_prefix}/{request_id}/" # Assuming folder structure like: s3_bucket_name/s3_folder_prefix/request_id/ local_folder_path = f"{OUTPUT_NAME}/{request_id}/" # Path to the local folder to save the downloaded files os.makedirs(OUTPUT_NAME, exist_ok=True) + if os.path.exists(local_folder_path): + continue os.makedirs(local_folder_path, exist_ok=True) From ddc9c0fff77952e62c2c4457cea783f752ce0b12 Mon Sep 17 00:00:00 2001 From: dx-tan Date: Mon, 1 Apr 2024 15:42:12 +0700 Subject: [PATCH 2/2] Add: Database change --- .../0188_subscriptionrequest_subsidiary.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 cope2n-api/fwd_api/migrations/0188_subscriptionrequest_subsidiary.py diff --git a/cope2n-api/fwd_api/migrations/0188_subscriptionrequest_subsidiary.py b/cope2n-api/fwd_api/migrations/0188_subscriptionrequest_subsidiary.py new file mode 100644 index 0000000..1f75501 --- /dev/null +++ b/cope2n-api/fwd_api/migrations/0188_subscriptionrequest_subsidiary.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2024-04-01 08:32 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('fwd_api', '0187_report_s3_dashboard_file_name'), + ] + + operations = [ + migrations.AddField( + model_name='subscriptionrequest', + name='subsidiary', + field=models.CharField(default='', max_length=200, null=True), + ), + ]