Merge pull request #104 from SDSRV-IDP/dev/sub_API

Dev/sub api

cope2n-api/fwd_api/api/ctel_view.py

@@ -198,6 +198,9 @@ class CtelViewSet(viewsets.ViewSet):
                 'redemption_ID': {
                     'type': 'string'
                 },
+                'subsidiary': {
+                    'type': 'string'
+                },
                 'is_test_request': {
                     'type': 'boolean',
                 },
@@ -245,6 +248,7 @@ class CtelViewSet(viewsets.ViewSet):
                                                             provider_code=provider_code,
                                                             subscription=sub,
                                                             redemption_id=validated_data["redemption_ID"],
+                                                            subsidiary=validated_data["subsidiary"],
                                                             is_test_request=is_test_request)
         new_request.save()
 

cope2n-api/fwd_api/celery_worker/internal_task.py

@@ -222,6 +222,7 @@ def process_pdf(rq_id, sub_id, p_type, user_id, files):
         file_meta["index_in_request"] = i
         file_meta["preprocessing_time"] = preprocessing_time
         file_meta["index_to_image_type"] = b_url["index_to_image_type"]
+        file_meta["subsidiary"] = new_request.subsidiary
         to_queue.append((fractorized_request_id, sub_id, [b_url], user_id, p_type, file_meta))
 
     # Send to next queue

cope2n-api/fwd_api/migrations/0188_subscriptionrequest_subsidiary.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.3 on 2024-04-01 08:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0187_report_s3_dashboard_file_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='subscriptionrequest',
+            name='subsidiary',
+            field=models.CharField(default='', max_length=200, null=True),
+        ),
+    ]

cope2n-api/fwd_api/models/SubscriptionRequest.py

@@ -36,3 +36,4 @@ class SubscriptionRequest(models.Model):
     total_memory = models.FloatField(default=-1)
     gpu_stats = models.CharField(max_length=100, null=True)
     is_reviewed = models.BooleanField(default=False)
+    subsidiary = models.CharField(default="", null=True, max_length=200)

cope2n-api/fwd_api/models/SubscriptionRequestFile.py

@@ -25,6 +25,7 @@ class SubscriptionRequestFile(models.Model):
     processing_time = models.FloatField(default=-1) # in milisecond
     reason = models.TextField(blank=True)
     counter_measures = models.TextField(blank=True)
+    # subsidiary = models.CharField(default="", null=True, max_length=200) # save already in SubscriptionRequest
 
     predict_result = models.JSONField(null=True)
     feedback_result = models.JSONField(null=True)

cope2n-api/fwd_api/utils/process.py

@@ -18,6 +18,7 @@ from fwd_api.exception.exceptions import NumberOfBoxLimitReachedException, \
     ServiceUnavailableException, DuplicateEntityException, LimitReachedException, BadGatewayException
 from fwd_api.utils import date as DateUtil
 from fwd_api.utils import file as FileUtils
+from fwd_api.utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long
 from ..constant.common import ProcessType, TEMPLATE_BOX_TYPE, EntityStatus
 from ..exception.exceptions import InvalidException, NotFoundException, \
     PermissionDeniedException, RequiredFieldException, InvalidException, InvalidDecompressedSizeException
@@ -144,6 +145,16 @@ def sbt_validate_ocr_request_and_get(request, subscription):
     validated_data['is_test_request'] = string_to_boolean(request.data.get('is_test_request', "false"))
     # print(f"[DEBUG]: is_test_request: ", validated_data['is_test_request'])
 
+    subsidiary = request.data.get("subsidiary", None)
+    valid_subs = list(settings.SUBS.keys())[:-2] # remove "ALL" and "SEAO"
+    # TODO: subsidiary will be a required field in the future
+    if not subsidiary:
+        validated_data['subsidiary'] = None
+    else:
+        if not subsidiary or subsidiary not in valid_subs:
+            raise InvalidException(excArgs="subsidiary")
+        validated_data['subsidiary'] = map_subsidiary_long_to_short(subsidiary)
+
     return validated_data
 
 def string_to_boolean(value):

scripts/crawl_database_by_time.py

@@ -13,9 +13,9 @@ load_dotenv("../.env_prod")
 
 tz = pytz.timezone('Asia/Singapore')
 
-OUTPUT_NAME = "Feb29"
-START_DATE = datetime(2024, 2, 29)
-END_DATE = datetime(2024, 3, 1)
+OUTPUT_NAME = "0303-0327"
+START_DATE = datetime(2024, 3, 3)
+END_DATE = datetime(2024, 3, 27)
 START_DATE = timezone.make_aware(START_DATE, tz)
 END_DATE = timezone.make_aware(END_DATE, tz)
 
@@ -83,6 +83,8 @@ for request_id in tqdm(request_ids):
     folder_key = f"{s3_folder_prefix}/{request_id}/"  # Assuming folder structure like: s3_bucket_name/s3_folder_prefix/request_id/
     local_folder_path = f"{OUTPUT_NAME}/{request_id}/"  # Path to the local folder to save the downloaded files
     os.makedirs(OUTPUT_NAME, exist_ok=True)
+    if os.path.exists(local_folder_path):
+        continue
     os.makedirs(local_folder_path, exist_ok=True)