Fix #54

cope2n-api/fwd_api/api/accuracy_view.py

@@ -19,6 +19,7 @@ from ..utils.accuracy import shadow_report, MonthReportAccumulate, first_of_list
 from ..utils.file import download_from_S3, dict2xlsx, save_report_to_S3, build_S3_url
 from ..utils.redis import RedisUtils
 from ..utils.process import string_to_boolean
+from ..utils.cache import get_cache, set_cache
 from ..request.ReportCreationSerializer import ReportCreationSerializer
 from ..utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long
 from ..utils.report import aggregate_overview
@@ -467,7 +468,7 @@ class AccuracyViewSet(viewsets.ViewSet):
 
             # Retrive data from Redis
             key =  f"{subsidiary}_{duration}"
-            data = json.loads(redis_client.get_specific_cache(settings.OVERVIEW_REPORT_ROOT, key)).get("data", [])
+            data = get_cache(key).get("data", [])
             response = {
                 'overview_data': data,
             }
@@ -510,7 +511,7 @@ class AccuracyViewSet(viewsets.ViewSet):
                 for sub in subsidiaries_to_include:
                     key =  f"{sub}_{duration}"
                     try:
-                        this_overview = json.loads(redis_client.get_specific_cache(settings.OVERVIEW_REPORT_ROOT, key)).get("data", [])
+                        this_overview = get_cache(key).get("data", [])
                         if sub != "seao":
                             this_overview = [d for d in this_overview if d.get("subs") != "+"]
                         else:
@@ -551,7 +552,7 @@ class AccuracyViewSet(viewsets.ViewSet):
             else:
                 # Retrive data from Redis
                 key =  f"{subsidiary}_{duration}"
-                data = json.loads(redis_client.get_specific_cache(settings.OVERVIEW_REPORT_ROOT, key)).get("data", [])
+                data = get_cache(key).get("data", [])
             response = {
                 'overview_data': data,
             }

cope2n-api/fwd_api/celery_worker/process_report_tasks.py

@@ -7,6 +7,7 @@ from ..utils.accuracy import update_temp_accuracy, IterAvg, calculate_and_save_s
 from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3, save_images_to_csv_briefly
 from ..utils import time_stuff
 from ..utils.redis import RedisUtils
+from ..utils.cache import set_cache, get_cache
 from django.utils import timezone
 from django.db.models import Q
 import json
@@ -315,7 +316,8 @@ def make_a_report_2(report_id, query_set):
             overview_filename = query_set["subsidiary"] + "_" + query_set["report_overview_duration"] + ".xlsx"
             local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT)
             s3_key=save_report_to_S3(report.report_id, local_workbook)
-            redis_client.set_cache(settings.OVERVIEW_REPORT_ROOT, overview_filename.replace(".xlsx", ""), json.dumps(save_data)) 
+            # redis_client.set_cache(settings.OVERVIEW_REPORT_ROOT, overview_filename.replace(".xlsx", ""), json.dumps(save_data)) 
+            set_cache(overview_filename.replace(".xlsx", ""), save_data)
 
     except IndexError as e:
         print(e)

cope2n-api/fwd_api/management/commands/migrate-csv.py

@@ -1,5 +1,5 @@
 # myapp/management/commands/mycustomcommand.py
-from io import StringIO
+from fwd_api.constant.common import FileCategory
 from django.core.management.base import BaseCommand
 from tqdm import tqdm
 from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest
@@ -37,7 +37,7 @@ class Command(BaseCommand):
                 "sold_to_party": ""
             }
 
-        images = SubscriptionRequestFile.objects.filter(request=request)
+        images = SubscriptionRequestFile.objects.filter(request=request, file_category=FileCategory.Origin.value)
         is_match = False
         try:
             for i, image in enumerate(images):

cope2n-api/fwd_api/migrations/0184_caching.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.3 on 2024-03-05 08:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('fwd_api', '0183_report_num_no_reviewed_report_num_not_reviewed_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Caching',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('key', models.CharField(max_length=200, unique=True)),
+                ('value', models.JSONField(null=True)),
+            ],
+        ),
+    ]

cope2n-api/fwd_api/models/Caching.py

@@ -0,0 +1,8 @@
+from django.db import models
+from django.utils import timezone
+from fwd_api.models.Subscription import Subscription
+
+class Caching(models.Model):
+    id = models.AutoField(primary_key=True)
+    key = models.CharField(max_length=200, unique=True)  # Change to request_id
+    value = models.JSONField(null=True)  # Change to request_id

cope2n-api/fwd_api/models/__init__.py

@@ -8,5 +8,4 @@ from .Subscription import Subscription
 from .FeedbackRequest import FeedbackRequest
 from .Report import Report
 from .ReportFile import ReportFile
-
+from .Caching import Caching
-

cope2n-api/fwd_api/utils/cache.py

@@ -0,0 +1,19 @@
+from fwd_api.models import Caching
+
+def set_cache(key, value):
+    cache = Caching.objects.filter(key=key)
+    if cache.count() == 0:
+        this_cache = Caching(key=key, value=value)
+    else:
+        this_cache = cache.first()
+        this_cache.value = value
+    this_cache.save()
+    return this_cache
+    
+
+def get_cache(key):
+    value = {}
+    cache = Caching.objects.filter(key=key)
+    if cache.count() > 0:
+        value = cache.first().value
+    return value

docker-compose-dev.yml

@@ -89,12 +89,12 @@ services:
     depends_on:
       db-sbt:
         condition: service_started
-    command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input &&
+    # command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input &&
-                  python manage.py makemigrations &&
+    #               python manage.py makemigrations &&
-                  python manage.py migrate &&
+    #               python manage.py migrate &&
-                  python manage.py compilemessages &&
+    #               python manage.py compilemessages &&
-                  gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
+    #               gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
-    # command: bash -c "tail -f > /dev/null"
+    command: bash -c "tail -f > /dev/null"
 
   minio:
     image: minio/minio
@@ -179,8 +179,8 @@ services:
       - ./cope2n-api:/app
 
     working_dir: /app
-    command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
+    # command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
-    # command: bash -c "tail -f > /dev/null"
+    command: bash -c "tail -f > /dev/null"
 
   # Back-end persistent
   db-sbt:
@@ -250,7 +250,6 @@ services:
     networks:
       - ctel-sbt
 
-
 volumes:
   db_data:
   BE_static: