tannedcung/sbt-idp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix #54

Browse Source
This commit is contained in:
dx-tan 2024-03-05 15:39:51 +07:00
parent 1d6bd417d9
commit 51623c3fa1
8 changed files with 66 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cope2n-api/fwd_api/api/accuracy_view.py
View File

@ -19,6 +19,7 @@ from ..utils.accuracy import shadow_report, MonthReportAccumulate, first_of_list
from ..utils.file import download_from_S3, dict2xlsx, save_report_to_S3, build_S3_url
from ..utils.redis import RedisUtils
from ..utils.process import string_to_boolean
from ..utils.cache import get_cache, set_cache
from ..request.ReportCreationSerializer import ReportCreationSerializer
from ..utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long
from ..utils.report import aggregate_overview
@ -467,7 +468,7 @@ class AccuracyViewSet(viewsets.ViewSet):
# Retrive data from Redis
key = f"{subsidiary}_{duration}"
data = json.loads(redis_client.get_specific_cache(settings.OVERVIEW_REPORT_ROOT, key)).get("data", [])
data = get_cache(key).get("data", [])
response = {
'overview_data': data,
}
@ -510,7 +511,7 @@ class AccuracyViewSet(viewsets.ViewSet):
for sub in subsidiaries_to_include:
key = f"{sub}_{duration}"
try:
this_overview = json.loads(redis_client.get_specific_cache(settings.OVERVIEW_REPORT_ROOT, key)).get("data", [])
this_overview = get_cache(key).get("data", [])
if sub != "seao":
this_overview = [d for d in this_overview if d.get("subs") != "+"]
else:
@ -551,7 +552,7 @@ class AccuracyViewSet(viewsets.ViewSet):
else:
# Retrive data from Redis
key = f"{subsidiary}_{duration}"
data = json.loads(redis_client.get_specific_cache(settings.OVERVIEW_REPORT_ROOT, key)).get("data", [])
data = get_cache(key).get("data", [])
response = {
'overview_data': data,
}

4
cope2n-api/fwd_api/celery_worker/process_report_tasks.py
View File

@ -7,6 +7,7 @@ from ..utils.accuracy import update_temp_accuracy, IterAvg, calculate_and_save_s
from ..utils.file import dict2xlsx, save_workbook_file, save_report_to_S3, save_images_to_csv_briefly
from ..utils import time_stuff
from ..utils.redis import RedisUtils
from ..utils.cache import set_cache, get_cache
from django.utils import timezone
from django.db.models import Q
import json
@ -315,7 +316,8 @@ def make_a_report_2(report_id, query_set):
overview_filename = query_set["subsidiary"] + "_" + query_set["report_overview_duration"] + ".xlsx"
local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT)
s3_key=save_report_to_S3(report.report_id, local_workbook)
redis_client.set_cache(settings.OVERVIEW_REPORT_ROOT, overview_filename.replace(".xlsx", ""), json.dumps(save_data))
# redis_client.set_cache(settings.OVERVIEW_REPORT_ROOT, overview_filename.replace(".xlsx", ""), json.dumps(save_data))
set_cache(overview_filename.replace(".xlsx", ""), save_data)
except IndexError as e:
print(e)

4
cope2n-api/fwd_api/management/commands/migrate-csv.py
View File

@ -1,5 +1,5 @@
# myapp/management/commands/mycustomcommand.py
from io import StringIO
from fwd_api.constant.common import FileCategory
from django.core.management.base import BaseCommand
from tqdm import tqdm
from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest
@ -37,7 +37,7 @@ class Command(BaseCommand):
"sold_to_party": ""
}
images = SubscriptionRequestFile.objects.filter(request=request)
images = SubscriptionRequestFile.objects.filter(request=request, file_category=FileCategory.Origin.value)
is_match = False
try:
for i, image in enumerate(images):

21
cope2n-api/fwd_api/migrations/0184_caching.py Normal file
View File

@ -0,0 +1,21 @@
# Generated by Django 4.1.3 on 2024-03-05 08:10
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('fwd_api', '0183_report_num_no_reviewed_report_num_not_reviewed_and_more'),
]
operations = [
migrations.CreateModel(
name='Caching',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
('key', models.CharField(max_length=200, unique=True)),
('value', models.JSONField(null=True)),
],
),
]

8
cope2n-api/fwd_api/models/Caching.py Normal file
View File

@ -0,0 +1,8 @@
from django.db import models
from django.utils import timezone
from fwd_api.models.Subscription import Subscription
class Caching(models.Model):
id = models.AutoField(primary_key=True)
key = models.CharField(max_length=200, unique=True) # Change to request_id
value = models.JSONField(null=True) # Change to request_id

3
cope2n-api/fwd_api/models/__init__.py
View File

@ -8,5 +8,4 @@ from .Subscription import Subscription
from .FeedbackRequest import FeedbackRequest
from .Report import Report
from .ReportFile import ReportFile
from .Caching import Caching

19
cope2n-api/fwd_api/utils/cache.py Normal file
View File

@ -0,0 +1,19 @@
from fwd_api.models import Caching
def set_cache(key, value):
cache = Caching.objects.filter(key=key)
if cache.count() == 0:
this_cache = Caching(key=key, value=value)
else:
this_cache = cache.first()
this_cache.value = value
this_cache.save()
return this_cache
def get_cache(key):
value = {}
cache = Caching.objects.filter(key=key)
if cache.count() > 0:
value = cache.first().value
return value

17
docker-compose-dev.yml
View File

@ -89,12 +89,12 @@ services:
depends_on:
db-sbt:
condition: service_started
command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input &&
python manage.py makemigrations &&
python manage.py migrate &&
python manage.py compilemessages &&
gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
# command: bash -c "tail -f > /dev/null"
# command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input &&
# python manage.py makemigrations &&
# python manage.py migrate &&
# python manage.py compilemessages &&
# gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod
command: bash -c "tail -f > /dev/null"
minio:
image: minio/minio
@ -179,8 +179,8 @@ services:
- ./cope2n-api:/app
working_dir: /app
command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
# command: bash -c "tail -f > /dev/null"
# command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5"
command: bash -c "tail -f > /dev/null"
# Back-end persistent
db-sbt:
@ -250,7 +250,6 @@ services:
networks:
- ctel-sbt
volumes:
db_data:
BE_static: