from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.response import Response from django.core.paginator import Paginator from django.http import JsonResponse from django.utils import timezone from django.db.models import Q import uuid from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiTypes # from drf_spectacular.types import OpenApiString import json from ..exception.exceptions import InvalidException, RequiredFieldException from ..models import SubscriptionRequest, Report, ReportFile from ..utils.accuracy import shadow_report, MonthReportAccumulate from ..utils.file import validate_report_list from ..utils.process import string_to_boolean def first_of_list(the_list): if not the_list: return None return the_list[0] class AccuracyViewSet(viewsets.ViewSet): lookup_field = "username" @extend_schema( parameters=[ OpenApiParameter( name='start_date', location=OpenApiParameter.QUERY, description='Start date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2023-01-02T00:00:00+0700', ), OpenApiParameter( name='end_date', location=OpenApiParameter.QUERY, description='End date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2024-01-10T00:00:00+0700', ), OpenApiParameter( name='include_test', location=OpenApiParameter.QUERY, description='Whether to include test record or not', type=OpenApiTypes.BOOL, ), OpenApiParameter( name='is_reviewed', location=OpenApiParameter.QUERY, description='Which records to be query', type=OpenApiTypes.STR, enum=['reviewed', 'not reviewed', 'all'], ), OpenApiParameter( name='request_id', location=OpenApiParameter.QUERY, description='Specific request id', type=OpenApiTypes.STR, ), OpenApiParameter( name='redemption_id', location=OpenApiParameter.QUERY, description='Specific redemption id', type=OpenApiTypes.STR, ), OpenApiParameter( name='page', location=OpenApiParameter.QUERY, description='Page number', type=OpenApiTypes.INT, required=False ), OpenApiParameter( name='page_size', location=OpenApiParameter.QUERY, description='Number of items per page', type=OpenApiTypes.INT, required=False ), ], responses=None, tags=['Accuracy'] ) @action(detail=False, url_path="request_list", methods=["GET"]) def get_request_list(self, request): if request.method == 'GET': start_date_str = request.GET.get('start_date') end_date_str = request.GET.get('end_date') page_number = int(request.GET.get('page', 1)) page_size = int(request.GET.get('page_size', 10)) request_id = request.GET.get('request_id', None) redemption_id = request.GET.get('redemption_id', None) is_reviewed = request.GET.get('is_reviewed', None) include_test = request.GET.get('include_test', False) try: start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%dT%H:%M:%S%z') end_date = timezone.datetime.strptime(end_date_str, '%Y-%m-%dT%H:%M:%S%z') except ValueError: raise InvalidException(excArgs="Date format") base_query = Q(created_at__range=(start_date, end_date)) if request_id: base_query &= Q(request_id=request_id) if redemption_id: base_query &= Q(redemption_id=redemption_id) base_query &= Q(is_test_request=False) if isinstance(include_test, str): include_test = True if include_test=="true" else False if include_test: # base_query = ~base_query base_query.children = base_query.children[:-1] elif isinstance(include_test, bool): if include_test: base_query = ~base_query if isinstance(is_reviewed, str): if is_reviewed == "reviewed": base_query &= Q(is_reviewed=True) elif is_reviewed == "not reviewed": base_query &= Q(is_reviewed=False) elif is_reviewed == "all": pass subscription_requests = SubscriptionRequest.objects.filter(base_query).order_by('created_at') paginator = Paginator(subscription_requests, page_size) page = paginator.get_page(page_number) data = [] for request in page: imeis = [] purchase_date = [] retailer = "" try: if request.reviewed_result is not None: imeis = request.reviewed_result.get("imei_number", []) purchase_date = request.reviewed_result.get("purchase_date", []) retailer = request.reviewed_result.get("retailername", "") elif request.feedback_result is not None : imeis = request.feedback_result.get("imei_number", []) purchase_date = request.feedback_result.get("purchase_date", []) retailer = request.feedback_result.get("retailername", "") elif request.predict_result is not None: if request.predict_result.get("status", 404) == 200: imeis = request.predict_result.get("content", {}).get("document", [])[0].get("content", [])[3].get("value", []) purchase_date = request.predict_result.get("content", {}).get("document", [])[0].get("content", [])[2].get("value", []) retailer = request.predict_result.get("content", {}).get("document", [])[0].get("content", [])[0].get("value", []) except Exception as e: print(f"[ERROR]: {e}") print(f"[ERROR]: {request}") data.append({ 'RequestID': request.request_id, 'RedemptionID': request.redemption_id, 'IMEIs': imeis, 'Purchase Date': purchase_date, 'Retailer': retailer, 'Client Request Time (ms)': request.client_request_time, 'Server Processing Time (ms)': request.preprocessing_time + request.ai_inference_time, 'Is Reviewed': request.is_reviewed, # 'Is Bad Quality': request.is_bad_image_quality, 'created_at': request.created_at.isoformat() }) response = { 'subscription_requests': data, 'page': { 'number': page.number, 'total_pages': page.paginator.num_pages, 'count': page.paginator.count, } } return JsonResponse(response) return JsonResponse({'error': 'Invalid request method.'}, status=405) @extend_schema( parameters=[ OpenApiParameter( name='is_daily_report', location=OpenApiParameter.QUERY, description='Whether to include test record or not', type=OpenApiTypes.BOOL, ), OpenApiParameter( name='start_date', location=OpenApiParameter.QUERY, description='Start date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2023-01-02T00:00:00+0700', ), OpenApiParameter( name='end_date', location=OpenApiParameter.QUERY, description='End date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2024-01-10T00:00:00+0700', ), OpenApiParameter( name='include_test', location=OpenApiParameter.QUERY, description='Whether to include test record or not', type=OpenApiTypes.BOOL, ), OpenApiParameter( name='is_reviewed', location=OpenApiParameter.QUERY, description='Which records to be query', type=OpenApiTypes.STR, enum=['reviewed', 'not reviewed', 'all'], ), OpenApiParameter( name='request_id', location=OpenApiParameter.QUERY, description='Specific request id', type=OpenApiTypes.STR, ), OpenApiParameter( name='redemption_id', location=OpenApiParameter.QUERY, description='Specific redemption id', type=OpenApiTypes.STR, ), OpenApiParameter( name='subsidiary', location=OpenApiParameter.QUERY, description='Subsidiary', type=OpenApiTypes.STR, ), ], responses=None, tags=['Accuracy'] ) @action(detail=False, url_path="make_report", methods=["GET"]) def make_report(self, request): if request.method == 'GET': start_date_str = request.GET.get('start_date') end_date_str = request.GET.get('end_date') request_id = request.GET.get('request_id', None) redemption_id = request.GET.get('redemption_id', None) is_reviewed = string_to_boolean(request.data.get('is_reviewed', "false")) include_test = string_to_boolean(request.data.get('include_test', "false")) subsidiary = request.GET.get("subsidiary", "all") is_daily_report = string_to_boolean(request.data.get('is_daily_report', "false")) try: start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%dT%H:%M:%S%z') end_date = timezone.datetime.strptime(end_date_str, '%Y-%m-%dT%H:%M:%S%z') except ValueError: raise InvalidException(excArgs="Date format") query_set = {"start_date_str": start_date_str, "end_date_str": end_date_str, "request_id": request_id, "redemption_id": redemption_id, "is_reviewed": is_reviewed, "include_test": include_test, "subsidiary": subsidiary, "is_daily_report": is_daily_report, } report_id = "report" + "_" + timezone.datetime.now().strftime("%Y%m%d%H%M%S%z") + "_" + uuid.uuid4().hex new_report: Report = Report( report_id=report_id, is_daily_report=is_daily_report, subsidiary=subsidiary.lower().replace(" ", ""), include_test=include_test, include_reviewed=is_reviewed, start_at=start_date, end_at=end_date, ) new_report.save() # Background job to calculate accuracy shadow_report(report_id, query_set) return JsonResponse(status=status.HTTP_200_OK, data={"report_id": report_id}) @extend_schema( parameters=[ OpenApiParameter( name='report_id', location=OpenApiParameter.QUERY, description='Specific report id', type=OpenApiTypes.STR, ), OpenApiParameter( name='page', location=OpenApiParameter.QUERY, description='Page number', type=OpenApiTypes.INT, required=False ), OpenApiParameter( name='page_size', location=OpenApiParameter.QUERY, description='Number of items per page', type=OpenApiTypes.INT, required=False ), ], responses=None, tags=['Accuracy'] ) @action(detail=False, url_path="report_detail_list", methods=["GET"]) def get_report_detail_list(self, request): if request.method == 'GET': report_id = request.GET.get('report_id', None) page_number = int(request.GET.get('page', 1)) page_size = int(request.GET.get('page_size', 10)) report = Report.objects.filter(report_id=report_id).first() report_files = ReportFile.objects.filter(report=report) paginator = Paginator(report_files, page_size) page = paginator.get_page(page_number) data = [] for report_file in page: data.append({ "Request ID": report_file.correspond_request_id, "Redemption Number": report_file.correspond_redemption_id, "Image type": report_file.doc_type, "IMEI_user submitted": first_of_list(report_file.feedback_result.get("imei_number", [None])), "IMEI_OCR retrieved": first_of_list(report_file.predict_result.get("imei_number", [None])), "IMEI1 Accuracy": first_of_list(report_file.feedback_accuracy.get("imei_number", [None])), "Invoice_Purchase Date_Consumer": report_file.feedback_result.get("purchase_date", None), "Invoice_Purchase Date_OCR": report_file.predict_result.get("purchase_date", []), "Invoice_Purchase Date Accuracy": first_of_list(report_file.feedback_accuracy.get("purchase_date", [None])), "Invoice_Retailer_Consumer": report_file.feedback_result.get("retailername", None), "Invoice_Retailer_OCR": report_file.predict_result.get("retailername", None), "Invoice_Retailer Accuracy": first_of_list(report_file.feedback_accuracy.get("retailername", [None])), "OCR Image Accuracy": report_file.acc, "OCR Image Speed (seconds)": report_file.time_cost, "Reviewed?": "No", "Bad Image Reasons": report_file.bad_image_reason, "Countermeasures": report_file.counter_measures, "IMEI_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("imei_number", [None])), "Purchase Date_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("purchase_date", [None])), "Retailer_Revised Accuracy": first_of_list(report_file.reviewed_accuracy.get("retailername", [None])), }) response = { 'report_detail': data, 'page': { 'number': page.number, 'total_pages': page.paginator.num_pages, 'count': page.paginator.count, } } return JsonResponse(response, status=200) return JsonResponse({'error': 'Invalid request method.'}, status=405) @extend_schema( parameters=[ OpenApiParameter( name='start_date', location=OpenApiParameter.QUERY, description='Start date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2023-01-02T00:00:00+0700', ), OpenApiParameter( name='end_date', location=OpenApiParameter.QUERY, description='End date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2024-01-10T00:00:00+0700', ), OpenApiParameter( name='daily_report_only', location=OpenApiParameter.QUERY, description='Specific report id', type=OpenApiTypes.BOOL, ), OpenApiParameter( name='page', location=OpenApiParameter.QUERY, description='Page number', type=OpenApiTypes.INT, required=False ), OpenApiParameter( name='page_size', location=OpenApiParameter.QUERY, description='Number of items per page', type=OpenApiTypes.INT, required=False ), ], responses=None, tags=['Accuracy'] ) @action(detail=False, url_path="report_list", methods=["GET"]) def get_report_list(self, request): if request.method == 'GET': daily_report_only = request.GET.get('daily_report_only', False) start_date_str = request.GET.get('start_date', "") end_date_str = request.GET.get('end_date', "") page_number = int(request.GET.get('page', 1)) page_size = int(request.GET.get('page_size', 10)) if not start_date_str or not end_date_str: reports = Report.objects.all() else: try: start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%dT%H:%M:%S%z') end_date = timezone.datetime.strptime(end_date_str, '%Y-%m-%dT%H:%M:%S%z') except ValueError: raise InvalidException(excArgs="Date format") base_query = Q(created_at__range=(start_date, end_date)) if daily_report_only: base_query &= Q(is_daily_report=True) reports = Report.objects.filter(base_query).order_by('created_at') paginator = Paginator(reports, page_size) page = paginator.get_page(page_number) data = [] for report in page: data.append({ "ID": report.id, "Created Date": report.created_at, "No. Requests": report.number_request, "Status": report.status, "Purchase Date Acc": report.reviewed_accuracy.get("purchase_date", None) if report.reviewed_accuracy else None, "Retailer Acc": report.feedback_accuracy.get("retailername", None) if report.reviewed_accuracy else None, "IMEI Acc": report.feedback_accuracy.get("imei_number", None) if report.reviewed_accuracy else None, "Avg. Accuracy": report.feedback_accuracy.get("avg", None) if report.reviewed_accuracy else None, "Avg. Client Request Time": report.average_client_time.get("avg", 0) if report.average_client_time else 0, "Avg. OCR Processing Time": report.average_OCR_time.get("avg", 0) if report.average_client_time else 0, "report_id": report.report_id, }) response = { 'report_detail': data, 'page': { 'number': page.number, 'total_pages': page.paginator.num_pages, 'count': page.paginator.count, } } return JsonResponse(response, status=200) return JsonResponse({'error': 'Invalid request method.'}, status=405) @extend_schema( parameters=[ OpenApiParameter( name='start_date', location=OpenApiParameter.QUERY, description='Start date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2023-01-02T00:00:00+0700', ), OpenApiParameter( name='end_date', location=OpenApiParameter.QUERY, description='End date (YYYY-mm-DDTHH:MM:SSZ)', type=OpenApiTypes.DATE, default='2024-01-10T00:00:00+0700', ), OpenApiParameter( name='subsidiary', location=OpenApiParameter.QUERY, description='Subsidiary', type=OpenApiTypes.STR, ), OpenApiParameter( name='page', location=OpenApiParameter.QUERY, description='Page number', type=OpenApiTypes.INT, required=False ), OpenApiParameter( name='page_size', location=OpenApiParameter.QUERY, description='Number of items per page', type=OpenApiTypes.INT, required=False ), ], responses=None, tags=['Accuracy'] ) @action(detail=False, url_path="overview", methods=["GET"]) def overview(self, request): if request.method == 'GET': subsidiary = request.GET.get('subsidiary', None) start_date_str = request.GET.get('start_date', "") end_date_str = request.GET.get('end_date', "") page_number = int(request.GET.get('page', 1)) page_size = int(request.GET.get('page_size', 10)) if not start_date_str or not end_date_str: reports = Report.objects.all() else: try: start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%dT%H:%M:%S%z') end_date = timezone.datetime.strptime(end_date_str, '%Y-%m-%dT%H:%M:%S%z') except ValueError: raise InvalidException(excArgs="Date format") base_query = Q(created_at__range=(start_date, end_date)) if subsidiary: base_query &= Q(subsidiary=subsidiary) base_query &= Q(is_daily_report=True) reports = Report.objects.filter(base_query).order_by('created_at') paginator = Paginator(reports, page_size) page = paginator.get_page(page_number) data = [] this_month_report = MonthReportAccumulate() for report in page: res = this_month_report.add(report) if not(res): _, _data, total = this_month_report() data += [total] data += _data this_month_report = MonthReportAccumulate() this_month_report.add(report) else: continue _, _data, total = this_month_report() data += [total] data += _data response = { 'overview_data': data, 'page': { 'number': page.number, 'total_pages': page.paginator.num_pages, 'count': page.paginator.count, } } return JsonResponse(response, status=200) return JsonResponse({'error': 'Invalid request method.'}, status=405) class RequestViewSet(viewsets.ViewSet): lookup_field = "username" @extend_schema(request = { 'multipart/form-data': { 'type': 'object', 'properties': { 'reviewed_result': { 'type': 'string', }, } }, }, responses=None, tags=['Request'] ) @action(detail=False, url_path=r"request/(?P[\w\-]+)", methods=["GET", "POST"]) def get_subscription_request(self, request, request_id=None): if request.method == 'GET': base_query = Q(request_id=request_id) subscription_request = SubscriptionRequest.objects.filter(base_query).first() data = [] imeis = [] purchase_date = [] retailer = "" try: if subscription_request.reviewed_result is not None: imeis = subscription_request.reviewed_result.get("imei_number", []) purchase_date = subscription_request.reviewed_result.get("purchase_date", []) retailer = subscription_request.reviewed_result.get("retailername", "") elif subscription_request.feedback_result is not None : imeis = subscription_request.feedback_result.get("imei_number", []) purchase_date = subscription_request.feedback_result.get("purchase_date", []) retailer = subscription_request.feedback_result.get("retailername", "") elif subscription_request.predict_result is not None: if subscription_request.predict_result.get("status", 404) == 200: imeis = subscription_request.predict_result.get("content", {}).get("document", [])[0].get("content", [])[3].get("value", []) purchase_date = subscription_request.predict_result.get("content", {}).get("document", [])[0].get("content", [])[2].get("value", []) retailer = subscription_request.predict_result.get("content", {}).get("document", [])[0].get("content", [])[0].get("value", []) except Exception as e: print(f"[ERROR]: {e}") print(f"[ERROR]: {subscription_request}") data.append({ 'RequestID': subscription_request.request_id, 'RedemptionID': subscription_request.redemption_id, 'IMEIs': imeis, 'Purchase Date': purchase_date, 'Retailer': retailer, 'Reviewed result': subscription_request.reviewed_result, 'Feedback result': subscription_request.feedback_result, 'Client Request Time (ms)': subscription_request.client_request_time, 'Server Processing Time (ms)': subscription_request.preprocessing_time + subscription_request.ai_inference_time, 'Is Reviewed': subscription_request.is_reviewed, # 'Is Bad Quality': subscription_request.is_bad_image_quality, 'created_at': subscription_request.created_at.isoformat(), 'updated_at': subscription_request.updated_at.isoformat() }) response = { 'subscription_requests': data } return JsonResponse(response) elif request.method == 'POST': data = request.data base_query = Q(request_id=request_id) subscription_request = SubscriptionRequest.objects.filter(base_query).first() reviewed_result = json.loads(data["reviewed_result"][1:-1]) for field in ['retailername', 'sold_to_party', 'purchase_date', 'imei_number']: if not field in reviewed_result.keys(): raise RequiredFieldException(excArgs=f'reviewed_result.{field}') subscription_request.reviewed_result = reviewed_result subscription_request.reviewed_result['request_id'] = request_id subscription_request.is_reviewed = True subscription_request.save() return JsonResponse({'message': 'success.'}, status=200) else: return JsonResponse({'error': 'Invalid request method.'}, status=405)