# myapp/management/commands/mycustomcommand.py from django.core.management.base import BaseCommand from tqdm import tqdm from fwd_api.models import SubscriptionRequestFile, SubscriptionRequest from fwd_api.exception.exceptions import InvalidException from fwd_api.utils.accuracy import predict_result_to_ready import traceback import copy from django.utils import timezone RETAILER_LIST = [ '', 'Amazon.sg', 'Gain City (TV/AV)', 'Harvey Norman (TV/AV)', 'KRIS SHOP', 'Lazada (Samsung Brand Store)', 'M1 Shop', 'Mohamed Mustafa & Samsuddin CO (TV/AV)', 'Parisilk (TV/AV)', 'Shopee (Samsung Brand Store)', 'StarHub Shop', 'M1 Shop', 'Ectri', 'Whandising', '3 Mobile', 'Mister Mobile', 'Lazada', 'Mister Mobile', 'Samsung Experience Store', 'A & Samsuddin Co.', 'Parisilk', 'Samsung Brand Store', 'Shopee', 'M1 Shop', 'Onephone Online', '3 Mobile', 'Samsung Experience Store', 'Challenger', 'Eas Marketing', 'Ing Mobile', 'Starhub Shop', 'Mister Mobile', 'Onephone Online', 'Starho' ] RETAILER_LIST = list(set(RETAILER_LIST)) class Command(BaseCommand): help = 'Refactor database for image level' def add_arguments(self, parser): # Add your command-line arguments here parser.add_argument('start', type=str, help='start date, sample: 2023-01-02T00:00:00+0700') parser.add_argument('end', type=str, help='end date, sample: 2023-01-03T00:00:00+0700') def process_request(self, request): if len(request.request_id.split(".")[0].split("_")) < 2: return images = SubscriptionRequestFile.objects.filter(request=request) time_cost = {"imei": [], "invoice": [], "all": []} if request.ai_inference_profile is None: time_cost["imei"] = [-1 for _ in range(len(images))] time_cost["invoice"] = [-1] time_cost["all"] = [-1] else: for k, v in request.ai_inference_profile.items(): time_cost[k.split("_")[0]].append(v["inference"][1][0] - v["inference"][0] + (v["postprocess"][1]-v["postprocess"][0])) for i, image in enumerate(images): # temp_imei_SAP_20240127223644_a493434edbf84fc08aeb87ef6cdde102_0.jpg try: # image.index_in_request = int(image.file_name.split(".")[0].split("_")[-1]) if len(image.file_name.split(".")[0].split("_")) > 4 else 0 image.doc_type = image.file_name.split(".")[0].split("_")[1] if len(image.file_name.split(".")[0].split("_")) > 4 else "all" image.processing_time = time_cost[image.doc_type][image.index_in_request] if not request.predict_result: raise KeyError(f"Key predict_result not found in {request.request_id}") if request.predict_result.get("status", 200) != 200: raise AttributeError(f"Failed request: {request.request_id}") _predict_result = copy.deepcopy(predict_result_to_ready(request.predict_result)) _feedback_result = copy.deepcopy(request.feedback_result) _reviewed_result = copy.deepcopy(request.reviewed_result) if image.doc_type == "invoice": _predict_result["imei_number"] = [] if _feedback_result: _feedback_result["imei_number"] = [] else: None if _reviewed_result: _reviewed_result["imei_number"] = [] else: None else: _predict_result = {"retailername": None, "sold_to_party": None, "purchase_date": [], "imei_number": [_predict_result["imei_number"][image.index_in_request]]} _feedback_result = {"retailername": None, "sold_to_party": None, "purchase_date": None, "imei_number": [_feedback_result["imei_number"][image.index_in_request]]} if _feedback_result else None if isinstance(_reviewed_result, dict) and (len(_reviewed_result.get("imei_number", [])) or any(element in RETAILER_LIST for element in _reviewed_result.get("imei_number", []))) : _reviewed_result = {"retailername": None, "sold_to_party": None, "purchase_date": None, "imei_number": [_reviewed_result["imei_number"][image.index_in_request]]} if _reviewed_result else None else: _reviewed_result = None request.reviewed_result = None request.is_reviewed = False request.save() image.predict_result = _predict_result # image.feedback_result = _feedback_result # image.reviewed_result = _reviewed_result image.save() except Exception as e: self.stdout.write(self.style.ERROR(f"Request: {request.request_id} failed with {e}")) print(traceback.format_exc()) continue def handle(self, *args, **options): start = options['start'] end = options['end'] white_list = [ ] if start or end: try: start_date = timezone.datetime.strptime(start, '%Y-%m-%dT%H:%M:%S%z') # We care only about day precision only end_date = timezone.datetime.strptime(end, '%Y-%m-%dT%H:%M:%S%z') # start_date = timezone.make_aware(start_date) # end_date = timezone.make_aware(end_date) except Exception as e: print(f"[INFO]: start: {start}") print(f"[INFO]: end: {end}") raise InvalidException(excArgs="Date format") subcription_iter = SubscriptionRequest.objects.filter(created_at__range=(start_date, end_date)) else: subcription_iter = SubscriptionRequest.objects.all() for request in tqdm(subcription_iter.iterator()): if request.request_id not in white_list: continue self.process_request(request) self.stdout.write(self.style.SUCCESS('Sample Django management command executed successfully!'))