diff --git a/cope2n-ai-fi/modules/sdsvkie_ b/cope2n-ai-fi/modules/sdsvkie_ new file mode 160000 index 0000000..8349a89 --- /dev/null +++ b/cope2n-ai-fi/modules/sdsvkie_ @@ -0,0 +1 @@ +Subproject commit 8349a89de7fd8c9e6958907047d16bdc23a77adf diff --git a/cope2n-ai-fi/modules/sdsvkv_oldu b/cope2n-ai-fi/modules/sdsvkv_oldu new file mode 160000 index 0000000..bdba044 --- /dev/null +++ b/cope2n-ai-fi/modules/sdsvkv_oldu @@ -0,0 +1 @@ +Subproject commit bdba044bb2eacac7c7cfe0e0f321196d03b681f6 diff --git a/cope2n-ai-fi/modules/sdsvtd_ b/cope2n-ai-fi/modules/sdsvtd_ new file mode 160000 index 0000000..a9a796f --- /dev/null +++ b/cope2n-ai-fi/modules/sdsvtd_ @@ -0,0 +1 @@ +Subproject commit a9a796f843f6ad99977a8dcba02d65fe75225574 diff --git a/cope2n-ai-fi/modules/sdsvtr_ b/cope2n-ai-fi/modules/sdsvtr_ new file mode 160000 index 0000000..2c788e9 --- /dev/null +++ b/cope2n-ai-fi/modules/sdsvtr_ @@ -0,0 +1 @@ +Subproject commit 2c788e9e2d7fe369869d474fbb22426a4ca84590 diff --git a/demo-ocr b/demo-ocr new file mode 160000 index 0000000..036226c --- /dev/null +++ b/demo-ocr @@ -0,0 +1 @@ +Subproject commit 036226c29ecd4aa0e28e89880c7b331523266caa diff --git a/deploy_images.py b/deploy_images.py new file mode 100644 index 0000000..e69de29 diff --git a/deploy_images.sh b/deploy_images.sh new file mode 100644 index 0000000..e69de29 diff --git a/speedtest_sync_periodically.py b/speedtest_sync_periodically.py new file mode 100644 index 0000000..abc58a2 --- /dev/null +++ b/speedtest_sync_periodically.py @@ -0,0 +1,163 @@ +import requests +import time +import argparse +import multiprocessing +import tqdm +import random +import traceback + + +parser = argparse.ArgumentParser() +parser.add_argument("--host", dest="host", default="https://sbt.idp.sdsrv.ai", required=False) +parser.add_argument("-u", "--username", help="Username to connect to server", required=True) +parser.add_argument("-p", "--password", help="Password to connect to server", required=True) +parser.add_argument("--num_requests", type=int, help="Number of requests", required=False, default=100) +parser.add_argument("--num_workers", type=int, help="Number of workers", required=False, default=3) +parser.add_argument("--checking_interval", type=float, help="Interval result checking time", required=False, default=0.5) +args = parser.parse_args() + +PROCESSING_TIMEOUT = 60 + + +# ================================================================= +# GET THE TOKEN +response = requests.post(f'{args.host}/api/ctel/login/', json={ + 'username': args.username, + 'password': args.password +}) +try: + token = response.json()['token'] +except: + print("Failed to login") + print(response.content) +# After the login, store the token in the memory (RAM) or DB +# Re-login to issue a new token after 6 days. +# ================================================================= + +def process_file(data): + files, token = data + num_files = len(files) + files.append( + ('processType', (None, 12)), + ) + # ================================================================= + # UPLOAD THE FILE + start_time = time.time() + try: + response = requests.post(f'{args.host}/api/ctel/images/process_sync/', headers={ + 'Authorization': token, + }, files=files, timeout=300) + except requests.exceptions.Timeout: + print("Timeout occurred while uploading") + return { + "success": False, + "status": "timeout", + "upload_time": 0, + "process_time": 0, + "num_files": 0, + } + except Exception as e: + print(e) + traceback.print_exc() + print("Unknown exception occurred while uploading") + return { + "success": False, + "status": "unknown error", + "upload_time": 0, + "process_time": 0, + "num_files": 0, + } + end_time = time.time() + upload_time = end_time - start_time + # ================================================================= + + try: + data = response.json() + data.pop("files", None) + print(data) + except: + print(response.content) + return { + "success": False, + "status": "timeout", + "upload_time": 0, + "process_time": 0, + "num_files": 0, + } + return { + "success": True, + "status": 200, + "upload_time": upload_time, + "process_time": upload_time, + "num_files": num_files, + } + +# invoice_files = [ +# ('invoice_file', ('invoice.pdf', open("test_samples/20220303025923NHNE_20220222_Starhub_Order_Confirmation_by_Email.pdf", "rb").read())), +# ] +invoice_files = [ + ('invoice_file', ('invoice.jpg', open("test_samples/sbt/invoice.jpg", "rb").read())), +] +imei_files = [ + ('imei_files', ("test_samples/sbt/imei1.jpg", open("test_samples/sbt/imei1.jpg", "rb").read())), + ('imei_files', ("test_samples/sbt/imei2.jpg", open("test_samples/sbt/imei2.jpg", "rb").read())), +] +# imei_files = [ +# ('imei_files', ("test_samples/sbt/imei1.jpg", open("test_samples/sbt/imei1.jpg", "rb").read())), +# ('imei_files', ("test_samples/sbt/imei2.jpg", open("test_samples/sbt/imei2.jpg", "rb").read())), +# ] +def get_imei_files(): + # num_files = random.randint(1, len(imei_files) + 1) + num_files = 2 + print("Num imeis", num_files) + files = imei_files[:num_files] + # print("Num of imei files:", len(files)) + return files +def get_files(): + return invoice_files + get_imei_files() +def gen_input(num_input): + last = time.time() + rate = 60/(12/3) + for _ in range(num_input): + interval = rate-(time.time()-last) + if interval > 0: + print(f"[INFO]: sleep for {interval}") + time.sleep(interval) + last = time.time() + yield (get_files(), token) +pool = multiprocessing.Pool(processes=args.num_workers*5) +results = [] +for result in tqdm.tqdm(pool.imap_unordered(process_file, gen_input(num_input=args.num_requests)), total=args.num_requests): + results.append(result) + +print("## TEST REPORT #################################") +print("Number of requests: {}".format(args.num_requests)) +print("Number of concurrent requests: {}".format(args.num_workers)) +print("Number of files: 1 invoice, 1-5 imei files (random)") +print("Query time interval for result: {:.3f}s ".format(args.checking_interval)) +print("--------------------------------------") +print("SUCCESS RATE") +counter = {} +for result in results: + counter[result["status"]] = counter.get(result["status"], 0) + 1 +total_requests = sum(counter.values()) +print("Success rate: {}".format(counter.get(200, 0) / total_requests if total_requests > 0 else -1)) +print("Statuses:", counter) +print("--------------------------------------") +print("TIME BY REQUEST") +uploading_time = [x["upload_time"] for x in results if x["success"]] +if len(uploading_time) == 0: + print("No valid uploading time") + print("Check the results!") +processing_time = [x["process_time"] for x in results if x["success"]] +print("Uploading time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(uploading_time) / len(uploading_time), min(uploading_time), max(uploading_time))) +print("Processing time (Avg / Min / Max): {:.3f}s {:.3f}s {:.3f}s".format(sum(processing_time) / len(processing_time), min(processing_time), max(processing_time))) +print("--------------------------------------") +print("TIME BY IMAGE") +uploading_time = [x["upload_time"] for x in results if x["success"]] +processing_time = [x["process_time"] for x in results if x["success"]] +num_images = sum(x["num_files"] for x in results if x["success"]) +print("Total images:", num_images) +print("Uploading time: {:.3f}s".format(sum(uploading_time) / num_images)) +print("Processing time: {:.3f}s".format(sum(processing_time) / num_images)) +print("--------------------------------------")