From 451381d5d743206d88c404ea69762f3c04af8c8b Mon Sep 17 00:00:00 2001 From: Viet Anh Nguyen <nv.anh5@samsung.com> Date: Wed, 6 Mar 2024 11:43:11 +0700 Subject: [PATCH 1/2] View pdf --- cope2n-fe/package-lock.json | 233 ++++++++++++++++++++++++++ cope2n-fe/package.json | 3 + cope2n-fe/src/index.tsx | 3 + cope2n-fe/src/pages/reviews/index.tsx | 142 +++++++++++----- 4 files changed, 337 insertions(+), 44 deletions(-) diff --git a/cope2n-fe/package-lock.json b/cope2n-fe/package-lock.json index e5db921..46eca9d 100644 --- a/cope2n-fe/package-lock.json +++ b/cope2n-fe/package-lock.json @@ -14,6 +14,8 @@ "@ant-design/pro-layout": "^7.10.3", "@babel/core": "^7.13.10", "@cyntler/react-doc-viewer": "^1.14.1", + "@react-pdf-viewer/core": "^3.12.0", + "@react-pdf-viewer/default-layout": "^3.12.0", "@tanstack/react-query": "^4.20.4", "antd": "^5.4.0", "axios": "^1.2.2", @@ -21,6 +23,7 @@ "history": "^5.3.0", "lodash-es": "^4.17.21", "mousetrap": "^1.6.5", + "pdfjs-dist": "^3.11.174", "process": "^0.11.10", "react": "^18.2.0", "react-chartjs-2": "^5.2.0", @@ -3939,6 +3942,236 @@ "react-dom": ">=16.9.0" } }, + "node_modules/@react-pdf-viewer/attachment": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/attachment/-/attachment-3.12.0.tgz", + "integrity": "sha512-mhwrYJSIpCvHdERpLUotqhMgSjhtF+BTY1Yb9Fnzpcq3gLZP+Twp5Rynq21tCrVdDizPaVY7SKu400GkgdMfZw==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/bookmark": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/bookmark/-/bookmark-3.12.0.tgz", + "integrity": "sha512-i7nEit8vIFMAES8RFGwprZ9cXOOZb9ZStPW6E6yuObJEXcvBj/ctsbBJGZxqUZOGklM0JoB7sjHyxAriHfe92A==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/core": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/core/-/core-3.12.0.tgz", + "integrity": "sha512-8MsdlQJ4jaw3GT+zpCHS33nwnvzpY0ED6DEahZg9WngG++A5RMhk8LSlxdHelwaFFHFiXBjmOaj2Kpxh50VQRg==", + "peerDependencies": { + "pdfjs-dist": "^2.16.105 || ^3.0.279", + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/default-layout": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/default-layout/-/default-layout-3.12.0.tgz", + "integrity": "sha512-K2fS4+TJynHxxCBFuIDiFuAw3nqOh4bkBgtVZ/2pGvnFn9lLg46YGLMnTXCQqtyZzzXYh696jmlFViun3is4pA==", + "dependencies": { + "@react-pdf-viewer/attachment": "3.12.0", + "@react-pdf-viewer/bookmark": "3.12.0", + "@react-pdf-viewer/core": "3.12.0", + "@react-pdf-viewer/thumbnail": "3.12.0", + "@react-pdf-viewer/toolbar": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/full-screen": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/full-screen/-/full-screen-3.12.0.tgz", + "integrity": "sha512-hQouJ26QUaRBCXNMU1aI1zpJn4l4PJRvlHhuE2dZYtLl37ycjl7vBCQYZW1FwnuxMWztZsY47R43DKaZORg0pg==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/get-file": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/get-file/-/get-file-3.12.0.tgz", + "integrity": "sha512-Uhq45n2RWlZ7Ec/BtBJ0WQESRciaYIltveDXHNdWvXgFdOS8XsvB+mnTh/wzm7Cfl9hpPyzfeezifdU9AkQgQg==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/open": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/open/-/open-3.12.0.tgz", + "integrity": "sha512-vhiDEYsiQLxvZkIKT9VPYHZ1BOnv46x9eCEmRWxO1DJ8fa/GRDTA9ivXmq/ap0dGEJs6t+epleCkCEfllLR/Yw==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/page-navigation": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/page-navigation/-/page-navigation-3.12.0.tgz", + "integrity": "sha512-tVEJ48Dd5kajV1nKkrPWijglJRNBiKBTyYDKVexhiRdTHUP1f6QQXiSyDgCUb0IGSZeJzOJb1h7ApKHe8OTtuw==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/print": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/print/-/print-3.12.0.tgz", + "integrity": "sha512-xJn76CgbU/M2iNaN7wLHTg+sdOekkRMfCakFLwPrE+SR7qD6NUF4vQQKJBSVCCK5bUijzb6cWfKGfo8VA72o4Q==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/properties": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/properties/-/properties-3.12.0.tgz", + "integrity": "sha512-dYTCHtVwFNkpDo7QxL2qk/8zAKndLwdD1FFxBftl6jIlQbtvNdxkFfkv1HcQING9Ic+7DBryOiD7W0ze4IERYg==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/rotate": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/rotate/-/rotate-3.12.0.tgz", + "integrity": "sha512-yaxaMYPChvNOjR8+AxRmj0kvojyJKPq4XHEcIB2lJJgBY1Zra3mliDUP3Nlb4yV8BS9+yBqWn9U9mtnopQD+tw==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/scroll-mode": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/scroll-mode/-/scroll-mode-3.12.0.tgz", + "integrity": "sha512-okII7Xqhl6cMvl1izdEvlXNJ+vJVq/qdg53hJIDYVgBCWskLk/cpjUg/ZonBxseG9lIDP3w2VO1McT8Gn11OAg==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/search": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/search/-/search-3.12.0.tgz", + "integrity": "sha512-jAkLpis49fsDDY/HrbUZIOIhzF5vynONQNA4INQKI38r/MjveblrkNv7qbr9j5lQ/WFic5+gD1e+Mtpf1/7DiA==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/selection-mode": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/selection-mode/-/selection-mode-3.12.0.tgz", + "integrity": "sha512-yysWEu2aCtBvzSgbhgI9kT5cq2hf0FU6Z+3B7MMXz14Kxyc3y18wUqxtgbvpFEfWF0bNUUq16JtWRljtxvZ83w==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/theme": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/theme/-/theme-3.12.0.tgz", + "integrity": "sha512-cdBi+wR1VOZ6URCcO9plmAZQu4ZGFcd7HJdBe7VIFiGyrvl9I/Of74ONLycnDImSuONt8D3uNjPBLieeaShVeg==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/thumbnail": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/thumbnail/-/thumbnail-3.12.0.tgz", + "integrity": "sha512-Vc8j3bO6wumWZV4o6pAbktPWKDSC9tQAzOCJ3cof541u4i44C11ccYC4W9aNcsMMUSO3bNwAGWtP8OFthV5akQ==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/toolbar": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/toolbar/-/toolbar-3.12.0.tgz", + "integrity": "sha512-qACTU3qXHgtNK8J+T13EWio+0liilj86SJ87BdapqXynhl720OKPlSKOQqskUGqg3oTUJAhrse9XG6SFdHJx+g==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0", + "@react-pdf-viewer/full-screen": "3.12.0", + "@react-pdf-viewer/get-file": "3.12.0", + "@react-pdf-viewer/open": "3.12.0", + "@react-pdf-viewer/page-navigation": "3.12.0", + "@react-pdf-viewer/print": "3.12.0", + "@react-pdf-viewer/properties": "3.12.0", + "@react-pdf-viewer/rotate": "3.12.0", + "@react-pdf-viewer/scroll-mode": "3.12.0", + "@react-pdf-viewer/search": "3.12.0", + "@react-pdf-viewer/selection-mode": "3.12.0", + "@react-pdf-viewer/theme": "3.12.0", + "@react-pdf-viewer/zoom": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@react-pdf-viewer/zoom": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/@react-pdf-viewer/zoom/-/zoom-3.12.0.tgz", + "integrity": "sha512-V0GUTyPM77+LzhoKX+T3XI10/HfGdqRTbgeP7ID60FCzcwu6kXWqJn5tzabjDKLTlFv8mJmn0aa/ppkIU97nfA==", + "dependencies": { + "@react-pdf-viewer/core": "3.12.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, "node_modules/@remix-run/router": { "version": "1.14.2", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.14.2.tgz", diff --git a/cope2n-fe/package.json b/cope2n-fe/package.json index 769f0c1..7798b18 100644 --- a/cope2n-fe/package.json +++ b/cope2n-fe/package.json @@ -32,6 +32,8 @@ "@ant-design/pro-layout": "^7.10.3", "@babel/core": "^7.13.10", "@cyntler/react-doc-viewer": "^1.14.1", + "@react-pdf-viewer/core": "^3.12.0", + "@react-pdf-viewer/default-layout": "^3.12.0", "@tanstack/react-query": "^4.20.4", "antd": "^5.4.0", "axios": "^1.2.2", @@ -39,6 +41,7 @@ "history": "^5.3.0", "lodash-es": "^4.17.21", "mousetrap": "^1.6.5", + "pdfjs-dist": "^3.11.174", "process": "^0.11.10", "react": "^18.2.0", "react-chartjs-2": "^5.2.0", diff --git a/cope2n-fe/src/index.tsx b/cope2n-fe/src/index.tsx index c3a45a1..3e7f2b1 100644 --- a/cope2n-fe/src/index.tsx +++ b/cope2n-fe/src/index.tsx @@ -6,6 +6,7 @@ import { queryClient } from 'queries'; import { Suspense } from 'react'; import { createRoot } from 'react-dom/client'; import AppRoutes from 'routes'; +import { Worker } from '@react-pdf-viewer/core'; import 'antd/dist/reset.css'; import './theme/compose.scss'; @@ -16,7 +17,9 @@ function App() { <QueryClientProvider client={queryClient}> <Internationalization> {/* <AbilityContext.Provider value={appAbilitiy}> */} + <Worker workerUrl="https://unpkg.com/pdfjs-dist@3.11.174/build/pdf.worker.min.js"> <AppRoutes /> + </Worker> {/* </AbilityContext.Provider> */} </Internationalization> <ReactQueryDevtools initialIsOpen={false} /> diff --git a/cope2n-fe/src/pages/reviews/index.tsx b/cope2n-fe/src/pages/reviews/index.tsx index 5bda53f..f9f2d2a 100644 --- a/cope2n-fe/src/pages/reviews/index.tsx +++ b/cope2n-fe/src/pages/reviews/index.tsx @@ -11,12 +11,15 @@ import { FullscreenExitOutlined, ClockCircleFilled, } from '@ant-design/icons'; -import FileViewer from '@cyntler/react-doc-viewer'; import styled from 'styled-components'; const { Sider, Content } = Layout; import { baseURL } from "request/api"; -import moment from 'moment'; import { useHotkeys } from "react-hotkeys-hook"; +// Import the main component +import { Viewer } from '@react-pdf-viewer/core'; + +// Import the styles +import '@react-pdf-viewer/core/lib/styles/index.css'; const siderStyle: React.CSSProperties = { @@ -133,32 +136,6 @@ const EditableCell: React.FC<EditableCellProps> = ({ // type EditableTableProps = Parameters<typeof Table>[0]; -const defaultColumns = [ - { - title: 'Key', - dataIndex: 'key', - key: 'key', - width: 200, - }, - { - title: 'Predicted', - dataIndex: 'predicted', - key: 'predicted', - }, - { - title: 'Submitted', - dataIndex: 'submitted', - key: 'submitted', - }, - { - title: 'Revised', - dataIndex: 'revised', - key: 'revised', - editable: true, - }, -]; - - const FileCard = ({ file, isSelected, onClick, setIsReasonModalOpen }) => { const fileName = file["File Name"]; @@ -311,9 +288,9 @@ const ReviewPage = () => { } }; - const loadCurrentRequest = (requestID) => { + const loadCurrentRequest = (requestIndex) => { setLoading(true); - fetchAllRequests(filterDateRange, filterSubsidiaries, filterReviewState, filterIncludeTests, requestID, 2).then((data) => { + fetchAllRequests(filterDateRange, filterSubsidiaries, filterReviewState, filterIncludeTests, requestIndex, 1).then((data) => { setRequests(data?.subscription_requests); setHasNextRequest(data?.subscription_requests.length > 1); setTotalPages(data?.page?.total_requests); @@ -364,7 +341,7 @@ const ReviewPage = () => { const reloadFilters = () => { setCurrentRequestIndex(1); - fetchAllRequests(filterDateRange, filterSubsidiaries, filterReviewState, filterIncludeTests, currentRequestIndex, 2).then((data) => { + fetchAllRequests(filterDateRange, filterSubsidiaries, filterReviewState, filterIncludeTests, 1, 2).then((data) => { setTotalPages(data?.page?.total_requests); setRequests(data?.subscription_requests); setHasNextRequest(data?.subscription_requests.length > 1); @@ -372,6 +349,9 @@ const ReviewPage = () => { firstRequest.then(async (data) => { if (data) setCurrentRequest(data); setAndLoadSelectedFile(data, 0); + setTimeout(() => { + loadCurrentRequest(1); + }, 500); }); }); @@ -379,7 +359,7 @@ const ReviewPage = () => { useEffect(() => { setCurrentRequestIndex(1); - fetchAllRequests(filterDateRange, filterSubsidiaries, filterReviewState, filterIncludeTests, currentRequestIndex, 2).then((data) => { + fetchAllRequests(filterDateRange, filterSubsidiaries, filterReviewState, filterIncludeTests, 1, 2).then((data) => { setTotalPages(data?.page?.total_requests); setRequests(data?.subscription_requests); setHasNextRequest(data?.subscription_requests.length > 1); @@ -446,6 +426,89 @@ const ReviewPage = () => { }) }; + const submitRevisedData = async () => { + const newData = [...dataSource]; + const newRevisedData = {}; + for (let i = 0; i < newData.length; i++) { + newRevisedData[newData[i].key] = newData[i].revised; + } + console.log(currentRequest) + updateRevisedData(newRevisedData).then(() => { + // "[Is Reviewed]" => true + setCurrentRequest({ + ...currentRequest, + ["Is Reviewed"]: true, + }) + }) + }; + + + const defaultColumns = [ + { + title: 'Key', + dataIndex: 'key', + key: 'key', + width: 200, + }, + { + title: 'Predicted', + dataIndex: 'predicted', + key: 'predicted', + }, + { + title: 'Submitted', + dataIndex: 'submitted', + key: 'submitted', + }, + { + title: (<div style={{ + width: 120, + display: 'flex', + lineHeight: '32px', + marginLeft: 10, + }}>Revised + <Button + onClick={() => { + if (!dataSource || !dataSource.length) return; + setDataSource(dataSource.map(item => { + item.revised = item.predicted; + return item; + })); + setTimeout(() => { + submitRevisedData(); + }, 1000); + }} + > + Copy Predicted + </Button> + <Button + onClick={() => { + if (!dataSource || !dataSource.length) return; + setDataSource(dataSource.map(item => { + item.revised = item.submitted; + return item; + })); + setTimeout(() => { + submitRevisedData(); + }, 1000); + }} + > + Copy Submitted + </Button> + </div>), + dataIndex: 'revised', + key: 'revised', + editable: true, + render: (text, record) => { + return <div style={{ + color: '#000', + fontWeight: 'bold', + }}>{text}</div>; + }, + }, + ]; + + const columns = defaultColumns.map((col) => { if (!col.editable) { return col; @@ -549,17 +612,9 @@ const ReviewPage = () => { flexGrow: 1, height: '100%', }}> - {selectedFileData === "FAILED_TO_LOAD_FILE" ? <p style={{ color: "#333" }}>Failed to load file.</p> : ( fileExtension === "pdf" ? (<FileViewer documents={[ - { uri: selectedFileData } - ]} config={{ - header: { - disableHeader: true, - disableFileName: true, - retainURLParams: true, - }, - csvDelimiter: ",", // "," as default, - pdfVerticalScrollByDefault: true, // false as default - }} />) : <div style={{ + {selectedFileData === "FAILED_TO_LOAD_FILE" ? <p style={{ color: "#333" }}>Failed to load file.</p> : (fileExtension === "pdf" ? (<Viewer + fileUrl={selectedFileData} + />) : <div style={{ height: "100%", width: "100%", overflow: "auto", @@ -777,7 +832,6 @@ const ReviewPage = () => { } } > - {currentRequest && JSON.stringify(currentRequest["Files"][selectedFileId])} <Form style={{ marginTop: 30, From adc8c8bbbd719cbb4cd380413357b8cb39fcd2c6 Mon Sep 17 00:00:00 2001 From: dx-tan <dx.tan@samsung.com> Date: Wed, 6 Mar 2024 15:09:29 +0700 Subject: [PATCH 2/2] Fix: #67 --- cope2n-api/fwd_api/api/accuracy_view.py | 5 +- .../celery_worker/process_report_tasks.py | 4 +- cope2n-api/fwd_api/utils/accuracy.py | 142 +++++++++--------- cope2n-api/fwd_api/utils/cache.py | 1 - cope2n-fe/src/utils/metric-format.ts | 2 +- docker-compose-dev.yml | 16 +- 6 files changed, 89 insertions(+), 81 deletions(-) diff --git a/cope2n-api/fwd_api/api/accuracy_view.py b/cope2n-api/fwd_api/api/accuracy_view.py index 9d14f51..9cc6676 100644 --- a/cope2n-api/fwd_api/api/accuracy_view.py +++ b/cope2n-api/fwd_api/api/accuracy_view.py @@ -20,6 +20,7 @@ from ..utils.file import download_from_S3, dict2xlsx, save_report_to_S3, build_S from ..utils.redis import RedisUtils from ..utils.process import string_to_boolean from ..utils.cache import get_cache, set_cache +from fwd_api.constant.common import FileCategory from ..request.ReportCreationSerializer import ReportCreationSerializer from ..utils.subsidiary import map_subsidiary_long_to_short, map_subsidiary_short_to_long from ..utils.report import aggregate_overview @@ -110,7 +111,7 @@ class AccuracyViewSet(viewsets.ViewSet): subsidiary = request.data.get("subsidiary", "all") subsidiary = map_subsidiary_long_to_short(subsidiary) - base_query = Q() + base_query = Q(status=200) if start_date_str or end_date_str: try: start_date = timezone.datetime.strptime(start_date_str, '%Y-%m-%d') # We care only about day precision only @@ -670,7 +671,7 @@ class AccuracyViewSet(viewsets.ViewSet): data = [] files = [] - subscription_request_files = SubscriptionRequestFile.objects.filter(request=subscription_request.id) + subscription_request_files = SubscriptionRequestFile.objects.filter(request=subscription_request.id, file_category=FileCategory.Origin.value) for subscription_request_file in subscription_request_files: sub = subscription_request.subscription diff --git a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py index aeab9b1..d56f978 100644 --- a/cope2n-api/fwd_api/celery_worker/process_report_tasks.py +++ b/cope2n-api/fwd_api/celery_worker/process_report_tasks.py @@ -261,6 +261,8 @@ def make_a_report_2(report_id, query_set): report.average_OCR_time = {"invoice": time_cost["invoice"](), "imei": time_cost["imei"](), "invoice_count": time_cost["invoice"].count, "imei_count": time_cost["imei"].count} + report.average_OCR_time["invoice"] = 0 if report.average_OCR_time["invoice"] is None else report.average_OCR_time["invoice"] + report.average_OCR_time["imei"] = 0 if report.average_OCR_time["imei"] is None else report.average_OCR_time["imei"] report.average_OCR_time["avg"] = (report.average_OCR_time["invoice"]*report.average_OCR_time["invoice_count"] + report.average_OCR_time["imei"]*report.average_OCR_time["imei_count"])/(report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) if (report.average_OCR_time["imei_count"] + report.average_OCR_time["invoice_count"]) > 0 else None report.number_imei_transaction = transaction_att.get("imei", 0) @@ -311,7 +313,7 @@ def make_a_report_2(report_id, query_set): for key in keys: if report_fine_data[i][key]: for x_key in report_fine_data[i][key].keys(): - report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 + report_fine_data[i][key][x_key] = report_fine_data[i][key][x_key]*100 if report_fine_data[i][key][x_key] is not None else None data_workbook = dict2xlsx(report_fine_data, _type='report') overview_filename = query_set["subsidiary"] + "_" + query_set["report_overview_duration"] + ".xlsx" local_workbook = save_workbook_file(overview_filename, report, data_workbook, settings.OVERVIEW_REPORT_ROOT) diff --git a/cope2n-api/fwd_api/utils/accuracy.py b/cope2n-api/fwd_api/utils/accuracy.py index 5651de3..fb17f13 100644 --- a/cope2n-api/fwd_api/utils/accuracy.py +++ b/cope2n-api/fwd_api/utils/accuracy.py @@ -115,84 +115,86 @@ class ReportAccumulateByRequest: @staticmethod def update_total(total, report_file): - total["total_images"] += 1 - total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 - total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 - doc_type = "imei" - if report_file.doc_type in ["imei", "invoice", "all"]: - doc_type = report_file.doc_type - else: - print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") - total["num_imei"] += 1 if doc_type == "imei" else 0 - total["num_invoice"] += 1 if doc_type == "invoice" else 0 + if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS: + total["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 + total["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 + total["total_images"] += 1 + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + total["num_imei"] += 1 if doc_type == "imei" else 0 + total["num_invoice"] += 1 if doc_type == "invoice" else 0 - if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : - total["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) - total["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) - total["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) - total["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) - elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: - total["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) - total["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) - total["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) - total["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) + if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0 : + total["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) + total["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) + total["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) + total["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) + elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: + total["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) + total["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) + total["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) + total["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - total["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - total["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + total["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + total["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) - if not total["average_processing_time"].get(report_file.doc_type, None): - print(f"[WARM]: Weird doctype: {report_file.doc_type}") - total["average_processing_time"][report_file.doc_type] = IterAvg() - total["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 + if not total["average_processing_time"].get(report_file.doc_type, None): + print(f"[WARM]: Weird doctype: {report_file.doc_type}") + total["average_processing_time"][report_file.doc_type] = IterAvg() + total["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 - doc_type = "imei" - if report_file.doc_type in ["imei", "invoice", "all"]: - doc_type = report_file.doc_type - else: - print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") - total["usage"]["imei"] += 1 if doc_type == "imei" else 0 - total["usage"]["invoice"] += 1 if doc_type == "invoice" else 0 - total["usage"]["total_images"] += 1 - total["review_progress"].append(report_file.review_status) + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + total["usage"]["imei"] += 1 if doc_type == "imei" else 0 + total["usage"]["invoice"] += 1 if doc_type == "invoice" else 0 + total["usage"]["total_images"] += 1 + total["review_progress"].append(report_file.review_status) return total @staticmethod def update_day(day_data, report_file): - day_data["total_images"] += 1 - day_data["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 - day_data["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 - doc_type = "imei" - if report_file.doc_type in ["imei", "invoice", "all"]: - doc_type = report_file.doc_type - else: - print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") - day_data["num_imei"] += 1 if doc_type == "imei" else 0 - day_data["num_invoice"] += 1 if doc_type == "invoice" else 0 - day_data["report_files"].append(report_file) - - if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0: - day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) - day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) - day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) - day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) - elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: - day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) - day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) - day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) - day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) + if report_file.bad_image_reason not in settings.ACC_EXCLUDE_RESEASONS: + day_data["images_quality"]["successful"] += 1 if not report_file.is_bad_image else 0 + day_data["images_quality"]["bad"] += 1 if report_file.is_bad_image else 0 + day_data["total_images"] += 1 + doc_type = "imei" + if report_file.doc_type in ["imei", "invoice", "all"]: + doc_type = report_file.doc_type + else: + print(f"[WARM]: Weird doc type {report_file.doc_type} if request id: {report_file.correspond_request_id}") + day_data["num_imei"] += 1 if doc_type == "imei" else 0 + day_data["num_invoice"] += 1 if doc_type == "invoice" else 0 + day_data["report_files"].append(report_file) + + if sum([len(report_file.reviewed_accuracy[x]) for x in report_file.reviewed_accuracy.keys() if "_count" not in x]) > 0: + day_data["average_accuracy_rate"]["imei"].add(report_file.reviewed_accuracy.get("imei_number", [])) + day_data["average_accuracy_rate"]["purchase_date"].add(report_file.reviewed_accuracy.get("purchase_date", [])) + day_data["average_accuracy_rate"]["retailer_name"].add(report_file.reviewed_accuracy.get("retailername", [])) + day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.reviewed_accuracy.get("sold_to_party", [])) + elif sum([len(report_file.feedback_accuracy[x]) for x in report_file.feedback_accuracy.keys() if "_count" not in x]) > 0: + day_data["average_accuracy_rate"]["imei"].add(report_file.feedback_accuracy.get("imei_number", [])) + day_data["average_accuracy_rate"]["purchase_date"].add(report_file.feedback_accuracy.get("purchase_date", [])) + day_data["average_accuracy_rate"]["retailer_name"].add(report_file.feedback_accuracy.get("retailername", [])) + day_data["average_accuracy_rate"]["sold_to_party"].add(report_file.feedback_accuracy.get("sold_to_party", [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) - for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: - day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + day_data["feedback_accuracy"][key].add(report_file.feedback_accuracy.get(key, [])) + for key in ["imei_number", "purchase_date", "retailername", "sold_to_party"]: + day_data["reviewed_accuracy"][key].add(report_file.reviewed_accuracy.get(key, [])) - if not day_data["average_processing_time"].get(report_file.doc_type, None): - print(f"[WARM]: Weird doctype: {report_file.doc_type}") - day_data["average_processing_time"][report_file.doc_type] = IterAvg() - day_data["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 - day_data["review_progress"].append(report_file.review_status) + if not day_data["average_processing_time"].get(report_file.doc_type, None): + print(f"[WARM]: Weird doctype: {report_file.doc_type}") + day_data["average_processing_time"][report_file.doc_type] = IterAvg() + day_data["average_processing_time"][report_file.doc_type].add_avg(report_file.time_cost, 1) if report_file.time_cost else 0 + day_data["review_progress"].append(report_file.review_status) return day_data def add(self, request, report_files): @@ -255,6 +257,8 @@ class ReportAccumulateByRequest: _average_OCR_time = {"invoice": self.data[month][1][day]["average_processing_time"]["invoice"](), "imei": self.data[month][1][day]["average_processing_time"]["imei"](), "invoice_count": self.data[month][1][day]["average_processing_time"]["invoice"].count, "imei_count": self.data[month][1][day]["average_processing_time"]["imei"].count} + _average_OCR_time["invoice"] = 0 if _average_OCR_time["invoice"] is None else _average_OCR_time["invoice"] + _average_OCR_time["imei"] = 0 if _average_OCR_time["imei"] is None else _average_OCR_time["imei"] _average_OCR_time["avg"] = (_average_OCR_time["invoice"]*_average_OCR_time["invoice_count"] + _average_OCR_time["imei"]*_average_OCR_time["imei_count"])/(_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) if (_average_OCR_time["imei_count"] + _average_OCR_time["invoice_count"]) > 0 else None acumulated_acc = {"feedback_accuracy": {}, "reviewed_accuracy": {}} @@ -527,6 +531,8 @@ class IterAvg: self.avg = (self.avg*(self.count-count) + avg*count)/(self.count) def __call__(self): + if self.count == 0: + return None return self.avg def validate_feedback_file(feedback, predict): diff --git a/cope2n-api/fwd_api/utils/cache.py b/cope2n-api/fwd_api/utils/cache.py index ea68137..4885d94 100644 --- a/cope2n-api/fwd_api/utils/cache.py +++ b/cope2n-api/fwd_api/utils/cache.py @@ -10,7 +10,6 @@ def set_cache(key, value): this_cache.save() return this_cache - def get_cache(key): value = {} cache = Caching.objects.filter(key=key) diff --git a/cope2n-fe/src/utils/metric-format.ts b/cope2n-fe/src/utils/metric-format.ts index c106edc..79a6340 100644 --- a/cope2n-fe/src/utils/metric-format.ts +++ b/cope2n-fe/src/utils/metric-format.ts @@ -1,5 +1,5 @@ export const formatPercent = (value: number, floatingPoint: number = 1) => { - if (value === 0) { + if (value === null || value === undefined) { return '-'; } if (value < 100.0) { diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index ffb2d60..3426a9d 100755 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -89,12 +89,12 @@ services: depends_on: db-sbt: condition: service_started - command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && - python manage.py makemigrations && - python manage.py migrate && - python manage.py compilemessages && - gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod - # command: bash -c "tail -f > /dev/null" + # command: sh -c "chmod -R 777 /app; sleep 5; python manage.py collectstatic --no-input && + # python manage.py makemigrations && + # python manage.py migrate && + # python manage.py compilemessages && + # gunicorn fwd.asgi:application -k uvicorn.workers.UvicornWorker --timeout 300 -b 0.0.0.0:9000" # pre-makemigrations on prod + command: bash -c "tail -f > /dev/null" minio: image: minio/minio @@ -179,8 +179,8 @@ services: - ./cope2n-api:/app working_dir: /app - command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" - # command: bash -c "tail -f > /dev/null" + # command: sh -c "celery -A fwd_api.celery_worker.worker worker -l INFO -c 5" + command: bash -c "tail -f > /dev/null" # Back-end persistent db-sbt: