From 7277f4595a6c4574e2abbb60529b41ff8da4a2f9 Mon Sep 17 00:00:00 2001 From: yashashk Date: Sun, 10 Nov 2024 15:36:00 +0530 Subject: [PATCH 01/17] #OBS-I304 : fixed file upload validation and added version key for read api --- .../src/components/Dropzone/MultiFile.tsx | 14 ++++--------- .../pages/StepsPages/Ingestion/Ingestion.tsx | 20 +++++++++++++++---- .../src/pages/StepsPages/StepperPage.tsx | 2 +- web-console-v2/src/services/dataset.ts | 3 ++- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/web-console-v2/src/components/Dropzone/MultiFile.tsx b/web-console-v2/src/components/Dropzone/MultiFile.tsx index 8fef3888..4b8e0235 100644 --- a/web-console-v2/src/components/Dropzone/MultiFile.tsx +++ b/web-console-v2/src/components/Dropzone/MultiFile.tsx @@ -63,25 +63,19 @@ const MultiFileUpload = ({ }; const dropzoneProps = useCustomDropzone({ - multiple: isMultiple, + multiple: false, onDrop: handleDrop, accept: { - 'application/json': ['.json'], - 'text/csv': ['.csv'], - 'application/xml': ['.xml'], - 'application/octet-stream': ['.parquet', '.avro', '.orc'] + 'application/json': ['.json'] }, maxSize: maxFileSize }); const otherDropzoneProps = useCustomDropzone({ - multiple: isMultiple, + multiple: false, onDrop: handleDrop, accept: { - 'application/json': ['.json'], - 'text/csv': ['.csv'], - 'application/xml': ['.xml'], - 'application/octet-stream': ['.parquet', '.avro', '.orc'] + 'application/json': ['.json'] }, maxSize: maxFileSize }); diff --git a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx index 538e5740..4319d289 100644 --- a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx +++ b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx @@ -111,7 +111,7 @@ const Ingestion = () => { const { data: fetchData, refetch } = useFetchDatasetsById({ datasetId: datasetId, - queryParams: 'status=Draft&mode=edit&fields=dataset_config,name,version_key' + queryParams: 'status=Draft&mode=edit&fields=dataset_config,name' }); const { @@ -379,6 +379,18 @@ const Ingestion = () => { setData(null); }; + const handleFileSelection = (selectedFiles:any) => { + if (selectedFiles?.length > 0) { + // Keep only the latest selected file + setFiles([selectedFiles[selectedFiles.length - 1]]); + } + }; + + // File removal logic to clear the selected file + const handleFileRemove = (fileToRemove:any) => { + setFiles([]); + }; + const nameRegex = /^[^!@#$%^&*()+{}[\]:;<>,?~\\|]*$/; const handleNameChange = (e: React.ChangeEvent) => { const newValue = e.target.value; @@ -480,7 +492,7 @@ const Ingestion = () => { { data={data} setData={setData} files={files} - setFiles={setFiles} + setFiles={handleFileSelection} allowSchema maxFileSize={maxFileSizeConfig} subscribeErrors={setFileErrors} @@ -533,7 +545,7 @@ const Ingestion = () => { )} diff --git a/web-console-v2/src/pages/StepsPages/StepperPage.tsx b/web-console-v2/src/pages/StepsPages/StepperPage.tsx index 1d7e9a72..077c4f31 100644 --- a/web-console-v2/src/pages/StepsPages/StepperPage.tsx +++ b/web-console-v2/src/pages/StepsPages/StepperPage.tsx @@ -58,7 +58,7 @@ const StepperPage = () => { const [steps, setSteps] = useState(stepData.steps); const [stateId, setStateId] = useState(uuidv4()); - const dataset = useFetchDatasetsById({datasetId, queryParams:'status=Draft&mode=edit&fields=connectors_config,validation_config,dataset_config,type'}); + const dataset = useFetchDatasetsById({datasetId, queryParams:'status=Draft&mode=edit&fields=connectors_config,validation_config,dataset_config,type,version_key'}); useEffect(() => { if(!dataset.data) { diff --git a/web-console-v2/src/services/dataset.ts b/web-console-v2/src/services/dataset.ts index e60a51ad..c5dbb008 100644 --- a/web-console-v2/src/services/dataset.ts +++ b/web-console-v2/src/services/dataset.ts @@ -128,7 +128,8 @@ export const useCreateDataset = () => }, onSuccess: (response, variables) => { const configDetail = { - version_key: _.get(response, 'version_key') + version_key: _.get(response, 'version_key'), + dataset_id: _.get(response, 'id') }; storeSessionStorageItem(configDetailKey, configDetail); From 6e9101f0a586d6dfbee29258ebb9fc942e72fe8a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Sun, 10 Nov 2024 16:42:32 +0530 Subject: [PATCH 02/17] #OBS-I304: fix: fixed update in processing page, sending dataset_id with update data --- web-console-v2/src/pages/StepsPages/Processing/Processing.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx index b2ebdcf4..ba2db96b 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx @@ -196,6 +196,7 @@ const Processing: React.FC = () => { const jsonData = _.get(datasetData, ['sample_data']) || {}; const [canProceed, setCanProceed] = useState(true); const processingData = mapDatasetToProcessingData(datasetData); + console.log('processingData', processingData); const { mutate: updateDataset } = useUpdateDataset(); @@ -217,7 +218,7 @@ const Processing: React.FC = () => { } }); } else { - updateDataset({ data: { [keyName]: data } }); + updateDataset({ data: { [keyName]: data, dataset_id: datasetId } }); } }; From 169460596ceb6dfa8ed6d84ed0419d18a27e9661 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Sun, 10 Nov 2024 16:43:01 +0530 Subject: [PATCH 03/17] #OBS-I304: updated live edit path --- web-console-v2/src/services/datasetV1.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/services/datasetV1.ts b/web-console-v2/src/services/datasetV1.ts index a2f72730..e618e986 100644 --- a/web-console-v2/src/services/datasetV1.ts +++ b/web-console-v2/src/services/datasetV1.ts @@ -417,7 +417,7 @@ export const createDraftversion = async ({ selection, navigateToPath, rollupRedi navigateToPath(`/home/datasets/management/${datasetResponse?.data?.result?.dataset_id}?status=${DatasetStatus.ReadyToPublish}`) return; } - navigateToPath(`/home/ingestion/schema-details/${datasetResponse?.data?.result?.dataset_id}`) + navigateToPath(`/dataset/edit/ingestion/schema/${datasetResponse?.data?.result?.dataset_id}`) _.set(versionKeyMap, "version_keys", { [selection]: _.get(datasetResponse, "data.result.version_key") }) return; } From 4698a43d7180c90bc48d53479dcddb04c134e3f5 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 11 Nov 2024 16:54:22 +0530 Subject: [PATCH 04/17] #OBS-I304: updated css for view dataset --- .../components/Accordian/AccordionSection.tsx | 3 ++- .../components/DatasetManagement.tsx | 8 ++++---- .../datasetV1/ReviewAllConfigurations/index.tsx | 16 +++++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/web-console-v2/src/components/Accordian/AccordionSection.tsx b/web-console-v2/src/components/Accordian/AccordionSection.tsx index ac167634..cd16ff22 100644 --- a/web-console-v2/src/components/Accordian/AccordionSection.tsx +++ b/web-console-v2/src/components/Accordian/AccordionSection.tsx @@ -12,6 +12,7 @@ import { import * as _ from 'lodash'; import config from 'data/initialConfig'; import MainCard from 'components/CustomCard/MainCard'; +import { borderColor } from '@mui/system'; const { spacing } = config; const FieldSection = (props: any) => { @@ -66,7 +67,7 @@ const FieldSection = (props: any) => { } contentSX={{ px: 3, background: 'inherit' }} - sx={{ border: '1px solid #D6D6D6' }} + sx={{ border: '1px solid #D6D6D6', borderColor: '#d6d6d6d' }} > {sectionDetails()} diff --git a/web-console-v2/src/pages/datasetManagement/components/DatasetManagement.tsx b/web-console-v2/src/pages/datasetManagement/components/DatasetManagement.tsx index 7a170072..53a25c6d 100644 --- a/web-console-v2/src/pages/datasetManagement/components/DatasetManagement.tsx +++ b/web-console-v2/src/pages/datasetManagement/components/DatasetManagement.tsx @@ -3,12 +3,13 @@ import AccordionSection from "components/Accordian/AccordionSection"; import _ from "lodash"; import { useParams, useSearchParams } from 'react-router-dom'; import { useEffect, useState } from "react"; -import { Typography } from "@mui/material"; +import { Box, Typography } from "@mui/material"; import { DatasetStatus } from "types/datasets"; import Loader from "components/Loader"; import ReviewAllCongurations from "pages/datasetV1/ReviewAllConfigurations"; import { useAlert } from "contexts/AlertContextProvider"; import { getDatasetState } from "services/datasetV1"; +import { margin } from "@mui/system"; const DatasetManagement = () => { const { datasetId } = useParams(); @@ -38,9 +39,8 @@ const DatasetManagement = () => { const liveSections = [ { - title: Review - {_.capitalize(datasetName)}, + title: Review - {_.capitalize(datasetName)}, id: datasetName, - componentType: 'box', component: } ] @@ -65,7 +65,7 @@ const DatasetManagement = () => { } return <> - {loading ? : } + {loading ? : } } diff --git a/web-console-v2/src/pages/datasetV1/ReviewAllConfigurations/index.tsx b/web-console-v2/src/pages/datasetV1/ReviewAllConfigurations/index.tsx index 1c865165..46c8ef8d 100644 --- a/web-console-v2/src/pages/datasetV1/ReviewAllConfigurations/index.tsx +++ b/web-console-v2/src/pages/datasetV1/ReviewAllConfigurations/index.tsx @@ -71,6 +71,7 @@ const ReviewAllCongurations = ({ master, datasetState }: any) => { component: , master: true, dataset: true, + componentType: 'box' }, { id: 'dataschema', @@ -78,7 +79,8 @@ const ReviewAllCongurations = ({ master, datasetState }: any) => { description: 'Details about data schema', component: , master: true, - dataset: true + dataset: true, + componentType: 'box' }, { id: 'transformations', @@ -87,13 +89,15 @@ const ReviewAllCongurations = ({ master, datasetState }: any) => { component: , master: true, dataset: true, + componentType: 'box' }, { id: 'denorm', title: 'Denormalization', description: 'Details about Denormalization', component: , - dataset: true + dataset: true, + componentType: 'box' } ] @@ -128,7 +132,7 @@ const ReviewAllCongurations = ({ master, datasetState }: any) => { return <> } @@ -136,14 +140,16 @@ const ReviewAllCongurations = ({ master, datasetState }: any) => { return <> - + {_.map(datasetActions, renderDatasetActions)} - {renderSections({ sections: sections })} + + {renderSections({ sections: sections })} + } From bd21a7637bbb7457813277af140cde1e0cbf9a39 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 11 Nov 2024 17:37:44 +0530 Subject: [PATCH 05/17] #OBS-I304: added version_key in ingestion page --- web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx index ccf6d887..6ef7591d 100644 --- a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx +++ b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx @@ -111,7 +111,7 @@ const Ingestion = () => { const { data: fetchData, refetch } = useFetchDatasetsById({ datasetId: datasetId, - queryParams: 'status=Draft&mode=edit&fields=dataset_config,name' + queryParams: 'status=Draft&mode=edit&fields=dataset_config,name,version_key' }); const { From d146b78801aad5b91c7bbd8127a9f149bbbba6f6 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 12 Nov 2024 12:21:05 +0530 Subject: [PATCH 06/17] #OBS-I304 : filtering out deleted schema fields before updating --- .../pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx | 2 ++ .../src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx b/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx index 249a52fa..51e94e8a 100644 --- a/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx +++ b/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx @@ -454,9 +454,11 @@ const SchemaDetails = (props: { showTableOnly?: boolean }) => { canExpand?: boolean; isExpanded?: boolean; description?: string; + isDeleted?: boolean; }, index ) => { + if (item?.isDeleted) return; const { subRows, oneof, diff --git a/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx b/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx index 9683aadf..35b66011 100644 --- a/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx +++ b/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx @@ -195,7 +195,7 @@ const AllConfigurations = () => { {connectorConfig && _.entries(connectorConfig.connector_config).map(([configKey, configValue]) => ( - {connectorMeta.ui_spec.properties[configKey].title} + {connectorMeta.ui_spec.properties[configKey]?.title} {String(configValue)} ))} From f26ffb231842cf28da5e064e1d0746551344fd44 Mon Sep 17 00:00:00 2001 From: Kartheek Palla Date: Tue, 12 Nov 2024 12:40:52 +0530 Subject: [PATCH 07/17] #OBS-I304: summary ui fix --- .../pages/StepsPages/PreviewAndSave/AllConfigurations.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx b/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx index 541c60f2..9cc44f18 100644 --- a/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx +++ b/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx @@ -203,7 +203,7 @@ const AllConfigurations = () => { {connectorConfig && _.entries(connectorConfig.connector_config).map(([configKey, configValue]) => ( - {connectorMeta.ui_spec.properties[configKey]?.title} + {connectorMeta.ui_spec?.properties[configKey]?.title} {String(configValue)} ))} @@ -336,9 +336,9 @@ const AllConfigurations = () => { } {dataDenormalizations.length > 0 && dataDenormalizations.map((value) => ( - {value.denorm_key} - {value.dataset_id} - {value.denorm_out_field} + {value?.denorm_key} + {value?.dataset_id} + {value?.denorm_out_field} ))} From 9b086a5b13e3bbec8ee508ae3e3de2f4e112520d Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Tue, 12 Nov 2024 15:19:25 +0530 Subject: [PATCH 08/17] #OBS-I304: Dataset name not showing issue fix --- web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx index 461f661e..247a5124 100644 --- a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx +++ b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx @@ -112,7 +112,7 @@ const Ingestion = () => { } = useGenerateJsonSchema(); let readDatasetQueryParams = 'status=Draft&fields=dataset_config,name,version_key,connectors_config'; - if(datasetIdParam === '') readDatasetQueryParams += '&mode=edit' + if(datasetIdParam !== '') readDatasetQueryParams += '&mode=edit' const { data: fetchData, refetch } = useFetchDatasetsById({ datasetId: datasetId, queryParams: readDatasetQueryParams From 5b9bff4d4afcc5e588142944b1b7d57a6da5f63f Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Tue, 12 Nov 2024 15:45:52 +0530 Subject: [PATCH 09/17] #OBS-I304: Dataset name exists fix --- web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx | 2 +- web-console-v2/src/services/dataset.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx index 247a5124..b4f059da 100644 --- a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx +++ b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx @@ -232,7 +232,7 @@ const Ingestion = () => { ); useEffect(() => { - if (datasetIdParam === '') { + if (datasetIdParam === '' && datasetId) { debouncedFetchDataset(datasetId); } }, [datasetId]); diff --git a/web-console-v2/src/services/dataset.ts b/web-console-v2/src/services/dataset.ts index 49da5256..26c682c3 100644 --- a/web-console-v2/src/services/dataset.ts +++ b/web-console-v2/src/services/dataset.ts @@ -198,7 +198,7 @@ export const useFetchDatasetDiff = ({ datasetId }: { datasetId: string }) => { export const useFetchDatasetExists = ({ datasetId }: { datasetId: string }) => { return useQuery({ - queryKey: ['fetchDatasetExists'], + queryKey: ['fetchDatasetExists', 'datasetId'], queryFn: () => http.get(`${ENDPOINTS.DATASET_EXISTS}/${datasetId}`).then((res) => res.data), enabled: !!datasetId }); From 1710297bc88e66fcaa2b18db97dd796272c8231f Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Tue, 12 Nov 2024 16:48:13 +0530 Subject: [PATCH 10/17] #OBS-I334 - ux refactoring --- .../components/AlertDialog/AlertDialog.tsx | 2 + .../components/CustomeTable/CustomTable.tsx | 3 +- .../src/components/filtering-table.tsx | 51 ++++---- .../PreviewAndSave/AllConfigurations.tsx | 112 +++++++++--------- .../Denormalization/DataDenormalization.tsx | 5 +- .../src/pages/dashboardV1/MoreOptions.tsx | 18 +-- .../dashboardV1/ReadyToPublishDatasets.tsx | 32 ++--- .../src/pages/dashboardV1/datasets.tsx | 11 +- .../src/pages/dashboardV1/datasetsList.tsx | 28 +---- .../pages/dashboardV1/draftDatasetsList.tsx | 34 ++---- web-console-v2/src/utils/react-table.tsx | 1 + 11 files changed, 121 insertions(+), 176 deletions(-) diff --git a/web-console-v2/src/components/AlertDialog/AlertDialog.tsx b/web-console-v2/src/components/AlertDialog/AlertDialog.tsx index 96cad7f8..dade9927 100644 --- a/web-console-v2/src/components/AlertDialog/AlertDialog.tsx +++ b/web-console-v2/src/components/AlertDialog/AlertDialog.tsx @@ -32,12 +32,14 @@ const AlertDialog = ({ handleClose, action = null, open = false, context = {} }: {context.show === true && (