diff --git a/cmflib/cmf.py b/cmflib/cmf.py index 0cf26e48..6df08508 100644 --- a/cmflib/cmf.py +++ b/cmflib/cmf.py @@ -1293,17 +1293,30 @@ def commit_metrics(self, metrics_name: str): def commit_existing_metrics(self, metrics_name: str, uri: str, custom_properties: t.Optional[t.Dict] = None): custom_props = {} if custom_properties is None else custom_properties - metrics = create_new_artifact_event_and_attribution( - store=self.store, - execution_id=self.execution.id, - context_id=self.child_context.id, - uri=uri, - name=metrics_name, - type_name="Step_Metrics", - event_type=mlpb.Event.Type.OUTPUT, - custom_properties=custom_props, - milliseconds_since_epoch=int(time.time() * 1000), - ) + c_hash = uri.strip() + existing_artifact = [] + existing_artifact.extend(self.store.get_artifacts_by_uri(c_hash)) + if (existing_artifact + and len(existing_artifact) != 0 ): + metrics = link_execution_to_artifact( + store=self.store, + execution_id=self.execution.id, + uri=c_hash, + input_name=metrics_name, + event_type=mlpb.Event.Type.OUTPUT, + ) + else: + metrics = create_new_artifact_event_and_attribution( + store=self.store, + execution_id=self.execution.id, + context_id=self.child_context.id, + uri=uri, + name=metrics_name, + type_name="Step_Metrics", + event_type=mlpb.Event.Type.OUTPUT, + custom_properties=custom_props, + milliseconds_since_epoch=int(time.time() * 1000), + ) if self.graph: self.driver.create_metrics_node( metrics_name, diff --git a/cmflib/cmfquery.py b/cmflib/cmfquery.py index 00d634fc..e87769b6 100644 --- a/cmflib/cmfquery.py +++ b/cmflib/cmfquery.py @@ -389,9 +389,63 @@ def get_all_exe_in_stage(self, stage_name: str) -> t.List[mlpb.Execution]: return self.store.get_executions_by_context(stage.id) return [] + def get_all_executions_by_ids_list(self, exe_ids: t.List[int]) -> pd.DataFrame: + """Return executions for given execution ids list as a pandas data frame. + + Args: + exe_ids: List of execution identifiers. + + Returns: + Data frame with all executions for the list of given execution identifiers. + """ + + df = pd.DataFrame() + executions = self.store.get_executions_by_id(exe_ids) + for exe in executions: + d1 = self._transform_to_dataframe(exe) + df = pd.concat([df, d1], sort=True, ignore_index=True) + return df + + def get_all_artifacts_by_context(self, pipeline_name: str) -> pd.DataFrame: + """Return artifacts for given pipeline name as a pandas data frame. + + Args: + pipeline_name: Name of the pipeline. + + Returns: + Data frame with all artifacts associated with given pipeline name. + """ + df = pd.DataFrame() + contexts = self.store.get_contexts_by_type("Parent_Context") + context_id = self.get_pipeline_id(pipeline_name) + for ctx in contexts: + if ctx.id == context_id: + child_contexts = self.store.get_children_contexts_by_context(ctx.id) + for cc in child_contexts: + artifacts = self.store.get_artifacts_by_context(cc.id) + for art in artifacts: + d1 = self.get_artifact_df(art) + df = pd.concat([df, d1], sort=True, ignore_index=True) + return df + + def get_all_artifacts_by_ids_list(self, artifact_ids: t.List[int]) -> pd.DataFrame: + """Return all artifacts for the given artifact ids list. + + Args: + artifact_ids: List of artifact identifiers + + Returns: + Data frame with all artifacts for the given artifact ids list. + """ + df = pd.DataFrame() + artifacts = self.store.get_artifacts_by_id(artifact_ids) + for art in artifacts: + d1 = self.get_artifact_df(art) + df = pd.concat([df, d1], sort=True, ignore_index=True) + return df + def get_all_executions_in_stage(self, stage_name: str) -> pd.DataFrame: """Return executions of the given stage as pandas data frame. - Args: stage_name: Stage name. See doc strings for the prev method. Returns: @@ -471,6 +525,16 @@ def get_all_artifacts_for_execution(self, execution_id: int) -> pd.DataFrame: ) return df + def get_all_artifact_types(self) -> t.List[str]: + """Return names of all artifact types. + + Returns: + List of all artifact types. + """ + artifact_list = self.store.get_artifact_types() + types=[i.name for i in artifact_list] + return types + def get_all_executions_for_artifact(self, artifact_name: str) -> pd.DataFrame: """Return executions that consumed and produced given artifact. @@ -491,6 +555,7 @@ def get_all_executions_for_artifact(self, artifact_name: str) -> pd.DataFrame: "Type": "INPUT" if event.type == mlpb.Event.Type.INPUT else "OUTPUT", "execution_id": event.execution_id, "execution_name": self.store.get_executions_by_id([event.execution_id])[0].name, + "execution_type_name":self.store.get_executions_by_id([event.execution_id])[0].properties['Execution_type_name'], "stage": stage_ctx.name, "pipeline": self.store.get_parent_contexts_by_context(stage_ctx.id)[0].name, } @@ -598,6 +663,7 @@ def find_producer_execution(self, artifact_name: str) -> t.Optional[mlpb.Executi executions_ids = set( event.execution_id for event in self.store.get_events_by_artifact_ids([artifact.id]) + if event.type == mlpb.Event.OUTPUT ) if not executions_ids: diff --git a/cmflib/commands/metadata/push.py b/cmflib/commands/metadata/push.py index 3f299be6..78e8fc5f 100644 --- a/cmflib/commands/metadata/push.py +++ b/cmflib/commands/metadata/push.py @@ -59,6 +59,8 @@ def run(self): attr_dict = CmfConfig.read_config(config_file_path) url = attr_dict.get("cmf-server-ip", "http://127.0.0.1:80") + print("metadata push started") + print("........................................") if self.args.pipeline_name in query.get_pipeline_names(): # Checks if pipeline name exists json_payload = query.dumptojson( diff --git a/cmflib/metadata_helper.py b/cmflib/metadata_helper.py index 095b221e..ef8436bc 100644 --- a/cmflib/metadata_helper.py +++ b/cmflib/metadata_helper.py @@ -305,6 +305,7 @@ def create_new_execution_in_existing_context( EXECUTION_CONTEXT_NAME_PROPERTY_NAME = "Context_Type" EXECUTION_CONTEXT_ID = "Context_ID" EXECUTION_EXECUTION = "Execution" +EXECUTION_EXECUTION_TYPE_NAME="Execution_type_name" EXECUTION_REPO = "Git_Repo" EXECUTION_START_COMMIT = "Git_Start_Commit" EXECUTION_END_COMMIT = "Git_End_Commit" @@ -402,6 +403,7 @@ def create_new_execution_in_existing_run_context( EXECUTION_CONTEXT_NAME_PROPERTY_NAME: metadata_store_pb2.STRING, EXECUTION_CONTEXT_ID: metadata_store_pb2.INT, EXECUTION_EXECUTION: metadata_store_pb2.STRING, + EXECUTION_EXECUTION_TYPE_NAME: metadata_store_pb2.STRING, EXECUTION_PIPELINE_TYPE: metadata_store_pb2.STRING, EXECUTION_PIPELINE_ID: metadata_store_pb2.INT, EXECUTION_REPO: metadata_store_pb2.STRING, @@ -415,6 +417,7 @@ def create_new_execution_in_existing_run_context( # Mistakenly used for grouping in the UX EXECUTION_CONTEXT_ID: metadata_store_pb2.Value(int_value=context_id), EXECUTION_EXECUTION: metadata_store_pb2.Value(string_value=execution), + EXECUTION_EXECUTION_TYPE_NAME: metadata_store_pb2.Value(string_value=execution_type_name), EXECUTION_PIPELINE_TYPE: metadata_store_pb2.Value(string_value=pipeline_type), EXECUTION_PIPELINE_ID: metadata_store_pb2.Value(int_value=pipeline_id), EXECUTION_REPO: metadata_store_pb2.Value(string_value=git_repo), diff --git a/server/app/get_data.py b/server/app/get_data.py index 31e9ca66..42f006a8 100644 --- a/server/app/get_data.py +++ b/server/app/get_data.py @@ -5,49 +5,89 @@ from server.app.query_visualization import query_visualization from fastapi.responses import FileResponse -def get_executions(mlmdfilepath, pipeline_name): +def get_executions_by_ids(mlmdfilepath, pipeline_name, exe_ids): query = cmfquery.CmfQuery(mlmdfilepath) - stages = query.get_pipeline_stages(pipeline_name) df = pd.DataFrame() - for stage in stages: - executions = query.get_all_executions_in_stage(stage) - if str(executions.Pipeline_Type[0]) == pipeline_name: - df = pd.concat([df, executions], sort=True, ignore_index=True) + executions = query.get_all_executions_by_ids_list(exe_ids) + df = pd.concat([df, executions], sort=True, ignore_index=True) + #df=df.drop('name',axis=1) return df +def get_all_exe_ids(mlmdfilepath): + query = cmfquery.CmfQuery(mlmdfilepath) + df = pd.DataFrame() + execution_ids = {} + names = query.get_pipeline_names() + for name in names: + stages = query.get_pipeline_stages(name) + for stage in stages: + executions = query.get_all_executions_in_stage(stage) + df = pd.concat([df, executions], sort=True, ignore_index=True) + if df.empty: + return + for name in names: + execution_ids[name] = df.loc[df['Pipeline_Type'] == name, ['id', 'Context_Type']] + return execution_ids + +def get_all_artifact_ids(mlmdfilepath): + # following is a dictionary of dictionary + # First level dictionary key is pipeline_name + # First level dicitonary value is nested dictionary + # Nested dictionary key is type i.e. Dataset, Model, etc. + # Nested dictionary value is ids i.e. set of integers + artifact_ids = {} + query = cmfquery.CmfQuery(mlmdfilepath) + names = query.get_pipeline_names() + for name in names: + df = pd.DataFrame() + artifacts = query.get_all_artifacts_by_context(name) + df = pd.concat([df, artifacts], sort=True, ignore_index=True) + if df.empty: + return + else: + artifact_ids[name] = {} + for art_type in df['type']: + filtered_values = df.loc[df['type'] == art_type, ['id', 'name']] + artifact_ids[name][art_type] = filtered_values + return artifact_ids -# This function fetches all the artifacts available in given mlmd -def get_artifacts(mlmdfilepath, pipeline_name, data): # get_artifacts return value (artifact_type or artifact_df) is - # determined by a data variable(). +def get_artifacts(mlmdfilepath, pipeline_name, art_type, artifact_ids): query = cmfquery.CmfQuery(mlmdfilepath) names = query.get_pipeline_names() # getting all pipeline names in mlmd - identifiers = [] - for name in names: - if name==pipeline_name: - stages = query.get_pipeline_stages(name) - for stage in stages: - executions = query.get_all_exe_in_stage(stage) - for exe in executions: - identifiers.append(exe.id) - name = [] - url = [] df = pd.DataFrame() - for identifier in identifiers: - get_artifacts = query.get_all_artifacts_for_execution( - identifier - ) # getting all artifacts - df = pd.concat([df, get_artifacts], sort=True, ignore_index=True) - df['event'] = df.groupby('id')['event'].transform(lambda x: ', '.join(x)) - df['name'] = df['name'].str.split(':').str[0] - df=df.drop_duplicates() - if data == "artifact_type": - tempout = list(set(df["type"])) - else: - df = df.loc[df["type"] == data] - result = df.to_json(orient="records") - tempout = json.loads(result) - return tempout + for name in names: + if name == pipeline_name: + df = query.get_all_artifacts_by_ids_list(artifact_ids) + if len(df) == 0: + return + df = df.drop_duplicates() + art_names = df['name'].tolist() + name_dict = {} + name_list = [] + exec_type_name_list = [] + exe_type_name = pd.DataFrame() + for name in art_names: + executions = query.get_all_executions_for_artifact(name) + exe_type_name = pd.concat([exe_type_name,executions],ignore_index=True) + execution_type_name = exe_type_name["execution_type_name"].drop_duplicates().tolist() + execution_type_name = [str(element).split('"')[1] for element in execution_type_name] + execution_type_name_str = ',\n '.join(map(str, execution_type_name)) + name_list.append(name) + exec_type_name_list.append(execution_type_name_str) + name_dict['name'] = name_list + name_dict['execution_type_name'] = exec_type_name_list + name_df = pd.DataFrame(name_dict) + merged_df = df.merge(name_df, on='name', how='left') + merged_df['name'] = merged_df['name'].apply(lambda x: x.split(':')[0] if ':' in x else x) + merged_df = merged_df.loc[merged_df["type"] == art_type] + result = merged_df.to_json(orient="records") + tempout = json.loads(result) + return tempout +def get_artifact_types(mlmdfilepath): + query = cmfquery.CmfQuery(mlmdfilepath) + artifact_types = query.get_all_artifact_types() + return artifact_types def create_unique_executions(server_store_path, req_info): mlmd_data = json.loads(req_info["json_payload"]) @@ -68,7 +108,7 @@ def create_unique_executions(server_store_path, req_info): executions_client = [] for i in mlmd_data['Pipeline'][0]["stages"]: # checks if given execution_id present in mlmd for j in i["executions"]: - if j['name'] != "":#If executions have name , they are reusable executions + if j['name'] != "": #If executions have name , they are reusable executions continue #which needs to be merged in irrespective of whether already #present or not so that new artifacts associated with it gets in. if 'Execution_uuid' in j['properties']: diff --git a/server/app/main.py b/server/app/main.py index f62bc124..b0f90bc7 100644 --- a/server/app/main.py +++ b/server/app/main.py @@ -1,23 +1,50 @@ # cmf-server api's -from fastapi import FastAPI, Request, APIRouter, status, HTTPException -import pandas as pd +from fastapi import FastAPI, Request, status, HTTPException, Query from fastapi.middleware.cors import CORSMiddleware -from cmflib import cmfquery, cmf_merger -from fastapi.encoders import jsonable_encoder -from fastapi.templating import Jinja2Templates from fastapi.responses import HTMLResponse, StreamingResponse from fastapi.staticfiles import StaticFiles -from server.app.get_data import get_executions, get_artifacts,get_lineage_img_path,create_unique_executions,get_mlmd_from_server + +from contextlib import asynccontextmanager +import pandas as pd + +from cmflib import cmfquery, cmf_merger +from server.app.get_data import ( + get_artifacts, + get_lineage_img_path, + create_unique_executions, + get_mlmd_from_server, + get_artifact_types, + get_all_artifact_ids, + get_all_exe_ids, + get_executions_by_ids +) from server.app.query_visualization import query_visualization -from server.app.schemas.dataframe import ExecutionDataFrame + from pathlib import Path import os import json -app = FastAPI(title="cmf-server") +server_store_path = "/cmf-server/data/mlmd" + +dict_of_art_ids = {} +dict_of_exe_ids = {} + +@asynccontextmanager +async def lifespan(app: FastAPI): + global dict_of_art_ids + global dict_of_exe_ids + if os.path.exists(server_store_path): + # loaded artifact ids into memory + dict_of_art_ids = get_all_artifact_ids(server_store_path) + # loaded execution ids with names into memory + dict_of_exe_ids = get_all_exe_ids(server_store_path) + yield + dict_of_art_ids.clear() + dict_of_exe_ids.clear() + +app = FastAPI(title="cmf-server", lifespan=lifespan) BASE_PATH = Path(__file__).resolve().parent -templates = Jinja2Templates(directory=str(BASE_PATH/"template")) app.mount("/cmf-server/data/static", StaticFiles(directory="/cmf-server/data/static"), name="static") server_store_path = "/cmf-server/data/mlmd" if os.environ.get("MYIP") != "127.0.0.1": @@ -39,6 +66,7 @@ allow_headers=["*"], ) + @app.get("/") def read_root(request: Request): return {"cmf-server"} @@ -47,8 +75,13 @@ def read_root(request: Request): # api to post mlmd file to cmf-server @app.post("/mlmd_push") async def mlmd_push(info: Request): + print("mlmd push started") + print("......................") req_info = await info.json() status= create_unique_executions(server_store_path,req_info) + # async function + await update_global_art_dict() + await update_global_exe_dict() return {"status": status, "data": req_info} @@ -64,16 +97,43 @@ async def mlmd_pull(info: Request, pipeline_name: str): json_payload = "" return json_payload - # api to display executions available in mlmd @app.get("/display_executions/{pipeline_name}") -async def display_exec(request: Request, pipeline_name: str): +async def display_exec( + request: Request, + pipeline_name: str, + page: int = Query(1, description="Page number", gt=0), + per_page: int = Query(5, description="Items per page", le=100), + sort_field: str = Query("Context_Type", description="Column to sort by"), + sort_order: str = Query("asc", description="Sort order (asc or desc)"), + filter_by: str = Query(None, description="Filter by column"), + filter_value: str = Query(None, description="Filter value"), + ): # checks if mlmd file exists on server if os.path.exists(server_store_path): - execution_df = get_executions(server_store_path, pipeline_name) - tempOut = execution_df.to_json(orient="records") - parsed = json.loads(tempOut) - return parsed + exe_ids_initial = dict_of_exe_ids[pipeline_name] + # Apply filtering if provided + if filter_by and filter_value: + exe_ids_initial = exe_ids_initial[exe_ids_initial[filter_by].str.contains(filter_value, case=False)] + # Apply sorting if provided + exe_ids_sorted = exe_ids_initial.sort_values(by=sort_field, ascending=(sort_order == "asc")) + exe_ids = exe_ids_sorted['id'].tolist() + total_items = len(exe_ids) + start_idx = (page - 1) * per_page + end_idx = start_idx + per_page + if total_items < end_idx: + end_idx = total_items + exe_ids_list = exe_ids[start_idx:end_idx] + executions_df = get_executions_by_ids(server_store_path, pipeline_name, exe_ids_list) + temp = executions_df.to_json(orient="records") + executions_parsed = json.loads(temp) + return { + "total_items": total_items, + "items": executions_parsed + } + else: + return + @app.get("/display_lineage/{pipeline_name}") async def display_lineage(request: Request, pipeline_name: str): @@ -88,17 +148,68 @@ async def display_lineage(request: Request, pipeline_name: str): return f"Pipeline name {pipeline_name} doesn't exist." else: - return 'mlmd doesnt exist' + return 'mlmd does not exist!!' + # api to display artifacts available in mlmd -@app.get("/display_artifact_type/{pipeline_name}/{data}") -async def display_artifact(request: Request, pipeline_name: str,data: str): +@app.get("/display_artifacts/{pipeline_name}/{type}") +async def display_artifact( + request: Request, + pipeline_name: str, + type: str, # type = artifact type + page: int = Query(1, description="Page number", gt=0), + per_page: int = Query(5, description="Items per page", le=100), + sort_field: str = Query("name", description="Column to sort by"), + sort_order: str = Query("asc", description="Sort order (asc or desc)"), + filter_by: str = Query(None, description="Filter by column"), + filter_value: str = Query(None, description="Filter value"), + ): + empty_df = pd.DataFrame() + art_ids_dict = {} + art_type = type # checks if mlmd file exists on server if os.path.exists(server_store_path): - artifact_df = get_artifacts(server_store_path, pipeline_name,data) - return artifact_df + art_ids_dict = dict_of_art_ids[pipeline_name] + if not art_ids_dict: + return + art_ids_initial = [] + if art_type in art_ids_dict: + art_ids_initial = art_ids_dict[art_type] + else: + return + # Apply filtering if provided + if filter_by and filter_value: + art_ids_initial = art_ids_initial[art_ids_initial[filter_by].str.contains(filter_value, case=False)] + # Apply sorting if provided + art_ids_sorted = art_ids_initial.sort_values(by=sort_field, ascending=(sort_order == "asc")) + art_ids = art_ids_sorted['id'].tolist() + total_items = len(art_ids) + start_idx = (page - 1) * per_page + end_idx = start_idx + per_page + if total_items < end_idx: + end_idx = total_items + artifact_id_list = list(art_ids)[start_idx:end_idx] + artifact_df = get_artifacts(server_store_path, pipeline_name, art_type, artifact_id_list) + data_paginated = artifact_df + return { + "total_items": total_items, + "items": data_paginated + } else: - artifact_df = "" + return f"{server_store_path} file doesn't exist." + + + +@app.get("/display_artifact_types") +async def display_artifact_types(request: Request): + # checks if mlmd file exists on server + if os.path.exists(server_store_path): + artifact_types = get_artifact_types(server_store_path) + return artifact_types + else: + artifact_types = "" + return + @app.get("/display_pipelines") async def display_list_of_pipelines(request: Request): @@ -112,3 +223,17 @@ async def display_list_of_pipelines(request: Request): pipeline_names = [] return pipeline_names + + +async def update_global_art_dict(): + global dict_of_art_ids + output_dict = get_all_artifact_ids(server_store_path) + dict_of_art_ids = output_dict + return + + +async def update_global_exe_dict(): + global dict_of_exe_ids + output_dict = get_all_exe_ids(server_store_path) + dict_of_exe_ids = output_dict + return diff --git a/server/requirements.txt b/server/requirements.txt index 98cd4761..3a1a76e6 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,4 +1,5 @@ fastapi +fastapi-pagination uvicorn pydantic minio diff --git a/ui/src/client.js b/ui/src/client.js index 823f81f5..47d888f2 100644 --- a/ui/src/client.js +++ b/ui/src/client.js @@ -23,9 +23,25 @@ class FastAPIClient { return client; } - async getArtifacts(pipelineName, type) { + async getArtifacts(pipelineName, type, page, sortField, sortOrder, filterBy, filterValue) { return this.apiClient - .get(`/display_artifact_type/${pipelineName}/${type}`) + .get(`/display_artifacts/${pipelineName}/${type}`, { + params: { + page: page, + sort_field: sortField, + sort_order: sortOrder, + filter_by: filterBy, + filter_value: filterValue, + }, + }) + .then(({ data }) => { + return data; + }); + } + + async getArtifactTypes() { + return this.apiClient + .get(`/display_artifact_types`) .then(({ data }) => { return data; }); @@ -41,9 +57,17 @@ class FastAPIClient { } } - getExecutions(pipelineName) { + async getExecutions(pipelineName, page, sortField, sortOrder , filterBy, filterValue) { return this.apiClient - .get(`/display_executions/${pipelineName}`) + .get(`/display_executions/${pipelineName}`, { + params: { + page: page, + sort_field: sortField, + sort_order: sortOrder, + filter_by: filterBy, + filter_value: filterValue, + }, + }) .then(({ data }) => { return data; }); diff --git a/ui/src/components/ArtifactTable/index.css b/ui/src/components/ArtifactTable/index.css index 900f5c34..45661af7 100644 --- a/ui/src/components/ArtifactTable/index.css +++ b/ui/src/components/ArtifactTable/index.css @@ -33,3 +33,7 @@ background-color: #1a365d; border-color: #1a365d; } +.arrow { + font-weight: bold; + font-size: 1.2em; /* Adjust the size as needed */ +} diff --git a/ui/src/components/ArtifactTable/index.jsx b/ui/src/components/ArtifactTable/index.jsx index 6a8b88cd..4ca0a797 100644 --- a/ui/src/components/ArtifactTable/index.jsx +++ b/ui/src/components/ArtifactTable/index.jsx @@ -1,66 +1,36 @@ // ArtifactTable.jsx -import React, { useState, useEffect } from 'react'; -import './index.css'; -const ArtifactTable = ({ artifacts }) => { +import React, { useState, useEffect } from "react"; +import "./index.css"; +const ArtifactTable = ({ artifacts, onSort, onFilter }) => { -const [searchQuery, setSearchQuery] = useState(''); -const [currentPage, setCurrentPage] = useState(1); -const [itemsPerPage] = useState(5); // Number of items to display per page -const [sortBy, setSortBy] = useState(null); // Property to sort by -const [sortOrder, setSortOrder] = useState('asc'); // Sort order ('asc' or 'desc') -const [expandedRow, setExpandedRow] = useState(null); -const handleSearchChange = (event) => { - setSearchQuery(event.target.value); - }; - -const handlePageChange = (page) => { - setCurrentPage(page); - }; - - -const handleSort = (property) => { - if (sortBy === property) { - // If currently sorted by the same property, toggle sort order - setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc'); - } else { - // If sorting by a new property, set it to ascending order by default - setSortBy(property); - setSortOrder('asc'); - } - }; - -const consistentColumns = []; + // Default sorting order + const [sortOrder, setSortOrder] = useState("Context_Type"); -const filteredData = artifacts.filter((item) => - (item.name && item.name.toLowerCase().includes(searchQuery.toLowerCase())) - || (item.type && item.type.toLowerCase().includes(searchQuery.toLowerCase())) - ); + // Local filter value state + const [filterValue, setFilterValue] = useState(""); -// eslint-disable-next-line -const sortedData = filteredData.sort((a, b) => { - const aValue = a[sortBy]; - const bValue = b[sortBy]; + const [expandedRow, setExpandedRow] = useState(null); - if (aValue < bValue) return sortOrder === 'asc' ? -1 : 1; - if (aValue > bValue) return sortOrder === 'asc' ? 1 : -1; - return 0; - }); + const consistentColumns = []; -const totalPages = Math.ceil(filteredData.length / itemsPerPage); -const indexOfLastItem = currentPage * itemsPerPage; -const indexOfFirstItem = indexOfLastItem - itemsPerPage; -const currentItems = filteredData.slice(indexOfFirstItem, indexOfLastItem); + useEffect(() => { + // Set initial sorting order when component mounts + setSortOrder("asc"); + }, []); -useEffect(() => { - setCurrentPage(1); // Reset current page to 1 when search query changes - }, [searchQuery]); - -useEffect(() => { - setCurrentPage(1); // Reset current page to 1 when search query changes - }, [artifacts]); + const handleSort = () => { + const newSortOrder = sortOrder === "asc" ? "desc" : "asc"; + setSortOrder(newSortOrder); + onSort("name", newSortOrder); // Notify parent component about sorting change + }; + const handleFilterChange = (event) => { + const value = event.target.value; + setFilterValue(value); + onFilter("name", value); // Notify parent component about filter change + }; -const toggleRow = (rowId) => { + const toggleRow = (rowId) => { if (expandedRow === rowId) { setExpandedRow(null); } else { @@ -68,79 +38,16 @@ const toggleRow = (rowId) => { } }; -// eslint-disable-next-line -const renderTableData = () => { - if (currentItems.length === 0) { - return ( - - No data available - - ); - } - - return currentItems.map((item) => ( - - {/* Render table row */} - - )); - }; - - -const renderPagination = () => { - if (totalPages === 1){ - return null; - } - - const totalPagesToShow = 5; // Number of pages to show in the pagination - - const startPage = Math.max(1, currentPage - Math.floor(totalPagesToShow / 2)); - const endPage = Math.min(totalPages, startPage + totalPagesToShow - 1); - - const pages = Array.from({ length: endPage - startPage + 1 }, (_, index) => startPage + index); - - return ( -
- - {pages.map((page) => ( - - ))} - - -
- ); - }; - - - -return ( + return (
- +
@@ -148,61 +55,88 @@ return ( - id - handleSort('name')} className="name px-6 py-3">name - {sortBy === 'name' && sortOrder === 'asc' && '▲'} - {sortBy === 'name' && sortOrder === 'desc' && '▼'} - Type - Url - Uri - Git_Repo - Commit + + id + + + name {sortOrder === "asc" && ↑} + {sortOrder === "desc" && ↓} + + + execution_type_name + + + Url + + + Uri + + + Git_Repo + + + Commit + - {currentItems.map((data, index) => ( + {artifacts.length > 0 && artifacts.map((data, index) => ( - toggleRow(index)} className="text-sm font-medium text-gray-800"> - {expandedRow === index ? '-' : '+'} - {data.id} - {data.name} - {data.type} - {data.url} - {data.uri} - {data.git_repo} - {data.Commit} - - {expandedRow === index && ( - - - - - {Object.entries(data).map(([key, value]) => { - if (!consistentColumns.includes(key) && value != null) { - return ( - - - - - - - ); - } - return null; - })} - -
{key}{value ? value :"Null"}
- - - )} - + toggleRow(index)} + className="text-sm font-medium text-gray-800" + > + + {expandedRow === index ? "-" : "+"} + + {data.id} + {data.name} + {data.execution_type_name} + {data.url} + {data.uri} + {data.git_repo} + {data.Commit} + + {expandedRow === index && ( + + + + + {Object.entries(data).map(([key, value]) => { + if ( + !consistentColumns.includes(key) && + value != null + ) { + return ( + + + + + + + ); + } + return null; + })} + +
{key} + {value ? value : "Null"} +
+ + + )} + ))} -
-
{renderPagination()}
-
+
+ ); }; diff --git a/ui/src/components/ExecutionTable/index.css b/ui/src/components/ExecutionTable/index.css index 7d98f9e0..d79cdb3e 100644 --- a/ui/src/components/ExecutionTable/index.css +++ b/ui/src/components/ExecutionTable/index.css @@ -9,3 +9,7 @@ background-color: #1a365d; border-color: #1a365d; } +.arrow { + font-weight: bold; + font-size: 1.2em; /* Adjust the size as needed */ +} diff --git a/ui/src/components/ExecutionTable/index.jsx b/ui/src/components/ExecutionTable/index.jsx index b1608a8b..0e2afbdb 100644 --- a/ui/src/components/ExecutionTable/index.jsx +++ b/ui/src/components/ExecutionTable/index.jsx @@ -1,67 +1,37 @@ //ExecutionTable.jsx -import React, { useState, useEffect } from 'react'; -import './index.css'; +import React, { useState, useEffect } from "react"; +import "./index.css"; -const ExecutionTable = ({ executions }) => { - -const [searchQuery, setSearchQuery] = useState(''); -const [currentPage, setCurrentPage] = useState(1); -const [itemsPerPage] = useState(5); // Number of items to display per page -const [sortBy, setSortBy] = useState(null); // Property to sort by -const [sortOrder, setSortOrder] = useState('asc'); // Sort order ('asc' or 'desc') -const [expandedRow, setExpandedRow] = useState(null); -const handleSearchChange = (event) => { - setSearchQuery(event.target.value); - }; - -const handlePageChange = (page) => { - setCurrentPage(page); - }; +const ExecutionTable = ({ executions, onSort, onFilter }) => { + // Default sorting order + const [sortOrder, setSortOrder] = useState("Context_Type"); -const handleSort = (property) => { - if (sortBy === property) { - // If currently sorted by the same property, toggle sort order - setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc'); - } else { - // If sorting by a new property, set it to ascending order by default - setSortBy(property); - setSortOrder('asc'); - } - }; -const consistentColumns = []; + // Local filter value state + const [filterValue, setFilterValue] = useState(""); -const filteredData = executions.filter((item) => - (item.Context_Type && item.Context_Type.toLowerCase().includes(searchQuery.toLowerCase())) - || (item.Execution && item.Execution.toLowerCase().includes(searchQuery.toLowerCase())) - ); + const [expandedRow, setExpandedRow] = useState(null); -// eslint-disable-next-line -const sortedData = filteredData.sort((a, b) => { - const aValue = a[sortBy]; - const bValue = b[sortBy]; + const consistentColumns = []; - if (aValue < bValue) return sortOrder === 'asc' ? -1 : 1; - if (aValue > bValue) return sortOrder === 'asc' ? 1 : -1; - return 0; - }); - -const totalPages = Math.ceil(filteredData.length / itemsPerPage); -const indexOfLastItem = currentPage * itemsPerPage; -const indexOfFirstItem = indexOfLastItem - itemsPerPage; -const currentItems = filteredData.slice(indexOfFirstItem, indexOfLastItem); - -useEffect(() => { - setCurrentPage(1); // Reset current page to 1 when search query changes - }, [searchQuery]); - -useEffect(() => { - setCurrentPage(1); // Reset current page to 1 when search query changes - }, [executions]); + useEffect(() => { + // Set initial sorting order when component mounts + setSortOrder("asc"); + }, []); + const handleSort = () => { + const newSortOrder = sortOrder === "asc" ? "desc" : "asc"; + setSortOrder(newSortOrder); + onSort("Context_Type", newSortOrder); // Notify parent component about sorting change + }; + const handleFilterChange = (event) => { + const value = event.target.value; + setFilterValue(value); + onFilter("Context_Type", value); // Notify parent component about filter change + }; -const toggleRow = (rowId) => { + const toggleRow = (rowId) => { if (expandedRow === rowId) { setExpandedRow(null); } else { @@ -69,79 +39,26 @@ const toggleRow = (rowId) => { } }; -// eslint-disable-next-line -const renderTableData = () => { - if (currentItems.length === 0) { - return ( - - No data available - - ); - } - - return currentItems.map((item) => ( - - {/* Render table row */} - - )); - }; - - -const renderPagination = () => { - if (totalPages === 1){ - return null; - } - - const totalPagesToShow = 5; // Number of pages to show in the pagination - - const startPage = Math.max(1, currentPage - Math.floor(totalPagesToShow / 2)); - const endPage = Math.min(totalPages, startPage + totalPagesToShow - 1); - - const pages = Array.from({ length: endPage - startPage + 1 }, (_, index) => startPage + index); - - return ( -
- - {pages.map((page) => ( - - ))} - - -
- ); - }; - - - -return ( + return (
-
- +
+
@@ -149,59 +66,81 @@ return ( - handleSort('Context_Type')} className="px-6 py-3 Context_Type">Context_Type - {sortBy === 'Context_Type' && sortOrder === 'asc' && '▲'} - {sortBy === 'Context_Type' && sortOrder === 'desc' && '▼'} - Execution - Git_Repo - Git_Start_Commit - Pipeline_Type + + Context_Type {sortOrder === "asc" && ↑} + {sortOrder === "desc" && ↓} + + + Execution + + + Git_Repo + + + Git_Start_Commit + + + Pipeline_Type + - {currentItems.map((data, index) => ( + {executions.map((data, index) => ( - toggleRow(index)} className="text-sm font-medium text-gray-800"> - {expandedRow === index ? '-' : '+'} - {data.Context_Type} - {data.Execution} - {data.Git_Repo} - {data.Git_Start_Commit} - {data.Pipeline_Type} - - {expandedRow === index && ( - - - - - {Object.entries(data).map(([key, value]) => { - if (!consistentColumns.includes(key) && value != null) { - return ( - - - - - - - ); - } - return null; - })} - -
{key}{value ? value :"Null"}
- - - )} - + toggleRow(index)} + className="text-sm font-medium text-gray-800" + > + + {expandedRow === index ? "-" : "+"} + + {data.Context_Type} + {data.Execution} + {data.Git_Repo} + {data.Git_Start_Commit} + {data.Pipeline_Type} + + {expandedRow === index && ( + + + + + {Object.entries(data).map(([key, value]) => { + if ( + !consistentColumns.includes(key) && + value != null + ) { + return ( + + + + + + + ); + } + return null; + })} + +
{key} + {value ? value : "Null"} +
+ + + )} + ))} -
-
{renderPagination()}
-
+
+ ); }; - export default ExecutionTable; diff --git a/ui/src/pages/artifacts/index.css b/ui/src/pages/artifacts/index.css index ada97ab9..97a4c34d 100644 --- a/ui/src/pages/artifacts/index.css +++ b/ui/src/pages/artifacts/index.css @@ -75,3 +75,16 @@ body { height: 5px; background: rgb(88, 147, 241); } + + +.active { + background-color: #2c5282; + color: #fff; + font-weight: bold; + border-color: #2c5282; +} + +.active:hover { + background-color: #1a365d; + border-color: #1a365d; +} diff --git a/ui/src/pages/artifacts/index.jsx b/ui/src/pages/artifacts/index.jsx index 0b2cfc67..0298f2e7 100644 --- a/ui/src/pages/artifacts/index.jsx +++ b/ui/src/pages/artifacts/index.jsx @@ -16,6 +16,17 @@ const Artifacts = () => { const [artifacts, setArtifacts] = useState([]); const [artifactTypes, setArtifactTypes] = useState([]); const [selectedArtifactType, setSelectedArtifactType] = useState(null); + const [totalItems, setTotalItems] = useState(0); + const [activePage, setActivePage] = useState(1); + const [clickedButton, setClickedButton] = useState("page"); + // Default sort field + const [sortField, setSortField] = useState("name"); + // Default sort order + const [sortOrder, setSortOrder] = useState("asc"); + // Default filter field + const [filterBy, setFilterBy] = useState(null); + // Default filter value + const [filterValue, setFilterValue] = useState(null); const fetchPipelines = () => { client.getPipelines("").then((data) => { @@ -24,55 +35,80 @@ const Artifacts = () => { }); }; - - useEffect(() => { - fetchPipelines(); - }, []); + useEffect(() => { + fetchPipelines(); + }, []); const handlePipelineClick = (pipeline) => { - setSelectedPipeline(pipeline) + setSelectedPipeline(pipeline); + setActivePage(1); }; const handleArtifactTypeClick = (artifactType) => { setSelectedArtifactType(artifactType); + setActivePage(1); }; - - const fetchArtifactTypes = (pipelineName) => { - client.getArtifacts(pipelineName, "artifact_type").then((types) => { + const fetchArtifactTypes = () => { + client.getArtifactTypes().then((types) => { setArtifactTypes(types); - handleArtifactTypeClick(types[0]) + handleArtifactTypeClick(types[0]); }); }; - - useEffect(() => { - if(selectedPipeline) { + useEffect(() => { + if (selectedPipeline) { fetchArtifactTypes(selectedPipeline); } // eslint-disable-next-line }, [selectedPipeline]); - - - const fetchArtifacts = (pipelineName, type) => { - client.getArtifacts(pipelineName, type).then((data) => { - setArtifacts(data); + const fetchArtifacts = (pipelineName, type, page, sortField, sortOrder, filterBy, filterValue) => { + client.getArtifacts(pipelineName, type, page, sortField, sortOrder, filterBy, filterValue).then((data) => { + setArtifacts(data.items); + setTotalItems(data.total_items); }); }; - + useEffect(() => { + if (selectedPipeline && selectedArtifactType) { + fetchArtifacts(selectedPipeline, selectedArtifactType, activePage, sortField, sortOrder, filterBy, filterValue); + } + }, [selectedPipeline, selectedArtifactType, activePage, sortField, sortOrder, filterBy, filterValue]); + + const handlePageClick = (page) => { + setActivePage(page); + setClickedButton("page"); + }; + const handlePrevClick = () => { + if (activePage > 1) { + setActivePage(activePage - 1); + setClickedButton("prev"); + handlePageClick(activePage - 1); + } + }; - useEffect(() => { - if(selectedPipeline && selectedArtifactType) { - fetchArtifacts(selectedPipeline, selectedArtifactType); + const handleNextClick = () => { + if (activePage < Math.ceil(totalItems / 5)) { + setActivePage(activePage + 1); + setClickedButton("next"); + handlePageClick(activePage + 1); } + }; + + const handleSort = (newSortField, newSortOrder) => { + setSortField(newSortField); + setSortOrder(newSortOrder); + }; - }, [selectedPipeline, selectedArtifactType]); + const handleFilter = (field, value) => { + setFilterBy(field); + setFilterValue(value); + }; -return ( + return ( <>
- +
-
+
{selectedPipeline !== null && ( - + )}
- {selectedPipeline !== null && selectedArtifactType !== null && ( - + )} +
+ {artifacts !== null && totalItems > 0 && ( + <> + + {Array.from({ length: Math.ceil(totalItems / 5) }).map( + (_, index) => { + const pageNumber = index + 1; + if ( + pageNumber === 1 || + pageNumber === Math.ceil(totalItems / 5) + ) { + return ( + + ); + } else if ( + (activePage <= 3 && pageNumber <= 6) || + (activePage >= Math.ceil(totalItems / 5) - 2 && + pageNumber >= Math.ceil(totalItems / 5) - 5) || + Math.abs(pageNumber - activePage) <= 2 + ) { + return ( + + ); + } else if ( + (pageNumber === 2 && activePage > 3) || + (pageNumber === Math.ceil(totalItems / 5) - 1 && + activePage < Math.ceil(totalItems / 5) - 3) + ) { + return ( + + ... + + ); + } + return null; + } + )} + + + )} +
diff --git a/ui/src/pages/executions/index.css b/ui/src/pages/executions/index.css index e8895220..73b44a4c 100644 --- a/ui/src/pages/executions/index.css +++ b/ui/src/pages/executions/index.css @@ -86,3 +86,15 @@ button { .active-content { display: block; } + +.active { + background-color: #2c5282; + color: #fff; + font-weight: bold; + border-color: #2c5282; +} + +.active:hover { + background-color: #1a365d; + border-color: #1a365d; +} diff --git a/ui/src/pages/executions/index.jsx b/ui/src/pages/executions/index.jsx index 74d79541..ba7eb991 100644 --- a/ui/src/pages/executions/index.jsx +++ b/ui/src/pages/executions/index.jsx @@ -10,10 +10,20 @@ import Sidebar from "../../components/Sidebar"; const client = new FastAPIClient(config); const Executions = () => { - const [pipelines, setPipelines] = useState([]); const [selectedPipeline, setSelectedPipeline] = useState(null); const [executions, setExecutions] = useState([]); + const [totalItems, setTotalItems] = useState(0); + const [activePage, setActivePage] = useState(1); + const [clickedButton, setClickedButton] = useState("page"); + // Default sort field + const [sortField, setSortField] = useState("Context_Type"); + // Default sort order + const [sortOrder, setSortOrder] = useState("asc"); + // Default filter field + const [filterBy, setFilterBy] = useState(null); + // Default filter value + const [filterValue, setFilterValue] = useState(null); useEffect(() => { fetchPipelines(); @@ -27,20 +37,52 @@ const Executions = () => { }; useEffect(() => { - if(selectedPipeline) { - fetchExecutions(selectedPipeline); + if (selectedPipeline) { + fetchExecutions(selectedPipeline, activePage, sortField, sortOrder, filterBy, filterValue); } + }, [selectedPipeline, activePage, sortField, sortOrder, filterBy, filterValue]); - }, [selectedPipeline]); - - const fetchExecutions = (pipelineName) => { - client.getExecutions(pipelineName).then((data) => { - setExecutions(data); + const fetchExecutions = (pipelineName, page, sortField, sortOrder, filterBy, filterValue) => { + client.getExecutions(pipelineName, page, sortField, sortOrder, filterBy, filterValue).then((data) => { + setExecutions(data.items); + setTotalItems(data.total_items); }); }; const handlePipelineClick = (pipeline) => { - setSelectedPipeline(pipeline) + setSelectedPipeline(pipeline); + setActivePage(1); + }; + + const handlePageClick = (page) => { + setActivePage(page); + setClickedButton("page"); + }; + + const handlePrevClick = () => { + if (activePage > 1) { + setActivePage(activePage - 1); + setClickedButton("prev"); + handlePageClick(activePage - 1); + } + }; + + const handleNextClick = () => { + if (activePage < Math.ceil(totalItems / 5)) { + setActivePage(activePage + 1); + setClickedButton("next"); + handlePageClick(activePage + 1); + } + }; + + const handleSort = (newSortField, newSortOrder) => { + setSortField(newSortField); + setSortOrder(newSortOrder); + }; + + const handleFilter = (field, value) => { + setFilterBy(field); + setFilterValue(value); }; return ( @@ -51,11 +93,92 @@ const Executions = () => { >
- +
{selectedPipeline !== null && ( - + + )} +
+
+ {executions !== null && totalItems > 0 && ( + <> + + {Array.from({ length: Math.ceil(totalItems / 5) }).map( + (_, index) => { + const pageNumber = index + 1; + if ( + pageNumber === 1 || + pageNumber === Math.ceil(totalItems / 5) + ) { + return ( + + ); + } else if ( + (activePage <= 3 && pageNumber <= 6) || + (activePage >= Math.ceil(totalItems / 5) - 2 && + pageNumber >= Math.ceil(totalItems / 5) - 5) || + Math.abs(pageNumber - activePage) <= 2 + ) { + return ( + + ); + } else if ( + (pageNumber === 2 && activePage > 3) || + (pageNumber === Math.ceil(totalItems / 5) - 1 && + activePage < Math.ceil(totalItems / 5) - 3) + ) { + return ( + + ... + + ); + } + return null; + } + )} + + )}