From 9b9ebed55925a257d59ccde5eb92663e4f3486e1 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Wed, 11 Dec 2024 10:05:11 +0800 Subject: [PATCH 01/33] add new panel under feature flag --- frontend/src/components/editor/chrome/wrapper/app-chrome.tsx | 2 ++ frontend/src/core/config/feature-flag.tsx | 2 ++ 2 files changed, 4 insertions(+) diff --git a/frontend/src/components/editor/chrome/wrapper/app-chrome.tsx b/frontend/src/components/editor/chrome/wrapper/app-chrome.tsx index b931b14867b..40d07bac762 100644 --- a/frontend/src/components/editor/chrome/wrapper/app-chrome.tsx +++ b/frontend/src/components/editor/chrome/wrapper/app-chrome.tsx @@ -30,6 +30,7 @@ import { IfCapability } from "@/core/config/if-capability"; import { PackagesPanel } from "../panels/packages-panel"; import { ChatPanel } from "@/components/chat/chat-panel"; import { TooltipProvider } from "@radix-ui/react-tooltip"; +import { TracingPanel } from "../panels/tracing-panel"; const LazyTerminal = React.lazy(() => import("@/components/terminal/terminal")); @@ -153,6 +154,7 @@ export const AppChrome: React.FC = ({ children }) => { {selectedPanel === "scratchpad" && } {selectedPanel === "chat" && } {selectedPanel === "logs" && } + {selectedPanel === "tracing" && } diff --git a/frontend/src/core/config/feature-flag.tsx b/frontend/src/core/config/feature-flag.tsx index c2823640c94..c8f38664358 100644 --- a/frontend/src/core/config/feature-flag.tsx +++ b/frontend/src/core/config/feature-flag.tsx @@ -12,6 +12,7 @@ export interface ExperimentalFeatures { scratchpad: boolean; multi_column: boolean; chat_sidebar: boolean; + tracing: boolean; // Add new feature flags here } @@ -21,6 +22,7 @@ const defaultValues: ExperimentalFeatures = { scratchpad: true, multi_column: import.meta.env.DEV, chat_sidebar: import.meta.env.DEV, + tracing: import.meta.env.DEV, }; export function getFeatureFlag( From 266d895b5ef845fd251f65c828c0ac72a55f4143 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sat, 14 Dec 2024 16:01:19 +0800 Subject: [PATCH 02/33] add experimental tracing panel --- .../src/components/editor/cell/CellStatus.tsx | 4 +- .../editor/chrome/panels/tracing-panel.tsx | 7 + .../src/components/editor/chrome/types.ts | 9 + .../src/components/editor/dynamic-favicon.tsx | 2 +- frontend/src/components/tracing/tracing.tsx | 391 ++++++++++++++++++ 5 files changed, 410 insertions(+), 3 deletions(-) create mode 100644 frontend/src/components/editor/chrome/panels/tracing-panel.tsx create mode 100644 frontend/src/components/tracing/tracing.tsx diff --git a/frontend/src/components/editor/cell/CellStatus.tsx b/frontend/src/components/editor/cell/CellStatus.tsx index ceeee2cb482..b748937d14c 100644 --- a/frontend/src/components/editor/cell/CellStatus.tsx +++ b/frontend/src/components/editor/cell/CellStatus.tsx @@ -321,7 +321,7 @@ export const CellStatusComponent: React.FC = ({ return null; }; -const ElapsedTime = (props: { elapsedTime: string }) => { +export const ElapsedTime = (props: { elapsedTime: string }) => { return ( {props.elapsedTime} ); @@ -346,7 +346,7 @@ const LastRanTime = (props: { lastRanTime: number }) => { ); }; -function formatElapsedTime(elapsedTime: number | null) { +export function formatElapsedTime(elapsedTime: number | null) { if (elapsedTime === null) { return ""; } diff --git a/frontend/src/components/editor/chrome/panels/tracing-panel.tsx b/frontend/src/components/editor/chrome/panels/tracing-panel.tsx new file mode 100644 index 00000000000..a80d3284c98 --- /dev/null +++ b/frontend/src/components/editor/chrome/panels/tracing-panel.tsx @@ -0,0 +1,7 @@ +/* Copyright 2024 Marimo. All rights reserved. */ +import { Tracing } from "@/components/tracing/tracing"; +import React from "react"; + +export const TracingPanel: React.FC = () => { + return ; +}; diff --git a/frontend/src/components/editor/chrome/types.ts b/frontend/src/components/editor/chrome/types.ts index 387e54edda1..42760482b4e 100644 --- a/frontend/src/components/editor/chrome/types.ts +++ b/frontend/src/components/editor/chrome/types.ts @@ -14,6 +14,7 @@ import { NotebookPenIcon, BoxIcon, BotMessageSquareIcon, + ActivityIcon, } from "lucide-react"; export type PanelType = @@ -22,6 +23,7 @@ export type PanelType = | "variables" | "outline" | "dependencies" + | "tracing" | "packages" | "documentation" | "snippets" @@ -94,6 +96,13 @@ export const PANELS: PanelDescriptor[] = [ tooltip: "Notebook logs", position: "sidebar", }, + { + type: "tracing", + Icon: ActivityIcon, + tooltip: "Tracing", + position: "sidebar", + hidden: !getFeatureFlag("tracing"), + }, { type: "snippets", Icon: SquareDashedBottomCodeIcon, diff --git a/frontend/src/components/editor/dynamic-favicon.tsx b/frontend/src/components/editor/dynamic-favicon.tsx index 9bcdfcfea37..572ab716585 100644 --- a/frontend/src/components/editor/dynamic-favicon.tsx +++ b/frontend/src/components/editor/dynamic-favicon.tsx @@ -4,7 +4,7 @@ import { useEventListener } from "@/hooks/useEventListener"; import { usePrevious } from "@dnd-kit/utilities"; import { useEffect } from "react"; -const FAVICONS = { +export const FAVICONS = { idle: "./favicon.ico", success: "./circle-check.ico", running: "./circle-play.ico", diff --git a/frontend/src/components/tracing/tracing.tsx b/frontend/src/components/tracing/tracing.tsx new file mode 100644 index 00000000000..506ffeaebe0 --- /dev/null +++ b/frontend/src/components/tracing/tracing.tsx @@ -0,0 +1,391 @@ +/* Copyright 2024 Marimo. All rights reserved. */ +import { FAVICONS } from "@/components/editor/dynamic-favicon"; +import React, { useRef, useState } from "react"; +import type { CellId } from "@/core/cells/ids"; +import { ElapsedTime, formatElapsedTime } from "../editor/cell/CellStatus"; +import { Tooltip } from "@/components/ui/tooltip"; +import { type Config, type TopLevelSpec, compile } from "vega-lite"; +import { ChevronRight, ChevronDown, SettingsIcon } from "lucide-react"; +import type { VisualizationSpec } from "react-vega"; + +const LazyVegaLite = React.lazy(() => + import("react-vega").then((m) => ({ default: m.VegaLite })), +); + +interface CellRun { + cellID: string; + code: string; + elapsedTime: number; + status: RunStatus; +} + +export interface Run { + runId: string; + cellRuns: CellRun[]; + runStartTime: string; +} + +// assumes the cellRuns are ordered correctly +const mockRuns: Run[] = [ + { + runId: "7", + cellRuns: [ + { + cellID: "1", + code: "import marimo as mo", + elapsedTime: 23_000, + status: "success", + }, + { + cellID: "2", + code: "def generate_some()", + elapsedTime: 46, + status: "success", + }, + { + cellID: "3", + code: "def generate_some()", + elapsedTime: 46, + status: "success", + }, + ], + runStartTime: "Dec 7 2.03pm", + }, + { + runId: "8", + cellRuns: [ + { + cellID: "1", + code: "import marimo as mo", + elapsedTime: 25, + status: "success", + }, + { + cellID: "2", + code: "def generate_some()", + elapsedTime: 100, + status: "success", + }, + { + cellID: "3", + code: "def generate_some()", + elapsedTime: 4600, + status: "error", + }, + { + cellID: "4", + code: "[print(i) for i in range(1, 100)]", + elapsedTime: 8, + status: "error", // should be disabled? + }, + ], + runStartTime: "Dec 7 2.01pm", + }, +]; + +interface Values { + cell: number; + startTimestamp: string; + endTimestamp: string; + elapsedTime: string; +} + +export interface Data { + values: Values[]; +} + +const sampleData: Data = { + values: [ + { + cell: 1, + startTimestamp: "2024-12-01 11:00:00.000", + endTimestamp: "2024-12-01 11:00:00.230", + elapsedTime: "23ms", + }, + { + cell: 2, + startTimestamp: "2024-12-01 11:00:00.230", + endTimestamp: "2024-12-01 11:00:00.466", + elapsedTime: "46ms", + }, + { + cell: 3, + startTimestamp: "2024-12-01 11:00:00.466", + endTimestamp: "2024-12-01 11:00:00.636", + elapsedTime: "46ms", + }, + ], +}; + +const baseSpec: TopLevelSpec = { + $schema: "https://vega.github.io/schema/vega-lite/v5.json", + mark: { + type: "bar", + cornerRadius: 2, + fill: "#37BE5F", // same colour as chrome's network tab + }, + height: { step: 23 }, + params: [ + { + name: "zoomAndPan", + select: "interval", + bind: "scales", + }, + { + name: "hoveredCellID", + bind: { element: "#hiddenInputElement" }, + }, + { + name: "cursor", + value: "grab", + }, + ], + encoding: { + y: { + field: "cell", + axis: null, + scale: { paddingInner: 0.2 }, + }, + x: { + field: "startTimestamp", + type: "temporal", + axis: { orient: "top", title: null }, + }, + x2: { field: "endTimestamp", type: "temporal" }, + tooltip: [ + { field: "cell", title: "Cell" }, + { + field: "startTimestamp", + type: "temporal", + timeUnit: "monthdatehoursminutessecondsmilliseconds", + title: "Start", + }, + { + field: "endTimestamp", + type: "temporal", + timeUnit: "monthdatehoursminutessecondsmilliseconds", + title: "End", + }, + ], + size: { + value: { expr: "hoveredCellID == toString(datum.cell) ? 25 : 20" }, + }, + }, +}; + +function createGanttVegaLiteSpec(data: Data): TopLevelSpec { + return { + ...baseSpec, + data, + }; +} + +const config: Config = { + view: { + stroke: "transparent", + }, +}; + +export const Tracing: React.FC = () => { + // TODO: Either we get the runs of cells from FE side or BE side + + const [chartPosition, setChartPosition] = + useState("sideBySide"); + + const toggleConfig = () => { + if (chartPosition === "above") { + setChartPosition("sideBySide"); + } else { + setChartPosition("above"); + } + }; + + return ( +
+ + + + +
+ {mockRuns.map((run) => ( + + ))} +
+
+ ); +}; + +type ChartPosition = "sideBySide" | "above"; + +interface ChartProps { + className?: string; + width: number; + height: number; + vegaSpec: VisualizationSpec; +} + +const Chart: React.FC = (props: ChartProps) => { + return ( +
+ +
+ ); +}; + +const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ + run, + chartPosition, +}) => { + // TODO: Initial one should be false, all the others are true + const [collapsed, setCollapsed] = useState(false); + + // Used to sync Vega charts and React components + // Note that this will only work for the first chart for now, until we create unique input elements + const [hoveredCellID, setHoveredCellID] = useState(); + const hiddenInputRef = useRef(null); + + const hoverOnCell = (cellID: number) => { + setHoveredCellID(cellID); + // dispatch input event to trigger vega's param to update + if (hiddenInputRef.current) { + hiddenInputRef.current.value = String(cellID); + hiddenInputRef.current.dispatchEvent( + new Event("input", { bubbles: true }), + ); + } + }; + + const ChevronComponent = () => { + const Icon = collapsed ? ChevronRight : ChevronDown; + return ; + }; + + const TraceTitle = ( + setCollapsed(!collapsed)} + > + Run - {run.runStartTime} + + + ); + + const vegaSpec = compile(createGanttVegaLiteSpec(sampleData), { + config, + }).spec; + + const TraceRows = ( +
+ + {run.cellRuns.map((cellRun) => ( + + ))} +
+ ); + + if (chartPosition === "above") { + return ( +
+ 
+          {TraceTitle}
+          {!collapsed && }
+          {!collapsed && TraceRows}
+
+
+ ); + } + + return ( +
+ 
+        {TraceTitle}
+        {!collapsed && TraceRows}
+
+ {!collapsed && ( + + )} +
+ ); +}; + +type RunStatus = "success" | "running" | "error"; + +interface TraceRowProps { + timestamp: string; + cellID: CellId; + code: string; + elapsedTime: number; + status: RunStatus; + hoverOnCell: (cellID: number) => void; +} + +const TraceRow: React.FC = (props: TraceRowProps) => { + const elapsedTimeStr = formatElapsedTime(props.elapsedTime); + const elapsedTimeTooltip = ( + + This cell took to run + + ); + + const handleMouseEnter = () => { + props.hoverOnCell(props.cellID); + }; + + return ( +
+ [{props.timestamp}] + + {/* () */} + (cell-1) + + {props.code} + +
+ + {elapsedTimeStr} + + + {/* TODO: Shouldn't use favicon. */} + + {`${props.status} + +
+
+ ); +}; From 466fa9993310dbbbac3c1c54a7c9eed2969b3e0c Mon Sep 17 00:00:00 2001 From: Myles Scolnick Date: Sat, 14 Dec 2024 13:29:18 -0500 Subject: [PATCH 03/33] run_id in the backend --- frontend/src/core/cells/runs.ts | 118 ++++++++++++++++++ .../src/core/websocket/useMarimoWebSocket.tsx | 9 +- marimo/_messaging/context.py | 23 ++++ marimo/_messaging/ops.py | 16 +++ marimo/_runtime/runtime.py | 26 ++-- openapi/api.yaml | 3 + openapi/src/api.ts | 1 + 7 files changed, 182 insertions(+), 14 deletions(-) create mode 100644 frontend/src/core/cells/runs.ts create mode 100644 marimo/_messaging/context.py diff --git a/frontend/src/core/cells/runs.ts b/frontend/src/core/cells/runs.ts new file mode 100644 index 00000000000..3910d989d87 --- /dev/null +++ b/frontend/src/core/cells/runs.ts @@ -0,0 +1,118 @@ +/* Copyright 2024 Marimo. All rights reserved. */ +import { createReducerAndAtoms } from "@/utils/createReducer"; +import type { CellMessage } from "../kernel/messages"; +import type { TypedString } from "@/utils/typed"; +import type { CellId } from "./ids"; +export type RunId = TypedString<"RunId">; + +export interface CellRun { + cellId: CellId; + code: string; + elapsedTime: number; + startTime: number; + status: "success" | "error"; +} + +export interface Run { + runId: RunId; + cellRuns: CellRun[]; + runStartTime: number; +} + +export interface RunsState { + runIds: RunId[]; + runMap: Map; +} + +function initialState(): RunsState { + return { + runIds: [], + runMap: new Map(), + }; +} + +const MAX_RUNS = 100; +const MAX_CODE_LENGTH = 200; + +const { + reducer, + createActions, + valueAtom: runsAtom, + useActions: useRunsActions, +} = createReducerAndAtoms(initialState, { + addCellOperation: ( + state, + opts: { cellOperation: CellMessage; code: string }, + ) => { + console.log("addCellOperation", opts); + const { cellOperation, code } = opts; + const runId = cellOperation.run_id as RunId | undefined; + if (!runId) { + return state; + } + let run = state.runMap.get(runId); + if (!run) { + run = { + runId: runId, + cellRuns: [], + runStartTime: cellOperation.timestamp, + }; + } + + const nextRuns: CellRun[] = []; + let found = false; + for (const cellRun of run.cellRuns) { + if (cellRun.cellId === cellOperation.cell_id) { + nextRuns.push({ + ...cellRun, + elapsedTime: cellOperation.timestamp - cellRun.startTime, + }); + found = true; + } else { + nextRuns.push(cellRun); + } + } + if (!found) { + nextRuns.push({ + cellId: cellOperation.cell_id as CellId, + code: code.slice(0, MAX_CODE_LENGTH), + elapsedTime: 0, + // TODO: not actually correct logic + status: cellOperation.status === "idle" ? "success" : "error", + startTime: cellOperation.timestamp, + }); + } + + const nextRunMap = new Map(state.runMap); + nextRunMap.set(runId, run); + + return { + ...state, + runIds: [runId, ...state.runIds.slice(0, MAX_RUNS)], + runMap: nextRunMap, + }; + }, + clearRuns: (state) => ({ + ...state, + runIds: [], + runMap: new Map(), + }), + removeRun: (state, runId: RunId) => { + const nextRunIds = state.runIds.filter((id) => id !== runId); + const nextRunMap = new Map(state.runMap); + nextRunMap.delete(runId); + return { + ...state, + runIds: nextRunIds, + runMap: nextRunMap, + }; + }, +}); + +export { runsAtom, useRunsActions }; + +export const exportedForTesting = { + reducer, + createActions, + initialState, +}; diff --git a/frontend/src/core/websocket/useMarimoWebSocket.tsx b/frontend/src/core/websocket/useMarimoWebSocket.tsx index 7af3d4f47f6..6136221e92d 100644 --- a/frontend/src/core/websocket/useMarimoWebSocket.tsx +++ b/frontend/src/core/websocket/useMarimoWebSocket.tsx @@ -4,7 +4,7 @@ import { useAtom, useSetAtom } from "jotai"; import { connectionAtom } from "../network/connection"; import { useWebSocket } from "@/core/websocket/useWebSocket"; import { logNever } from "@/utils/assertNever"; -import { useCellActions } from "@/core/cells/cells"; +import { getNotebook, useCellActions } from "@/core/cells/cells"; import { AUTOCOMPLETER } from "@/core/codemirror/completion/Autocompleter"; import type { OperationMessage } from "@/core/kernel/messages"; import type { CellData } from "../cells/types"; @@ -40,6 +40,7 @@ import { focusAndScrollCellOutputIntoView } from "../cells/scrollCellIntoView"; import { capabilitiesAtom } from "../config/capabilities"; import { UI_ELEMENT_REGISTRY } from "../dom/uiregistry"; import { reloadSafe } from "@/utils/reload-safe"; +import { useRunsActions } from "../cells/runs"; /** * WebSocket that connects to the Marimo kernel and handles incoming messages. @@ -55,6 +56,7 @@ export function useMarimoWebSocket(opts: { const { showBoundary } = useErrorBoundary(); const { handleCellMessage, setCellCodes, setCellIds } = useCellActions(); + const { addCellOperation } = useRunsActions(); const setAppConfig = useSetAppConfig(); const { setVariables, setMetadata } = useVariablesActions(); const { addColumnPreview } = useDatasetsActions(); @@ -110,9 +112,12 @@ export function useMarimoWebSocket(opts: { msg.data, ); return; - case "cell-op": + case "cell-op": { handleCellOperation(msg.data, handleCellMessage); + const cellData = getNotebook().cellData[msg.data.cell_id as CellId]; + addCellOperation({ cellOperation: msg.data, code: cellData.code }); return; + } case "variables": setVariables( diff --git a/marimo/_messaging/context.py b/marimo/_messaging/context.py new file mode 100644 index 00000000000..af7a8edab96 --- /dev/null +++ b/marimo/_messaging/context.py @@ -0,0 +1,23 @@ +import uuid +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Any, Optional + +RunId_t = str +RUN_ID_CTX = ContextVar[Optional[RunId_t]]("run_id") + + +@dataclass +class run_id_context: + """Context manager for setting and unsetting the run ID.""" + + run_id: RunId_t + + def __init__(self) -> None: + self.run_id = str(uuid.uuid4()) + + def __enter__(self) -> None: + self.token = RUN_ID_CTX.set(self.run_id) + + def __exit__(self, *_: Any) -> None: + RUN_ID_CTX.reset(self.token) diff --git a/marimo/_messaging/ops.py b/marimo/_messaging/ops.py index 7d84e623f5f..0337abada8f 100644 --- a/marimo/_messaging/ops.py +++ b/marimo/_messaging/ops.py @@ -32,6 +32,7 @@ from marimo._dependencies.dependencies import DependencyManager from marimo._messaging.cell_output import CellChannel, CellOutput from marimo._messaging.completion_option import CompletionOption +from marimo._messaging.context import RUN_ID_CTX, RunId_t from marimo._messaging.errors import ( Error, MarimoInternalError, @@ -127,8 +128,23 @@ class CellOp(Op): console: Optional[Union[CellOutput, List[CellOutput]]] = None status: Optional[RuntimeStateType] = None stale_inputs: Optional[bool] = None + run_id: Optional[RunId_t] = None timestamp: float = field(default_factory=lambda: time.time()) + def __post_init__(self) -> None: + try: + self.run_id = RUN_ID_CTX.get() + except LookupError: + # Be specific about the exception we're catching + # The context variable hasn't been set yet + # TODO: where are these warnings coming from? + # good enough to silence for now? + LOGGER.warning("No run_id context found, setting to None") + self.run_id = None + except Exception as e: + LOGGER.error("Error getting run id: %s", str(e)) + self.run_id = None + @staticmethod def maybe_truncate_output( mimetype: KnownMimeType, data: str diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py index a94138e0f17..4b241ae664c 100644 --- a/marimo/_runtime/runtime.py +++ b/marimo/_runtime/runtime.py @@ -31,6 +31,7 @@ ) from marimo._dependencies.dependencies import DependencyManager from marimo._messaging.cell_output import CellChannel +from marimo._messaging.context import run_id_context from marimo._messaging.errors import ( Error, MarimoInterruptionError, @@ -1129,18 +1130,19 @@ def mutate_graph( async def _run_cells(self, cell_ids: set[CellId_t]) -> None: """Run cells and any state updates they trigger""" - # This patch is an attempt to mitigate problems caused by the fact - # that in run mode, kernels run in threads and share the same - # sys.modules. Races can still happen, but this should help in most - # common cases. We could also be more aggressive and run this before - # every cell, or even before pickle.dump/pickle.dumps() - with patches.patch_main_module_context(self._module): - while cell_ids := await self._run_cells_internal(cell_ids): - LOGGER.debug("Running state updates ...") - if self.lazy() and cell_ids: - self.graph.set_stale(cell_ids, prune_imports=True) - break - LOGGER.debug("Finished run.") + with run_id_context(): + # This patch is an attempt to mitigate problems caused by the fact + # that in run mode, kernels run in threads and share the same + # sys.modules. Races can still happen, but this should help in most + # common cases. We could also be more aggressive and run this before + # every cell, or even before pickle.dump/pickle.dumps() + with patches.patch_main_module_context(self._module): + while cell_ids := await self._run_cells_internal(cell_ids): + LOGGER.debug("Running state updates ...") + if self.lazy() and cell_ids: + self.graph.set_stale(cell_ids, prune_imports=True) + break + LOGGER.debug("Finished run.") async def _if_autorun_then_run_cells( self, cell_ids: set[CellId_t] diff --git a/openapi/api.yaml b/openapi/api.yaml index 1de7bc31467..5f8eb53db5c 100644 --- a/openapi/api.yaml +++ b/openapi/api.yaml @@ -184,6 +184,9 @@ components: output: $ref: '#/components/schemas/CellOutput' nullable: true + run_id: + nullable: true + type: string stale_inputs: nullable: true type: boolean diff --git a/openapi/src/api.ts b/openapi/src/api.ts index d097f59267c..9e181116af6 100644 --- a/openapi/src/api.ts +++ b/openapi/src/api.ts @@ -2158,6 +2158,7 @@ export interface components { /** @enum {string} */ name: "cell-op"; output?: components["schemas"]["CellOutput"]; + run_id?: string | null; stale_inputs?: boolean | null; status?: components["schemas"]["RuntimeState"]; timestamp: number; From b20bffdef9aea2916c00179b64aca2cf3b471072 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Sun, 15 Dec 2024 23:26:54 +0800 Subject: [PATCH 04/33] hook up the frontend and add some helper funcs --- .../src/components/tracing/tracing.test.tsx | 17 ++ frontend/src/components/tracing/tracing.tsx | 206 ++++++++---------- frontend/src/core/cells/runs.ts | 45 +++- marimo/_messaging/ops.py | 3 +- openapi/.gitignore | 2 + openapi/Makefile | 4 + openapi/README.md | 6 + 7 files changed, 162 insertions(+), 121 deletions(-) create mode 100644 frontend/src/components/tracing/tracing.test.tsx create mode 100644 openapi/.gitignore create mode 100644 openapi/Makefile create mode 100644 openapi/README.md diff --git a/frontend/src/components/tracing/tracing.test.tsx b/frontend/src/components/tracing/tracing.test.tsx new file mode 100644 index 00000000000..897244c3367 --- /dev/null +++ b/frontend/src/components/tracing/tracing.test.tsx @@ -0,0 +1,17 @@ +/* Copyright 2024 Marimo. All rights reserved. */ +import { describe, it, expect } from "vitest"; +import { formatChartTime } from "./tracing"; + +describe("formatUTCTime", () => { + it("should format a timestamp correctly", () => { + const timestamp = 1_700_000_000; // Example timestamp + const formattedTime = formatChartTime(timestamp); + expect(formattedTime).toBe("2023-11-15 06:13:20.000"); + }); + + it("should return an empty string for invalid timestamp", () => { + const invalidTimestamp = Number.NaN; + const formattedTime = formatChartTime(invalidTimestamp); + expect(formattedTime).toBe(""); + }); +}); diff --git a/frontend/src/components/tracing/tracing.tsx b/frontend/src/components/tracing/tracing.tsx index 506ffeaebe0..37a930d71fd 100644 --- a/frontend/src/components/tracing/tracing.tsx +++ b/frontend/src/components/tracing/tracing.tsx @@ -7,84 +7,25 @@ import { Tooltip } from "@/components/ui/tooltip"; import { type Config, type TopLevelSpec, compile } from "vega-lite"; import { ChevronRight, ChevronDown, SettingsIcon } from "lucide-react"; import type { VisualizationSpec } from "react-vega"; - +import { + type RunId, + runsAtom, + type CellRun, + type Run, +} from "@/core/cells/runs"; +import { useAtomValue } from "jotai"; +import { CellLink } from "../editor/links/cell-link"; +import { formatLogTimestamp } from "@/core/cells/logs"; +import { useCellIds } from "@/core/cells/cells"; + +// TODO: There are a few components like this in the codebase, maybe remove the redundancy const LazyVegaLite = React.lazy(() => import("react-vega").then((m) => ({ default: m.VegaLite })), ); -interface CellRun { - cellID: string; - code: string; - elapsedTime: number; - status: RunStatus; -} - -export interface Run { - runId: string; - cellRuns: CellRun[]; - runStartTime: string; -} - -// assumes the cellRuns are ordered correctly -const mockRuns: Run[] = [ - { - runId: "7", - cellRuns: [ - { - cellID: "1", - code: "import marimo as mo", - elapsedTime: 23_000, - status: "success", - }, - { - cellID: "2", - code: "def generate_some()", - elapsedTime: 46, - status: "success", - }, - { - cellID: "3", - code: "def generate_some()", - elapsedTime: 46, - status: "success", - }, - ], - runStartTime: "Dec 7 2.03pm", - }, - { - runId: "8", - cellRuns: [ - { - cellID: "1", - code: "import marimo as mo", - elapsedTime: 25, - status: "success", - }, - { - cellID: "2", - code: "def generate_some()", - elapsedTime: 100, - status: "success", - }, - { - cellID: "3", - code: "def generate_some()", - elapsedTime: 4600, - status: "error", - }, - { - cellID: "4", - code: "[print(i) for i in range(1, 100)]", - elapsedTime: 8, - status: "error", // should be disabled? - }, - ], - runStartTime: "Dec 7 2.01pm", - }, -]; - interface Values { - cell: number; + cell: CellId; + cellNum: number; startTimestamp: string; endTimestamp: string; elapsedTime: string; @@ -153,17 +94,17 @@ const baseSpec: TopLevelSpec = { }, x2: { field: "endTimestamp", type: "temporal" }, tooltip: [ - { field: "cell", title: "Cell" }, + { field: "cellNum", title: "Cell" }, { field: "startTimestamp", type: "temporal", - timeUnit: "monthdatehoursminutessecondsmilliseconds", + timeUnit: "hoursminutessecondsmilliseconds", title: "Start", }, { field: "endTimestamp", type: "temporal", - timeUnit: "monthdatehoursminutessecondsmilliseconds", + timeUnit: "hoursminutessecondsmilliseconds", title: "End", }, ], @@ -187,7 +128,7 @@ const config: Config = { }; export const Tracing: React.FC = () => { - // TODO: Either we get the runs of cells from FE side or BE side + const { runIds: newestToOldestRunIds, runMap } = useAtomValue(runsAtom); const [chartPosition, setChartPosition] = useState("sideBySide"); @@ -213,9 +154,18 @@ export const Tracing: React.FC = () => {
- {mockRuns.map((run) => ( - - ))} + {newestToOldestRunIds.map((runId: RunId) => { + const run = runMap.get(runId); + if (run) { + return ( + + ); + } + })}
); @@ -251,14 +201,14 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ // Used to sync Vega charts and React components // Note that this will only work for the first chart for now, until we create unique input elements - const [hoveredCellID, setHoveredCellID] = useState(); + const [hoveredCellId, setHoveredCellId] = useState(); const hiddenInputRef = useRef(null); - const hoverOnCell = (cellID: number) => { - setHoveredCellID(cellID); + const hoverOnCell = (cellId: CellId) => { + setHoveredCellId(cellId); // dispatch input event to trigger vega's param to update if (hiddenInputRef.current) { - hiddenInputRef.current.value = String(cellID); + hiddenInputRef.current.value = String(cellId); hiddenInputRef.current.dispatchEvent( new Event("input", { bubbles: true }), ); @@ -275,12 +225,26 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ className="text-sm cursor-pointer" onClick={() => setCollapsed(!collapsed)} > - Run - {run.runStartTime} + Run - {formatLogTimestamp(run.runStartTime)} ); - const vegaSpec = compile(createGanttVegaLiteSpec(sampleData), { + const cellIds = useCellIds(); + + const data: Data = { + values: run.cellRuns.map((cellRun) => { + return { + cell: cellRun.cellId, + cellNum: cellIds.inOrderIds.indexOf(cellRun.cellId), + startTimestamp: formatChartTime(cellRun.startTime), + endTimestamp: formatChartTime(cellRun.startTime + cellRun.elapsedTime), + elapsedTime: formatElapsedTime(cellRun.elapsedTime * 1000), + }; + }), + }; + + const vegaSpec = compile(createGanttVegaLiteSpec(data), { config, }).spec; @@ -289,18 +253,14 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ {run.cellRuns.map((cellRun) => ( ))} @@ -337,19 +297,16 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ ); }; -type RunStatus = "success" | "running" | "error"; - interface TraceRowProps { - timestamp: string; - cellID: CellId; - code: string; - elapsedTime: number; - status: RunStatus; - hoverOnCell: (cellID: number) => void; + cellRun: CellRun; + hoverOnCell: (cellId: CellId) => void; } -const TraceRow: React.FC = (props: TraceRowProps) => { - const elapsedTimeStr = formatElapsedTime(props.elapsedTime); +const TraceRow: React.FC = ({ + cellRun, + hoverOnCell, +}: TraceRowProps) => { + const elapsedTimeStr = formatElapsedTime(cellRun.elapsedTime * 1000); const elapsedTimeTooltip = ( This cell took to run @@ -357,7 +314,7 @@ const TraceRow: React.FC = (props: TraceRowProps) => { ); const handleMouseEnter = () => { - props.hoverOnCell(props.cellID); + hoverOnCell(cellRun.cellId); }; return ( @@ -365,12 +322,13 @@ const TraceRow: React.FC = (props: TraceRowProps) => { className="flex flex-row gap-2 py-1 px-1 opacity-70 hover:bg-[var(--gray-3)] hover:opacity-100" onMouseEnter={handleMouseEnter} > - [{props.timestamp}] - {/* () */} - (cell-1) + [{formatLogTimestamp(cellRun.startTime)}] + + + () - {props.code} + {cellRun.code}
@@ -378,14 +336,40 @@ const TraceRow: React.FC = (props: TraceRowProps) => { {/* TODO: Shouldn't use favicon. */} - + {`${props.status}
); }; + +export function formatChartTime(timestamp: number): string { + try { + // Create a Date object from the timestamp + // Multiply by 1000 to convert seconds to milliseconds + const date = new Date(timestamp * 1000); + + // Extract date components in local time + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, "0"); // getMonth() is 0-indexed + const day = String(date.getDate()).padStart(2, "0"); + + // Extract time components in local time + const hours = String(date.getHours()).padStart(2, "0"); + const minutes = String(date.getMinutes()).padStart(2, "0"); + const seconds = String(date.getSeconds()).padStart(2, "0"); + + // Extract milliseconds + const milliseconds = String(date.getMilliseconds()).padStart(3, "0"); + + // Combine into the desired format + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${milliseconds}`; + } catch { + return ""; + } +} diff --git a/frontend/src/core/cells/runs.ts b/frontend/src/core/cells/runs.ts index 3910d989d87..066735dbc21 100644 --- a/frontend/src/core/cells/runs.ts +++ b/frontend/src/core/cells/runs.ts @@ -50,45 +50,72 @@ const { if (!runId) { return state; } + let runIds: RunId[]; + let run = state.runMap.get(runId); - if (!run) { + if (run) { + runIds = state.runIds; + } else { run = { runId: runId, cellRuns: [], runStartTime: cellOperation.timestamp, }; + + runIds = [runId, ...state.runIds]; + if (runIds.length > MAX_RUNS) { + const oldestRunId = runIds.pop(); + if (oldestRunId) { + state.runMap.delete(oldestRunId); + } + } } + // TODO: is this ideal? + const erroredOutput = + cellOperation.output && + (cellOperation.output.channel === "marimo-error" || + cellOperation.output.channel === "stderr"); + const nextRuns: CellRun[] = []; let found = false; - for (const cellRun of run.cellRuns) { - if (cellRun.cellId === cellOperation.cell_id) { + for (const existingCellRun of run.cellRuns) { + if (existingCellRun.cellId === cellOperation.cell_id) { + const hasErroredPreviously = existingCellRun.status === "error"; + const status: CellRun["status"] = + hasErroredPreviously || erroredOutput ? "error" : "success"; + + // TODO: Need to update the cell run in place, to maintain order nextRuns.push({ - ...cellRun, - elapsedTime: cellOperation.timestamp - cellRun.startTime, + ...existingCellRun, + elapsedTime: cellOperation.timestamp - existingCellRun.startTime, + status: status, }); found = true; } else { - nextRuns.push(cellRun); + nextRuns.push(existingCellRun); } } if (!found) { + const status: CellRun["status"] = erroredOutput ? "error" : "success"; + nextRuns.push({ cellId: cellOperation.cell_id as CellId, code: code.slice(0, MAX_CODE_LENGTH), elapsedTime: 0, - // TODO: not actually correct logic - status: cellOperation.status === "idle" ? "success" : "error", + status: status, startTime: cellOperation.timestamp, }); } + run.cellRuns = nextRuns; + const nextRunMap = new Map(state.runMap); nextRunMap.set(runId, run); return { ...state, - runIds: [runId, ...state.runIds.slice(0, MAX_RUNS)], + runIds: runIds, runMap: nextRunMap, }; }, diff --git a/marimo/_messaging/ops.py b/marimo/_messaging/ops.py index 0337abada8f..ea40ff6ac4a 100644 --- a/marimo/_messaging/ops.py +++ b/marimo/_messaging/ops.py @@ -107,13 +107,14 @@ def serialize(self) -> dict[str, Any]: class CellOp(Op): """Op to transition a cell. - A CellOp's data has three optional fields: + A CellOp's data has some optional fields: output - a CellOutput console - a CellOutput (console msg to append), or a list of CellOutputs status - execution status stale_inputs - whether the cell has stale inputs (variables, modules, ...) + run_id - the run associated with this cell. Omitting a field means that its value should be unchanged! diff --git a/openapi/.gitignore b/openapi/.gitignore new file mode 100644 index 00000000000..a7554c2fcc8 --- /dev/null +++ b/openapi/.gitignore @@ -0,0 +1,2 @@ +# dependencies +/node_modules \ No newline at end of file diff --git a/openapi/Makefile b/openapi/Makefile new file mode 100644 index 00000000000..ecdb3cdea7f --- /dev/null +++ b/openapi/Makefile @@ -0,0 +1,4 @@ +.PHONY: generate + +generate: + pnpm openapi-typescript ./api.yaml -o ./src/api.ts \ No newline at end of file diff --git a/openapi/README.md b/openapi/README.md new file mode 100644 index 00000000000..611804f66e6 --- /dev/null +++ b/openapi/README.md @@ -0,0 +1,6 @@ +# Modifying the API Specification + +1. Run `pnpm install` inside /openapi folder +2. Update the `api.yaml` file +3. Run `make generate` to generate the API spec. +4. Run `make fe` in the root directory to pick up the changes. \ No newline at end of file From ce5a1962bd80583c5b34c6d4ef3f3209633ee5a7 Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Mon, 16 Dec 2024 00:06:30 +0800 Subject: [PATCH 05/33] remove comments --- frontend/src/components/tracing/tracing.tsx | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frontend/src/components/tracing/tracing.tsx b/frontend/src/components/tracing/tracing.tsx index 37a930d71fd..b0118d11adc 100644 --- a/frontend/src/components/tracing/tracing.tsx +++ b/frontend/src/components/tracing/tracing.tsx @@ -350,24 +350,19 @@ const TraceRow: React.FC = ({ export function formatChartTime(timestamp: number): string { try { - // Create a Date object from the timestamp // Multiply by 1000 to convert seconds to milliseconds const date = new Date(timestamp * 1000); - // Extract date components in local time const year = date.getFullYear(); const month = String(date.getMonth() + 1).padStart(2, "0"); // getMonth() is 0-indexed const day = String(date.getDate()).padStart(2, "0"); - // Extract time components in local time const hours = String(date.getHours()).padStart(2, "0"); const minutes = String(date.getMinutes()).padStart(2, "0"); const seconds = String(date.getSeconds()).padStart(2, "0"); - // Extract milliseconds const milliseconds = String(date.getMilliseconds()).padStart(3, "0"); - // Combine into the desired format return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${milliseconds}`; } catch { return ""; From 25d7c22e000dab568bb918a80c750a4e1832fc0a Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Mon, 16 Dec 2024 01:15:42 +0800 Subject: [PATCH 06/33] correct timestamp logic and run ordering --- .../src/components/editor/dynamic-favicon.tsx | 1 + frontend/src/components/tracing/tracing.tsx | 26 ++-------------- frontend/src/core/cells/runs.ts | 31 ++++++++++++++++--- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/frontend/src/components/editor/dynamic-favicon.tsx b/frontend/src/components/editor/dynamic-favicon.tsx index 572ab716585..c95006e5e42 100644 --- a/frontend/src/components/editor/dynamic-favicon.tsx +++ b/frontend/src/components/editor/dynamic-favicon.tsx @@ -9,6 +9,7 @@ export const FAVICONS = { success: "./circle-check.ico", running: "./circle-play.ico", error: "./circle-x.ico", + queued: "./favicon.ico", }; interface Props { diff --git a/frontend/src/components/tracing/tracing.tsx b/frontend/src/components/tracing/tracing.tsx index b0118d11adc..2b9e8ea4405 100644 --- a/frontend/src/components/tracing/tracing.tsx +++ b/frontend/src/components/tracing/tracing.tsx @@ -35,29 +35,6 @@ export interface Data { values: Values[]; } -const sampleData: Data = { - values: [ - { - cell: 1, - startTimestamp: "2024-12-01 11:00:00.000", - endTimestamp: "2024-12-01 11:00:00.230", - elapsedTime: "23ms", - }, - { - cell: 2, - startTimestamp: "2024-12-01 11:00:00.230", - endTimestamp: "2024-12-01 11:00:00.466", - elapsedTime: "46ms", - }, - { - cell: 3, - startTimestamp: "2024-12-01 11:00:00.466", - endTimestamp: "2024-12-01 11:00:00.636", - elapsedTime: "46ms", - }, - ], -}; - const baseSpec: TopLevelSpec = { $schema: "https://vega.github.io/schema/vega-lite/v5.json", mark: { @@ -86,6 +63,7 @@ const baseSpec: TopLevelSpec = { field: "cell", axis: null, scale: { paddingInner: 0.2 }, + sort: { field: "cellNum" }, }, x: { field: "startTimestamp", @@ -109,7 +87,7 @@ const baseSpec: TopLevelSpec = { }, ], size: { - value: { expr: "hoveredCellID == toString(datum.cell) ? 25 : 20" }, + value: { expr: "hoveredCellID == toString(datum.cell) ? 22 : 20" }, }, }, }; diff --git a/frontend/src/core/cells/runs.ts b/frontend/src/core/cells/runs.ts index 066735dbc21..05ef0ab8c5f 100644 --- a/frontend/src/core/cells/runs.ts +++ b/frontend/src/core/cells/runs.ts @@ -10,7 +10,7 @@ export interface CellRun { code: string; elapsedTime: number; startTime: number; - status: "success" | "error"; + status: "success" | "error" | "queued" | "running"; } export interface Run { @@ -82,12 +82,23 @@ const { for (const existingCellRun of run.cellRuns) { if (existingCellRun.cellId === cellOperation.cell_id) { const hasErroredPreviously = existingCellRun.status === "error"; - const status: CellRun["status"] = - hasErroredPreviously || erroredOutput ? "error" : "success"; + let status: CellRun["status"]; + let startTime = existingCellRun.startTime; + + if (hasErroredPreviously || erroredOutput) { + status = "error"; + } else if (cellOperation.status === "queued") { + status = "queued"; + } else if (cellOperation.status === "running") { + status = "running"; + startTime = cellOperation.timestamp; + } else { + status = "success"; + } - // TODO: Need to update the cell run in place, to maintain order nextRuns.push({ ...existingCellRun, + startTime: startTime, elapsedTime: cellOperation.timestamp - existingCellRun.startTime, status: status, }); @@ -97,7 +108,17 @@ const { } } if (!found) { - const status: CellRun["status"] = erroredOutput ? "error" : "success"; + let status: CellRun["status"]; + + if (erroredOutput) { + status = "error"; + } else if (cellOperation.status === "queued") { + status = "queued"; + } else if (cellOperation.status === "running") { + status = "running"; + } else { + status = "success"; + } nextRuns.push({ cellId: cellOperation.cell_id as CellId, From cef77c25e04d98cd8a3804f8e77893981de2fdcb Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Mon, 16 Dec 2024 09:35:32 +0800 Subject: [PATCH 07/33] consolidate docs --- CONTRIBUTING.md | 1 + development_docs/openapi.md | 7 +++++++ openapi/Makefile | 4 ---- openapi/README.md | 6 ------ 4 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 openapi/Makefile delete mode 100644 openapi/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 98cf8d41356..4e50d02cad3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,6 +95,7 @@ NODE_OPTIONS=--max_old_space_size=8192 NODE_ENV=development make fe -B | `py` | Setup | Editable python install; only need to run once | | `install-all` | Setup | Install everything; takes a long time due to editable install | | `fe` | Build | Package frontend into `marimo/` | +| `fe-codegen` | Build | Build [openapi spec](./development_docs/openapi.md) | | `wheel` | Build | Build wheel | | `check` | Test | Run all checks | | `check-test` | Test | Run all checks and tests | diff --git a/development_docs/openapi.md b/development_docs/openapi.md index 63c1f4fae2f..f576e0b5665 100644 --- a/development_docs/openapi.md +++ b/development_docs/openapi.md @@ -24,3 +24,10 @@ marimo development openapi | openapi-spec-validator - ```bash make fe-codegen ``` + +You will then need to reinstall the package in `/frontend`: + +```bash +cd frontend +pnpm update @marimo-team/marimo-api +``` \ No newline at end of file diff --git a/openapi/Makefile b/openapi/Makefile deleted file mode 100644 index ecdb3cdea7f..00000000000 --- a/openapi/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -.PHONY: generate - -generate: - pnpm openapi-typescript ./api.yaml -o ./src/api.ts \ No newline at end of file diff --git a/openapi/README.md b/openapi/README.md deleted file mode 100644 index 611804f66e6..00000000000 --- a/openapi/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Modifying the API Specification - -1. Run `pnpm install` inside /openapi folder -2. Update the `api.yaml` file -3. Run `make generate` to generate the API spec. -4. Run `make fe` in the root directory to pick up the changes. \ No newline at end of file From f413faa98098882f578dac6439c8687e6758a72a Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Tue, 17 Dec 2024 18:47:20 +0800 Subject: [PATCH 08/33] dynamic spec generation and refactor some ui components --- .../src/components/buttons/clear-button.tsx | 23 +++ .../editor/chrome/panels/logs-panel.tsx | 9 +- .../src/components/tracing/tracing-spec.tsx | 74 +++++++++ frontend/src/components/tracing/tracing.tsx | 153 ++++++++---------- frontend/src/core/cells/runs.ts | 9 +- 5 files changed, 171 insertions(+), 97 deletions(-) create mode 100644 frontend/src/components/buttons/clear-button.tsx create mode 100644 frontend/src/components/tracing/tracing-spec.tsx diff --git a/frontend/src/components/buttons/clear-button.tsx b/frontend/src/components/buttons/clear-button.tsx new file mode 100644 index 00000000000..28e9bd1629c --- /dev/null +++ b/frontend/src/components/buttons/clear-button.tsx @@ -0,0 +1,23 @@ +/* Copyright 2024 Marimo. All rights reserved. */ + +import { cn } from "@/utils/cn"; + +interface ClearButtonProps { + className?: string; + dataTestId?: string; + onClick: () => void; +} + +export const ClearButton: React.FC = (props) => ( + +); diff --git a/frontend/src/components/editor/chrome/panels/logs-panel.tsx b/frontend/src/components/editor/chrome/panels/logs-panel.tsx index dd100f838ef..81df21fbe56 100644 --- a/frontend/src/components/editor/chrome/panels/logs-panel.tsx +++ b/frontend/src/components/editor/chrome/panels/logs-panel.tsx @@ -6,6 +6,7 @@ import React from "react"; import { FileTextIcon } from "lucide-react"; import { CellLink } from "../../links/cell-link"; import { PanelEmptyState } from "./empty-state"; +import { ClearButton } from "@/components/buttons/clear-button"; interface Props { className?: string; @@ -35,13 +36,7 @@ export const LogsPanel: React.FC = () => { return ( <>
- +
diff --git a/frontend/src/components/tracing/tracing-spec.tsx b/frontend/src/components/tracing/tracing-spec.tsx new file mode 100644 index 00000000000..45e8375bc91 --- /dev/null +++ b/frontend/src/components/tracing/tracing-spec.tsx @@ -0,0 +1,74 @@ +/* Copyright 2024 Marimo. All rights reserved. */ +import type { Field } from "@/plugins/impl/vega/types"; +import type { TopLevelSpec } from "vega-lite"; +import type { PositionDef } from "vega-lite/build/src/channeldef"; +import type { TopLevelParameter } from "vega-lite/build/src/spec/toplevel"; + +export function createBaseSpec( + showYAxis: boolean, + ...additionalParams: TopLevelParameter[] +): TopLevelSpec { + const yAxis: PositionDef | Partial> = { + field: "cellNum", + scale: { paddingInner: 0.2 }, + sort: { field: "cellNum" }, + title: "cell", + }; + if (!showYAxis) { + yAxis.axis = null; + } + + return { + $schema: "https://vega.github.io/schema/vega-lite/v5.json", + mark: { + type: "bar", + cornerRadius: 2, + fill: "#37BE5F", // same colour as chrome's network tab + }, + params: [ + ...additionalParams, + { + name: "zoomAndPan", + select: "interval", + bind: "scales", + }, + { + name: "cursor", + value: "grab", + }, + ], + height: { step: 23 }, + encoding: { + y: yAxis, + x: { + field: "startTimestamp", + type: "temporal", + axis: { orient: "top", title: null }, + }, + x2: { field: "endTimestamp", type: "temporal" }, + tooltip: [ + { field: "cellNum", title: "Cell" }, + { + field: "startTimestamp", + type: "temporal", + timeUnit: "hoursminutessecondsmilliseconds", + title: "Start", + }, + { + field: "endTimestamp", + type: "temporal", + timeUnit: "hoursminutessecondsmilliseconds", + title: "End", + }, + ], + size: { + value: { expr: "hoveredCellId == toString(datum.cell) ? 20 : 18" }, + }, + }, + config: { + view: { + stroke: "transparent", + }, + }, + }; +} diff --git a/frontend/src/components/tracing/tracing.tsx b/frontend/src/components/tracing/tracing.tsx index 2b9e8ea4405..6f1b7dc2dee 100644 --- a/frontend/src/components/tracing/tracing.tsx +++ b/frontend/src/components/tracing/tracing.tsx @@ -4,19 +4,22 @@ import React, { useRef, useState } from "react"; import type { CellId } from "@/core/cells/ids"; import { ElapsedTime, formatElapsedTime } from "../editor/cell/CellStatus"; import { Tooltip } from "@/components/ui/tooltip"; -import { type Config, type TopLevelSpec, compile } from "vega-lite"; -import { ChevronRight, ChevronDown, SettingsIcon } from "lucide-react"; +import { type TopLevelSpec, compile } from "vega-lite"; +import { ChevronRight, ChevronDown } from "lucide-react"; import type { VisualizationSpec } from "react-vega"; import { type RunId, runsAtom, type CellRun, type Run, + useRunsActions, } from "@/core/cells/runs"; import { useAtomValue } from "jotai"; import { CellLink } from "../editor/links/cell-link"; import { formatLogTimestamp } from "@/core/cells/logs"; import { useCellIds } from "@/core/cells/cells"; +import { createBaseSpec } from "./tracing-spec"; +import { ClearButton } from "../buttons/clear-button"; // TODO: There are a few components like this in the codebase, maybe remove the redundancy const LazyVegaLite = React.lazy(() => @@ -35,78 +38,30 @@ export interface Data { values: Values[]; } -const baseSpec: TopLevelSpec = { - $schema: "https://vega.github.io/schema/vega-lite/v5.json", - mark: { - type: "bar", - cornerRadius: 2, - fill: "#37BE5F", // same colour as chrome's network tab - }, - height: { step: 23 }, - params: [ - { - name: "zoomAndPan", - select: "interval", - bind: "scales", - }, - { - name: "hoveredCellID", - bind: { element: "#hiddenInputElement" }, - }, - { - name: "cursor", - value: "grab", - }, - ], - encoding: { - y: { - field: "cell", - axis: null, - scale: { paddingInner: 0.2 }, - sort: { field: "cellNum" }, - }, - x: { - field: "startTimestamp", - type: "temporal", - axis: { orient: "top", title: null }, - }, - x2: { field: "endTimestamp", type: "temporal" }, - tooltip: [ - { field: "cellNum", title: "Cell" }, - { - field: "startTimestamp", - type: "temporal", - timeUnit: "hoursminutessecondsmilliseconds", - title: "Start", - }, - { - field: "endTimestamp", - type: "temporal", - timeUnit: "hoursminutessecondsmilliseconds", - title: "End", - }, - ], - size: { - value: { expr: "hoveredCellID == toString(datum.cell) ? 22 : 20" }, - }, - }, -}; +function createGanttVegaLiteSpec( + data: Data, + hiddenInputElementId: string, + chartPosition: ChartPosition, +): TopLevelSpec { + const hoverParam = { + name: "hoveredCellId", + bind: { element: `#${hiddenInputElementId}` }, + }; + + const showYAxis = chartPosition !== "sideBySide"; + const baseSpec = createBaseSpec(showYAxis, hoverParam); -function createGanttVegaLiteSpec(data: Data): TopLevelSpec { return { ...baseSpec, - data, + data: data, }; } -const config: Config = { - view: { - stroke: "transparent", - }, -}; +type ChartPosition = "sideBySide" | "above"; export const Tracing: React.FC = () => { const { runIds: newestToOldestRunIds, runMap } = useAtomValue(runsAtom); + const { clearRuns } = useRunsActions(); const [chartPosition, setChartPosition] = useState("sideBySide"); @@ -119,19 +74,36 @@ export const Tracing: React.FC = () => { } }; + // {/* + // + // */} + return ( -
- - - - -
+
+
+
+ + +
+ + +
+ +
{newestToOldestRunIds.map((runId: RunId) => { const run = runMap.get(runId); if (run) { @@ -149,8 +121,6 @@ export const Tracing: React.FC = () => { ); }; -type ChartPosition = "sideBySide" | "above"; - interface ChartProps { className?: string; width: number; @@ -212,25 +182,27 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ const data: Data = { values: run.cellRuns.map((cellRun) => { + const elapsedTime = cellRun.elapsedTime ?? 0; return { cell: cellRun.cellId, cellNum: cellIds.inOrderIds.indexOf(cellRun.cellId), startTimestamp: formatChartTime(cellRun.startTime), - endTimestamp: formatChartTime(cellRun.startTime + cellRun.elapsedTime), - elapsedTime: formatElapsedTime(cellRun.elapsedTime * 1000), + endTimestamp: formatChartTime(cellRun.startTime + elapsedTime), + elapsedTime: formatElapsedTime(elapsedTime * 1000), }; }), }; - const vegaSpec = compile(createGanttVegaLiteSpec(data), { - config, - }).spec; + const hiddenInputElementId = `hiddenInputElement-${run.runId}`; + const vegaSpec = compile( + createGanttVegaLiteSpec(data, hiddenInputElementId, chartPosition), + ).spec; const TraceRows = (
= ({ 
           {TraceTitle}
-          {!collapsed && }
+          {!collapsed && }
           {!collapsed && TraceRows}
@@ -284,11 +256,16 @@ const TraceRow: React.FC = ({ cellRun, hoverOnCell, }: TraceRowProps) => { - const elapsedTimeStr = formatElapsedTime(cellRun.elapsedTime * 1000); - const elapsedTimeTooltip = ( + const elapsedTimeStr = cellRun.elapsedTime + ? formatElapsedTime(cellRun.elapsedTime * 1000) + : "-"; + + const elapsedTimeTooltip = cellRun.elapsedTime ? ( This cell took to run + ) : ( + This cell has not been run ); const handleMouseEnter = () => { diff --git a/frontend/src/core/cells/runs.ts b/frontend/src/core/cells/runs.ts index 05ef0ab8c5f..423d78473a2 100644 --- a/frontend/src/core/cells/runs.ts +++ b/frontend/src/core/cells/runs.ts @@ -8,7 +8,7 @@ export type RunId = TypedString<"RunId">; export interface CellRun { cellId: CellId; code: string; - elapsedTime: number; + elapsedTime?: number; startTime: number; status: "success" | "error" | "queued" | "running"; } @@ -96,10 +96,15 @@ const { status = "success"; } + let elapsedTime: number | undefined = undefined; + if (status === "success" || status === "error") { + elapsedTime = cellOperation.timestamp - existingCellRun.startTime; + } + nextRuns.push({ ...existingCellRun, startTime: startTime, - elapsedTime: cellOperation.timestamp - existingCellRun.startTime, + elapsedTime: elapsedTime, status: status, }); found = true; From 94d8cf21b52fe448b2b418d420b5321a82fc76ac Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Tue, 17 Dec 2024 22:35:59 +0800 Subject: [PATCH 09/33] add signals from vega to react component --- .../src/components/tracing/tracing-spec.tsx | 15 ++- frontend/src/components/tracing/tracing.tsx | 96 +++++++++++++------ 2 files changed, 81 insertions(+), 30 deletions(-) diff --git a/frontend/src/components/tracing/tracing-spec.tsx b/frontend/src/components/tracing/tracing-spec.tsx index 45e8375bc91..1c84e306989 100644 --- a/frontend/src/components/tracing/tracing-spec.tsx +++ b/frontend/src/components/tracing/tracing-spec.tsx @@ -4,6 +4,9 @@ import type { TopLevelSpec } from "vega-lite"; import type { PositionDef } from "vega-lite/build/src/channeldef"; import type { TopLevelParameter } from "vega-lite/build/src/spec/toplevel"; +export const REACT_HOVERED_CELLID = "hoveredCellId"; +export const VEGA_HOVER_SIGNAL = "cellHover"; + export function createBaseSpec( showYAxis: boolean, ...additionalParams: TopLevelParameter[] @@ -36,6 +39,14 @@ export function createBaseSpec( name: "cursor", value: "grab", }, + { + name: VEGA_HOVER_SIGNAL, + select: { + type: "point", + on: "pointerover", + fields: ["cell"], + }, + }, ], height: { step: 23 }, encoding: { @@ -62,7 +73,9 @@ export function createBaseSpec( }, ], size: { - value: { expr: "hoveredCellId == toString(datum.cell) ? 20 : 18" }, + value: { + expr: `${REACT_HOVERED_CELLID} == toString(datum.cell) ? 20 : 18`, + }, }, }, config: { diff --git a/frontend/src/components/tracing/tracing.tsx b/frontend/src/components/tracing/tracing.tsx index 6f1b7dc2dee..1ab0e0071e7 100644 --- a/frontend/src/components/tracing/tracing.tsx +++ b/frontend/src/components/tracing/tracing.tsx @@ -6,7 +6,7 @@ import { ElapsedTime, formatElapsedTime } from "../editor/cell/CellStatus"; import { Tooltip } from "@/components/ui/tooltip"; import { type TopLevelSpec, compile } from "vega-lite"; import { ChevronRight, ChevronDown } from "lucide-react"; -import type { VisualizationSpec } from "react-vega"; +import type { SignalListeners, VisualizationSpec } from "react-vega"; import { type RunId, runsAtom, @@ -18,15 +18,20 @@ import { useAtomValue } from "jotai"; import { CellLink } from "../editor/links/cell-link"; import { formatLogTimestamp } from "@/core/cells/logs"; import { useCellIds } from "@/core/cells/cells"; -import { createBaseSpec } from "./tracing-spec"; +import { + createBaseSpec, + REACT_HOVERED_CELLID, + VEGA_HOVER_SIGNAL, +} from "./tracing-spec"; import { ClearButton } from "../buttons/clear-button"; +import { cn } from "@/utils/cn"; // TODO: There are a few components like this in the codebase, maybe remove the redundancy const LazyVegaLite = React.lazy(() => import("react-vega").then((m) => ({ default: m.VegaLite })), ); -interface Values { +interface ChartValues { cell: CellId; cellNum: number; startTimestamp: string; @@ -34,17 +39,13 @@ interface Values { elapsedTime: string; } -export interface Data { - values: Values[]; -} - function createGanttVegaLiteSpec( - data: Data, + chartValues: ChartValues[], hiddenInputElementId: string, chartPosition: ChartPosition, ): TopLevelSpec { const hoverParam = { - name: "hoveredCellId", + name: REACT_HOVERED_CELLID, bind: { element: `#${hiddenInputElementId}` }, }; @@ -53,7 +54,9 @@ function createGanttVegaLiteSpec( return { ...baseSpec, - data: data, + data: { + values: chartValues, + }, }; } @@ -126,6 +129,7 @@ interface ChartProps { width: number; height: number; vegaSpec: VisualizationSpec; + signalListeners: SignalListeners; } const Chart: React.FC = (props: ChartProps) => { @@ -135,11 +139,17 @@ const Chart: React.FC = (props: ChartProps) => { spec={props.vegaSpec} width={props.width} height={props.height} + signalListeners={props.signalListeners} />
); }; +interface VegaHoverCellSignal { + cell: string[]; + vlPoint: unknown; +} + const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ run, chartPosition, @@ -149,10 +159,10 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ // Used to sync Vega charts and React components // Note that this will only work for the first chart for now, until we create unique input elements - const [hoveredCellId, setHoveredCellId] = useState(); + const [hoveredCellId, setHoveredCellId] = useState(); const hiddenInputRef = useRef(null); - const hoverOnCell = (cellId: CellId) => { + const hoverOnCell = (cellId: CellId | null) => { setHoveredCellId(cellId); // dispatch input event to trigger vega's param to update if (hiddenInputRef.current) { @@ -163,6 +173,17 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ } }; + const handleVegaSignal = { + [VEGA_HOVER_SIGNAL]: (name: string, value: unknown) => { + const signalValue = value as VegaHoverCellSignal; + if (signalValue.cell && signalValue.cell.length > 0) { + setHoveredCellId(signalValue.cell[0] as CellId); + } else { + setHoveredCellId(null); + } + }, + }; + const ChevronComponent = () => { const Icon = collapsed ? ChevronRight : ChevronDown; return ; @@ -180,22 +201,20 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ const cellIds = useCellIds(); - const data: Data = { - values: run.cellRuns.map((cellRun) => { - const elapsedTime = cellRun.elapsedTime ?? 0; - return { - cell: cellRun.cellId, - cellNum: cellIds.inOrderIds.indexOf(cellRun.cellId), - startTimestamp: formatChartTime(cellRun.startTime), - endTimestamp: formatChartTime(cellRun.startTime + elapsedTime), - elapsedTime: formatElapsedTime(elapsedTime * 1000), - }; - }), - }; + const chartValues: ChartValues[] = run.cellRuns.map((cellRun) => { + const elapsedTime = cellRun.elapsedTime ?? 0; + return { + cell: cellRun.cellId, + cellNum: cellIds.inOrderIds.indexOf(cellRun.cellId), + startTimestamp: formatChartTime(cellRun.startTime), + endTimestamp: formatChartTime(cellRun.startTime + elapsedTime), + elapsedTime: formatElapsedTime(elapsedTime * 1000), + }; + }); const hiddenInputElementId = `hiddenInputElement-${run.runId}`; const vegaSpec = compile( - createGanttVegaLiteSpec(data, hiddenInputElementId, chartPosition), + createGanttVegaLiteSpec(chartValues, hiddenInputElementId, chartPosition), ).spec; const TraceRows = ( @@ -203,7 +222,7 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ @@ -211,6 +230,7 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ ))} @@ -222,7 +242,14 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ 
           {TraceTitle}
-          {!collapsed && }
+          {!collapsed && (
+            
+          )}
           {!collapsed && TraceRows}
@@ -241,6 +268,7 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ vegaSpec={vegaSpec} width={240} height={100} + signalListeners={handleVegaSignal} /> )}
@@ -249,11 +277,13 @@ const TraceBlock: React.FC<{ run: Run; chartPosition: ChartPosition }> = ({ interface TraceRowProps { cellRun: CellRun; - hoverOnCell: (cellId: CellId) => void; + hovered: boolean; + hoverOnCell: (cellId: CellId | null) => void; } const TraceRow: React.FC = ({ cellRun, + hovered, hoverOnCell, }: TraceRowProps) => { const elapsedTimeStr = cellRun.elapsedTime @@ -272,10 +302,18 @@ const TraceRow: React.FC = ({ hoverOnCell(cellRun.cellId); }; + const handleMouseLeave = () => { + hoverOnCell(null); + }; + return (
[{formatLogTimestamp(cellRun.startTime)}] From 67287ff5f312cd61e9fb8aff98f0e68924ed6b0f Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Tue, 17 Dec 2024 23:25:55 +0800 Subject: [PATCH 10/33] some refactoring and style changes --- .../src/components/tracing/tracing-spec.tsx | 2 +- frontend/src/components/tracing/tracing.tsx | 42 +++++++------------ frontend/src/core/cells/runs.ts | 1 - 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/frontend/src/components/tracing/tracing-spec.tsx b/frontend/src/components/tracing/tracing-spec.tsx index 1c84e306989..abf2e8c66e8 100644 --- a/frontend/src/components/tracing/tracing-spec.tsx +++ b/frontend/src/components/tracing/tracing-spec.tsx @@ -74,7 +74,7 @@ export function createBaseSpec( ], size: { value: { - expr: `${REACT_HOVERED_CELLID} == toString(datum.cell) ? 20 : 18`, + expr: `${REACT_HOVERED_CELLID} == toString(datum.cell) ? 19.5 : 18`, }, }, }, diff --git a/frontend/src/components/tracing/tracing.tsx b/frontend/src/components/tracing/tracing.tsx index 1ab0e0071e7..36ba4ba9265 100644 --- a/frontend/src/components/tracing/tracing.tsx +++ b/frontend/src/components/tracing/tracing.tsx @@ -69,7 +69,7 @@ export const Tracing: React.FC = () => { const [chartPosition, setChartPosition] = useState("sideBySide"); - const toggleConfig = () => { + const toggleChartPosition = () => { if (chartPosition === "above") { setChartPosition("sideBySide"); } else { @@ -77,33 +77,24 @@ export const Tracing: React.FC = () => { } }; - // {/* - // - // */} - return (
-
-
-