From 73329f2d804a172b15f908a0abde0f8b5a5e973a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Fri, 5 Jul 2024 15:53:09 +0200 Subject: [PATCH 01/11] Remove non-functional multipart upload bits --- packages/cypress/src/index.ts | 1 + packages/playwright/src/reporter.ts | 1 + packages/test-utils/src/legacy-cli/client.ts | 2 +- .../test-utils/src/legacy-cli/launchdarkly.ts | 102 ------------------ packages/test-utils/src/legacy-cli/main.ts | 74 ++----------- packages/test-utils/src/legacy-cli/upload.ts | 94 +--------------- 6 files changed, 14 insertions(+), 260 deletions(-) delete mode 100644 packages/test-utils/src/legacy-cli/launchdarkly.ts diff --git a/packages/cypress/src/index.ts b/packages/cypress/src/index.ts index 57c52c35..af5ecaf6 100644 --- a/packages/cypress/src/index.ts +++ b/packages/cypress/src/index.ts @@ -263,6 +263,7 @@ const plugin = ( ) => { setUserAgent(`${packageName}/${packageVersion}`); + // TODO: enable launchDarkly initLogger(packageName, packageVersion); mixpanelAPI.initialize({ accessToken: getAuthKey(config), diff --git a/packages/playwright/src/reporter.ts b/packages/playwright/src/reporter.ts index cc5f6ff6..d70a895e 100644 --- a/packages/playwright/src/reporter.ts +++ b/packages/playwright/src/reporter.ts @@ -85,6 +85,7 @@ export default class ReplayPlaywrightReporter implements Reporter { constructor(config: ReplayPlaywrightConfig) { setUserAgent(`${packageName}/${packageVersion}`); + // TODO: enable launchDarkly initLogger(packageName, packageVersion); mixpanelAPI.initialize({ accessToken: getAccessToken(config), diff --git a/packages/test-utils/src/legacy-cli/client.ts b/packages/test-utils/src/legacy-cli/client.ts index c7f4bb9a..80b6d967 100644 --- a/packages/test-utils/src/legacy-cli/client.ts +++ b/packages/test-utils/src/legacy-cli/client.ts @@ -1,5 +1,5 @@ import WebSocket from "ws"; -import { defer, maybeLogToConsole } from "./utils"; +import { defer } from "./utils"; import { Agent } from "http"; import { logger } from "@replay-cli/shared/logger"; diff --git a/packages/test-utils/src/legacy-cli/launchdarkly.ts b/packages/test-utils/src/legacy-cli/launchdarkly.ts deleted file mode 100644 index be0ffd70..00000000 --- a/packages/test-utils/src/legacy-cli/launchdarkly.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { logger } from "@replay-cli/shared/logger"; -import { initialize, LDClient, LDLogger } from "launchdarkly-node-client-sdk"; - -type UserFeatureProfile = { - type: "user"; - id: string; -}; - -type AnonymousFeatureProfile = { - type: "anonymous"; - id: "anonymous"; -}; - -type FeatureProfile = AnonymousFeatureProfile | UserFeatureProfile; - -class NoOpLogger implements LDLogger { - error() {} - warn() {} - info() {} - debug() {} -} - -class LaunchDarkly { - private client: LDClient | undefined; - - public initialize() { - const key = "60ca05fb43d6f10d234bb3cf"; - const defaultProfile = { type: "anonymous", id: "anonymous" }; - this.client = initialize( - key, - { - kind: "user", - key: defaultProfile.id, - anonymous: defaultProfile.type === "anonymous", - }, - { - logger: new NoOpLogger(), - } - ); - return this; - } - - public async identify(profile: FeatureProfile): Promise { - if (!this.client) { - return; - } - try { - await this.client.waitForInitialization(); - } catch (error) { - logger.error("LaunchDarklyIdentify:InitializationFailed", { error }); - return; - } - - await this.client.identify({ - kind: "user", - key: profile.id, - anonymous: profile.type === "anonymous", - }); - } - - public async isEnabled(flag: string, defaultValue: boolean): Promise { - if (!this.client) { - return defaultValue; - } - return await this.variant(flag, defaultValue); - } - - public async variant(name: string, defaultValue: T): Promise { - if (!this.client) { - return defaultValue; - } - try { - await this.client.waitForInitialization(); - } catch (error) { - logger.error("LaunchDarklyVariant:WaitForInitializationFailed", { error }); - return defaultValue; - } - - const val = await this.client.variation(name, defaultValue); - return val; - } - - public async close() { - if (!this.client) { - return; - } - try { - await this.client.close(); - } catch (error) { - logger.error("LaunchDarklyClose:Failed", { error }); - } - } -} - -let launchDarkly: LaunchDarkly | undefined; -export const getLaunchDarkly = () => { - if (launchDarkly) { - return launchDarkly; - } - launchDarkly = new LaunchDarkly(); - return launchDarkly; -}; diff --git a/packages/test-utils/src/legacy-cli/main.ts b/packages/test-utils/src/legacy-cli/main.ts index f02fe6e6..013d4bb7 100644 --- a/packages/test-utils/src/legacy-cli/main.ts +++ b/packages/test-utils/src/legacy-cli/main.ts @@ -1,7 +1,6 @@ import { retryWithExponentialBackoff } from "@replay-cli/shared/async/retryOnFailure"; import fs from "fs"; import { getHttpAgent } from "./utils"; -import assert from "node:assert/strict"; // requiring v4 explicitly because it's the last version with commonjs support. // Should be upgraded to the latest when converting this code to es modules. @@ -10,7 +9,6 @@ import pMap from "p-map"; import { Agent, AgentOptions } from "http"; import jsonata from "jsonata"; import { ProtocolError } from "./client"; -import { getLaunchDarkly } from "./launchdarkly"; import { addRecordingEvent, readRecordings, removeRecordingFromLog } from "./recordingLog"; import { FilterOptions, @@ -207,44 +205,6 @@ async function setMetadata( } } -const MIN_MULTIPART_UPLOAD_SIZE = 5 * 1024 * 1024; -async function multipartUploadRecording( - server: string, - client: ReplayClient, - recording: RecordingEntry, - metadata: RecordingMetadata | null, - size: number, - strict: boolean, - verbose: boolean, - agentOptions?: AgentOptions -) { - const requestPartChunkSize = - parseInt(process.env.REPLAY_MULTIPART_UPLOAD_CHUNK || "", 10) || undefined; - const { recordingId, uploadId, partLinks, chunkSize } = - await client.connectionBeginRecordingMultipartUpload( - recording.id, - recording.buildId!, - size, - requestPartChunkSize - ); - await setMetadata(client, recordingId, metadata, strict, verbose); - addRecordingEvent("uploadStarted", recording.id, { - server, - recordingId, - }); - const eTags = await client.uploadRecordingInParts( - recording.path!, - partLinks, - chunkSize, - agentOptions - ); - - assert(eTags.length === partLinks.length, "Mismatched eTags and partLinks"); - - await client.connectionEndRecordingMultipartUpload(recording.id, uploadId, eTags); - return recordingId; -} - async function directUploadRecording( server: string, client: ReplayClient, @@ -355,30 +315,16 @@ async function doUploadRecording( const metadata = await validateMetadata(client, recording.metadata, verbose); let recordingId: string; - const isMultipartEnabled = await getLaunchDarkly().isEnabled("cli-multipart-upload", false); try { - if (size > MIN_MULTIPART_UPLOAD_SIZE && isMultipartEnabled) { - recordingId = await multipartUploadRecording( - server, - client, - recording, - metadata, - size, - strict, - verbose, - agentOptions - ); - } else { - recordingId = await directUploadRecording( - server, - client, - recording, - metadata, - size, - strict, - verbose - ); - } + recordingId = await directUploadRecording( + server, + client, + recording, + metadata, + size, + strict, + verbose + ); } catch (err) { const errorMessage = err instanceof ProtocolError ? err.protocolMessage : String(err); logger.error("DoUploadRecording:ProtocolError", { @@ -386,7 +332,7 @@ async function doUploadRecording( server, strict, errorMessage, - wasMultipartUpload: size > MIN_MULTIPART_UPLOAD_SIZE && isMultipartEnabled, + wasMultipartUpload: false, }); handleUploadingError(errorMessage, strict, verbose, err); return null; diff --git a/packages/test-utils/src/legacy-cli/upload.ts b/packages/test-utils/src/legacy-cli/upload.ts index f482c039..2ef7d309 100644 --- a/packages/test-utils/src/legacy-cli/upload.ts +++ b/packages/test-utils/src/legacy-cli/upload.ts @@ -1,11 +1,7 @@ -import { retryWithLinearBackoff } from "@replay-cli/shared/async/retryOnFailure"; import crypto from "crypto"; import fs from "fs"; -import type { Agent, AgentOptions } from "http"; +import type { Agent } from "http"; import fetch from "node-fetch"; -import pMap from "p-map"; -import path from "path"; -import { Worker } from "worker_threads"; import ProtocolClient from "./client"; import { sanitize as sanitizeMetadata } from "./metadata"; import { Options, OriginalSourceEntry, RecordingMetadata, SourceMapEntry } from "./types"; @@ -71,31 +67,6 @@ class ReplayClient { return { recordingId, uploadLink }; } - async connectionBeginRecordingMultipartUpload( - id: string, - buildId: string, - size: number, - multiPartChunkSize?: number - ) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - const { recordingId, uploadId, chunkSize, partLinks } = await this.client.sendCommand<{ - recordingId: string; - uploadId: string; - partLinks: string[]; - chunkSize: number; - }>("Internal.beginRecordingMultipartUpload", { - buildId, - // 3/22/2022: Older builds use integers instead of UUIDs for the recording - // IDs written to disk. These are not valid to use as recording IDs when - // uploading recordings to the backend. - recordingId: isValidUUID(id) ? id : undefined, - recordingSize: size, - chunkSize: multiPartChunkSize, - }); - return { recordingId, uploadId, chunkSize, partLinks }; - } - async buildRecordingMetadata( metadata: Record, _opts: Options = {} @@ -185,69 +156,6 @@ class ReplayClient { } } - async uploadPart( - link: string, - partMeta: { filePath: string; start: number; end: number }, - size: number, - agentOptions?: AgentOptions - ): Promise { - return new Promise((resolve, reject) => { - const worker = new Worker(path.join(__dirname, "./uploadWorker.js")); - - worker.on("message", resolve); - worker.on("error", reject); - worker.on("exit", code => { - if (code !== 0) { - reject(new Error(`Worker stopped with exit code ${code}`)); - } - }); - - worker.postMessage({ link, partMeta, size, agentOptions }); - }); - } - - async uploadRecordingInParts( - filePath: string, - partUploadLinks: string[], - partSize: number, - agentOptions?: AgentOptions - ) { - const stats = fs.statSync(filePath); - const totalSize = stats.size; - const results = await pMap( - partUploadLinks, - async (url, index) => { - return retryWithLinearBackoff( - async () => { - const partNumber = index + 1; - const start = index * partSize; - const end = Math.min(start + partSize, totalSize) - 1; // -1 because end is inclusive - - logger.info("UploadRecordingInParts:UploadingPart", { - partNumber, - start, - end, - totalSize, - partSize, - }); - - return this.uploadPart(url, { filePath, start, end }, end - start + 1, agentOptions); - }, - error => { - logger.error("UploadRecordingInParts:WillRetryPart", { - partNumber: index + 1, - error, - }); - }, - 10 - ); - }, - { concurrency: 10 } - ); - - return results; - } - async connectionEndRecordingUpload(recordingId: string) { if (!this.client) throw new Error("Protocol client is not initialized"); From c6fcebfbf2fe48235cefd68866ae07365a52b2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 11:46:31 +0200 Subject: [PATCH 02/11] Introduce uploadWorker --- .../playwright.config.ts | 4 +- .../src/utils/recordings/uploadRecordings.ts | 214 ++++++++ .../shared/src/protocol/ProtocolClient.ts | 2 +- .../src/recording/createSettledDeferred.ts | 4 +- .../printDeferredRecordingActions.ts | 59 --- .../src/recording/printViewRecordingLinks.ts | 40 -- .../src/recording/upload/uploadRecordings.ts | 141 ----- .../src/recording/upload/uploadWorker.ts | 73 +++ packages/test-utils/src/legacy-cli/client.ts | 143 ----- packages/test-utils/src/legacy-cli/index.ts | 1 - .../src/legacy-cli/listAllRecordings.ts | 73 +++ packages/test-utils/src/legacy-cli/main.ts | 492 ------------------ .../test-utils/src/legacy-cli/recordingLog.ts | 54 +- .../test-utils/src/legacy-cli/updateStatus.ts | 14 - packages/test-utils/src/legacy-cli/upload.ts | 249 --------- packages/test-utils/src/legacy-cli/utils.ts | 48 +- packages/test-utils/src/reporter.ts | 2 +- 17 files changed, 385 insertions(+), 1228 deletions(-) create mode 100644 packages/replayio/src/utils/recordings/uploadRecordings.ts delete mode 100644 packages/shared/src/recording/printDeferredRecordingActions.ts delete mode 100644 packages/shared/src/recording/printViewRecordingLinks.ts delete mode 100644 packages/shared/src/recording/upload/uploadRecordings.ts create mode 100644 packages/shared/src/recording/upload/uploadWorker.ts delete mode 100644 packages/test-utils/src/legacy-cli/client.ts delete mode 100644 packages/test-utils/src/legacy-cli/index.ts create mode 100644 packages/test-utils/src/legacy-cli/listAllRecordings.ts delete mode 100644 packages/test-utils/src/legacy-cli/main.ts delete mode 100644 packages/test-utils/src/legacy-cli/updateStatus.ts delete mode 100644 packages/test-utils/src/legacy-cli/upload.ts diff --git a/examples/create-react-app-typescript/playwright.config.ts b/examples/create-react-app-typescript/playwright.config.ts index c6780f1b..f20c87ee 100644 --- a/examples/create-react-app-typescript/playwright.config.ts +++ b/examples/create-react-app-typescript/playwright.config.ts @@ -16,7 +16,9 @@ export default defineConfig({ reporter: [ // replicating Playwright's defaults process.env.CI ? (["dot"] as const) : (["list"] as const), - replayReporter({}), + replayReporter({ + upload: true, + }), ], projects: [ { diff --git a/packages/replayio/src/utils/recordings/uploadRecordings.ts b/packages/replayio/src/utils/recordings/uploadRecordings.ts new file mode 100644 index 00000000..13a7fc3f --- /dev/null +++ b/packages/replayio/src/utils/recordings/uploadRecordings.ts @@ -0,0 +1,214 @@ +import { Deferred, STATUS_RESOLVED } from "@replay-cli/shared/async/createDeferred"; +import { disableAnimatedLog, replayAppHost } from "@replay-cli/shared/config"; +import { logger } from "@replay-cli/shared/logger"; +import { logUpdate } from "@replay-cli/shared/logUpdate"; +import { createAsyncFunctionWithTracking } from "@replay-cli/shared/mixpanel/createAsyncFunctionWithTracking"; +import { printTable } from "@replay-cli/shared/printTable"; +import { exitProcess } from "@replay-cli/shared/process/exitProcess"; +import { + AUTHENTICATION_REQUIRED_ERROR_CODE, + ProtocolError, +} from "@replay-cli/shared/protocol/ProtocolError"; +import { canUpload } from "@replay-cli/shared/recording/canUpload"; +import { formatRecording } from "@replay-cli/shared/recording/formatRecording"; +import type { LocalRecording } from "@replay-cli/shared/recording/types"; +import type { ProcessingBehavior } from "@replay-cli/shared/recording/upload/types"; +import { createUploadWorker } from "@replay-cli/shared/recording/upload/uploadWorker"; +import { + dim, + highlight, + link, + statusFailed, + statusPending, + statusSuccess, +} from "@replay-cli/shared/theme"; +import { dots } from "cli-spinners"; +import strip from "strip-ansi"; + +async function printDeferredRecordingActions( + deferredActions: Deferred[], + { + renderTitle, + renderExtraColumns, + renderFailedSummary, + }: { + renderTitle: (options: { done: boolean }) => string; + renderExtraColumns: (recording: LocalRecording) => string[]; + renderFailedSummary: (failedRecordings: LocalRecording[]) => string; + } +) { + let dotIndex = 0; + + const print = (done = false) => { + const dot = dots.frames[++dotIndex % dots.frames.length]; + const title = renderTitle({ done }); + const table = printTable({ + rows: deferredActions.map(deferred => { + let status = disableAnimatedLog ? "" : statusPending(dot); + if (deferred.resolution === true) { + status = statusSuccess("✔"); + } else if (deferred.resolution === false) { + status = statusFailed("✘"); + } + const recording = deferred.data; + const { date, duration, id, title } = formatRecording(recording); + return [status, id, title, date, duration, ...renderExtraColumns(recording)]; + }), + }); + + logUpdate(title + "\n" + table); + }; + + print(); + + const interval = disableAnimatedLog ? undefined : setInterval(print, dots.interval); + + await Promise.all(deferredActions.map(deferred => deferred.promise)); + + clearInterval(interval); + print(true); + logUpdate.done(); + + const failedActions = deferredActions.filter(deferred => deferred.status !== STATUS_RESOLVED); + if (failedActions.length > 0) { + const failedSummary = renderFailedSummary(failedActions.map(action => action.data)); + console.log(statusFailed(`${failedSummary}`) + "\n"); + } +} + +function printViewRecordingLinks(recordings: LocalRecording[]) { + switch (recordings.length) { + case 0: { + break; + } + case 1: { + const recording = recordings[0]; + const url = `${replayAppHost}/recording/${recording.id}`; + + console.log("View recording at:"); + console.log(link(url)); + break; + } + default: { + console.log("View recording(s) at:"); + + for (const recording of recordings) { + const { processType, title } = formatRecording(recording); + + const url = `${replayAppHost}/recording/${recording.id}`; + + const formatted = processType ? `${title} ${processType}` : title; + + let text = `${formatted}: ${link(url)}`; + if (strip(text).length > process.stdout.columns) { + text = `${formatted}:\n${link(url)}`; + } + + console.log(text); + } + break; + } + } +} + +export const uploadRecordings = createAsyncFunctionWithTracking( + async function uploadRecordings( + recordings: LocalRecording[], + { + silent = false, + ...options + }: { + deleteOnSuccess?: boolean; + processingBehavior: ProcessingBehavior; + silent?: boolean; + } + ) { + recordings = recordings.filter(recording => { + if (!canUpload(recording)) { + logger.debug(`Cannot upload recording ${recording.id}`, { recording }); + return false; + } + + return true; + }); + + if (recordings.length === 0) { + return []; + } + + const worker = createUploadWorker(options); + const deferredActions = recordings.map(recording => worker.upload(recording)); + + if (!silent) { + printDeferredRecordingActions(deferredActions, { + renderTitle: ({ done }) => (done ? "Uploaded recordings" : `Uploading recordings...`), + renderExtraColumns: recording => { + let status: string | undefined; + if (recording.processingStatus) { + switch (recording.processingStatus) { + case "processing": + status = "(processing…)"; + break; + case "processed": + status = "(uploaded+processed)"; + break; + } + } else { + switch (recording.uploadStatus) { + case "failed": + status = "(failed)"; + break; + case "uploading": + status = "(uploading…)"; + break; + case "uploaded": + status = "(uploaded)"; + break; + } + } + return [status ? dim(status) : ""]; + }, + renderFailedSummary: failedRecordings => + `${failedRecordings.length} recording(s) did not upload successfully`, + }); + } + + try { + recordings = await worker.onEnd(); + } catch (error) { + if ( + error instanceof ProtocolError && + error.protocolCode === AUTHENTICATION_REQUIRED_ERROR_CODE + ) { + let message = `${statusFailed("✘")} Authentication failed.`; + if (process.env.REPLAY_API_KEY || process.env.RECORD_REPLAY_API_KEY) { + const name = process.env.REPLAY_API_KEY ? "REPLAY_API_KEY" : "RECORD_REPLAY_API_KEY"; + message += ` Please check your ${highlight(name)}.`; + } else { + message += ` Please try to ${highlight("replay login")} again.`; + } + console.error(message); + await exitProcess(1); + } + throw error; + } + + if (!silent) { + const uploadedRecordings = recordings.filter( + recording => recording.uploadStatus === "uploaded" + ); + printViewRecordingLinks(uploadedRecordings); + } + + return recordings; + }, + "upload.results", + recordings => { + return { + failedCount: + recordings?.filter(recording => recording.uploadStatus !== "uploaded").length ?? 0, + uploadedCount: + recordings?.filter(recording => recording.uploadStatus === "uploaded").length ?? 0, + }; + } +); diff --git a/packages/shared/src/protocol/ProtocolClient.ts b/packages/shared/src/protocol/ProtocolClient.ts index f4f829a3..9eed7972 100644 --- a/packages/shared/src/protocol/ProtocolClient.ts +++ b/packages/shared/src/protocol/ProtocolClient.ts @@ -110,7 +110,7 @@ export default class ProtocolClient { private onSocketClose = () => { if (this.deferredAuthenticated.status === STATUS_PENDING) { - this.deferredAuthenticated.resolve(false); + this.deferredAuthenticated.reject(new Error("Socket closed before authentication completed")); } }; diff --git a/packages/shared/src/recording/createSettledDeferred.ts b/packages/shared/src/recording/createSettledDeferred.ts index 0846945d..c7aeb08c 100644 --- a/packages/shared/src/recording/createSettledDeferred.ts +++ b/packages/shared/src/recording/createSettledDeferred.ts @@ -1,10 +1,10 @@ import { createDeferred } from "../async/createDeferred"; import { logger } from "../logger"; -export function createSettledDeferred(data: Data, promise: Promise) { +export function createSettledDeferred(data: Data, task: () => Promise) { const deferred = createDeferred(data); - promise.then( + task().then( () => { deferred.resolve(true); }, diff --git a/packages/shared/src/recording/printDeferredRecordingActions.ts b/packages/shared/src/recording/printDeferredRecordingActions.ts deleted file mode 100644 index a47f053b..00000000 --- a/packages/shared/src/recording/printDeferredRecordingActions.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { dots } from "cli-spinners"; -import { Deferred, STATUS_RESOLVED } from "../async/createDeferred"; -import { disableAnimatedLog } from "../config"; -import { logUpdate } from "../logUpdate"; -import { printTable } from "../printTable"; -import { statusFailed, statusPending, statusSuccess } from "../theme"; -import { formatRecording } from "./formatRecording"; -import { LocalRecording } from "./types"; - -export async function printDeferredRecordingActions( - deferredActions: Deferred[], - { - renderTitle, - renderExtraColumns, - renderFailedSummary, - }: { - renderTitle: (options: { done: boolean }) => string; - renderExtraColumns: (recording: LocalRecording) => string[]; - renderFailedSummary: (failedRecordings: LocalRecording[]) => string; - } -) { - let dotIndex = 0; - - const print = (done = false) => { - const dot = dots.frames[++dotIndex % dots.frames.length]; - const title = renderTitle({ done }); - const table = printTable({ - rows: deferredActions.map(deferred => { - let status = disableAnimatedLog ? "" : statusPending(dot); - if (deferred.resolution === true) { - status = statusSuccess("✔"); - } else if (deferred.resolution === false) { - status = statusFailed("✘"); - } - const recording = deferred.data; - const { date, duration, id, title } = formatRecording(recording); - return [status, id, title, date, duration, ...renderExtraColumns(recording)]; - }), - }); - - logUpdate(title + "\n" + table); - }; - - print(); - - const interval = disableAnimatedLog ? undefined : setInterval(print, dots.interval); - - await Promise.all(deferredActions.map(deferred => deferred.promise)); - - clearInterval(interval); - print(true); - logUpdate.done(); - - const failedActions = deferredActions.filter(deferred => deferred.status !== STATUS_RESOLVED); - if (failedActions.length > 0) { - const failedSummary = renderFailedSummary(failedActions.map(action => action.data)); - console.log(statusFailed(`${failedSummary}`) + "\n"); - } -} diff --git a/packages/shared/src/recording/printViewRecordingLinks.ts b/packages/shared/src/recording/printViewRecordingLinks.ts deleted file mode 100644 index af328e00..00000000 --- a/packages/shared/src/recording/printViewRecordingLinks.ts +++ /dev/null @@ -1,40 +0,0 @@ -import strip from "strip-ansi"; -import { replayAppHost } from "../config"; -import { link } from "../theme"; -import { formatRecording } from "./formatRecording"; -import { LocalRecording } from "./types"; - -export function printViewRecordingLinks(recordings: LocalRecording[]) { - switch (recordings.length) { - case 0: { - break; - } - case 1: { - const recording = recordings[0]; - const url = `${replayAppHost}/recording/${recording.id}`; - - console.log("View recording at:"); - console.log(link(url)); - break; - } - default: { - console.log("View recording(s) at:"); - - for (const recording of recordings) { - const { processType, title } = formatRecording(recording); - - const url = `${replayAppHost}/recording/${recording.id}`; - - const formatted = processType ? `${title} ${processType}` : title; - - let text = `${formatted}: ${link(url)}`; - if (strip(text).length > process.stdout.columns) { - text = `${formatted}:\n${link(url)}`; - } - - console.log(text); - } - break; - } - } -} diff --git a/packages/shared/src/recording/upload/uploadRecordings.ts b/packages/shared/src/recording/upload/uploadRecordings.ts deleted file mode 100644 index 1b8ff17b..00000000 --- a/packages/shared/src/recording/upload/uploadRecordings.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { getFeatureFlagValue } from "../../launch-darkly/getFeatureFlagValue"; -import { logger } from "../../logger"; -import { createAsyncFunctionWithTracking } from "../../mixpanel/createAsyncFunctionWithTracking"; -import { exitProcess } from "../../process/exitProcess"; -import ProtocolClient from "../../protocol/ProtocolClient"; -import { AUTHENTICATION_REQUIRED_ERROR_CODE, ProtocolError } from "../../protocol/ProtocolError"; -import { dim, highlight, statusFailed } from "../../theme"; -import { canUpload } from "../canUpload"; -import { createSettledDeferred } from "../createSettledDeferred"; -import { printDeferredRecordingActions } from "../printDeferredRecordingActions"; -import { printViewRecordingLinks } from "../printViewRecordingLinks"; -import { removeFromDisk } from "../removeFromDisk"; -import { LocalRecording } from "../types"; -import { ProcessingBehavior } from "./types"; -import { uploadCrashedData } from "./uploadCrashData"; -import { uploadRecording } from "./uploadRecording"; - -export const uploadRecordings = createAsyncFunctionWithTracking( - async function uploadRecordings( - recordings: LocalRecording[], - options: { - deleteOnSuccess?: boolean; - processingBehavior: ProcessingBehavior; - silent?: boolean; - } - ) { - const { deleteOnSuccess = true, processingBehavior, silent = false } = options; - - recordings = recordings.filter(recording => { - if (!canUpload(recording)) { - logger.debug(`Cannot upload recording ${recording.id}`, { recording }); - return false; - } - - return true; - }); - - if (recordings.length === 0) { - return []; - } - - const multiPartUpload = await getFeatureFlagValue("cli-multipart-upload", false); - const client = new ProtocolClient(); - try { - await client.waitUntilAuthenticated(); - } catch (error) { - if ( - error instanceof ProtocolError && - error.protocolCode === AUTHENTICATION_REQUIRED_ERROR_CODE - ) { - let message = `${statusFailed("✘")} Authentication failed.`; - if (process.env.REPLAY_API_KEY || process.env.RECORD_REPLAY_API_KEY) { - const name = process.env.REPLAY_API_KEY ? "REPLAY_API_KEY" : "RECORD_REPLAY_API_KEY"; - message += ` Please check your ${highlight(name)}.`; - } else { - message += ` Please try to ${highlight("replay login")} again.`; - } - console.error(message); - await exitProcess(1); - } - throw error; - } - - const deferredActions = recordings.map(recording => { - if (recording.recordingStatus === "crashed") { - return createSettledDeferred( - recording, - uploadCrashedData(client, recording) - ); - } else { - return createSettledDeferred( - recording, - uploadRecording(client, recording, { multiPartUpload, processingBehavior }) - ); - } - }); - - if (!silent) { - printDeferredRecordingActions(deferredActions, { - renderTitle: ({ done }) => (done ? "Uploaded recordings" : `Uploading recordings...`), - renderExtraColumns: recording => { - let status: string | undefined; - if (recording.processingStatus) { - switch (recording.processingStatus) { - case "processing": - status = "(processing…)"; - break; - case "processed": - status = "(uploaded+processed)"; - break; - } - } else { - switch (recording.uploadStatus) { - case "failed": - status = "(failed)"; - break; - case "uploading": - status = "(uploading…)"; - break; - case "uploaded": - status = "(uploaded)"; - break; - } - } - return [status ? dim(status) : ""]; - }, - renderFailedSummary: failedRecordings => - `${failedRecordings.length} recording(s) did not upload successfully`, - }); - } - - await Promise.all(deferredActions.map(deferred => deferred.promise)); - - const uploadedRecordings = recordings.filter( - recording => recording.uploadStatus === "uploaded" - ); - - if (!silent) { - printViewRecordingLinks(uploadedRecordings); - } - - if (deleteOnSuccess) { - uploadedRecordings.forEach(recording => { - removeFromDisk(recording.id); - }); - } - - client.close(); - - return deferredActions.map(action => action.data); - }, - "upload.results", - recordings => { - return { - failedCount: - recordings?.filter(recording => recording.uploadStatus !== "uploaded").length ?? 0, - uploadedCount: - recordings?.filter(recording => recording.uploadStatus === "uploaded").length ?? 0, - }; - } -); diff --git a/packages/shared/src/recording/upload/uploadWorker.ts b/packages/shared/src/recording/upload/uploadWorker.ts new file mode 100644 index 00000000..eeb6bc5b --- /dev/null +++ b/packages/shared/src/recording/upload/uploadWorker.ts @@ -0,0 +1,73 @@ +import { createDeferred, Deferred } from "../../async/createDeferred"; +import { getFeatureFlagValue } from "../../launch-darkly/getFeatureFlagValue"; +import ProtocolClient from "../../protocol/ProtocolClient"; +import { createSettledDeferred } from "../createSettledDeferred"; +import { removeFromDisk } from "../removeFromDisk"; +import { LocalRecording } from "../types"; +import { ProcessingBehavior } from "./types"; +import { uploadCrashedData } from "./uploadCrashData"; +import { uploadRecording } from "./uploadRecording"; + +export function createUploadWorker({ + deleteOnSuccess, + processingBehavior, +}: { + deleteOnSuccess?: boolean; + processingBehavior: ProcessingBehavior; +}) { + const client = new ProtocolClient(); + const deferredAuthenticated = createDeferred(); + const deferredActions: Deferred[] = []; + + let multiPartUpload = false; + + (async () => { + multiPartUpload = await getFeatureFlagValue("cli-multipart-upload", false); + try { + await client.waitUntilAuthenticated(); + deferredAuthenticated.resolve(true); + } catch (error: any) { + deferredAuthenticated.reject(error); + } + })(); + + return { + upload: (recording: LocalRecording) => { + const deferred = createSettledDeferred(recording, async () => { + await deferredAuthenticated.promise; + + if (recording.recordingStatus === "crashed") { + await uploadCrashedData(client, recording); + } else { + await uploadRecording(client, recording, { multiPartUpload, processingBehavior }); + } + }); + deferredActions.push(deferred); + return deferred; + }, + onEnd: async () => { + try { + await deferredAuthenticated.promise; + } catch (err) { + client.close(); + throw err; + } + + await Promise.all(deferredActions.map(deferred => deferred.promise)); + + client.close(); + + const recordings = deferredActions.map(action => action.data); + + if (deleteOnSuccess) { + recordings + .filter(recording => recording.uploadStatus === "uploaded") + .forEach(recording => { + removeFromDisk(recording.id); + }); + } + + return recordings; + }, + }; +} diff --git a/packages/test-utils/src/legacy-cli/client.ts b/packages/test-utils/src/legacy-cli/client.ts deleted file mode 100644 index 80b6d967..00000000 --- a/packages/test-utils/src/legacy-cli/client.ts +++ /dev/null @@ -1,143 +0,0 @@ -import WebSocket from "ws"; -import { defer } from "./utils"; -import { Agent } from "http"; -import { logger } from "@replay-cli/shared/logger"; - -// Simple protocol client for use in writing standalone applications. - -interface Callbacks { - onOpen: (socket: WebSocket) => void; - onClose: (socket: WebSocket) => void; - onError: (socket: WebSocket) => void; -} - -type ErrorDataValue = string | number | boolean | null; -type ErrorData = Record; -type ProtocolErrorBase = { - code: number; - message: string; - data: ErrorData; -}; - -export class ProtocolError extends Error { - readonly protocolCode: number; - readonly protocolMessage: string; - readonly protocolData: unknown; - - constructor(err: ProtocolErrorBase) { - super(`protocol error ${err.code}: ${err.message}`); - this.protocolCode = err.code; - this.protocolMessage = err.message; - this.protocolData = err.data ?? {}; - } - - toString() { - return `Protocol error ${this.protocolCode}: ${this.protocolMessage}`; - } -} - -class ProtocolClient { - socket: WebSocket; - callbacks: Callbacks; - pendingMessages = new Map(); - eventListeners = new Map(); - nextMessageId = 1; - - constructor(address: string, callbacks: Callbacks, agent?: Agent) { - logger.info("ProtocolClient:WillInitialize", { websocketAddress: address, agent }); - this.socket = new WebSocket(address, { - agent: agent, - }); - this.callbacks = callbacks; - - this.socket.on("open", callbacks.onOpen); - this.socket.on("close", callbacks.onClose); - this.socket.on("error", callbacks.onError); - this.socket.on("message", message => this.onMessage(message)); - logger.info("ProtocolClient:DidInitialize", { websocketAddress: address, agent }); - } - - close() { - this.socket.close(); - } - - async setAccessToken(accessToken?: string) { - accessToken = accessToken || process.env.REPLAY_API_KEY || process.env.RECORD_REPLAY_API_KEY; - - if (!accessToken) { - throw new Error( - "Access token must be passed or set via the REPLAY_API_KEY environment variable." - ); - } - - return this.sendCommand("Authentication.setAccessToken", { - accessToken, - }); - } - - async sendCommand>( - method: string, - params: P, - data?: any, - sessionId?: string, - callback?: (err?: Error) => void - ) { - const id = this.nextMessageId++; - logger.info("SendCommand:Started", { id, sessionId, method }); - - this.socket.send( - JSON.stringify({ - id, - method, - params, - binary: data ? true : undefined, - sessionId, - }), - error => { - if (!error && data) { - this.socket.send(data, callback); - } else { - if (error) { - logger.error("SendCommand:ReceivedSocketError", { - id, - params, - sessionId, - error, - }); - } - callback?.(error); - } - } - ); - const waiter = defer(); - this.pendingMessages.set(id, waiter); - return waiter.promise; - } - - setEventListener(method: string, callback: (params: any) => void) { - this.eventListeners.set(method, callback); - } - - onMessage(contents: WebSocket.RawData) { - const msg = JSON.parse(String(contents)); - logger.info("OnMessage:ReceivedMessage", { msg }); - - if (msg.id) { - const { resolve, reject } = this.pendingMessages.get(msg.id); - this.pendingMessages.delete(msg.id); - if (msg.result) { - resolve(msg.result); - } else if (msg.error) { - reject(new ProtocolError(msg.error)); - } else { - reject(`Channel error: ${JSON.stringify(msg)}`); - } - } else if (this.eventListeners.has(msg.method)) { - this.eventListeners.get(msg.method)(msg.params); - } else { - logger.info("OnMessage:ReceivedEventWithoutListener", { msg }); - } - } -} - -export default ProtocolClient; diff --git a/packages/test-utils/src/legacy-cli/index.ts b/packages/test-utils/src/legacy-cli/index.ts deleted file mode 100644 index 98d2fd18..00000000 --- a/packages/test-utils/src/legacy-cli/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { listAllRecordings, removeRecording, uploadRecording } from "./main"; diff --git a/packages/test-utils/src/legacy-cli/listAllRecordings.ts b/packages/test-utils/src/legacy-cli/listAllRecordings.ts new file mode 100644 index 00000000..11fe0c06 --- /dev/null +++ b/packages/test-utils/src/legacy-cli/listAllRecordings.ts @@ -0,0 +1,73 @@ +import jsonata from "jsonata"; +import { readRecordings } from "./recordingLog"; +import { + FilterOptions, + ListOptions, + Options, + RecordingEntry, + type ExternalRecordingEntry, +} from "./types"; +import { logger } from "@replay-cli/shared/logger"; + +function filterRecordings( + recordings: RecordingEntry[], + filter: FilterOptions["filter"], + includeCrashes: FilterOptions["includeCrashes"] +) { + let filteredRecordings = recordings; + logger.info("FilterRecordings:Started", { + numRecordingLogReplays: recordings.length, + filterType: filter ? typeof filter : undefined, + }); + if (filter && typeof filter === "string") { + const exp = jsonata(`$filter($, ${filter})[]`); + filteredRecordings = exp.evaluate(recordings) || []; + + logger.info("FilterRecordings:UsedString", { + filteredRecordingsLength: filteredRecordings.length, + filter, + }); + } else if (typeof filter === "function") { + filteredRecordings = recordings.filter(filter); + + logger.info("FilterRecordings:UsedFunction", { + filteredRecordingsLength: filteredRecordings.length, + }); + } + + if (includeCrashes) { + recordings.forEach(r => { + if (r.status === "crashed" && !filteredRecordings.includes(r)) { + filteredRecordings.push(r); + } + }); + logger.info("FilterRecordings:IncludedCrashes", { + filteredRecordingsLength: filteredRecordings.length, + }); + } + + return filteredRecordings; +} + +// Convert a recording into a format for listing. +function listRecording(recording: RecordingEntry): ExternalRecordingEntry { + // Remove properties we only use internally. + const { buildId, crashData, ...recordingWithoutInternalProperties } = recording; + return recordingWithoutInternalProperties; +} + +export function listAllRecordings(opts: Options & ListOptions = {}) { + logger.info("ListAllRecordings:Started"); + const recordings = readRecordings(); + + if (opts.all) { + return filterRecordings(recordings, opts.filter, opts.includeCrashes).map(listRecording); + } + + const uploadableRecordings = recordings.filter(recording => + ["onDisk", "startedWrite", "crashed"].includes(recording.status) + ); + return filterRecordings(uploadableRecordings, opts.filter, opts.includeCrashes).map( + listRecording + ); +} diff --git a/packages/test-utils/src/legacy-cli/main.ts b/packages/test-utils/src/legacy-cli/main.ts deleted file mode 100644 index 013d4bb7..00000000 --- a/packages/test-utils/src/legacy-cli/main.ts +++ /dev/null @@ -1,492 +0,0 @@ -import { retryWithExponentialBackoff } from "@replay-cli/shared/async/retryOnFailure"; -import fs from "fs"; -import { getHttpAgent } from "./utils"; - -// requiring v4 explicitly because it's the last version with commonjs support. -// Should be upgraded to the latest when converting this code to es modules. -import pMap from "p-map"; - -import { Agent, AgentOptions } from "http"; -import jsonata from "jsonata"; -import { ProtocolError } from "./client"; -import { addRecordingEvent, readRecordings, removeRecordingFromLog } from "./recordingLog"; -import { - FilterOptions, - ListOptions, - Options, - RecordingEntry, - RecordingMetadata, - SourceMapEntry, - UploadOptions, - type ExternalRecordingEntry, -} from "./types"; -import { ReplayClient } from "./upload"; -import { maybeLogToConsole } from "./utils"; -import { logger } from "@replay-cli/shared/logger"; -export type { RecordingEntry } from "./types"; -export { updateStatus } from "./updateStatus"; - -function filterRecordings( - recordings: RecordingEntry[], - filter: FilterOptions["filter"], - includeCrashes: FilterOptions["includeCrashes"] -) { - let filteredRecordings = recordings; - logger.info("FilterRecordings:Started", { - numRecordingLogReplays: recordings.length, - filterType: filter ? typeof filter : undefined, - }); - if (filter && typeof filter === "string") { - const exp = jsonata(`$filter($, ${filter})[]`); - filteredRecordings = exp.evaluate(recordings) || []; - - logger.info("FilterRecordings:UsedString", { - filteredRecordingsLength: filteredRecordings.length, - filter, - }); - } else if (typeof filter === "function") { - filteredRecordings = recordings.filter(filter); - - logger.info("FilterRecordings:UsedFunction", { - filteredRecordingsLength: filteredRecordings.length, - }); - } - - if (includeCrashes) { - recordings.forEach(r => { - if (r.status === "crashed" && !filteredRecordings.includes(r)) { - filteredRecordings.push(r); - } - }); - logger.info("FilterRecordings:IncludedCrashes", { - filteredRecordingsLength: filteredRecordings.length, - }); - } - - return filteredRecordings; -} - -// Convert a recording into a format for listing. -function listRecording(recording: RecordingEntry): ExternalRecordingEntry { - // Remove properties we only use internally. - const { buildId, crashData, ...recordingWithoutInternalProperties } = recording; - return recordingWithoutInternalProperties; -} - -function listAllRecordings(opts: Options & ListOptions = {}) { - logger.info("ListAllRecordings:Started"); - const recordings = readRecordings(); - - if (opts.all) { - return filterRecordings(recordings, opts.filter, opts.includeCrashes).map(listRecording); - } - - const uploadableRecordings = recordings.filter(recording => - ["onDisk", "startedWrite", "crashed"].includes(recording.status) - ); - return filterRecordings(uploadableRecordings, opts.filter, opts.includeCrashes).map( - listRecording - ); -} - -function uploadSkipReason(recording: RecordingEntry) { - // Status values where there is something worth uploading. - const canUploadStatus = ["onDisk", "startedWrite", "startedUpload", "crashed"]; - if (!canUploadStatus.includes(recording.status)) { - return `wrong recording status ${recording.status}`; - } - if (!recording.path && recording.status != "crashed") { - return "recording not saved to disk"; - } - return null; -} - -function getServer(opts: Options) { - return ( - opts.server || - process.env.RECORD_REPLAY_SERVER || - process.env.REPLAY_SERVER || - "wss://dispatch.replay.io" - ); -} - -async function doUploadCrash( - server: string, - recording: RecordingEntry, - verbose?: boolean, - apiKey?: string, - agent?: Agent -) { - const client = new ReplayClient(); - logger.info("DoUploadCrash:Started", { recordingId: recording.id }); - maybeLogToConsole(verbose, `Starting crash data upload for ${recording.id}...`); - - if (!(await client.initConnection(server, apiKey, verbose, agent))) { - logger.error("DoUploadCrash:CannotConnectToServer", { recordingId: recording.id, server }); - maybeLogToConsole(verbose, `Crash data upload failed: can't connect to server ${server}`); - return null; - } - - const crashData = recording.crashData || []; - crashData.push({ - kind: "recordingMetadata", - recordingId: recording.id, - }); - - await Promise.all( - crashData.map(async data => { - await client.connectionReportCrash(data); - }) - ); - addRecordingEvent("crashUploaded", recording.id, { server }); - maybeLogToConsole(verbose, `Crash data upload finished.`); - logger.info("DoUploadCrash:Successful", { recordingId: recording.id, server }); - client.closeConnection(); -} - -class RecordingUploadError extends Error { - interiorError?: any; - - constructor(message?: string, interiorError?: any) { - super(message); - this.name = "RecordingUploadError"; - this.interiorError = interiorError; - Object.setPrototypeOf(this, new.target.prototype); // Restore error prototype chain. - } -} - -function handleUploadingError( - err: string, - strict: boolean, - verbose?: boolean, - interiorError?: any -) { - maybeLogToConsole(verbose, `Upload failed: ${err}`); - - if (strict) { - throw new RecordingUploadError(err, interiorError); - } -} - -async function validateMetadata( - client: ReplayClient, - metadata: Record | null, - verbose: boolean | undefined -): Promise { - return metadata ? await client.buildRecordingMetadata(metadata, { verbose }) : null; -} - -async function setMetadata( - client: ReplayClient, - recordingId: string, - metadata: RecordingMetadata | null, - strict: boolean, - verbose: boolean -) { - if (metadata) { - try { - await retryWithExponentialBackoff( - () => client.setRecordingMetadata(recordingId, metadata), - error => { - logger.error("SetMetadata:WillRetry", { - recordingId, - error, - }); - } - ); - } catch (error) { - logger.error("SetMetadata:Failed", { - recordingId, - strict, - error, - }); - handleUploadingError(`Failed to set recording metadata ${error}`, strict, verbose, error); - } - } -} - -async function directUploadRecording( - server: string, - client: ReplayClient, - recording: RecordingEntry, - metadata: RecordingMetadata | null, - size: number, - strict: boolean, - verbose: boolean -) { - const { recordingId, uploadLink } = await client.connectionBeginRecordingUpload( - recording.id, - recording.buildId!, - size - ); - await setMetadata(client, recordingId, metadata, strict, verbose); - addRecordingEvent("uploadStarted", recording.id, { - server, - recordingId, - }); - await retryWithExponentialBackoff( - () => client.uploadRecording(recording.path!, uploadLink, size), - error => { - logger.error("DirectUploadRecording:WillRetry", { - recordingId, - error, - }); - } - ); - - logger.info("DoUploadRecording:Succeeded", { recordingId: recording.id, sizeInBytes: size }); - - await client.connectionEndRecordingUpload(recording.id); - return recordingId; -} - -async function doUploadRecording( - server: string, - recording: RecordingEntry, - verbose: boolean = false, - apiKey: string, - agentOptions?: AgentOptions, - removeAssets: boolean = false, - strict: boolean = false -) { - logger.info("DoUploadRecording:Started", { recordingId: recording.id, server }); - maybeLogToConsole(verbose, `Starting upload for ${recording.id}...`); - - if (recording.status == "uploaded" && recording.recordingId) { - logger.info("DoUploadRecording:AlreadyUploaded", { recordingId: recording.id }); - maybeLogToConsole(verbose, `Already uploaded: ${recording.recordingId}`); - - return recording.recordingId; - } - - const reason = uploadSkipReason(recording); - if (reason) { - logger.error("DoUploadRecording:Failed", { - recordingId: recording.id, - server, - uploadSkipReason, - strict, - }); - - handleUploadingError(reason, strict, verbose); - return null; - } - - const agent = getHttpAgent(server, agentOptions); - - if (recording.status == "crashed") { - logger.info("DoUploadRecording:WillUploadCrashReport", { - recordingId: recording.id, - recordingStatus: recording.status, - }); - await doUploadCrash(server, recording, verbose, apiKey, agent); - logger.info("DoUploadRecording:CrashReportUploaded", { - recordingId: recording.id, - }); - maybeLogToConsole(verbose, `Crash report uploaded for ${recording.id}`); - - if (removeAssets) { - removeRecordingAssets(recording); - logger.info("DoUploadRecording:RemovedRecordingAssets", { - recordingId: recording.id, - }); - } - return recording.id; - } - - const { size } = await fs.promises.stat(recording.path!); - - logger.info("DoUploadRecording:WillUpload", { - recording, - }); - - const client = new ReplayClient(); - if (!(await client.initConnection(server, apiKey, verbose, agent))) { - logger.error("DoUploadRecording:ServerConnectionError", { - recording, - server, - strict, - }); - handleUploadingError(`Cannot connect to server ${server}`, strict, verbose); - return null; - } - - // validate metadata before uploading so invalid data can block the upload - const metadata = await validateMetadata(client, recording.metadata, verbose); - - let recordingId: string; - try { - recordingId = await directUploadRecording( - server, - client, - recording, - metadata, - size, - strict, - verbose - ); - } catch (err) { - const errorMessage = err instanceof ProtocolError ? err.protocolMessage : String(err); - logger.error("DoUploadRecording:ProtocolError", { - recording, - server, - strict, - errorMessage, - wasMultipartUpload: false, - }); - handleUploadingError(errorMessage, strict, verbose, err); - return null; - } - - await pMap( - recording.sourcemaps, - async (sourcemap: SourceMapEntry) => { - try { - logger.info("DoUploadRecording:WillUploadSourcemaps", { - recordingId: recording.id, - sourcemapPath: sourcemap.path, - }); - - const contents = fs.readFileSync(sourcemap.path, "utf8"); - const sourcemapId = await client.connectionUploadSourcemap( - recordingId, - sourcemap, - contents - ); - await pMap( - sourcemap.originalSources, - originalSource => { - logger.info("DoUploadRecording:WillUploadOriginalSources", { - recordingId: recording.id, - sourcemapPath: sourcemap.path, - }); - - const contents = fs.readFileSync(originalSource.path, "utf8"); - return client.connectionUploadOriginalSource( - recordingId, - sourcemapId, - originalSource, - contents - ); - }, - { concurrency: 5, stopOnError: false } - ); - } catch (error) { - logger.error("DoUploadRecording:CannotUploadSourcemapFromDisk", { - recordingId: recording.id, - sourcemapPath: sourcemap.path, - error, - }); - - handleUploadingError( - `Cannot upload sourcemap ${sourcemap.path} from disk: ${error}`, - strict, - verbose, - error - ); - } - }, - { concurrency: 10, stopOnError: false } - ); - - if (removeAssets) { - removeRecordingAssets(recording); - } - - addRecordingEvent("uploadFinished", recording.id); - const replayUrl = ` https://app.replay.io/recording/${recordingId}`; - - maybeLogToConsole(verbose, `Upload finished! View your Replay at: ${replayUrl}`); - - logger.info("DoUploadRecording:Succeeded", { - recordingId: recording.id, - replayUrl, - }); - - client.closeConnection(); - return recordingId; -} - -async function uploadRecording(id: string, opts: UploadOptions) { - const server = getServer(opts); - const recordings = readRecordings(); - const recording = recordings.find(r => r.id == id); - - if (!recording) { - maybeLogToConsole(opts.verbose, `Unknown recording ${id}`); - logger.error("UploadRecording:UnknownRecording", { - id, - }); - - return null; - } - - return doUploadRecording( - server, - recording, - opts.verbose, - opts.apiKey, - opts.agentOptions, - opts.removeAssets ?? true, - opts.strict - ); -} - -function maybeRemoveAssetFile(asset?: string) { - if (asset) { - try { - if (fs.existsSync(asset)) { - logger.info("MaybeRemoveAssetFile:Removing", { asset }); - fs.unlinkSync(asset); - } - } catch (error) { - logger.error("MaybeRemoveAssetFile:Failed", { asset, error }); - } - } -} - -function removeRecording(id: string, opts: Options = {}) { - const recordings = readRecordings(); - const recording = recordings.find(r => r.id == id); - if (!recording) { - logger.error("RemoveRecording:UnknownRecording", { - id, - }); - maybeLogToConsole(opts.verbose, `Unknown recording ${id}`); - return false; - } - removeRecordingAssets(recording); - removeRecordingFromLog(id); - return true; -} - -function getRecordingAssetFiles(recording: RecordingEntry) { - const assetFiles: string[] = []; - if (recording.path) { - assetFiles.push(recording.path); - } - - recording.sourcemaps.forEach(sm => { - assetFiles.push(sm.path); - assetFiles.push(sm.path.replace(/\.map$/, ".lookup")); - sm.originalSources.forEach(o => assetFiles.push(o.path)); - }); - - return assetFiles; -} - -function removeRecordingAssets(recording: RecordingEntry) { - const localRecordings = listAllRecordings({ - filter: r => r.status !== "uploaded" && r.status !== "crashUploaded" && r.id !== recording.id, - }); - - const localRecordingAssetFiles = new Set(localRecordings.flatMap(getRecordingAssetFiles)); - const assetFiles = getRecordingAssetFiles(recording); - assetFiles.forEach(file => { - if (!localRecordingAssetFiles.has(file)) { - maybeRemoveAssetFile(file); - } - }); -} - -export { listAllRecordings, removeRecording, uploadRecording }; diff --git a/packages/test-utils/src/legacy-cli/recordingLog.ts b/packages/test-utils/src/legacy-cli/recordingLog.ts index 2372b441..61227b63 100644 --- a/packages/test-utils/src/legacy-cli/recordingLog.ts +++ b/packages/test-utils/src/legacy-cli/recordingLog.ts @@ -1,8 +1,6 @@ import fs from "node:fs"; import { RecordingEntry } from "./types"; import { generateDefaultTitle } from "./generateDefaultTitle"; -import { updateStatus } from "./updateStatus"; -import { logger } from "@replay-cli/shared/logger"; import { recordingLogPath } from "@replay-cli/shared/recording/config"; function readRecordingFile() { @@ -12,14 +10,12 @@ function readRecordingFile() { return fs.readFileSync(recordingLogPath, "utf8").split("\n"); } -function writeRecordingFile(lines: string[]) { - // Add a trailing newline so the driver can safely append logs - fs.writeFileSync(recordingLogPath, lines.join("\n") + "\n"); -} + function getBuildRuntime(buildId: string) { const match = /.*?-(.*?)-/.exec(buildId); return match ? match[1] : "unknown"; } + const RECORDING_LOG_KIND = [ "createRecording", "addMetadata", @@ -34,10 +30,25 @@ const RECORDING_LOG_KIND = [ "crashData", "crashUploaded", ] as const; + interface RecordingLogEntry { [key: string]: any; kind: (typeof RECORDING_LOG_KIND)[number]; } + +function updateStatus(recording: RecordingEntry, status: RecordingEntry["status"]) { + // Once a recording enters an unusable or crashed status, don't change it + // except to mark crashes as uploaded. + if ( + recording.status == "unusable" || + recording.status == "crashUploaded" || + (recording.status == "crashed" && status != "crashUploaded") + ) { + return; + } + recording.status = status; +} + export function readRecordings(includeHidden = false) { const recordings: RecordingEntry[] = []; const lines = readRecordingFile() @@ -205,34 +216,3 @@ export function readRecordings(includeHidden = false) { // when a recording process starts if it will ever do anything interesting. return recordings.filter(r => !(r.unusableReason || "").includes("No interesting content")); } - -function addRecordingEvent(kind: string, id: string, tags = {}) { - const event = { - kind, - id, - timestamp: Date.now(), - ...tags, - }; - logger.info("AddRecordingEvent:Started", { event, kind }); - const lines = readRecordingFile(); - lines.push(JSON.stringify(event)); - writeRecordingFile(lines); -} - -function removeRecordingFromLog(id: string) { - const lines = readRecordingFile().filter(line => { - try { - const obj = JSON.parse(line); - if (obj.id == id) { - return false; - } - } catch (e) { - return false; - } - return true; - }); - - writeRecordingFile(lines); -} - -export { readRecordingFile, removeRecordingFromLog, addRecordingEvent }; diff --git a/packages/test-utils/src/legacy-cli/updateStatus.ts b/packages/test-utils/src/legacy-cli/updateStatus.ts deleted file mode 100644 index 1984e6c1..00000000 --- a/packages/test-utils/src/legacy-cli/updateStatus.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { RecordingEntry } from "./types"; - -export function updateStatus(recording: RecordingEntry, status: RecordingEntry["status"]) { - // Once a recording enters an unusable or crashed status, don't change it - // except to mark crashes as uploaded. - if ( - recording.status == "unusable" || - recording.status == "crashUploaded" || - (recording.status == "crashed" && status != "crashUploaded") - ) { - return; - } - recording.status = status; -} diff --git a/packages/test-utils/src/legacy-cli/upload.ts b/packages/test-utils/src/legacy-cli/upload.ts deleted file mode 100644 index 2ef7d309..00000000 --- a/packages/test-utils/src/legacy-cli/upload.ts +++ /dev/null @@ -1,249 +0,0 @@ -import crypto from "crypto"; -import fs from "fs"; -import type { Agent } from "http"; -import fetch from "node-fetch"; -import ProtocolClient from "./client"; -import { sanitize as sanitizeMetadata } from "./metadata"; -import { Options, OriginalSourceEntry, RecordingMetadata, SourceMapEntry } from "./types"; -import { defer, isValidUUID, maybeLogToConsole } from "./utils"; -import { getUserAgent } from "@replay-cli/shared/userAgent"; -import { logger } from "@replay-cli/shared/logger"; - -function sha256(text: string) { - return crypto.createHash("sha256").update(text).digest("hex"); -} - -class ReplayClient { - client: ProtocolClient | undefined; - clientReady = defer(); - - async initConnection(server: string, accessToken?: string, verbose?: boolean, agent?: Agent) { - if (!this.client) { - let { resolve } = this.clientReady; - this.client = new ProtocolClient( - server, - { - onOpen: async () => { - try { - await this.client!.setAccessToken(accessToken); - resolve(true); - } catch (err) { - logger.error("ProtocolClient:ServerAuthFailed", { error: err }); - - maybeLogToConsole(verbose, `Error authenticating with server: ${err}`); - resolve(false); - } - }, - onClose() { - resolve(false); - }, - onError(e) { - maybeLogToConsole(verbose, `Error connecting to server: ${e}`); - resolve(false); - logger.error("ProtocolClient:Error", { error: e }); - }, - }, - agent - ); - } - - return this.clientReady.promise; - } - - async connectionBeginRecordingUpload(id: string, buildId: string, size: number) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - const { recordingId, uploadLink } = await this.client.sendCommand<{ - recordingId: string; - uploadLink: string; - }>("Internal.beginRecordingUpload", { - buildId, - // 3/22/2022: Older builds use integers instead of UUIDs for the recording - // IDs written to disk. These are not valid to use as recording IDs when - // uploading recordings to the backend. - recordingId: isValidUUID(id) ? id : undefined, - recordingSize: size, - }); - return { recordingId, uploadLink }; - } - - async buildRecordingMetadata( - metadata: Record, - _opts: Options = {} - ): Promise { - // extract the "standard" metadata and route the `rest` through the sanitizer - const { duration, url, uri, title, operations, ...rest } = metadata; - - const metadataUrl = url || uri; - - return { - recordingData: { - duration: typeof duration === "number" ? duration : 0, - url: typeof metadataUrl === "string" ? metadataUrl : "", - title: typeof title === "string" ? title : "", - operations: - operations && typeof operations === "object" - ? operations - : { - scriptDomains: [], - }, - lastScreenData: "", - lastScreenMimeType: "", - }, - metadata: await sanitizeMetadata(rest), - }; - } - - async setRecordingMetadata(id: string, metadata: RecordingMetadata) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - metadata.recordingData.id = id; - await this.client.sendCommand("Internal.setRecordingMetadata", metadata); - } - - connectionProcessRecording(recordingId: string) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - this.client.sendCommand("Recording.processRecording", { recordingId }); - } - - async connectionWaitForProcessed(recordingId: string) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - const { sessionId } = await this.client.sendCommand<{ sessionId: string }>( - "Recording.createSession", - { - recordingId, - } - ); - const waiter = defer(); - - this.client.setEventListener("Recording.sessionError", ({ message }) => - waiter.resolve(`session error ${sessionId}: ${message}`) - ); - - this.client.setEventListener("Session.unprocessedRegions", () => {}); - - this.client - .sendCommand("Session.ensureProcessed", { level: "basic" }, null, sessionId) - .then(() => waiter.resolve(null)); - - const error = await waiter.promise; - return error; - } - - async connectionReportCrash(data: any) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - await this.client.sendCommand("Internal.reportCrash", { data }); - } - - async uploadRecording(path: string, uploadLink: string, size: number) { - const file = fs.createReadStream(path); - const resp = await fetch(uploadLink, { - method: "PUT", - headers: { "Content-Length": size.toString(), "User-Agent": getUserAgent() }, - body: file, - }); - - if (resp.status !== 200) { - logger.error("ReplayClientUploadRecording:Failed", { - responseText: await resp.text(), - responseStatus: resp.status, - responseStatusText: resp.statusText, - }); - throw new Error(`Failed to upload recording. Response was ${resp.status} ${resp.statusText}`); - } - } - - async connectionEndRecordingUpload(recordingId: string) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - await this.client.sendCommand<{ recordingId: string }>("Internal.endRecordingUpload", { - recordingId, - }); - } - - async connectionEndRecordingMultipartUpload( - recordingId: string, - uploadId: string, - eTags: string[] - ) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - await this.client.sendCommand<{ recordingId: string; uploadId: string; partETags: string[] }>( - "Internal.endRecordingMultipartUpload", - { - recordingId, - uploadId, - partIds: eTags, - } - ); - } - - async connectionUploadSourcemap(recordingId: string, metadata: SourceMapEntry, content: string) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - const resource = await this.createResource(content); - - const { baseURL, targetContentHash, targetURLHash, targetMapURLHash } = metadata; - const result = await this.client.sendCommand<{ id: string }>("Recording.addSourceMap", { - recordingId, - resource, - baseURL, - targetContentHash, - targetURLHash, - targetMapURLHash, - }); - return result.id; - } - - async connectionUploadOriginalSource( - recordingId: string, - parentId: string, - metadata: OriginalSourceEntry, - content: string - ) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - const resource = await this.createResource(content); - - const { parentOffset } = metadata; - await this.client.sendCommand("Recording.addOriginalSource", { - recordingId, - resource, - parentId, - parentOffset, - }); - } - - async createResource(content: string) { - if (!this.client) throw new Error("Protocol client is not initialized"); - - const hash = "sha256:" + sha256(content); - const { token } = await this.client.sendCommand<{ token: string }>("Resource.token", { hash }); - let resource = { - token, - saltedHash: "sha256:" + sha256(token + content), - }; - - const { exists } = await this.client.sendCommand<{ exists: boolean }>("Resource.exists", { - resource, - }); - if (!exists) { - ({ resource } = await this.client.sendCommand("Resource.create", { content })); - } - - return resource; - } - - closeConnection() { - if (this.client) { - this.client.close(); - this.client = undefined; - this.clientReady = defer(); - } - } -} - -export { ReplayClient }; diff --git a/packages/test-utils/src/legacy-cli/utils.ts b/packages/test-utils/src/legacy-cli/utils.ts index 85c1331b..836fcbd4 100644 --- a/packages/test-utils/src/legacy-cli/utils.ts +++ b/packages/test-utils/src/legacy-cli/utils.ts @@ -1,53 +1,7 @@ -// This module is meant to be somewhat browser-friendly. -// It can't lead to importing node builtin modules like like worker_threads. -// Cypress bundles this file and runs it in the browser, -// some imports like path and http are OK because they are aliased~ by their webpack config: -// https://github.com/cypress-io/cypress/blob/fb87950d6337ba99d13cb5fa3ce129e5f5cac02b/npm/webpack-batteries-included-preprocessor/index.js#L151 -// TODO: decouple this more so we never run into problems with this - we shouldn't rely on implementation details of Cypress bundling -import { AgentOptions, Agent as HttpAgent } from "http"; -import { Agent as HttpsAgent } from "https"; - -function defer() { - let resolve: (value: T) => void = () => {}; - let reject: (reason?: any) => void = () => {}; - const promise = new Promise((res, rej) => { - resolve = res; - reject = rej; - }); - return { promise, resolve, reject }; -} - function maybeLogToConsole(verbose: boolean | undefined, str: string) { if (verbose) { console.log(str); } } -function isValidUUID(str: unknown) { - if (typeof str != "string" || str.length != 36) { - return false; - } - for (let i = 0; i < str.length; i++) { - if ("0123456789abcdef-".indexOf(str[i]) == -1) { - return false; - } - } - return true; -} - -function getHttpAgent(server: string, agentOptions?: AgentOptions) { - const serverURL = new URL(server); - if (!agentOptions) { - return; - } - - if (["wss:", "https:"].includes(serverURL.protocol)) { - return new HttpsAgent(agentOptions); - } else if (["ws:", "http:"].includes(serverURL.protocol)) { - return new HttpAgent(agentOptions); - } - - throw new Error(`Unsupported protocol: ${serverURL.protocol} for URL ${serverURL}`); -} - -export { defer, getHttpAgent, isValidUUID, maybeLogToConsole }; +export { maybeLogToConsole }; diff --git a/packages/test-utils/src/reporter.ts b/packages/test-utils/src/reporter.ts index 36d44220..77fcfb43 100644 --- a/packages/test-utils/src/reporter.ts +++ b/packages/test-utils/src/reporter.ts @@ -10,7 +10,7 @@ import assert from "node:assert/strict"; import { dirname } from "path"; import { v4 as uuid } from "uuid"; import { getAccessToken } from "./getAccessToken"; -import { listAllRecordings, removeRecording, uploadRecording } from "./legacy-cli"; +import { listAllRecordings } from "./legacy-cli/listAllRecordings"; import { add, source as sourceMetadata, test as testMetadata } from "./legacy-cli/metadata"; import type { TestMetadataV2 } from "./legacy-cli/metadata/test"; import { log } from "./logging"; From dde23da4642bf6f1fd51baebe6630cc268e0cf9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 13:24:34 +0200 Subject: [PATCH 03/11] use upload worker in the test utils reporter --- .../src/utils/recordings/uploadRecordings.ts | 8 +- .../shared/src/protocol/ProtocolClient.ts | 11 +- .../shared/src/recording/getRecordings.ts | 1 + packages/shared/src/recording/types.ts | 1 + .../src/recording/upload/uploadCrashData.ts | 19 +- .../src/recording/upload/uploadRecording.ts | 8 + .../src/recording/upload/uploadWorker.ts | 8 +- packages/test-utils/src/reporter.ts | 211 +++++++++--------- 8 files changed, 143 insertions(+), 124 deletions(-) diff --git a/packages/replayio/src/utils/recordings/uploadRecordings.ts b/packages/replayio/src/utils/recordings/uploadRecordings.ts index 13a7fc3f..98c4d7fd 100644 --- a/packages/replayio/src/utils/recordings/uploadRecordings.ts +++ b/packages/replayio/src/utils/recordings/uploadRecordings.ts @@ -1,4 +1,5 @@ import { Deferred, STATUS_RESOLVED } from "@replay-cli/shared/async/createDeferred"; +import { getAccessToken } from "@replay-cli/shared/authentication/getAccessToken"; import { disableAnimatedLog, replayAppHost } from "@replay-cli/shared/config"; import { logger } from "@replay-cli/shared/logger"; import { logUpdate } from "@replay-cli/shared/logUpdate"; @@ -23,6 +24,7 @@ import { statusSuccess, } from "@replay-cli/shared/theme"; import { dots } from "cli-spinners"; +import assert from "node:assert/strict"; import strip from "strip-ansi"; async function printDeferredRecordingActions( @@ -136,7 +138,9 @@ export const uploadRecordings = createAsyncFunctionWithTracking( return []; } - const worker = createUploadWorker(options); + const { accessToken } = await getAccessToken(); + assert(accessToken, "No access token found"); + const worker = createUploadWorker({ accessToken, ...options }); const deferredActions = recordings.map(recording => worker.upload(recording)); if (!silent) { @@ -174,7 +178,7 @@ export const uploadRecordings = createAsyncFunctionWithTracking( } try { - recordings = await worker.onEnd(); + recordings = await worker.end(); } catch (error) { if ( error instanceof ProtocolError && diff --git a/packages/shared/src/protocol/ProtocolClient.ts b/packages/shared/src/protocol/ProtocolClient.ts index 9eed7972..8b52ba7c 100644 --- a/packages/shared/src/protocol/ProtocolClient.ts +++ b/packages/shared/src/protocol/ProtocolClient.ts @@ -2,7 +2,6 @@ import { SessionId, sessionError } from "@replayio/protocol"; import assert from "node:assert/strict"; import WebSocket from "ws"; import { Deferred, STATUS_PENDING, createDeferred } from "../async/createDeferred"; -import { getAccessToken } from "../authentication/getAccessToken"; import { replayWsServer } from "../config"; import { logger } from "../logger"; import { ProtocolError } from "./ProtocolError"; @@ -22,10 +21,12 @@ export default class ProtocolClient { private nextMessageId = 1; private pendingCommands: Map> = new Map(); private socket: WebSocket; + private accessToken: string; - constructor() { + constructor(accessToken: string) { logger.debug(`Creating WebSocket for ${replayWsServer}`); + this.accessToken = accessToken; this.socket = new WebSocket(replayWsServer); this.socket.on("close", this.onSocketClose); @@ -153,11 +154,7 @@ export default class ProtocolClient { private onSocketOpen = async () => { try { - const { accessToken } = await getAccessToken(); - assert(accessToken, "No access token found"); - - await setAccessToken(this, { accessToken }); - + await setAccessToken(this, { accessToken: this.accessToken }); this.deferredAuthenticated.resolve(true); } catch (error) { logger.debug("Error authenticating", { error }); diff --git a/packages/shared/src/recording/getRecordings.ts b/packages/shared/src/recording/getRecordings.ts index 5b6c9137..19f65c98 100644 --- a/packages/shared/src/recording/getRecordings.ts +++ b/packages/shared/src/recording/getRecordings.ts @@ -99,6 +99,7 @@ export function getRecordings(processGroupIdFilter?: string): LocalRecording[] { recordingStatus: "recording", unusableReason: undefined, uploadStatus: undefined, + uploadError: undefined, }; idToRecording[entry.id] = recording; diff --git a/packages/shared/src/recording/types.ts b/packages/shared/src/recording/types.ts index 2f2c29ed..267e684c 100644 --- a/packages/shared/src/recording/types.ts +++ b/packages/shared/src/recording/types.ts @@ -89,5 +89,6 @@ export type LocalRecording = { processingStatus: "failed" | "processed" | "processing" | undefined; recordingStatus: "crashed" | "finished" | "recording" | "unusable"; unusableReason: string | undefined; + uploadError: Error | undefined; uploadStatus: "failed" | "uploading" | "uploaded" | undefined; }; diff --git a/packages/shared/src/recording/upload/uploadCrashData.ts b/packages/shared/src/recording/upload/uploadCrashData.ts index 4fb9dfb5..4f6d8d88 100644 --- a/packages/shared/src/recording/upload/uploadCrashData.ts +++ b/packages/shared/src/recording/upload/uploadCrashData.ts @@ -6,7 +6,7 @@ import { LocalRecording, RECORDING_LOG_KIND } from "../types"; import { updateRecordingLog } from "../updateRecordingLog"; export async function uploadCrashedData(client: ProtocolClient, recording: LocalRecording) { - logger.debug("Uploading crash data for recording", { recording }); + logger.info("UploadCrashedData:Started", { recordingId: recording.id }); const crashData = recording.crashData?.slice() ?? []; crashData.push({ @@ -14,12 +14,17 @@ export async function uploadCrashedData(client: ProtocolClient, recording: Local recordingId: recording.id, }); - await Promise.all(crashData.map(async data => reportCrash(client, { data }))); + try { + await Promise.all(crashData.map(async data => reportCrash(client, { data }))); - updateRecordingLog(recording, { - kind: RECORDING_LOG_KIND.crashUploaded, - server: replayWsServer, - }); + updateRecordingLog(recording, { + kind: RECORDING_LOG_KIND.crashUploaded, + server: replayWsServer, + }); - recording.uploadStatus = "uploaded"; + recording.uploadStatus = "uploaded"; + } catch (error) { + recording.uploadStatus = "failed"; + recording.uploadError = error as Error; + } } diff --git a/packages/shared/src/recording/upload/uploadRecording.ts b/packages/shared/src/recording/upload/uploadRecording.ts index 1a48d89a..0dd8cdc2 100644 --- a/packages/shared/src/recording/upload/uploadRecording.ts +++ b/packages/shared/src/recording/upload/uploadRecording.ts @@ -31,6 +31,7 @@ export async function uploadRecording( processingBehavior: ProcessingBehavior; } ) { + logger.info("UploadRecording:Started", { recordingId: recording.id }); const { buildId, id, path } = recording; assert(path, "Recording path is required"); @@ -116,7 +117,13 @@ export async function uploadRecording( kind: RECORDING_LOG_KIND.uploadFailed, }); + logger.error("UploadRecording:Failed", { + error, + recordingId: recording.id, + buildId: recording.buildId, + }); recording.uploadStatus = "failed"; + recording.uploadError = error as Error; throw error; } @@ -133,6 +140,7 @@ export async function uploadRecording( server: replayWsServer, }); + logger.info("UploadRecording:Succeeded", { recording: recording.id }); recording.uploadStatus = "uploaded"; switch (processingBehavior) { diff --git a/packages/shared/src/recording/upload/uploadWorker.ts b/packages/shared/src/recording/upload/uploadWorker.ts index eeb6bc5b..ece6bda6 100644 --- a/packages/shared/src/recording/upload/uploadWorker.ts +++ b/packages/shared/src/recording/upload/uploadWorker.ts @@ -9,13 +9,15 @@ import { uploadCrashedData } from "./uploadCrashData"; import { uploadRecording } from "./uploadRecording"; export function createUploadWorker({ + accessToken, deleteOnSuccess, processingBehavior, }: { + accessToken: string; deleteOnSuccess?: boolean; processingBehavior: ProcessingBehavior; }) { - const client = new ProtocolClient(); + const client = new ProtocolClient(accessToken); const deferredAuthenticated = createDeferred(); const deferredActions: Deferred[] = []; @@ -45,7 +47,7 @@ export function createUploadWorker({ deferredActions.push(deferred); return deferred; }, - onEnd: async () => { + end: async () => { try { await deferredAuthenticated.promise; } catch (err) { @@ -71,3 +73,5 @@ export function createUploadWorker({ }, }; } + +export type UploadWorker = ReturnType; diff --git a/packages/test-utils/src/reporter.ts b/packages/test-utils/src/reporter.ts index 77fcfb43..2a933f93 100644 --- a/packages/test-utils/src/reporter.ts +++ b/packages/test-utils/src/reporter.ts @@ -2,14 +2,17 @@ import { retryWithExponentialBackoff } from "@replay-cli/shared/async/retryOnFai import { getAuthInfo } from "@replay-cli/shared/graphql/getAuthInfo"; import { queryGraphQL } from "@replay-cli/shared/graphql/queryGraphQL"; import { logger } from "@replay-cli/shared/logger"; -import { Properties, mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI"; -import { UnstructuredMetadata } from "@replay-cli/shared/recording/types"; +import { mixpanelAPI, Properties } from "@replay-cli/shared/mixpanel/mixpanelAPI"; +import { getRecordings } from "@replay-cli/shared/recording/getRecordings"; +import { LocalRecording, UnstructuredMetadata } from "@replay-cli/shared/recording/types"; +import { createUploadWorker, UploadWorker } from "@replay-cli/shared/recording/upload/uploadWorker"; import { spawnSync } from "child_process"; import { mkdirSync, writeFileSync } from "fs"; import assert from "node:assert/strict"; import { dirname } from "path"; import { v4 as uuid } from "uuid"; import { getAccessToken } from "./getAccessToken"; +import { getErrorMessage } from "./legacy-cli/error"; import { listAllRecordings } from "./legacy-cli/listAllRecordings"; import { add, source as sourceMetadata, test as testMetadata } from "./legacy-cli/metadata"; import type { TestMetadataV2 } from "./legacy-cli/metadata/test"; @@ -18,7 +21,6 @@ import { getMetadataFilePath } from "./metadata"; import { pingTestMetrics } from "./metrics"; import { buildTestId, generateOpaqueId } from "./testId"; import type { RecordingEntry, ReplayReporterConfig, UploadStatusThreshold } from "./types"; -import { getErrorMessage } from "./legacy-cli/error"; function last(arr: T[]): T | undefined { return arr[arr.length - 1]; @@ -70,12 +72,11 @@ export type TestResult = TestMetadataV2.TestResult; export type TestError = TestMetadataV2.TestError; export type TestRun = TestMetadataV2.TestRun; -type PendingWorkType = "test-run" | "test-run-tests" | "post-test" | "upload"; +type PendingWorkType = "test-run" | "test-run-tests" | "post-test"; export type PendingWorkError = TErrorData & { type: K; error: Error; }; -export type PendingUploadError = Extract; type PendingWorkEntry = | PendingWorkError @@ -88,15 +89,6 @@ type TestRunPendingWork = PendingWorkEntry< } >; type TestRunTestsPendingWork = PendingWorkEntry<"test-run-tests">; -type UploadPendingWork = PendingWorkEntry< - "upload", - { - recording: RecordingEntry; - }, - { - recording: RecordingEntry; - } ->; type PostTestPendingWork = PendingWorkEntry< "post-test", { @@ -104,23 +96,37 @@ type PostTestPendingWork = PendingWorkEntry< testRun: TestRun; } >; -export type PendingWork = - | TestRunPendingWork - | TestRunTestsPendingWork - | UploadPendingWork - | PostTestPendingWork; +export type PendingWork = TestRunPendingWork | TestRunTestsPendingWork | PostTestPendingWork; function logPendingWorkErrors(errors: PendingWorkError[]) { return errors.map(e => ` - ${e.error.message}`); } -function getTestResult(recording: RecordingEntry): TestRun["result"] { - const test = recording.metadata.test as TestRun | undefined; +function getTestResult( + recording: LocalRecording, + metadatas: Map< + string, + { + test: TestMetadataV2.TestRun; + title: string; + } + > +): TestRun["result"] { + const test = metadatas.get(recording.id)?.test; return !test ? "unknown" : test.result; } -function getTestResultEmoji(recording: RecordingEntry) { - const result = getTestResult(recording); +function getTestResultEmoji( + recording: LocalRecording, + metadatas: Map< + string, + { + test: TestMetadataV2.TestRun; + title: string; + } + > +) { + const result = getTestResult(recording, metadatas); switch (result) { case "unknown": return "﹖"; @@ -136,10 +142,20 @@ function getTestResultEmoji(recording: RecordingEntry) { const resultOrder = ["failed", "timedOut", "passed", "skipped", "unknown"]; -function sortRecordingsByResult(recordings: RecordingEntry[]) { +function sortRecordingsByResult( + recordings: LocalRecording[], + metadatas: Map< + string, + { + test: TestMetadataV2.TestRun; + title: string; + } + > +) { return [...recordings].sort((a, b) => { return ( - resultOrder.indexOf(getTestResult(a)) - resultOrder.indexOf(getTestResult(b)) || + resultOrder.indexOf(getTestResult(a, metadatas)) - + resultOrder.indexOf(getTestResult(b, metadatas)) || ((a.metadata.title as string) || "").localeCompare((b.metadata.title as string) || "") ); }); @@ -229,8 +245,16 @@ export default class ReplayReporter< private _uploadableResults: Map> = new Map(); private _testRunShardIdPromise: Promise | null = null; private _uploadStatusThreshold: UploadStatusThresholdInternal = "none"; + private _uploadWorker: UploadWorker | undefined; private _cacheAuthIdsPromise: Promise | null = null; private _uploadedRecordings = new Set(); + private _recordingMetadatas = new Map< + string, + { + test: TestMetadataV2.TestRun; + title: string; + } + >(); constructor( runner: TestRunner, @@ -332,18 +356,23 @@ export default class ReplayReporter< ) { this._apiKey = getAccessToken(config); this._upload = "upload" in config ? !!config.upload : !!process.env.REPLAY_UPLOAD; - if (this._upload && !this._apiKey) { - throw new Error( - `\`@replayio/${this._runner.name}/reporter\` requires an API key to upload recordings. Either pass a value to the apiKey plugin configuration or set the REPLAY_API_KEY environment variable` - ); - } if (this._upload) { + if (!this._apiKey) { + throw new Error( + `\`@replayio/${this._runner.name}/reporter\` requires an API key to upload recordings. Either pass a value to the apiKey plugin configuration or set the REPLAY_API_KEY environment variable` + ); + } if (typeof config.upload === "object") { this._minimizeUploads = !!config.upload.minimizeUploads; this._uploadStatusThreshold = config.upload.statusThreshold ?? "all"; } else { this._uploadStatusThreshold = "all"; } + this._uploadWorker = createUploadWorker({ + accessToken: this._apiKey, + deleteOnSuccess: true, + processingBehavior: "do-not-process", + }); } // always favor environment variables over config so the config can be @@ -724,10 +753,8 @@ export default class ReplayReporter< ); } - private async _uploadRecording( - recording: RecordingEntry - ): Promise { - if (this._uploadStatusThreshold === "none" || !this._apiKey) { + private async _uploadRecording(recording: RecordingEntry) { + if (this._uploadStatusThreshold === "none" || !this._apiKey || !this._uploadWorker) { return; } // Cypress retries are on the same recordings, we only want to upload a single recording once @@ -737,40 +764,15 @@ export default class ReplayReporter< }); return; } - this._uploadedRecordings.add(recording.id); - logger.info("UploadRecording:Started", { recordingId: recording.id }); - - try { - await uploadRecording(recording.id, { - apiKey: this._apiKey, - // Per TT-941, we want to throw on any error so it can be caught below - // and reported back to the user rather than just returning null - strict: true, - // uploads are enqueued in this reporter asap - // but the extra assets should be removed after all of them are uploaded - removeAssets: false, - }); - - logger.info("UploadRecording:Succeeded", { recording: recording.id }); - - const recordings = listAllRecordings({ filter: r => r.id === recording.id, all: true }); - - return { - type: "upload", - recording: recordings[0], - }; - } catch (error) { - logger.error("UploadRecording:Failed", { - error, + const uploadableRecording = getRecordings().find(r => r.id === recording.id); + if (!uploadableRecording) { + logger.info("UploadRecording:NoUploadableRecording", { recordingId: recording.id, - buildId: recording.buildId, }); - return { - type: "upload", - recording, - error: new Error(getErrorMessage(error)), - }; + return; } + this._uploadedRecordings.add(uploadableRecording.id); + this._uploadWorker.upload(uploadableRecording); } getRecordingsForTest(tests: { executionId: string }[]) { @@ -868,7 +870,10 @@ export default class ReplayReporter< }); } - recordings.forEach(rec => add(rec.id, mergedMetadata)); + recordings.forEach(rec => { + this._recordingMetadatas.set(rec.id, mergedMetadata); + add(rec.id, mergedMetadata); + }); // Re-fetch recordings so we have the most recent metadata const allRecordings = listAllRecordings({ all: true }) as RecordingEntry[]; @@ -916,12 +921,12 @@ export default class ReplayReporter< logger.info("EnqueuePostTestWork:WillSkipAddTests"); } - const testRun = this._buildTestMetadata(tests, specFile); + const testMetadata = this._buildTestMetadata(tests, specFile); if (recordings.length > 0) { const recordingsWithMetadata = await this._setRecordingMetadata( recordings, - testRun, + testMetadata, replayTitle, extraMetadata ); @@ -945,13 +950,13 @@ export default class ReplayReporter< firstRecording?.id, this._baseId, { - id: testRun.source.path + "#" + testRun.source.title, - source: testRun.source, - approximateDuration: testRun.approximateDuration, + id: testMetadata.source.path + "#" + testMetadata.source.title, + source: testMetadata.source, + approximateDuration: testMetadata.approximateDuration, recorded: firstRecording !== undefined, runtime: parseRuntime(firstRecording?.runtime), runner: this._runner.name, - result: testRun.result, + result: testMetadata.result, }, this._apiKey ); @@ -959,7 +964,7 @@ export default class ReplayReporter< return { type: "post-test", recordings, - testRun, + testRun: testMetadata, }; } catch (error) { logger.error("EnqueuePostTestWork:Failed", { error }); @@ -1091,12 +1096,13 @@ export default class ReplayReporter< } } - this._pendingWork.push( - ...toUpload - .flatMap(result => result.recordings) - .filter(r => (this._filter ? this._filter(r) : true)) - .map(r => this._uploadRecording(r)) - ); + const filteredRecordings = toUpload + .flatMap(result => result.recordings) + .filter(r => (this._filter ? this._filter(r) : true)); + + for (const recording of filteredRecordings) { + this._uploadRecording(recording); + } } private _assignAggregateStatus( @@ -1180,16 +1186,11 @@ export default class ReplayReporter< "post-test": [] as Extract[], "test-run": [] as Extract[], "test-run-tests": [] as Extract[], - upload: [] as Extract[], }; - let uploads: RecordingEntry[] = []; + const uploads = await this._uploadWorker?.end(); for (const r of results) { if ("error" in r) { errors[r.type].push(r as any); - } else { - if (r.type === "upload") { - uploads.push(r.recording); - } } } @@ -1204,39 +1205,37 @@ export default class ReplayReporter< output.push(...logPendingWorkErrors(errors["test-run"])); } - if (errors["upload"].length > 0) { - output.push(`\n❌ Failed to upload ${errors["upload"].length} recordings:\n`); - - errors["upload"].forEach(err => { - if ("recording" in err) { - const r = err.recording; - output.push(` ${(r.metadata.title as string | undefined) || "Unknown"}`); - output.push(` ${getErrorMessage(err.error)}\n`); - } - }); - } - let numCrashed = 0; let numUploaded = 0; - if (uploads.length > 0) { - const recordingIds = uploads.map(u => u.recordingId).filter(isNonNullable); - for (const recordingId of recordingIds) { - removeRecording(recordingId); - } + if (uploads?.length) { + const failedUploads = uploads.filter(u => u.uploadStatus === "failed"); + const uploaded = uploads.filter( + u => u.recordingStatus !== "crashed" && u.uploadStatus === "uploaded" + ); + const crashed = uploads.filter( + u => u.recordingStatus === "crashed" && u.uploadStatus === "uploaded" + ); - const uploaded = uploads.filter(u => u.status === "uploaded"); - const crashed = uploads.filter(u => u.status === "crashUploaded"); + if (failedUploads.length > 0) { + output.push(`\n❌ Failed to upload ${failedUploads.length} recordings:\n`); + failedUploads.forEach(recording => { + output.push(` ${(recording.metadata.title as string | undefined) || "Unknown"}`); + output.push(` ${getErrorMessage(recording.uploadError)}\n`); + }); + } numCrashed = crashed.length; numUploaded = uploaded.length; if (uploaded.length > 0) { output.push(`\n🚀 Successfully uploaded ${uploads.length} recordings:`); - const sortedUploads = sortRecordingsByResult(uploads); + const sortedUploads = sortRecordingsByResult(uploads, this._recordingMetadatas); sortedUploads.forEach(r => { output.push( - `\n ${getTestResultEmoji(r)} ${(r.metadata.title as string | undefined) || "Unknown"}` + `\n ${getTestResultEmoji(r, this._recordingMetadatas)} ${ + (r.metadata.title as string | undefined) || "Unknown" + }` ); output.push( ` ${process.env.REPLAY_VIEW_HOST || "https://app.replay.io"}/recording/${r.id}` From e65fbc2e7c727388bedcf294fd4de6949dc3b286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 13:27:43 +0200 Subject: [PATCH 04/11] revert unrelated change --- examples/create-react-app-typescript/playwright.config.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/create-react-app-typescript/playwright.config.ts b/examples/create-react-app-typescript/playwright.config.ts index f20c87ee..c6780f1b 100644 --- a/examples/create-react-app-typescript/playwright.config.ts +++ b/examples/create-react-app-typescript/playwright.config.ts @@ -16,9 +16,7 @@ export default defineConfig({ reporter: [ // replicating Playwright's defaults process.env.CI ? (["dot"] as const) : (["list"] as const), - replayReporter({ - upload: true, - }), + replayReporter({}), ], projects: [ { From 416a493e7070d47e19c5ae443e57552e26ab6d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 13:35:47 +0200 Subject: [PATCH 05/11] use proper uploadRecordings --- packages/replayio/src/commands/record.ts | 4 ++-- packages/replayio/src/commands/upload.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/replayio/src/commands/record.ts b/packages/replayio/src/commands/record.ts index 23cdb4ae..f6b6952b 100644 --- a/packages/replayio/src/commands/record.ts +++ b/packages/replayio/src/commands/record.ts @@ -1,4 +1,5 @@ import { ProcessError } from "@replay-cli/shared/ProcessError"; +import { logger } from "@replay-cli/shared/logger"; import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI"; import { exitProcess } from "@replay-cli/shared/process/exitProcess"; import { canUpload } from "@replay-cli/shared/recording/canUpload"; @@ -6,7 +7,6 @@ import { getRecordings } from "@replay-cli/shared/recording/getRecordings"; import { printRecordings } from "@replay-cli/shared/recording/printRecordings"; import { selectRecordings } from "@replay-cli/shared/recording/selectRecordings"; import { LocalRecording } from "@replay-cli/shared/recording/types"; -import { uploadRecordings } from "@replay-cli/shared/recording/upload/uploadRecordings"; import { dim, statusFailed } from "@replay-cli/shared/theme"; import debug from "debug"; import { v4 as uuid } from "uuid"; @@ -16,7 +16,7 @@ import { launchBrowser } from "../utils/browser/launchBrowser"; import { reportBrowserCrash } from "../utils/browser/reportBrowserCrash"; import { registerCommand } from "../utils/commander/registerCommand"; import { confirm } from "../utils/confirm"; -import { logger } from "@replay-cli/shared/logger"; +import { uploadRecordings } from "../utils/recordings/uploadRecordings"; registerCommand("record", { checkForRuntimeUpdate: true, requireAuthentication: true }) .argument("[url]", `URL to open (default: "about:blank")`) diff --git a/packages/replayio/src/commands/upload.ts b/packages/replayio/src/commands/upload.ts index 6c819f72..197cf56d 100644 --- a/packages/replayio/src/commands/upload.ts +++ b/packages/replayio/src/commands/upload.ts @@ -1,12 +1,12 @@ import { exitProcess } from "@replay-cli/shared/process/exitProcess"; -import { registerCommand } from "../utils/commander/registerCommand"; import { findRecordingsWithShortIds } from "@replay-cli/shared/recording/findRecordingsWithShortIds"; import { getRecordings } from "@replay-cli/shared/recording/getRecordings"; import { printRecordings } from "@replay-cli/shared/recording/printRecordings"; import { selectRecordings } from "@replay-cli/shared/recording/selectRecordings"; import { LocalRecording } from "@replay-cli/shared/recording/types"; -import { uploadRecordings } from "@replay-cli/shared/recording/upload/uploadRecordings"; import { dim } from "@replay-cli/shared/theme"; +import { registerCommand } from "../utils/commander/registerCommand"; +import { uploadRecordings } from "../utils/recordings/uploadRecordings"; registerCommand("upload", { requireAuthentication: true }) .argument("[ids...]", `Recording ids ${dim("(comma-separated)")}`, value => value.split(",")) From 41aac5aab24e8ad1d1242fe1a09a15fc83847f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 14:11:56 +0200 Subject: [PATCH 06/11] add temp changeset --- .changeset/silver-geckos-smoke.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/silver-geckos-smoke.md diff --git a/.changeset/silver-geckos-smoke.md b/.changeset/silver-geckos-smoke.md new file mode 100644 index 00000000..70f3fa53 --- /dev/null +++ b/.changeset/silver-geckos-smoke.md @@ -0,0 +1,6 @@ +--- +"@replayio/playwright": patch +"replayio": patch +--- + +Stuff From c91878453d883bb47aef5c13a402e04065a7bb54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 15:17:02 +0200 Subject: [PATCH 07/11] initialize launchdarkly in test reporters --- packages/cypress/src/index.ts | 9 +++++++-- packages/playwright/src/reporter.ts | 10 +++++++--- .../shared/src/launch-darkly/getLaunchDarklyClient.ts | 9 +++++++-- .../launch-darkly/initLaunchDarklyFromAccessToken.ts | 4 ++-- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/cypress/src/index.ts b/packages/cypress/src/index.ts index af5ecaf6..f4ee7cad 100644 --- a/packages/cypress/src/index.ts +++ b/packages/cypress/src/index.ts @@ -1,5 +1,6 @@ /// +import { initLaunchDarklyFromAccessToken } from "@replay-cli/shared/launch-darkly/initLaunchDarklyFromAccessToken"; import { initLogger, logger } from "@replay-cli/shared/logger"; import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI"; import { getRuntimePath } from "@replay-cli/shared/runtime/getRuntimePath"; @@ -263,13 +264,17 @@ const plugin = ( ) => { setUserAgent(`${packageName}/${packageVersion}`); - // TODO: enable launchDarkly + const accessToken = getAuthKey(config); + initLogger(packageName, packageVersion); mixpanelAPI.initialize({ - accessToken: getAuthKey(config), + accessToken, packageName, packageVersion, }); + if (accessToken) { + initLaunchDarklyFromAccessToken(accessToken); + } cypressReporter = new CypressReporter(config, options); diff --git a/packages/playwright/src/reporter.ts b/packages/playwright/src/reporter.ts index d70a895e..a33a964b 100644 --- a/packages/playwright/src/reporter.ts +++ b/packages/playwright/src/reporter.ts @@ -5,6 +5,7 @@ import type { TestError, TestResult, } from "@playwright/test/reporter"; +import { initLaunchDarklyFromAccessToken } from "@replay-cli/shared/launch-darkly/initLaunchDarklyFromAccessToken"; import { initLogger, logger } from "@replay-cli/shared/logger"; import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI"; import { getRuntimePath } from "@replay-cli/shared/runtime/getRuntimePath"; @@ -85,14 +86,17 @@ export default class ReplayPlaywrightReporter implements Reporter { constructor(config: ReplayPlaywrightConfig) { setUserAgent(`${packageName}/${packageVersion}`); - // TODO: enable launchDarkly + const accessToken = getAccessToken(config); + initLogger(packageName, packageVersion); mixpanelAPI.initialize({ - accessToken: getAccessToken(config), + accessToken, packageName, packageVersion, }); - + if (accessToken) { + initLaunchDarklyFromAccessToken(accessToken); + } if (!config || typeof config !== "object") { mixpanelAPI.trackEvent("error.invalid-reporter-config", { config }); diff --git a/packages/shared/src/launch-darkly/getLaunchDarklyClient.ts b/packages/shared/src/launch-darkly/getLaunchDarklyClient.ts index 36bc440f..4b197ec3 100644 --- a/packages/shared/src/launch-darkly/getLaunchDarklyClient.ts +++ b/packages/shared/src/launch-darkly/getLaunchDarklyClient.ts @@ -1,4 +1,8 @@ -import { LDClient, LDUser, initialize as initializeLDClient } from "launchdarkly-node-client-sdk"; +import { + LDClient, + LDSingleKindContext, + initialize as initializeLDClient, +} from "launchdarkly-node-client-sdk"; import { getReplayPath } from "../getReplayPath"; let client: LDClient; @@ -14,8 +18,9 @@ export function getLaunchDarklyClient(initialize: boolean = true) { client = initializeLDClient( "60ca05fb43d6f10d234bb3cf", { + kind: "user", anonymous: true, - } satisfies LDUser, + } satisfies LDSingleKindContext, { localStoragePath: getReplayPath("launchdarkly-user-cache"), logger: { diff --git a/packages/shared/src/launch-darkly/initLaunchDarklyFromAccessToken.ts b/packages/shared/src/launch-darkly/initLaunchDarklyFromAccessToken.ts index a406dccc..d4a80e11 100644 --- a/packages/shared/src/launch-darkly/initLaunchDarklyFromAccessToken.ts +++ b/packages/shared/src/launch-darkly/initLaunchDarklyFromAccessToken.ts @@ -4,14 +4,14 @@ import { identifyUserProfile } from "./identifyUserProfile"; export async function initLaunchDarklyFromAccessToken( accessToken: string, - abortSignal: AbortSignal + abortSignal?: AbortSignal ) { logger.debug("Initializing LaunchDarkly profile"); try { const authInfo = await getAuthInfo(accessToken); - if (abortSignal.aborted) { + if (abortSignal?.aborted) { return; } From f98f61fe11c984e8e6e1ee07846897885c3ff863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 15:22:06 +0200 Subject: [PATCH 08/11] add extra logs --- packages/shared/src/recording/upload/uploadCrashData.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/shared/src/recording/upload/uploadCrashData.ts b/packages/shared/src/recording/upload/uploadCrashData.ts index 4f6d8d88..3182e04d 100644 --- a/packages/shared/src/recording/upload/uploadCrashData.ts +++ b/packages/shared/src/recording/upload/uploadCrashData.ts @@ -22,8 +22,14 @@ export async function uploadCrashedData(client: ProtocolClient, recording: Local server: replayWsServer, }); + logger.info("UploadCrashedData:Succeeded", { recording: recording.id }); recording.uploadStatus = "uploaded"; } catch (error) { + logger.error("UploadCrashedData:Failed", { + error, + recordingId: recording.id, + buildId: recording.buildId, + }); recording.uploadStatus = "failed"; recording.uploadError = error as Error; } From bd1cda5bc45943739c6bb9eebdc6b345e5351276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Mon, 8 Jul 2024 16:21:22 +0200 Subject: [PATCH 09/11] loggerify multipart upload --- .../shared/src/protocol/ProtocolClient.ts | 5 ++- .../src/recording/upload/uploadRecording.ts | 35 +++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/packages/shared/src/protocol/ProtocolClient.ts b/packages/shared/src/protocol/ProtocolClient.ts index 8b52ba7c..c6f23a30 100644 --- a/packages/shared/src/protocol/ProtocolClient.ts +++ b/packages/shared/src/protocol/ProtocolClient.ts @@ -116,7 +116,7 @@ export default class ProtocolClient { }; private onSocketError = (error: any) => { - logger.debug("Socket error", { error }); + logger.error("ProtocolClient:Error", { error }); if (this.deferredAuthenticated.status === STATUS_PENDING) { this.deferredAuthenticated.reject(error); @@ -157,8 +157,7 @@ export default class ProtocolClient { await setAccessToken(this, { accessToken: this.accessToken }); this.deferredAuthenticated.resolve(true); } catch (error) { - logger.debug("Error authenticating", { error }); - + logger.error("ProtocolClient:ServerAuthFailed", { error }); this.socket.close(); this.deferredAuthenticated.reject(error as Error); } diff --git a/packages/shared/src/recording/upload/uploadRecording.ts b/packages/shared/src/recording/upload/uploadRecording.ts index 0d75890c..414741fe 100644 --- a/packages/shared/src/recording/upload/uploadRecording.ts +++ b/packages/shared/src/recording/upload/uploadRecording.ts @@ -279,15 +279,36 @@ async function uploadPart( }, abortSignal: AbortSignal ): Promise { - logger.debug("Uploading chunk", { recordingPath, size, start, end }); + logger.info("UploadRecording:UploadPart:Started", { + recordingPath, + size, + start, + end, + }); const stream = createReadStream(recordingPath, { start, end }); - const response = await uploadRecordingReadStream(stream, { url, size }, abortSignal); - - const etag = response.headers.get("etag"); - assert(etag, "Etag has to be returned in the response headers"); - logger.debug("Etag received", { etag, recordingPath, size, start, end }); + try { + const response = await uploadRecordingReadStream(stream, { url, size }, abortSignal); + const etag = response.headers.get("etag"); + + logger.info("UploadRecording:UploadPart:Succeeded", { + recordingPath, + size, + start, + end, + etag, + }); - return etag; + assert(etag, "Etag has to be returned in the response headers"); + return etag; + } catch (error) { + logger.error("UploadRecording:UploadPart:Failed", { + recordingPath, + size, + start, + end, + }); + throw error; + } } async function uploadRecordingReadStream( From 8392ef980dadac754bd6a34defbbb87fa5950bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Wed, 10 Jul 2024 17:20:09 +0200 Subject: [PATCH 10/11] revert LD changes --- packages/cypress/src/index.ts | 4 ---- packages/playwright/src/reporter.ts | 4 ---- 2 files changed, 8 deletions(-) diff --git a/packages/cypress/src/index.ts b/packages/cypress/src/index.ts index f4ee7cad..3c8677cb 100644 --- a/packages/cypress/src/index.ts +++ b/packages/cypress/src/index.ts @@ -1,6 +1,5 @@ /// -import { initLaunchDarklyFromAccessToken } from "@replay-cli/shared/launch-darkly/initLaunchDarklyFromAccessToken"; import { initLogger, logger } from "@replay-cli/shared/logger"; import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI"; import { getRuntimePath } from "@replay-cli/shared/runtime/getRuntimePath"; @@ -272,9 +271,6 @@ const plugin = ( packageName, packageVersion, }); - if (accessToken) { - initLaunchDarklyFromAccessToken(accessToken); - } cypressReporter = new CypressReporter(config, options); diff --git a/packages/playwright/src/reporter.ts b/packages/playwright/src/reporter.ts index a33a964b..a22a47cd 100644 --- a/packages/playwright/src/reporter.ts +++ b/packages/playwright/src/reporter.ts @@ -5,7 +5,6 @@ import type { TestError, TestResult, } from "@playwright/test/reporter"; -import { initLaunchDarklyFromAccessToken } from "@replay-cli/shared/launch-darkly/initLaunchDarklyFromAccessToken"; import { initLogger, logger } from "@replay-cli/shared/logger"; import { mixpanelAPI } from "@replay-cli/shared/mixpanel/mixpanelAPI"; import { getRuntimePath } from "@replay-cli/shared/runtime/getRuntimePath"; @@ -94,9 +93,6 @@ export default class ReplayPlaywrightReporter implements Reporter { packageName, packageVersion, }); - if (accessToken) { - initLaunchDarklyFromAccessToken(accessToken); - } if (!config || typeof config !== "object") { mixpanelAPI.trackEvent("error.invalid-reporter-config", { config }); From 22b5d6ec8e156ed4031837eacff8ca0f262d5354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Burzy=C5=84ski?= Date: Wed, 10 Jul 2024 17:23:19 +0200 Subject: [PATCH 11/11] remove dummy changeset --- .changeset/silver-geckos-smoke.md | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .changeset/silver-geckos-smoke.md diff --git a/.changeset/silver-geckos-smoke.md b/.changeset/silver-geckos-smoke.md deleted file mode 100644 index 70f3fa53..00000000 --- a/.changeset/silver-geckos-smoke.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -"@replayio/playwright": patch -"replayio": patch ---- - -Stuff