diff --git a/application.meta.ts b/application.meta.ts index c891960..21cd298 100644 --- a/application.meta.ts +++ b/application.meta.ts @@ -21,7 +21,7 @@ const Application = { name: "pup", - version: "1.0.0-rc.32", + version: "1.0.0-rc.33", url: "jsr:@pup/pup@$VERSION", canary_url: "https://raw.githubusercontent.com/Hexagon/pup/main/pup.ts", deno: null, /* Minimum stable version of Deno required to run Pup (without --unstable-* flags) */ diff --git a/deno.json b/deno.json index e406fcb..706547f 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@pup/pup", - "version": "1.0.0-rc.32", + "version": "1.0.0-rc.33", "exports": { ".": "./pup.ts", @@ -41,17 +41,18 @@ "imports": { "@cross/deepmerge": "jsr:@cross/deepmerge@^1.0.0", "@cross/env": "jsr:@cross/env@^1.0.2", - "@cross/fs": "jsr:@cross/fs@^0.0.9", + "@cross/fs": "jsr:@cross/fs@^0.0.10", "@cross/jwt": "jsr:@cross/jwt@^0.4.1", "@cross/runtime": "jsr:@cross/runtime@^1.0.0", "@cross/service": "jsr:@cross/service@^1.0.3", "@cross/test": "jsr:@cross/test@^0.0.9", - "@cross/utils": "jsr:@cross/utils@^0.11.0", + "@cross/utils": "jsr:@cross/utils@^0.12.0", "@hexagon/croner": "jsr:@hexagon/croner@^8.0.1", "@oak/oak": "jsr:@oak/oak@^15.0.0", "@pup/api-client": "jsr:@pup/api-client@^1.0.0", "@pup/api-definitions": "jsr:@pup/api-definitions@^1.0.0", "@pup/common": "jsr:@pup/common@^1.0.0", + "@pup/plugin": "jsr:@pup/plugin@^1.0.0", "@std/assert": "jsr:@std/assert@^0.223.0", "@std/async": "jsr:@std/async@^0.223.0", "@std/encoding": "jsr:@std/encoding@^0.223.0", diff --git a/docs/src/_data.json b/docs/src/_data.json index b18fd67..9b58703 100644 --- a/docs/src/_data.json +++ b/docs/src/_data.json @@ -6,7 +6,7 @@ "description": "Universal Process Manager" }, "substitute": { - "$PUP_VERSION": "1.0.0-rc.32" + "$PUP_VERSION": "1.0.0-rc.33" }, "top_links": [ { diff --git a/docs/src/changelog.md b/docs/src/changelog.md index c558605..4622d7a 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -9,6 +9,12 @@ nav_order: 13 All notable changes to this project will be documented in this section. +## [1.0.0-rc.33] - 2024-04-25 + +## Added + +- Added support for Plugins based on + ## [1.0.0-rc.32] - 2024-04-23 **Breaking:** Any plugins or packages importing telemetry from `@pup/pup/telemetry.ts` needs to be updated to import from `@pup/telemetry` instead. diff --git a/docs/src/examples/basic-webinterface/pup.jsonc b/docs/src/examples/basic-webinterface/pup.jsonc index 43e6d62..018172b 100644 --- a/docs/src/examples/basic-webinterface/pup.jsonc +++ b/docs/src/examples/basic-webinterface/pup.jsonc @@ -49,8 +49,8 @@ ], "plugins": [ { - // Use full uri to plugin, e.g. jsr:@pup/pup@$VERSION/plugins/web-interface - "url": "../../plugins/web-interface/mod.ts", + // Use full uri to plugin, e.g. jsr:@pup/plugin-web-interface + "url": "../../../pup-plugin-web-interface/mod.ts", "options": { "port": 8002 } diff --git a/docs/src/examples/cluster/pup.json b/docs/src/examples/cluster/pup.json index 27c4b00..a7b9448 100644 --- a/docs/src/examples/cluster/pup.json +++ b/docs/src/examples/cluster/pup.json @@ -17,13 +17,5 @@ "cron": "*/20 * * * * *", "restartDelayMs": 3000 } - ], - "plugins": [ - { - "url": "../../plugins/web-interface/mod.ts", - "options": { - "port": 5000 - } - } ] } diff --git a/docs/src/examples/docker/Dockerfile b/docs/src/examples/docker/Dockerfile index 6ce24b1..fcc3c91 100644 --- a/docs/src/examples/docker/Dockerfile +++ b/docs/src/examples/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM denoland/deno:debian-1.31.1 +FROM denoland/deno:debian-1.42.4 # Copy application RUN mkdir /app diff --git a/docs/src/examples/plugins/README.md b/docs/src/examples/plugins/README.md index 98f0113..22b4c0a 100644 --- a/docs/src/examples/plugins/README.md +++ b/docs/src/examples/plugins/README.md @@ -6,6 +6,8 @@ nav_order: 2 # Plugin development +**Note! This is outdated, need to be updated asap.** Refer to for up-to-date info. + --- ### Getting started diff --git a/docs/src/examples/splunk/README.md b/docs/src/examples/splunk/README.md index 964a05c..545462f 100644 --- a/docs/src/examples/splunk/README.md +++ b/docs/src/examples/splunk/README.md @@ -6,6 +6,8 @@ nav_order: 6 # Server with Splunk Logging +**Note! This is outdated, need to be updated asap.** + --- The example at [/docs/src/examples/splunk](https://github.com/Hexagon/pup/tree/main/docs/src/examples/splunk) runs a Deno server script as a continuously monitored and restarted process, logging all diff --git a/docs/src/examples/telemetry/README.md b/docs/src/examples/telemetry/README.md index d69b60c..f1d32a2 100644 --- a/docs/src/examples/telemetry/README.md +++ b/docs/src/examples/telemetry/README.md @@ -13,24 +13,20 @@ nav_order: 6 This example demonstrates the telemetry feature of pup, which ... - Automatically report process metrics, such as memory usage, back to Pup. -- Opens a channel for managed processes to communicate using a slow polling IPC - mechanism. +- Opens a channel for managed processes to communicate using a slow polling IPC mechanism. -The simplest use case, where you only want to monitor your client metrics is -used like this: +The simplest use case, where you only want to monitor your client metrics is used like this: ```ts -import { PupTelemetry } from "jsr:@pup/telemetry"; +import { PupTelemetry } from "jsr:@pup/telemetry" -new PupTelemetry(); +new PupTelemetry() // The rest of your application ``` -This will make your process report memory usage and current working directory -back to the Pup main process, no further configuration needed. Now you can see -memory usage for all processes running telemetry (including main process) using -`status` on the cli. +This will make your process report memory usage and current working directory back to the Pup main process, no further configuration needed. Now you can see memory usage for all processes running +telemetry (including main process) using `status` on the cli. **To use the IPC mechanism:** @@ -38,8 +34,8 @@ memory usage for all processes running telemetry (including main process) using // PupTelemetry is a singleton, so it can be imported one or many times in your application // - Installation instructions for different runtimes available at https://jsr.io/@pup/telemetry // This example imports directly from jsr.io using Deno jsr:-specifier -import { PupTelemetry } from "jsr:@pup/telemetry"; -const telemetry = new PupTelemetry(); +import { PupTelemetry } from "jsr:@pup/telemetry" +const telemetry = new PupTelemetry() // One part of your application ... @@ -47,35 +43,29 @@ const telemetry = new PupTelemetry(); telemetry.on("my-event", (data) => { console.log( `Another process triggered 'my-event' with data ${JSON.stringify(data)}`, - ); -}); + ) +}) // Send ipc events -telemetry.emit("another-process-id", "my-event", { data: { to: "send" } }); +telemetry.emit("another-process-id", "my-event", { data: { to: "send" } }) // ... another part of your application ``` ## Files -- [pup.json](https://github.com/Hexagon/pup/tree/main/docs/src/examples/telemetry/pup.json) - - Pup configuration, sets up `task-with-telemetry.ts` to run forever. - `server.ts` to be kept alive forever. -- [task-with-telemetry-1.ts](https://github.com/Hexagon/pup/tree/main/docs/src/examples/telemetry/task-with-telemetry-1.ts) - - "task-1", script sending telemetry data to main process, and sending messages - over IPC to task-2 -- [task-with-telemetry-2.ts](https://github.com/Hexagon/pup/tree/main/docs/src/examples/telemetry/task-with-telemetry-2.ts) - - "task-2", script sending telemetry data to main process, and receiving +- [pup.json](https://github.com/Hexagon/pup/tree/main/docs/src/examples/telemetry/pup.json) - Pup configuration, sets up `task-with-telemetry.ts` to run forever. `server.ts` to be kept alive forever. +- [task-with-telemetry-1.ts](https://github.com/Hexagon/pup/tree/main/docs/src/examples/telemetry/task-with-telemetry-1.ts) - "task-1", script sending telemetry data to main process, and sending + messages over IPC to task-2 +- [task-with-telemetry-2.ts](https://github.com/Hexagon/pup/tree/main/docs/src/examples/telemetry/task-with-telemetry-2.ts) - "task-2", script sending telemetry data to main process, and receiving messages over IPC from task-1 ## Running `cd` to `docs/examples/telemetry` directory. -Start example by running `pup run` if pup is installed, or something like -`deno run -A ../../../pup.ts run` if not. +Start example by running `pup run` if pup is installed, or something like `deno run -A ../../../pup.ts run` if not. -Now open another terminal and issue `pup status`, a brief overview of current -status is shown, including memory usage. +Now open another terminal and issue `pup status`, a brief overview of current status is shown, including memory usage. Success! diff --git a/docs/src/examples/telemetry/task-with-telemetry-1.ts b/docs/src/examples/telemetry/task-with-telemetry-1.ts index f96d115..a70f728 100644 --- a/docs/src/examples/telemetry/task-with-telemetry-1.ts +++ b/docs/src/examples/telemetry/task-with-telemetry-1.ts @@ -1,20 +1,18 @@ // See docs/examples/telemetry/README.md for full documentation on telemetry, including using the IPC // - Pin this to the latest version of pup, or include in import map -import { PupTelemetry } from "jsr:@pup/telemetry"; -const telemetry = new PupTelemetry(1); +import { PupTelemetry } from "jsr:@pup/telemetry" +const telemetry = new PupTelemetry(1) // The task -let i = 0; -console.log("Task running"); +let i = 0 +console.log("Task running") // Send data every 5th second setInterval(() => { - i += 1; + i += 1 telemetry.emit( "task-2", "message", - `${ - Deno.env.get("PUP_PROCESS_ID") - } sending "Hello" to 'task-2', iteration ${i}`, - ); -}, 2000); + `${Deno.env.get("PUP_PROCESS_ID")} sending "Hello" to 'task-2', iteration ${i}`, + ) +}, 2000) diff --git a/docs/src/examples/telemetry/task-with-telemetry-2.ts b/docs/src/examples/telemetry/task-with-telemetry-2.ts index d5cc767..7bea559 100644 --- a/docs/src/examples/telemetry/task-with-telemetry-2.ts +++ b/docs/src/examples/telemetry/task-with-telemetry-2.ts @@ -1,18 +1,18 @@ // See docs/examples/telemetry/README.md for full documentation on telemetry, including using the IPC // - Pin this to the latest version of pup, or include in import map -import { PupTelemetry } from "jsr:@pup/telemetry"; -const telemetry = new PupTelemetry(1); +import { PupTelemetry } from "jsr:@pup/telemetry" +const telemetry = new PupTelemetry(1) // The task -console.log("Process running"); +console.log("Process running") // Receive data // deno-lint-ignore no-explicit-any telemetry.on("message", (data: any) => { - console.log(`task-2 received: ${data}`); -}); + console.log(`task-2 received: ${data}`) +}) // Wait 5 minutes setTimeout(() => { - console.log("Done!"); -}, 300_000); + console.log("Done!") +}, 300_000) diff --git a/lib/common/token.ts b/lib/common/token.ts index 43fcb59..92061a8 100644 --- a/lib/common/token.ts +++ b/lib/common/token.ts @@ -13,7 +13,7 @@ export interface PupTokenPayload { exp: number | undefined } -export async function GenerateToken(secret: string, data: unknown, expMs: number | undefined) { +export async function GenerateToken(secret: string, data: unknown, expMs: number | undefined): Promise { const key = await generateKey(secret, DEFAULT_SECRET_KEY_ALGORITHM) // Require a consumer diff --git a/lib/core/api.ts b/lib/core/api.ts index bde676a..48cbd16 100644 --- a/lib/core/api.ts +++ b/lib/core/api.ts @@ -42,7 +42,10 @@ export class PupApi { return statuses } public applicationState(): ApiApplicationState { - return this._pup.status.applicationState(this._pup.allProcesses(), this._pup.port) as ApiApplicationState + return this._pup.status.applicationState( + this._pup.allProcesses(), + this._pup.port, + ) as ApiApplicationState } // Global actions @@ -65,7 +68,9 @@ export class PupApi { } public async stop(id: string, reason: string): Promise { const processesToStart = (id === "all") ? this.allProcessStates() : [this.allProcessStates().find((p) => p.status.id === id)] - const results = await Promise.all([processesToStart.map((process) => this._pup.stop(process!.status.id, reason))]) + const results = await Promise.all([ + processesToStart.map((process) => this._pup.stop(process!.status.id, reason)), + ]) return results.filter((r) => r).length > 0 } public block(id: string, reason: string): boolean { diff --git a/lib/core/configuration.ts b/lib/core/configuration.ts index 1728eff..b667622 100644 --- a/lib/core/configuration.ts +++ b/lib/core/configuration.ts @@ -6,6 +6,7 @@ */ import { z } from "zod" +import { PluginConfiguration } from "@pup/plugin" // Logger constants export const DEFAULT_INTERNAL_LOG_HOURS = 48 @@ -32,6 +33,7 @@ interface Configuration { logger?: GlobalLoggerConfiguration watcher?: GlobalWatcherConfiguration processes: ProcessConfiguration[] + plugins?: PluginConfiguration[] terminateTimeout?: number terminateGracePeriod?: number } @@ -165,6 +167,14 @@ const ConfigurationSchema = z.object({ ), }).strict(), ), + plugins: z.optional( + z.array( + z.object({ + url: z.string(), + options: z.optional(z.any()), + }), + ), + ), }).strict() function validateConfiguration(unsafeConfiguration: unknown): Configuration { diff --git a/lib/core/plugin.ts b/lib/core/plugin.ts new file mode 100644 index 0000000..44e1fe2 --- /dev/null +++ b/lib/core/plugin.ts @@ -0,0 +1,41 @@ +import { PluginConfiguration, PluginImplementation } from "@pup/plugin" + +/** + * Internal representation of a plugin + * - Used by @pup/pup internally! + */ +export class Plugin { + private config: PluginConfiguration + private apiUrl: string + private apiToken: string + public impl?: PluginImplementation + constructor(config: PluginConfiguration, apiUrl: string, apiToken: string) { + this.config = config + this.apiUrl = apiUrl + this.apiToken = apiToken + } + /** + * Will throw on any error + */ + public async load() { + const { PupPlugin } = await import(this.config.url) + this.impl = new PupPlugin(this.config, this.apiUrl, this.apiToken) as PluginImplementation + } + public verify() { + if (!this.impl || this.impl.meta.name === "unset") { + throw new Error("Plugin missing meta.name") + } + if (!this.impl || this.impl.meta.repository === "unset") { + throw new Error("Plugin missing meta.repository") + } + if (!this.impl || this.impl.meta.version === "unset") { + throw new Error("Plugin missing meta.version") + } + if (!this.impl || this.impl.meta.api === "unset") { + throw new Error("Plugin missing meta.api") + } + } + async terminate() { + if (this.impl?.cleanup) await this.impl?.cleanup() + } +} diff --git a/lib/core/pup.ts b/lib/core/pup.ts index dafe0fa..234a403 100644 --- a/lib/core/pup.ts +++ b/lib/core/pup.ts @@ -27,6 +27,8 @@ import { Prop } from "../common/prop.ts" import type { ApiTelemetryData } from "@pup/api-definitions" import { rm } from "@cross/fs" import { findFreePort } from "./port.ts" +import { Plugin } from "./plugin.ts" +import { GenerateToken } from "../common/token.ts" interface InstructionResponse { success: boolean @@ -42,7 +44,7 @@ class Pup { public restApi?: RestApi public processes: (Process | Cluster)[] = [] - + public plugins: Plugin[] = [] private requestTerminate = false private watchdogTimer?: number @@ -99,7 +101,9 @@ class Pup { this.status = new Status(statusFile) // Initialize API secret - if (secretFile) this.secret = new Prop(secretFile, DEFAULT_SECRET_FILE_PERMISSIONS) + if (secretFile) { + this.secret = new Prop(secretFile, DEFAULT_SECRET_FILE_PERMISSIONS) + } // Initialize API port if (portFile) this.port = new Prop(portFile) @@ -127,17 +131,53 @@ class Pup { // Initialize api await this.api() + // Initialize plugins + if (this.configuration.plugins) { + const secret = await this.secret?.load() + const pluginToken = await GenerateToken(secret!, { consumer: "plugin" }, Date.now() + 100000) + for (const plugin of this.configuration.plugins) { + const newPlugin = new Plugin(plugin, `${this.restApi?.hostname}:${this.restApi?.port}`, pluginToken) + let success = true + try { + this.logger.log("plugins", `Loading plugin from '${plugin.url}'`) + await newPlugin.load() + } catch (e) { + this.logger.error("plugins", `Failed to load plugin '${plugin.url}: ${e.message}'`) + success = false + } + try { + this.logger.log("plugins", `Verifying plugin from '${plugin.url}'`) + newPlugin.verify() + } catch (e) { + this.logger.error("plugins", `Failed to verify plugin '${plugin.url}': ${e.message}`) + success = false + } + + if (success) { + this.plugins.push(newPlugin) + this.logger.log("plugins", `Plugin '${newPlugin.impl?.meta.name}@${newPlugin.impl?.meta.version}' loaded from '${plugin.url}'`) + } + } + } + // Attach logger to events - this.logger.attach((severity: string, category: string, text: string, process?: ProcessConfiguration): boolean => { - this.events.emit("log", { - timeStamp: Date.now(), - severity, - category, - text, - processId: process?.id, - }) - return false - }) + this.logger.attach( + ( + severity: string, + category: string, + text: string, + process?: ProcessConfiguration, + ): boolean => { + this.events.emit("log", { + timeStamp: Date.now(), + severity, + category, + text, + processId: process?.id, + }) + return false + }, + ) // Create processes if (this.configuration.processes) { @@ -305,7 +345,10 @@ class Pup { this.restApi = new RestApi(this, this.configuration.api?.hostname, parseInt(port!, 10), secret) this.restApi.start() } catch (e) { - this.logger.error("rest", `An error occured while inizializing the rest api: ${e.message}`) + this.logger.error( + "rest", + `An error occured while inizializing the rest api: ${e.message}`, + ) } } @@ -411,6 +454,11 @@ class Pup { return } + // Terminate all plugins + for (const plugin of this.plugins) { + await plugin.terminate() + } + // Stop watchdog this.requestTerminate = true @@ -447,7 +495,10 @@ class Pup { private registerGlobalErrorHandler() { addEventListener("error", (event) => { - this.logger.error("fatal", `Unhandled error caught by core: ${event.error.message}`) + this.logger.error( + "fatal", + `Unhandled error caught by core: ${event.error.message}`, + ) event.preventDefault() }) } diff --git a/lib/core/rest.ts b/lib/core/rest.ts index 41a70d5..294c460 100644 --- a/lib/core/rest.ts +++ b/lib/core/rest.ts @@ -10,6 +10,7 @@ import { Pup } from "./pup.ts" import { generateKey, JWTPayload } from "@cross/jwt" import { DEFAULT_REST_API_HOSTNAME, DEFAULT_SECRET_KEY_ALGORITHM } from "./configuration.ts" import { ValidateToken } from "../common/token.ts" +import { EventHandler } from "@pup/common/eventemitter" const ALLOWED_SEVERITIES = ["log", "info", "warn", "error"] @@ -90,7 +91,9 @@ export class RestApi { } private setupRoutes() { + // New websocket connection this.router.get("/wss", (ctx) => { + // Upgrade if (!ctx.isUpgradable) { ctx.throw(501) } @@ -98,15 +101,31 @@ export class RestApi { if (!ctx.isUpgradable) { ctx.throw(501) } - const proxyFn = (d: unknown) => { - ws.send(JSON.stringify({ t: "log", d: d })) - } - this.pupApi.events.on("log", proxyFn) + + /* + Handle incoming message ws.onmessage = (m) => { ws.send(m.data as string) } - ws.onclose = (_e) => { - this.pupApi.events.off("log", proxyFn) + */ + + // Expose events to the API Consumer + const fnsToProxy = ["log", "process_status_changed"] + const proxyFns: Record>[] = [] + const proxyFnFactory = (evtName: string) => { + const proxyFn = (d: unknown) => { + ws.send(JSON.stringify({ t: evtName, d: d })) + } + proxyFns.push({ evtName, proxyFn }) + return proxyFn + } + for (const evtName of fnsToProxy) { + this.pupApi.events.on(evtName, proxyFnFactory(evtName)) + } + ws.onclose = () => { + for (const evt of proxyFns) { + this.pupApi.events.off(evt.evtName as string, evt.proxyFn as EventHandler) + } } }) @@ -257,7 +276,11 @@ export class RestApi { return } - this.pupApi.log(parsedBody.severity, parsedBody.plugin, parsedBody.message) + this.pupApi.log( + parsedBody.severity, + parsedBody.plugin, + parsedBody.message, + ) ctx.response.status = Status.Created } catch (err) { ctx.response.status = Status.InternalServerError @@ -304,7 +327,12 @@ export class RestApi { } const nRowsCapped = (!nRows || (nRows && nRows > 100)) ? 100 : nRows - let logContents = await this.pupApi.getLogs(processId || undefined, startTimeStamp, endTimeStamp, nRowsCapped) + let logContents = await this.pupApi.getLogs( + processId || undefined, + startTimeStamp, + endTimeStamp, + nRowsCapped, + ) if (severity) { const severityLower = severity.toLowerCase() @@ -314,18 +342,34 @@ export class RestApi { context.response.body = { data: logContents } } catch (error) { context.response.status = 500 - context.response.body = { error: "Internal Server Error", message: error.message } + context.response.body = { + error: "Internal Server Error", + message: error.message, + } } }) } public async start(): Promise { - this.app.use(generateAuthMiddleware(await this.setupKey(), this.pupApi.getConfiguration().api?.revoked)) + this.app.use( + generateAuthMiddleware( + await this.setupKey(), + this.pupApi.getConfiguration().api?.revoked, + ), + ) this.app.use(this.router.routes()) this.app.use(this.router.allowedMethods()) this.pupApi.log("info", "rest", `Starting the Rest API`) - await this.app.listen({ port: this.port, hostname: this.hostname, signal: this.appAbortController.signal }) - this.pupApi.log("info", "rest", `Rest API running, available on ${this.hostname}:${this.port}`) + await this.app.listen({ + port: this.port, + hostname: this.hostname, + signal: this.appAbortController.signal, + }) + this.pupApi.log( + "info", + "rest", + `Rest API running, available on ${this.hostname}:${this.port}`, + ) return this.port } diff --git a/mod.ts b/mod.ts index c6ca12a..7eba9aa 100644 --- a/mod.ts +++ b/mod.ts @@ -15,10 +15,3 @@ export type { Configuration, GlobalLoggerConfiguration, ProcessConfiguration } f /** Exports types for Logger */ export type { AttachedLogger } from "./lib/core/logger.ts" - -/** Export everything needed for Plugins */ -export { PupApi } from "./lib/core/api.ts" -export { Process } from "./lib/core/process.ts" -export type { ProcessScheduledEvent, ProcessStateChangedEvent, ProcessWatchEvent } from "./lib/core/process.ts" - -export type { LogEvent } from "./lib/core/logger.ts"