From 1dd2845d02c4bc9f8da6ffc42e05edb5e9600786 Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Fri, 20 Oct 2023 19:29:17 +0100 Subject: [PATCH] feat(service-spec-importers): expose importers via DatabaseBuilder --- .../src/build-database.ts | 125 ------------ .../src/cli/import-db.ts | 8 +- .../service-spec-importers/src/db-builder.ts | 188 ++++++++++++++++++ .../src/full-database.ts | 34 ++++ .../{ => importers}/import-augmentations.ts | 0 .../{ => importers}/import-canned-metrics.ts | 4 +- .../import-cloudformation-docs.ts | 2 +- .../import-cloudformation-registry.ts | 8 +- .../{ => importers}/import-resource-spec.ts | 4 +- .../src/{ => importers}/import-sam.ts | 4 +- .../src/{ => importers}/import-scrutinies.ts | 0 .../import-stateful-resources.ts | 2 +- .../service-spec-importers/src/index.ts | 7 +- .../src/{loading => loaders}/index.ts | 0 .../src/loaders/load-cloudformation-docs.ts | 16 ++ .../load-cloudformation-registry.ts | 15 +- ...ad-cloudwatch-console-service-directory.ts | 15 +- .../load-resource-spec.ts | 14 +- .../{loading => loaders}/load-sam-schema.ts | 12 +- .../src/loaders/load-sam-spec.ts | 22 ++ .../src/loaders/load-stateful-resources.ts | 16 ++ .../src/{loading => loaders}/loader.ts | 11 +- .../src/loading/load-cloudformation-docs.ts | 21 -- .../src/loading/load-sam-spec.ts | 22 -- .../src/loading/load-stateful-resources.ts | 14 -- .../src/tools/validate-registry-resources.ts | 51 ----- 26 files changed, 325 insertions(+), 290 deletions(-) delete mode 100644 packages/@aws-cdk/service-spec-importers/src/build-database.ts create mode 100644 packages/@aws-cdk/service-spec-importers/src/db-builder.ts create mode 100644 packages/@aws-cdk/service-spec-importers/src/full-database.ts rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-augmentations.ts (100%) rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-canned-metrics.ts (95%) rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-cloudformation-docs.ts (96%) rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-cloudformation-registry.ts (98%) rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-resource-spec.ts (99%) rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-sam.ts (98%) rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-scrutinies.ts (100%) rename packages/@aws-cdk/service-spec-importers/src/{ => importers}/import-stateful-resources.ts (88%) rename packages/@aws-cdk/service-spec-importers/src/{loading => loaders}/index.ts (100%) create mode 100644 packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-docs.ts rename packages/@aws-cdk/service-spec-importers/src/{loading => loaders}/load-cloudformation-registry.ts (86%) rename packages/@aws-cdk/service-spec-importers/src/{loading => loaders}/load-cloudwatch-console-service-directory.ts (57%) rename packages/@aws-cdk/service-spec-importers/src/{loading => loaders}/load-resource-spec.ts (68%) rename packages/@aws-cdk/service-spec-importers/src/{loading => loaders}/load-sam-schema.ts (55%) create mode 100644 packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-spec.ts create mode 100644 packages/@aws-cdk/service-spec-importers/src/loaders/load-stateful-resources.ts rename packages/@aws-cdk/service-spec-importers/src/{loading => loaders}/loader.ts (96%) delete mode 100644 packages/@aws-cdk/service-spec-importers/src/loading/load-cloudformation-docs.ts delete mode 100644 packages/@aws-cdk/service-spec-importers/src/loading/load-sam-spec.ts delete mode 100644 packages/@aws-cdk/service-spec-importers/src/loading/load-stateful-resources.ts delete mode 100644 packages/@aws-cdk/service-spec-importers/src/tools/validate-registry-resources.ts diff --git a/packages/@aws-cdk/service-spec-importers/src/build-database.ts b/packages/@aws-cdk/service-spec-importers/src/build-database.ts deleted file mode 100644 index bbf7b9ee9..000000000 --- a/packages/@aws-cdk/service-spec-importers/src/build-database.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { emptyDatabase, SpecDatabase } from '@aws-cdk/service-spec-types'; -import { assertSuccess, Result } from '@cdklabs/tskb'; -import { Augmentations } from './import-augmentations'; -import { importCannedMetrics } from './import-canned-metrics'; -import { importCloudFormationDocumentation } from './import-cloudformation-docs'; -import { importCloudFormationRegistryResource } from './import-cloudformation-registry'; -import { ResourceSpecImporter, SAMSpecImporter } from './import-resource-spec'; -import { SamResources } from './import-sam'; -import { Scrutinies } from './import-scrutinies'; -import { importStatefulResources } from './import-stateful-resources'; -import { - loadDefaultCloudFormationDocs, - loadDefaultCloudFormationRegistryResources, - loadDefaultCloudWatchConsoleServiceDirectory, - loadDefaultResourceSpecification, - loadDefaultStatefulResources, - LoadResult, - loadSamSchema, - loadSamSpec, -} from './loading'; -import { ProblemReport, ReportAudience } from './report'; - -export interface BuildDatabaseOptions { - readonly mustValidate?: boolean; - readonly quiet?: boolean; -} - -export class DatabaseBuilder { - public static buildDatabase(db: SpecDatabase = emptyDatabase(), options: BuildDatabaseOptions) { - return new DatabaseBuilder(db, options).build(); - } - public readonly report = new ProblemReport(); - - constructor(public readonly db: SpecDatabase, private readonly options: BuildDatabaseOptions) {} - - public async build() { - await this.importCloudFormationResourceSpec(); - await this.importSamResourceSpec(); - await this.importCloudFormationRegistryResources(); - await this.importSamJsonSchema(); - await this.importEnhancements(); - - return { - db: this.db, - report: this.report, - }; - } - - /** - * Import the (legacy) resource spec - */ - private async importCloudFormationResourceSpec() { - const resourceSpec = this.loadResult( - await loadDefaultResourceSpecification(this.options.mustValidate, this.options.quiet), - ); - - ResourceSpecImporter.importTypes({ - db: this.db, - specification: resourceSpec, - }); - } - - /** - * Import the (legacy) resource spec for SAM, from GoFormation - */ - private async importSamResourceSpec() { - const samSpec = this.loadResult(await loadSamSpec(this.options.mustValidate, this.options.quiet)); - SAMSpecImporter.importTypes({ - db: this.db, - specification: samSpec, - }); - } - - /** - * Import the (modern) registry spec from CloudFormation - */ - private async importCloudFormationRegistryResources() { - const regions = await loadDefaultCloudFormationRegistryResources(this.report, this.options.mustValidate); - for (const region of regions) { - for (const resource of region.resources) { - importCloudFormationRegistryResource({ - db: this.db, - resource, - report: this.report, - region: region.regionName, - }); - } - } - } - - /** - * Import the (modern) JSON schema spec from SAM - */ - private async importSamJsonSchema() { - const samSchema = this.loadResult(await loadSamSchema()); - new SamResources({ db: this.db, samSchema, report: this.report }).import(); - } - - /** - * Import various additions on top of the base specs - */ - private async importEnhancements() { - const docs = await loadDefaultCloudFormationDocs(this.report); - importCloudFormationDocumentation(this.db, docs); - - const stateful = this.loadResult(await loadDefaultStatefulResources()); - importStatefulResources(this.db, stateful); - - const cloudWatchServiceDirectory = this.loadResult(await loadDefaultCloudWatchConsoleServiceDirectory()); - importCannedMetrics(this.db, cloudWatchServiceDirectory, this.report); - - new Scrutinies(this.db).import(); - new Augmentations(this.db).import(); - } - - private loadResult(x: Result>): A { - assertSuccess(x); - - // We might need to handle these issues earlier so that we can push them to appropriate teams - this.report.reportFailure(ReportAudience.cdkTeam(), 'loading', ...x.warnings); - this.report.reportPatch(ReportAudience.cdkTeam(), ...x.patchesApplied); - - return x.value; - } -} diff --git a/packages/@aws-cdk/service-spec-importers/src/cli/import-db.ts b/packages/@aws-cdk/service-spec-importers/src/cli/import-db.ts index e957d4ca3..e4c3f2497 100644 --- a/packages/@aws-cdk/service-spec-importers/src/cli/import-db.ts +++ b/packages/@aws-cdk/service-spec-importers/src/cli/import-db.ts @@ -4,8 +4,8 @@ import * as zlib from 'node:zlib'; import { SpecDatabase, emptyDatabase, loadDatabase } from '@aws-cdk/service-spec-types'; import { Command } from 'commander'; import { CliError, handleFailure } from './util'; -import { DatabaseBuilder } from '../build-database'; import { ProblemReport } from '../report'; +import { FullDatabase } from '../full-database'; async function main() { const program = new Command(); @@ -34,10 +34,10 @@ async function main() { const baseDb = await database(options.input); process.stdout.write('Importing sources... '); - const { db, report } = await DatabaseBuilder.buildDatabase(baseDb, { + const { db, report } = await FullDatabase.buildDatabase(baseDb, { // FIXME: Switch this to 'true' at some point - mustValidate: false, - quiet: !options.debug, + validate: false, + debug: options.debug, }); const numProblems = countProblems(report); diff --git a/packages/@aws-cdk/service-spec-importers/src/db-builder.ts b/packages/@aws-cdk/service-spec-importers/src/db-builder.ts new file mode 100644 index 000000000..c58cd3de4 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/db-builder.ts @@ -0,0 +1,188 @@ +import { emptyDatabase, SpecDatabase } from '@aws-cdk/service-spec-types'; +import { assertSuccess, Result } from '@cdklabs/tskb'; +import { Augmentations } from './importers/import-augmentations'; +import { importCannedMetrics } from './importers/import-canned-metrics'; +import { importCloudFormationDocumentation } from './importers/import-cloudformation-docs'; +import { importCloudFormationRegistryResource } from './importers/import-cloudformation-registry'; +import { ResourceSpecImporter, SAMSpecImporter } from './importers/import-resource-spec'; +import { SamResources } from './importers/import-sam'; +import { Scrutinies } from './importers/import-scrutinies'; +import { importStatefulResources } from './importers/import-stateful-resources'; +import { + loadDefaultCloudFormationDocs, + loadDefaultCloudFormationRegistryResources, + loadDefaultCloudWatchConsoleServiceDirectory, + loadDefaultResourceSpecification, + loadDefaultStatefulResources, + LoadResult, + loadSamSchema, + loadSamSpec, +} from './loaders'; +import { ProblemReport, ReportAudience } from './report'; + +export interface DatabaseBuilderOptions { + /** + * Fail if we detect schema validations with the data source + * @default true + */ + readonly validate?: boolean; + + /** + * Print additional debug information + * @default false + */ + readonly debug?: boolean; +} + +export type SourceImporter = (db: SpecDatabase, report: ProblemReport) => Promise; + +export class DatabaseBuilder { + private readonly sourceImporters = new Array(); + + constructor( + protected readonly db: SpecDatabase = emptyDatabase(), + private readonly options: DatabaseBuilderOptions, + ) {} + + /** + * Add a SourceImporter to the database builder + */ + public addSourceImporter(sourceImporter: SourceImporter): DatabaseBuilder { + this.sourceImporters.push(sourceImporter); + return this; + } + + /** + * Apply all source importers + */ + public async build(): Promise<{ + db: SpecDatabase; + report: ProblemReport; + }> { + const report = new ProblemReport(); + + for (const sourceImporter of this.sourceImporters) { + await sourceImporter(this.db, report); + } + + return { + db: this.db, + report: report, + }; + } + + /** + * Import the (legacy) resource spec + */ + public importCloudFormationResourceSpec(specDirectory: string) { + return this.addSourceImporter(async (db, report) => { + const resourceSpec = this.loadResult(await loadDefaultResourceSpecification(specDirectory, this.options), report); + + ResourceSpecImporter.importTypes({ + db, + specification: resourceSpec, + }); + }); + } + + /** + * Import the (legacy) resource spec for SAM, from GoFormation + */ + public importSamResourceSpec(filePath: string) { + return this.addSourceImporter(async (db, report) => { + const samSpec = this.loadResult(await loadSamSpec(filePath, this.options), report); + SAMSpecImporter.importTypes({ + db, + specification: samSpec, + }); + }); + } + + /** + * Import the (modern) registry spec from CloudFormation + */ + public importCloudFormationRegistryResources(schemaDirectory: string) { + return this.addSourceImporter(async (db, report) => { + const regions = await loadDefaultCloudFormationRegistryResources(schemaDirectory, report, this.options); + for (const region of regions) { + for (const resource of region.resources) { + importCloudFormationRegistryResource({ + db, + resource, + report, + region: region.regionName, + }); + } + } + }); + } + + /** + * Import the (modern) JSON schema spec from SAM + */ + public importSamJsonSchema(filePath: string) { + return this.addSourceImporter(async (db, report) => { + const samSchema = this.loadResult(await loadSamSchema(filePath, this.options), report); + new SamResources({ db, samSchema, report }).import(); + }); + } + + /** + * Import the CloudFormation Documentation + */ + public importCloudFormationDocs(filePath: string) { + return this.addSourceImporter(async (db, report) => { + const docs = this.loadResult(await loadDefaultCloudFormationDocs(filePath, this.options), report); + importCloudFormationDocumentation(db, docs); + }); + } + + /** + * Import stateful resource information + */ + public importStatefulResources(filePath: string) { + return this.addSourceImporter(async (db, report) => { + const stateful = this.loadResult(await loadDefaultStatefulResources(filePath, this.options), report); + importStatefulResources(db, stateful); + }); + } + + /** + * Import canned metrics from the CloudWatch Console Service Directory + */ + public importCannedMetrics(filePath: string) { + return this.addSourceImporter(async (db, report) => { + const cloudWatchServiceDirectory = this.loadResult( + await loadDefaultCloudWatchConsoleServiceDirectory(filePath, this.options), + report, + ); + importCannedMetrics(db, cloudWatchServiceDirectory, report); + }); + } + + /** + * Import Augmentations + */ + public importAugmentations() { + return this.addSourceImporter(async (db) => new Augmentations(db).import()); + } + + /** + * Import Scrutinies + */ + public importScrutinies() { + return this.addSourceImporter(async (db) => new Scrutinies(db).import()); + } + + /** + * Look at a load result and report problems + */ + private loadResult(result: Result>, report: ProblemReport): A { + assertSuccess(result); + + report.reportFailure(ReportAudience.cdkTeam(), 'loading', ...result.warnings); + report.reportPatch(ReportAudience.cdkTeam(), ...result.patchesApplied); + + return result.value; + } +} diff --git a/packages/@aws-cdk/service-spec-importers/src/full-database.ts b/packages/@aws-cdk/service-spec-importers/src/full-database.ts new file mode 100644 index 000000000..a27af3f62 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/full-database.ts @@ -0,0 +1,34 @@ +import { SpecDatabase, emptyDatabase } from '@aws-cdk/service-spec-types'; +import { DatabaseBuilder, DatabaseBuilderOptions } from './db-builder'; +import path from 'node:path'; + +const SOURCES = path.join(__dirname, '../../../../sources'); + +export class FullDatabase { + public static buildDatabase(db: SpecDatabase = emptyDatabase(), options: DatabaseBuilderOptions) { + return new FullDatabase(db, options).build(); + } + + private builder: DatabaseBuilder; + + constructor(db: SpecDatabase, options: DatabaseBuilderOptions) { + this.builder = new DatabaseBuilder(db, options); + } + + public async build() { + this.builder + .importCloudFormationResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification')) + .importSamResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification/us-east-1/100_sam')) + .importCloudFormationRegistryResources(path.join(SOURCES, 'CloudFormationSchema')) + .importSamJsonSchema(path.join(SOURCES, 'SAMSpec/sam.schema.json')) + .importCloudFormationDocs(path.join(SOURCES, 'CloudFormationDocumentation/CloudFormationDocumentation.json')) + .importStatefulResources(path.join(SOURCES, 'StatefulResources/StatefulResources.json')) + .importCannedMetrics( + path.join(SOURCES, 'CloudWatchConsoleServiceDirectory/CloudWatchConsoleServiceDirectory.json'), + ) + .importScrutinies() + .importAugmentations(); + + return this.builder.build(); + } +} diff --git a/packages/@aws-cdk/service-spec-importers/src/import-augmentations.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-augmentations.ts similarity index 100% rename from packages/@aws-cdk/service-spec-importers/src/import-augmentations.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-augmentations.ts diff --git a/packages/@aws-cdk/service-spec-importers/src/import-canned-metrics.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-canned-metrics.ts similarity index 95% rename from packages/@aws-cdk/service-spec-importers/src/import-canned-metrics.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-canned-metrics.ts index 01085006f..d1e8f3ec4 100644 --- a/packages/@aws-cdk/service-spec-importers/src/import-canned-metrics.ts +++ b/packages/@aws-cdk/service-spec-importers/src/importers/import-canned-metrics.ts @@ -1,8 +1,8 @@ import { createHash } from 'crypto'; import { SpecDatabase } from '@aws-cdk/service-spec-types'; import { Entity, failure, Plain } from '@cdklabs/tskb'; -import { ProblemReport, ReportAudience } from './report'; -import { CloudWatchConsoleServiceDirectory } from './types'; +import { ProblemReport, ReportAudience } from '../report'; +import { CloudWatchConsoleServiceDirectory } from '../types'; /** * Returns a deduplicatable entity diff --git a/packages/@aws-cdk/service-spec-importers/src/import-cloudformation-docs.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-cloudformation-docs.ts similarity index 96% rename from packages/@aws-cdk/service-spec-importers/src/import-cloudformation-docs.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-cloudformation-docs.ts index 5eb5e3920..149da41d7 100644 --- a/packages/@aws-cdk/service-spec-importers/src/import-cloudformation-docs.ts +++ b/packages/@aws-cdk/service-spec-importers/src/importers/import-cloudformation-docs.ts @@ -1,5 +1,5 @@ import { SpecDatabase } from '@aws-cdk/service-spec-types'; -import { CloudFormationDocumentation, cfndocs } from './types'; +import { CloudFormationDocumentation, cfndocs } from '../types'; export function importCloudFormationDocumentation(db: SpecDatabase, docs: CloudFormationDocumentation) { for (const [typeName, typeDocs] of Object.entries(docs.Types)) { diff --git a/packages/@aws-cdk/service-spec-importers/src/import-cloudformation-registry.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-cloudformation-registry.ts similarity index 98% rename from packages/@aws-cdk/service-spec-importers/src/import-cloudformation-registry.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-cloudformation-registry.ts index 3046ca1f0..715ecefdf 100644 --- a/packages/@aws-cdk/service-spec-importers/src/import-cloudformation-registry.ts +++ b/packages/@aws-cdk/service-spec-importers/src/importers/import-cloudformation-registry.ts @@ -1,14 +1,14 @@ import { PropertyType, RichPropertyType, SpecDatabase, TagVariant } from '@aws-cdk/service-spec-types'; import { locateFailure, Fail, failure, isFailure, Result, tryCatch, using, ref, isSuccess } from '@cdklabs/tskb'; -import { ProblemReport, ReportAudience } from './report'; -import { PropertyBagBuilder, SpecBuilder } from './resource-builder'; -import { unionSchemas } from './schema-manipulation/unify-schemas'; +import { ProblemReport, ReportAudience } from '../report'; +import { PropertyBagBuilder, SpecBuilder } from '../resource-builder'; +import { unionSchemas } from '../schema-manipulation/unify-schemas'; import { CloudFormationRegistryResource, ImplicitJsonSchemaRecord, jsonschema, simplePropNameFromJsonPtr, -} from './types'; +} from '../types'; export interface LoadCloudFormationRegistryResourceOptions { readonly db: SpecDatabase; diff --git a/packages/@aws-cdk/service-spec-importers/src/import-resource-spec.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-resource-spec.ts similarity index 99% rename from packages/@aws-cdk/service-spec-importers/src/import-resource-spec.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-resource-spec.ts index a2c62f429..5d0091144 100644 --- a/packages/@aws-cdk/service-spec-importers/src/import-resource-spec.ts +++ b/packages/@aws-cdk/service-spec-importers/src/importers/import-resource-spec.ts @@ -1,7 +1,7 @@ import { PropertyType, SpecDatabase, TypeDefinition } from '@aws-cdk/service-spec-types'; import { ref } from '@cdklabs/tskb'; -import { PropertyBagBuilder, ResourceBuilder, SpecBuilder } from './resource-builder'; -import { CloudFormationResourceSpecification, SAMResourceSpecification, resourcespec } from './types'; +import { PropertyBagBuilder, ResourceBuilder, SpecBuilder } from '../resource-builder'; +import { CloudFormationResourceSpecification, SAMResourceSpecification, resourcespec } from '../types'; ////////////////////////////////////////////////////////////////////// diff --git a/packages/@aws-cdk/service-spec-importers/src/import-sam.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-sam.ts similarity index 98% rename from packages/@aws-cdk/service-spec-importers/src/import-sam.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-sam.ts index cd250df77..60869e06b 100644 --- a/packages/@aws-cdk/service-spec-importers/src/import-sam.ts +++ b/packages/@aws-cdk/service-spec-importers/src/importers/import-sam.ts @@ -1,8 +1,8 @@ import { Region, Service, SpecDatabase } from '@aws-cdk/service-spec-types'; import { chain, failure, liftUndefined, unpackOr } from '@cdklabs/tskb'; import { importCloudFormationRegistryResource } from './import-cloudformation-registry'; -import { ProblemReport } from './report'; -import { CloudFormationRegistryResource, SamTemplateSchema, jsonschema } from './types'; +import { ProblemReport } from '../report'; +import { CloudFormationRegistryResource, SamTemplateSchema, jsonschema } from '../types'; export interface SamResourcesOptions { readonly db: SpecDatabase; diff --git a/packages/@aws-cdk/service-spec-importers/src/import-scrutinies.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-scrutinies.ts similarity index 100% rename from packages/@aws-cdk/service-spec-importers/src/import-scrutinies.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-scrutinies.ts diff --git a/packages/@aws-cdk/service-spec-importers/src/import-stateful-resources.ts b/packages/@aws-cdk/service-spec-importers/src/importers/import-stateful-resources.ts similarity index 88% rename from packages/@aws-cdk/service-spec-importers/src/import-stateful-resources.ts rename to packages/@aws-cdk/service-spec-importers/src/importers/import-stateful-resources.ts index 71b6f9051..08525f6d7 100644 --- a/packages/@aws-cdk/service-spec-importers/src/import-stateful-resources.ts +++ b/packages/@aws-cdk/service-spec-importers/src/importers/import-stateful-resources.ts @@ -1,5 +1,5 @@ import { SpecDatabase } from '@aws-cdk/service-spec-types'; -import { StatefulResources } from './types'; +import { StatefulResources } from '../types'; export function importStatefulResources(db: SpecDatabase, stateful: StatefulResources) { for (const [typeName, _] of Object.entries(stateful.ResourceTypes)) { diff --git a/packages/@aws-cdk/service-spec-importers/src/index.ts b/packages/@aws-cdk/service-spec-importers/src/index.ts index 342883275..0b114c234 100644 --- a/packages/@aws-cdk/service-spec-importers/src/index.ts +++ b/packages/@aws-cdk/service-spec-importers/src/index.ts @@ -1,8 +1,3 @@ -export * from './build-database'; +export * from './db-builder'; export * from './db-diff'; -export * from './types'; -export * from './schema-manipulation/unify-schemas'; -export * from './patching/format-patch-report'; -export * from './patching/patching'; -export * from './loading'; export * from './report'; diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/index.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/index.ts similarity index 100% rename from packages/@aws-cdk/service-spec-importers/src/loading/index.ts rename to packages/@aws-cdk/service-spec-importers/src/loaders/index.ts diff --git a/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-docs.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-docs.ts new file mode 100644 index 000000000..5eaac3b55 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-docs.ts @@ -0,0 +1,16 @@ +import { assertSuccess } from '@cdklabs/tskb'; +import { Loader, LoadResult, LoadSourceOptions as LoadSourceOptions } from './loader'; +import { CloudFormationDocumentation } from '../types'; + +export async function loadDefaultCloudFormationDocs( + filePath: string, + options: LoadSourceOptions = {}, +): Promise> { + const loader = await Loader.fromSchemaFile('CloudFormationDocumentation.schema.json', { + mustValidate: options.validate, + }); + + const result = await loader.loadFile(filePath); + assertSuccess(result); + return result; +} diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/load-cloudformation-registry.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-registry.ts similarity index 86% rename from packages/@aws-cdk/service-spec-importers/src/loading/load-cloudformation-registry.ts rename to packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-registry.ts index fef6e9268..f5563afd0 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loading/load-cloudformation-registry.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-registry.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as util from 'util'; import { isSuccess, Result } from '@cdklabs/tskb'; import * as _glob from 'glob'; -import { Loader, LoadResult } from './loader'; +import { Loader, LoadResult, LoadSourceOptions } from './loader'; import { patchCloudFormationRegistry } from '../patches/registry-patches'; import { ProblemReport, ReportAudience } from '../report'; import { CloudFormationRegistryResource } from '../types'; @@ -34,20 +34,15 @@ export interface CloudFormationRegistryResources { } export async function loadDefaultCloudFormationRegistryResources( + schemaDir: string, report: ProblemReport, - mustValidate = true, + options: LoadSourceOptions = {}, ): Promise { - const errorRootDirectory = path.join(__dirname, '../../../../../sources/CloudFormationSchema'); - const files = await glob(`${errorRootDirectory}/*`); + const files = await glob(`${schemaDir}/*`); return Promise.all( files.map(async (directoryName) => { const regionName = path.basename(directoryName); - const resources = await loadCloudFormationRegistryDirectory( - directoryName, - report, - mustValidate, - errorRootDirectory, - ); + const resources = await loadCloudFormationRegistryDirectory(directoryName, report, options.validate, schemaDir); return { regionName, resources }; }), diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/load-cloudwatch-console-service-directory.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudwatch-console-service-directory.ts similarity index 57% rename from packages/@aws-cdk/service-spec-importers/src/loading/load-cloudwatch-console-service-directory.ts rename to packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudwatch-console-service-directory.ts index 0e493d89f..3daf7a926 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loading/load-cloudwatch-console-service-directory.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudwatch-console-service-directory.ts @@ -1,22 +1,17 @@ -import * as path from 'path'; import { assertSuccess } from '@cdklabs/tskb'; -import { Loader, LoadResult } from './loader'; +import { Loader, LoadResult, LoadSourceOptions } from './loader'; import { CloudWatchConsoleServiceDirectory } from '../types'; export async function loadDefaultCloudWatchConsoleServiceDirectory( - mustValidate = true, + filePath: string, + options: LoadSourceOptions = {}, ): Promise> { const loader = await Loader.fromSchemaFile( 'CloudWatchConsoleServiceDirectory.schema.json', - { mustValidate }, + { mustValidate: options.validate }, ); - const result = await loader.loadFile( - path.join( - __dirname, - '../../../../../sources/CloudWatchConsoleServiceDirectory/CloudWatchConsoleServiceDirectory.json', - ), - ); + const result = await loader.loadFile(filePath); assertSuccess(result); return result; } diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/load-resource-spec.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-resource-spec.ts similarity index 68% rename from packages/@aws-cdk/service-spec-importers/src/loading/load-resource-spec.ts rename to packages/@aws-cdk/service-spec-importers/src/loaders/load-resource-spec.ts index 9c318e9d8..2d859fac2 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loading/load-resource-spec.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-resource-spec.ts @@ -1,19 +1,19 @@ import * as path from 'path'; -import { combineLoadResults, Loader, LoadResult, mapLoadResult } from './loader'; +import { combineLoadResults, Loader, LoadResult, LoadSourceOptions, mapLoadResult } from './loader'; import { applyPatchSet } from '../patching/json-patch-set'; import { CloudFormationResourceSpecification } from '../types'; export async function loadDefaultResourceSpecification( - mustValidate = true, - quiet = false, + specDir: string, + options: LoadSourceOptions = {}, ): Promise> { const loader = await Loader.fromSchemaFile('ResourceSpecification.schema.json', { - mustValidate, + mustValidate: options.validate, }); + const quiet = !options.debug; - const cfnSpecDir = path.join(__dirname, '../../../../../sources/CloudFormationResourceSpecification'); - const usEast1 = await applyPatchSet(path.join(cfnSpecDir, 'us-east-1', '000_cloudformation'), { quiet }); - const usWest2 = await applyPatchSet(path.join(cfnSpecDir, 'us-west-2', '000_cloudformation'), { quiet }); + const usEast1 = await applyPatchSet(path.join(specDir, 'us-east-1', '000_cloudformation'), { quiet }); + const usWest2 = await applyPatchSet(path.join(specDir, 'us-west-2', '000_cloudformation'), { quiet }); const usEast1Result = await loader.load(usEast1); const usWest2Result = await loader.load(usWest2); diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/load-sam-schema.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-schema.ts similarity index 55% rename from packages/@aws-cdk/service-spec-importers/src/loading/load-sam-schema.ts rename to packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-schema.ts index 723453ddb..cc06722c9 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loading/load-sam-schema.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-schema.ts @@ -1,19 +1,21 @@ -import * as path from 'path'; import { assertSuccess } from '@cdklabs/tskb'; -import { Loader, LoadResult } from './loader'; +import { Loader, LoadResult, LoadSourceOptions } from './loader'; import { patchSamTemplateSpec } from '../patches/sam-patches'; import { SamTemplateSchema } from '../types'; /** * Load the new SAM (json) schema */ -export async function loadSamSchema(mustValidate = true): Promise> { +export async function loadSamSchema( + filePath: string, + options: LoadSourceOptions = {}, +): Promise> { const loader = await Loader.fromSchemaFile('SamTemplateSchema.schema.json', { - mustValidate, + mustValidate: options.validate, patcher: patchSamTemplateSpec, }); - const result = await loader.loadFile(path.join(__dirname, '../../../../../sources/SAMSpec/sam.schema.json')); + const result = await loader.loadFile(filePath); assertSuccess(result); return result; } diff --git a/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-spec.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-spec.ts new file mode 100644 index 000000000..ebd330cf6 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-spec.ts @@ -0,0 +1,22 @@ +import { assertSuccess } from '@cdklabs/tskb'; +import { Loader, LoadResult, LoadSourceOptions } from './loader'; +import { applyPatchSet } from '../patching/json-patch-set'; +import { SAMResourceSpecification } from '../types'; + +/** + * Load the old SAM spec (CloudFormation spec + extensions) + */ +export async function loadSamSpec( + filePath: string, + options: LoadSourceOptions, +): Promise> { + const loader = await Loader.fromSchemaFile('SAMResourceSpecification.schema.json', { + mustValidate: options.validate, + }); + + const spec = await applyPatchSet(filePath, { quiet: !options.debug }); + const result = await loader.load(spec); + assertSuccess(result); + + return result; +} diff --git a/packages/@aws-cdk/service-spec-importers/src/loaders/load-stateful-resources.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-stateful-resources.ts new file mode 100644 index 000000000..5a1ff768a --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-stateful-resources.ts @@ -0,0 +1,16 @@ +import { assertSuccess } from '@cdklabs/tskb'; +import { Loader, LoadResult, LoadSourceOptions } from './loader'; +import { StatefulResources } from '../types'; + +export async function loadDefaultStatefulResources( + filePath: string, + options: LoadSourceOptions = {}, +): Promise> { + const loader = await Loader.fromSchemaFile('StatefulResources.schema.json', { + mustValidate: options.validate, + }); + + const result = await loader.loadFile(filePath); + assertSuccess(result); + return result; +} diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/loader.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/loader.ts similarity index 96% rename from packages/@aws-cdk/service-spec-importers/src/loading/loader.ts rename to packages/@aws-cdk/service-spec-importers/src/loaders/loader.ts index 6d67e264e..029550352 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loading/loader.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/loader.ts @@ -10,7 +10,12 @@ import * as _glob from 'glob'; import { applyPatcher, JsonLensPatcher, PatchReport } from '../patching'; import { BoundProblemReport } from '../report'; -export interface LoadOptions { +export interface LoadSourceOptions { + readonly validate?: boolean; + readonly debug?: boolean; +} + +export interface LoaderOptions { /** * Fail if we detect schema validations with the data source * @@ -35,7 +40,7 @@ export interface LoadOptions { } export class Loader { - public static async fromSchemaFile(fileName: string, options: LoadOptions): Promise> { + public static async fromSchemaFile(fileName: string, options: LoaderOptions): Promise> { const ajv = new Ajv({ verbose: true }); const cfnSchemaJson = JSON.parse( await fs.readFile(path.join(__dirname, `../../schemas/${fileName}`), { encoding: 'utf-8' }), @@ -44,7 +49,7 @@ export class Loader { return new Loader(validator, options); } - private constructor(private readonly validator: Ajv.ValidateFunction, private readonly options: LoadOptions) {} + private constructor(private readonly validator: Ajv.ValidateFunction, private readonly options: LoaderOptions) {} /** * Validate the given object diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/load-cloudformation-docs.ts b/packages/@aws-cdk/service-spec-importers/src/loading/load-cloudformation-docs.ts deleted file mode 100644 index 90f9ec12c..000000000 --- a/packages/@aws-cdk/service-spec-importers/src/loading/load-cloudformation-docs.ts +++ /dev/null @@ -1,21 +0,0 @@ -import * as path from 'path'; -import { assertSuccess } from '@cdklabs/tskb'; -import { Loader } from './loader'; -import { ProblemReport, ReportAudience } from '../report'; -import { CloudFormationDocumentation } from '../types'; - -export async function loadDefaultCloudFormationDocs( - report: ProblemReport, - mustValidate = true, -): Promise { - const loader = await Loader.fromSchemaFile('CloudFormationDocumentation.schema.json', { - mustValidate, - report: report.forAudience(ReportAudience.cdkTeam()), - }); - - const result = await loader.loadFile( - path.join(__dirname, '../../../../../sources/CloudFormationDocumentation/CloudFormationDocumentation.json'), - ); - assertSuccess(result); - return result.value; -} diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/load-sam-spec.ts b/packages/@aws-cdk/service-spec-importers/src/loading/load-sam-spec.ts deleted file mode 100644 index 7466be6de..000000000 --- a/packages/@aws-cdk/service-spec-importers/src/loading/load-sam-spec.ts +++ /dev/null @@ -1,22 +0,0 @@ -import * as path from 'path'; -import { assertSuccess } from '@cdklabs/tskb'; -import { Loader, LoadResult } from './loader'; -import { applyPatchSet } from '../patching/json-patch-set'; -import { SAMResourceSpecification } from '../types'; - -/** - * Load the old SAM spec (CloudFormation spec + extensions) - */ -export async function loadSamSpec(mustValidate = true, quiet = false): Promise> { - const loader = await Loader.fromSchemaFile('SAMResourceSpecification.schema.json', { - mustValidate, - }); - - const cfnSpecDir = path.join(__dirname, '../../../../../sources/CloudFormationResourceSpecification'); - const usEast1 = await applyPatchSet(path.join(cfnSpecDir, 'us-east-1', '100_sam'), { quiet }); - - const usEast1Result = await loader.load(usEast1); - assertSuccess(usEast1Result); - - return usEast1Result; -} diff --git a/packages/@aws-cdk/service-spec-importers/src/loading/load-stateful-resources.ts b/packages/@aws-cdk/service-spec-importers/src/loading/load-stateful-resources.ts deleted file mode 100644 index 2afdf7e47..000000000 --- a/packages/@aws-cdk/service-spec-importers/src/loading/load-stateful-resources.ts +++ /dev/null @@ -1,14 +0,0 @@ -import * as path from 'path'; -import { assertSuccess } from '@cdklabs/tskb'; -import { Loader, LoadResult } from './loader'; -import { StatefulResources } from '../types'; - -export async function loadDefaultStatefulResources(mustValidate = true): Promise> { - const loader = await Loader.fromSchemaFile('StatefulResources.schema.json', { mustValidate }); - - const result = await loader.loadFile( - path.join(__dirname, '../../../../../sources/StatefulResources/StatefulResources.json'), - ); - assertSuccess(result); - return result; -} diff --git a/packages/@aws-cdk/service-spec-importers/src/tools/validate-registry-resources.ts b/packages/@aws-cdk/service-spec-importers/src/tools/validate-registry-resources.ts deleted file mode 100644 index 294f7bc1d..000000000 --- a/packages/@aws-cdk/service-spec-importers/src/tools/validate-registry-resources.ts +++ /dev/null @@ -1,51 +0,0 @@ -// A build tool to validate that our type definitions cover all resources -// -// Not a lot of thought given to where this needs to live yet. -import * as path from 'path'; -import { loadDefaultCloudFormationRegistryResources } from '../loading'; -import { formatPatchReport, PatchReport } from '../patching'; -import { ProblemReport } from '../report'; - -async function main() { - const report = new ProblemReport(); - await loadDefaultCloudFormationRegistryResources(report, false); - - if (report.patchesApplied.length > 0) { - const patches = uniqueReports(report.patchesApplied); - console.log(`${patches.length} patches applied to sources`); - console.log('==========================================='); - console.log(); - - for (const patch of patches) { - console.log(formatPatchReport(patch) + '\n'); - } - } - - if (report.counts.interpreting + report.counts.loading > 0) { - console.log(`${report.counts.interpreting + report.counts.loading} problems remaining after patching`); - console.log('==========================================='); - console.log(); - process.exitCode = 1; - } -} - -/** - * We have the same schema files in multiple directories. Do not report redundant patch reports. - */ -function uniqueReports(reports: PatchReport[]) { - const seen = new Set(); - const ret = new Array(); - for (const report of reports) { - const key = `${path.basename(report.fileName)}|${report.reason}|${report.path}`; - if (!seen.has(key)) { - ret.push(report); - seen.add(key); - } - } - return ret; -} - -main().catch((e) => { - console.log(e); - process.exitCode = 1; -});