feat(service-spec-importers): expose importers via DatabaseBuilder

packages/@aws-cdk/service-spec-importers/src/cli/import-db.ts

@@ -4,8 +4,8 @@ import * as zlib from 'node:zlib';
 import { SpecDatabase, emptyDatabase, loadDatabase } from '@aws-cdk/service-spec-types';
 import { Command } from 'commander';
 import { CliError, handleFailure } from './util';
-import { DatabaseBuilder } from '../build-database';
 import { ProblemReport } from '../report';
+import { FullDatabase } from '../full-database';
 
 async function main() {
   const program = new Command();
@@ -34,10 +34,10 @@ async function main() {
   const baseDb = await database(options.input);
 
   process.stdout.write('Importing sources... ');
-  const { db, report } = await DatabaseBuilder.buildDatabase(baseDb, {
+  const { db, report } = await FullDatabase.buildDatabase(baseDb, {
     // FIXME: Switch this to 'true' at some point
-    mustValidate: false,
-    quiet: !options.debug,
+    validate: false,
+    debug: options.debug,
   });
 
   const numProblems = countProblems(report);

packages/@aws-cdk/service-spec-importers/src/db-builder.ts

@@ -0,0 +1,188 @@
+import { emptyDatabase, SpecDatabase } from '@aws-cdk/service-spec-types';
+import { assertSuccess, Result } from '@cdklabs/tskb';
+import { Augmentations } from './importers/import-augmentations';
+import { importCannedMetrics } from './importers/import-canned-metrics';
+import { importCloudFormationDocumentation } from './importers/import-cloudformation-docs';
+import { importCloudFormationRegistryResource } from './importers/import-cloudformation-registry';
+import { ResourceSpecImporter, SAMSpecImporter } from './importers/import-resource-spec';
+import { SamResources } from './importers/import-sam';
+import { Scrutinies } from './importers/import-scrutinies';
+import { importStatefulResources } from './importers/import-stateful-resources';
+import {
+  loadDefaultCloudFormationDocs,
+  loadDefaultCloudFormationRegistryResources,
+  loadDefaultCloudWatchConsoleServiceDirectory,
+  loadDefaultResourceSpecification,
+  loadDefaultStatefulResources,
+  LoadResult,
+  loadSamSchema,
+  loadSamSpec,
+} from './loaders';
+import { ProblemReport, ReportAudience } from './report';
+
+export interface DatabaseBuilderOptions {
+  /**
+   * Fail if we detect schema validations with the data source
+   * @default true
+   */
+  readonly validate?: boolean;
+
+  /**
+   * Print additional debug information
+   * @default false
+   */
+  readonly debug?: boolean;
+}
+
+export type SourceImporter = (db: SpecDatabase, report: ProblemReport) => Promise<void>;
+
+export class DatabaseBuilder {
+  private readonly sourceImporters = new Array<SourceImporter>();
+
+  constructor(
+    protected readonly db: SpecDatabase = emptyDatabase(),
+    private readonly options: DatabaseBuilderOptions,
+  ) {}
+
+  /**
+   * Add a SourceImporter to the database builder
+   */
+  public addSourceImporter(sourceImporter: SourceImporter): DatabaseBuilder {
+    this.sourceImporters.push(sourceImporter);
+    return this;
+  }
+
+  /**
+   * Apply all source importers
+   */
+  public async build(): Promise<{
+    db: SpecDatabase;
+    report: ProblemReport;
+  }> {
+    const report = new ProblemReport();
+
+    for (const sourceImporter of this.sourceImporters) {
+      await sourceImporter(this.db, report);
+    }
+
+    return {
+      db: this.db,
+      report: report,
+    };
+  }
+
+  /**
+   * Import the (legacy) resource spec
+   */
+  public importCloudFormationResourceSpec(specDirectory: string) {
+    return this.addSourceImporter(async (db, report) => {
+      const resourceSpec = this.loadResult(await loadDefaultResourceSpecification(specDirectory, this.options), report);
+
+      ResourceSpecImporter.importTypes({
+        db,
+        specification: resourceSpec,
+      });
+    });
+  }
+
+  /**
+   * Import the (legacy) resource spec for SAM, from GoFormation
+   */
+  public importSamResourceSpec(filePath: string) {
+    return this.addSourceImporter(async (db, report) => {
+      const samSpec = this.loadResult(await loadSamSpec(filePath, this.options), report);
+      SAMSpecImporter.importTypes({
+        db,
+        specification: samSpec,
+      });
+    });
+  }
+
+  /**
+   * Import the (modern) registry spec from CloudFormation
+   */
+  public importCloudFormationRegistryResources(schemaDirectory: string) {
+    return this.addSourceImporter(async (db, report) => {
+      const regions = await loadDefaultCloudFormationRegistryResources(schemaDirectory, report, this.options);
+      for (const region of regions) {
+        for (const resource of region.resources) {
+          importCloudFormationRegistryResource({
+            db,
+            resource,
+            report,
+            region: region.regionName,
+          });
+        }
+      }
+    });
+  }
+
+  /**
+   * Import the (modern) JSON schema spec from SAM
+   */
+  public importSamJsonSchema(filePath: string) {
+    return this.addSourceImporter(async (db, report) => {
+      const samSchema = this.loadResult(await loadSamSchema(filePath, this.options), report);
+      new SamResources({ db, samSchema, report }).import();
+    });
+  }
+
+  /**
+   * Import the CloudFormation Documentation
+   */
+  public importCloudFormationDocs(filePath: string) {
+    return this.addSourceImporter(async (db, report) => {
+      const docs = this.loadResult(await loadDefaultCloudFormationDocs(filePath, this.options), report);
+      importCloudFormationDocumentation(db, docs);
+    });
+  }
+
+  /**
+   * Import stateful resource information
+   */
+  public importStatefulResources(filePath: string) {
+    return this.addSourceImporter(async (db, report) => {
+      const stateful = this.loadResult(await loadDefaultStatefulResources(filePath, this.options), report);
+      importStatefulResources(db, stateful);
+    });
+  }
+
+  /**
+   * Import canned metrics from the CloudWatch Console Service Directory
+   */
+  public importCannedMetrics(filePath: string) {
+    return this.addSourceImporter(async (db, report) => {
+      const cloudWatchServiceDirectory = this.loadResult(
+        await loadDefaultCloudWatchConsoleServiceDirectory(filePath, this.options),
+        report,
+      );
+      importCannedMetrics(db, cloudWatchServiceDirectory, report);
+    });
+  }
+
+  /**
+   * Import Augmentations
+   */
+  public importAugmentations() {
+    return this.addSourceImporter(async (db) => new Augmentations(db).import());
+  }
+
+  /**
+   * Import Scrutinies
+   */
+  public importScrutinies() {
+    return this.addSourceImporter(async (db) => new Scrutinies(db).import());
+  }
+
+  /**
+   * Look at a load result and report problems
+   */
+  private loadResult<A>(result: Result<LoadResult<A>>, report: ProblemReport): A {
+    assertSuccess(result);
+
+    report.reportFailure(ReportAudience.cdkTeam(), 'loading', ...result.warnings);
+    report.reportPatch(ReportAudience.cdkTeam(), ...result.patchesApplied);
+
+    return result.value;
+  }
+}

packages/@aws-cdk/service-spec-importers/src/full-database.ts

@@ -0,0 +1,34 @@
+import { SpecDatabase, emptyDatabase } from '@aws-cdk/service-spec-types';
+import { DatabaseBuilder, DatabaseBuilderOptions } from './db-builder';
+import path from 'node:path';
+
+const SOURCES = path.join(__dirname, '../../../../sources');
+
+export class FullDatabase {
+  public static buildDatabase(db: SpecDatabase = emptyDatabase(), options: DatabaseBuilderOptions) {
+    return new FullDatabase(db, options).build();
+  }
+
+  private builder: DatabaseBuilder;
+
+  constructor(db: SpecDatabase, options: DatabaseBuilderOptions) {
+    this.builder = new DatabaseBuilder(db, options);
+  }
+
+  public async build() {
+    this.builder
+      .importCloudFormationResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification'))
+      .importSamResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification/us-east-1/100_sam'))
+      .importCloudFormationRegistryResources(path.join(SOURCES, 'CloudFormationSchema'))
+      .importSamJsonSchema(path.join(SOURCES, 'SAMSpec/sam.schema.json'))
+      .importCloudFormationDocs(path.join(SOURCES, 'CloudFormationDocumentation/CloudFormationDocumentation.json'))
+      .importStatefulResources(path.join(SOURCES, 'StatefulResources/StatefulResources.json'))
+      .importCannedMetrics(
+        path.join(SOURCES, 'CloudWatchConsoleServiceDirectory/CloudWatchConsoleServiceDirectory.json'),
+      )
+      .importScrutinies()
+      .importAugmentations();
+
+    return this.builder.build();
+  }
+}

packages/@aws-cdk/service-spec-importers/src/import-canned-metrics.ts → packages/@aws-cdk/service-spec-importers/src/importers/import-canned-metrics.ts

@@ -1,8 +1,8 @@
 import { createHash } from 'crypto';
 import { SpecDatabase } from '@aws-cdk/service-spec-types';
 import { Entity, failure, Plain } from '@cdklabs/tskb';
-import { ProblemReport, ReportAudience } from './report';
-import { CloudWatchConsoleServiceDirectory } from './types';
+import { ProblemReport, ReportAudience } from '../report';
+import { CloudWatchConsoleServiceDirectory } from '../types';
 
 /**
  * Returns a deduplicatable entity

packages/@aws-cdk/service-spec-importers/src/import-cloudformation-docs.ts → packages/@aws-cdk/service-spec-importers/src/importers/import-cloudformation-docs.ts

@@ -1,5 +1,5 @@
 import { SpecDatabase } from '@aws-cdk/service-spec-types';
-import { CloudFormationDocumentation, cfndocs } from './types';
+import { CloudFormationDocumentation, cfndocs } from '../types';
 
 export function importCloudFormationDocumentation(db: SpecDatabase, docs: CloudFormationDocumentation) {
   for (const [typeName, typeDocs] of Object.entries(docs.Types)) {