diff --git a/src/destination/credentials/credentials.component.ts b/src/destination/credentials/credentials.component.ts index e4e5e1b..1427f67 100644 --- a/src/destination/credentials/credentials.component.ts +++ b/src/destination/credentials/credentials.component.ts @@ -26,6 +26,7 @@ import {DataService} from "../../data.service"; import {MigrateComponent} from "../../migrate/migrate.component"; import {UpdateableAlert} from "../../utils/UpdateableAlert"; import {delay} from "../../utils/utils"; +import {Migration} from "../../migrate/migration.service"; @Component({ templateUrl: './credentials.component.html' @@ -100,7 +101,7 @@ export class CredentialsComponent { alrt.update(`Downloading Migration Tool... ${(progress * 100).toFixed(0)}%`); }); // TODO: progress alrt.update('Uploading to tenant...'); - const newApp = MigrateComponent.appMigrationToApp({ // TODO: shouldn't be accessing this as a static method, should be in a service or util file + const newApp = Migration.appMigrationToApp({ newName: 'Migration Tool', newContextPath: 'migration-tool', newAppKey: 'migration-tool-application-key', diff --git a/src/migrate/migrate.component.ts b/src/migrate/migrate.component.ts index ed99e71..bed9464 100644 --- a/src/migrate/migrate.component.ts +++ b/src/migrate/migrate.component.ts @@ -19,35 +19,21 @@ import {Component, TemplateRef, ViewChild} from '@angular/core'; import {IApplication, IManagedObject} from '@c8y/client'; import {DataService} from "../data.service"; import {SelectionService} from "../selection.service"; -import { - getDashboardName, - getIdPathsFromDashboard, - getSimulatorId, - isSimulatorDevice, - setFromPath -} from "../utils/utils"; +import {getDashboardName} from "../utils/utils"; import {AlertService} from "@c8y/ngx-components"; import download from "downloadjs"; import {FileDataClient} from "../FileDataClient"; -import _ from "lodash"; -import objectScan from "object-scan"; import {UpdateableAlert} from "../utils/UpdateableAlert"; -import {IExternalId} from "../c8y-interfaces/IExternalId"; -import {ISmartRuleConfig} from "../c8y-interfaces/ISmartRuleConfig"; - -interface ApplicationMigration { - newName: string, - newContextPath: string, - newAppKey: string, - application: IApplication & { binary:IManagedObject }, - updateExisting?: IApplication -} +import { + ApplicationMigration, + ManagedObjectMigration, + Migration, + MigrationLogEvent, + MigrationLogLevel +} from "./migration.service"; +import {filter} from 'rxjs/operators'; +import {BehaviorSubject} from "rxjs"; -interface ManagedObjectMigration { - newName: string, - managedObject: IManagedObject, - updateExisting?: IManagedObject -} @Component({ templateUrl: './migrate.component.html' @@ -234,145 +220,29 @@ export class MigrateComponent { const sourceClient = this.dataService.getSourceDataClient(); const destinationClient = this.dataService.getDestinationDataClient(); - try { - // Migrate the standard managedObjects - alrt.update("Migrating Groups, Devices, and Other ManagedObjects..."); - const [simulatorDeviceMigrations, nonSimulatorDeviceMigrations] = _.partition(this.deviceMigrations, deviceMigration => { - if (!isSimulatorDevice(deviceMigration.managedObject)) return false; - const simulatorId = getSimulatorId(deviceMigration.managedObject as (IManagedObject & {externalIds: IExternalId[]})); - return this.simulatorMigrations.some(simMigration => simMigration.managedObject.id.toString() === simulatorId); - }); + const migration = new Migration(sourceClient, destinationClient); + + const infoLogger = migration.log$.pipe( + filter(log => log.level == MigrationLogLevel.Info) + ).subscribe(log => alrt.update(log.description)); + + const consoleLogger = migration.log$.subscribe(log => { + switch(log.level) { + case MigrationLogLevel.Verbose: + case MigrationLogLevel.Info: + console.info(log.description); + return; + case MigrationLogLevel.Error: + console.error(log.description); + return; + } + }); + + const lastLogMessage = new BehaviorSubject(undefined); + const lastLogSubscriber = migration.log$.subscribe(lastLogMessage); - const oldIdsToNewIds = []; - - oldIdsToNewIds.push(...await Promise.all( - [...this.groupMigrations, ...nonSimulatorDeviceMigrations, ...this.otherMigrations] - .map(async (moMigration) => { - if (moMigration.updateExisting) { - const mo = this.managedObjectMigrationToManagedObject(moMigration); - mo.id = moMigration.updateExisting.id; - return [moMigration.managedObject.id.toString(), await destinationClient.updateManagedObject(mo)] - } else { - return [moMigration.managedObject.id.toString(), await destinationClient.createManagedObject(this.managedObjectMigrationToManagedObject(moMigration))]; - } - }))); - - // Migrate the simulators - alrt.update("Migrating Simulators..."); - const simulatorDeviceMigrationsBySimulatorId = simulatorDeviceMigrations.reduce((acc, simulatorDeviceMigration) => { - const simulatorDevice = simulatorDeviceMigration.managedObject as (IManagedObject & {externalIds: IExternalId[]}); - const simulatorId = getSimulatorId(simulatorDevice); - if (simulatorId) { - if (!acc.has(simulatorId)) { - acc.set(simulatorId, []); - } - acc.get(simulatorId).push(simulatorDeviceMigration); - } else { - this.alertService.danger('Unable to migrate' + JSON.stringify(simulatorDevice)); - } - return acc; - }, new Map()); - - oldIdsToNewIds.push(..._.flatten(await Promise.all(Array.from(simulatorDeviceMigrationsBySimulatorId).map(async ([simulatorId, simDeviceMigrations]) => { - const simulatorMigration = this.simulatorMigrations.find(simMigration => simMigration.managedObject.id.toString() === simulatorId); - if (simulatorMigration) { - const simulatorConfig = _.omit(_.cloneDeep(simulatorMigration.managedObject.c8y_DeviceSimulator), 'id'); - simulatorConfig.name = simulatorMigration.newName; - - let newSimulatorId: string|number; - let deviceIds: (string|number)[]; - if (simulatorMigration.updateExisting) { - simulatorConfig.id = simulatorMigration.updateExisting.id; - simulatorConfig.instances = Math.max(simDeviceMigrations.length, simulatorConfig.instances); - ({simulatorId: newSimulatorId, deviceIds: deviceIds} = await destinationClient.updateSimulator(simulatorConfig)); - } else { - // Create the simulator (which creates the simulator devices - simulatorConfig.instances = simDeviceMigrations.length; - ({simulatorId: newSimulatorId, deviceIds: deviceIds} = await destinationClient.createSimulator(simulatorConfig)); - } - - // Update the simulator devices - const oldIdsToNewIds = await Promise.all(_.zip(simDeviceMigrations, deviceIds).map(async ([simDeviceMigration, newDeviceId]) => { - const update = this.managedObjectMigrationToManagedObject(simDeviceMigration); - update.id = newDeviceId.toString(); - await destinationClient.updateManagedObject(update); - return [simDeviceMigration.managedObject.id.toString(), newDeviceId]; - })); - - return [ - [simulatorId.toString(), newSimulatorId], - ...oldIdsToNewIds - ]; - } else { - this.alertService.danger('Unable to find simulator' + simulatorId); - return []; - } - })))); - - // Migrate the SmartRules - alrt.update("Migrating Smart Rules..."); - oldIdsToNewIds.push(...await Promise.all( - this.smartRuleMigrations - .map(async (srMigration) => { - if (srMigration.updateExisting) { - const srConfig = _.omit(this.smartRuleMigrationToSmartRuleConfig(srMigration, new Map(oldIdsToNewIds)),'type'); - srConfig.id = srMigration.updateExisting.id; - return [srMigration.managedObject.id.toString(), await destinationClient.updateSmartRule(srConfig)]; - } else { - return [srMigration.managedObject.id.toString(), await destinationClient.createSmartRule(this.smartRuleMigrationToSmartRuleConfig(srMigration, new Map(oldIdsToNewIds)))]; - } - }))); - - // Migrate the Binaries - alrt.update("Migrating Binaries..."); - oldIdsToNewIds.push(...await Promise.all( - this.binaryMigrations - .map(async (bMigration) => { - const blob = await sourceClient.getBinaryBlob(bMigration.managedObject); - if (bMigration.updateExisting) { - const mo = this.managedObjectMigrationToManagedObject(bMigration); - mo.id = bMigration.updateExisting.id; - return [bMigration.managedObject.id.toString(), await destinationClient.updateBinary(mo, blob)] - } else { - return [bMigration.managedObject.id.toString(), await destinationClient.createBinary(this.managedObjectMigrationToManagedObject(bMigration), blob)]; - } - }))); - - // Migrate the dashboards - alrt.update("Migrating Dashboards..."); - oldIdsToNewIds.push(...await Promise.all( - this.dashboardMigrations - .map(async (dashboardMigration) => { - if (dashboardMigration.updateExisting) { - const db = this.dashboardMigrationToManagedObject(dashboardMigration, new Map(oldIdsToNewIds)); - db.id = dashboardMigration.updateExisting.id; - return [dashboardMigration.managedObject.id.toString(), await destinationClient.updateManagedObject(db)] - } else { - return [dashboardMigration.managedObject.id.toString(), await destinationClient.createManagedObject(this.dashboardMigrationToManagedObject(dashboardMigration, new Map(oldIdsToNewIds)))] - } - }) - )); - - const oldIdsToNewIdsMap = new Map(oldIdsToNewIds); - - // Create the parent child linkages - alrt.update("Migrating Parent/Child linkages..."); - await Promise.all([...this.dashboardMigrations, ...this.groupMigrations, ...this.deviceMigrations, ...this.otherMigrations] - .map(moMigration => - destinationClient.createLinkages(oldIdsToNewIdsMap.get(moMigration.managedObject.id.toString()).toString(), this.managedObjectMigrationToManagedObjectLinkages(moMigration, oldIdsToNewIdsMap)))); - - // Migrate the applications - alrt.update("Migrating Applications..."); - await Promise.all(this.appMigrations.map(async (appMigration) => { - const blob = await sourceClient.getApplicationBlob(appMigration.application); - if (appMigration.updateExisting) { - const app = MigrateComponent.appMigrationToApp(appMigration, oldIdsToNewIdsMap); - app.id = appMigration.updateExisting.id; - return destinationClient.updateApplication(app, blob); - } else { - return destinationClient.createApplication(MigrateComponent.appMigrationToApp(appMigration, oldIdsToNewIdsMap), blob); - } - })); + try { + await migration.migrate(this.deviceMigrations, this.simulatorMigrations, this.groupMigrations, this.otherMigrations, this.smartRuleMigrations, this.dashboardMigrations, this.binaryMigrations, this.appMigrations); if (destinationClient instanceof FileDataClient) { alrt.update("Opening..."); @@ -384,204 +254,29 @@ export class MigrateComponent { } this.dirty = false; } catch(e) { - if (e instanceof Error) { - alrt.update(`${e.name || 'Error'}: ${e.message}`, 'danger'); - } else if (e.data && e.data.error) { - alrt.update(`${e.data.error}: ${e.data.message}`, 'danger'); + if (lastLogMessage.getValue() != undefined) { + alrt.update(`Failed to migrate!\nFailed at: ${lastLogMessage.getValue().description}\n${this.getErrorMessage(e)}\nCheck browser console for more details`, 'danger'); } else { - alrt.update('Error: Check browser console for details', 'danger'); + alrt.update(`Failed to migrate!\n${this.getErrorMessage(e)}\nCheck browser console for more details`, 'danger'); } + throw(e); } finally { + infoLogger.unsubscribe(); + consoleLogger.unsubscribe(); + lastLogSubscriber.unsubscribe(); this.reset(); } } - static appMigrationToApp(appMigration: ApplicationMigration, oldIdsToNewIds: Map): IApplication { - const result: IApplication & {applicationBuilder?: any} = {}; - - // Blacklist certain fields - function isBlacklistedKey(key) { - if (key.length) { - switch(key[0]) { - case 'downloading': // we added downloading so remove it - case 'binary': // we added the binary key so remove it - case 'activeVersionId': - return true; - } - - switch(key[key.length - 1]) { - case 'owner': - case 'self': - return true; - } - } - return false; - } - - const paths = objectScan(['**.*'], {useArraySelector: false, joined: false, breakFn: (key, value) => isBlacklistedKey(key), filterFn: (key, value) => { - if (isBlacklistedKey(key)) { - return false; - } - - // We want to copy the leaf nodes (and will create their paths) so we skip anything that isn't a leaf node - // In other words: only values (string or number) and empty objects or arrays - return _.isString(value) || _.isNumber(value) || _.isNull(value) || Object.keys(value).length === 0; - }})(appMigration.application); - - paths.forEach(path => { - _.set(result, path, _.get(appMigration.application, path)); - }); - - if (result.applicationBuilder) { - result.externalUrl = appMigration.application.externalUrl.split(appMigration.application.id.toString()).join('UNKNOWN-APP-ID'); - // Update application builder dashboard ids - if (result.applicationBuilder.dashboards) { - result.applicationBuilder.dashboards.forEach(dashboard => { - if (oldIdsToNewIds.has(dashboard.id.toString())) { - dashboard.id = oldIdsToNewIds.get(dashboard.id.toString()); - } else { - // TODO: add to warning - } - }) - } - } - - // Update the application with the user provided changes... - result.contextPath = appMigration.newContextPath; - result.name = appMigration.newName; - result.key = appMigration.newAppKey; - - return result; - } - - dashboardMigrationToManagedObject(dashboardMigration: ManagedObjectMigration, oldDeviceIdToNew: Map): IManagedObject { - let result = this.managedObjectMigrationToManagedObject(dashboardMigration); - - // Update all of the device/group ids to be the new ones - const idPaths = getIdPathsFromDashboard(result); - idPaths.forEach(path => { - if (_.has(result, path)) { - const oldId = _.get(result, path); - if (oldDeviceIdToNew.has(oldId.toString())) { - _.set(result, path, oldDeviceIdToNew.get(oldId.toString())); - } else { - // TODO: add to warning - } - } - }); - - // Update the c8y_Dashboard!device!1263673 property to point at the new id - const dashboardRegex = /^c8y_Dashboard!(group|device)!(\d+)$/; - const key = Object.keys(result).find(key => dashboardRegex.test(key)); - if (key) { - const match = key.match(dashboardRegex); - if (oldDeviceIdToNew.has(match[2])) { - const value = result[key]; - result = _.omit(result, key) as IManagedObject; - result[`c8y_Dashboard!${match[1]}!${oldDeviceIdToNew.get(match[2])}`] = value; - } else { - // TODO: add to warning - } + getErrorMessage(e) { + if (e instanceof Error) { + return `${e.name || 'Error'}: ${e.message}`; + } else if (e.data && e.data.error) { + return `Error: ${e.data.error} - ${e.data.message}`; } else { - // TODO: warn? - } - - return result; - } - - managedObjectMigrationToManagedObject(managedObjectMigration: ManagedObjectMigration): IManagedObject { - const result: IManagedObject = {} as any; - - function isBlacklistedKey(key) { - if (key.length) { - switch(key[0]) { - case 'id': - case 'lastUpdated': - case 'additionParents': - case 'assetParents': - case 'childAdditions': - case 'childAssets': - case 'childDevices': - case 'c8y_Availability': - case 'c8y_Connection': - case 'c8y_ActiveAlarmsStatus': - case 'externalIds': // we added the externalIds key so remove it - return true; - } - - switch(key[key.length - 1]) { - case 'owner': - case 'self': - return true; - } - } - return false; - } - - const paths = objectScan(['**.*'], {useArraySelector: false, joined: false, breakFn: (key, value) => isBlacklistedKey(key), filterFn: (key, value) => { - if (isBlacklistedKey(key)) { - return false; - } - - // We want to copy the leaf nodes (and will create their paths) so we skip anything that isn't a leaf node - // In other words: only values (string or number) and empty objects or arrays - return _.isString(value) || _.isNumber(value) || _.isNull(value) || Object.keys(value).length === 0; - }})(managedObjectMigration.managedObject); - - paths.forEach(path => { - setFromPath(result, path, _.get(managedObjectMigration.managedObject, path)); - }); - - // Update the managedObject with the user provided changes... - if (result.name != null) { - result.name = managedObjectMigration.newName; + return 'Error: An Unknown Error occurred'; } - - return result; - } - - managedObjectMigrationToManagedObjectLinkages(managedObjectMigration: ManagedObjectMigration, oldDeviceIdToNew: Map): - { - additionParents: string[], - childAdditions: string[], - assetParents: string[], - childAssets: string[], - childDevices: string[], - deviceParents: string[] - } - { - return { - // TODO: add missing to warnings - additionParents: _.flatMap(_.get(managedObjectMigration.managedObject, 'additionParents.references', []), - reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), - childAdditions: _.flatMap(_.get(managedObjectMigration.managedObject, 'childAdditions.references', []), - reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), - assetParents: _.flatMap(_.get(managedObjectMigration.managedObject, 'assetParents.references', []), - reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), - childAssets: _.flatMap(_.get(managedObjectMigration.managedObject, 'childAssets.references', []), - reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), - childDevices: _.flatMap(_.get(managedObjectMigration.managedObject, 'childDevices.references', []), - reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), - deviceParents: _.flatMap(_.get(managedObjectMigration.managedObject, 'deviceParents.references', []), - reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []) - } - } - - smartRuleMigrationToSmartRuleConfig(smartRuleMigration: ManagedObjectMigration, oldDeviceIdToNew: Map): ISmartRuleConfig { - const managedObject = this.managedObjectMigrationToManagedObject(smartRuleMigration); - - const result: ISmartRuleConfig = _.pick(managedObject, ['c8y_Context', 'config', 'enabled', 'enabledSources', 'name', 'ruleTemplateName', 'type']); - - if (result.c8y_Context && result.c8y_Context.id && oldDeviceIdToNew.has(result.c8y_Context.id)) { - result.c8y_Context.id = oldDeviceIdToNew.get(result.c8y_Context.id).toString(); - } - // TODO: warn about missing? - if (_.isArray(result.enabledSources)) { - result.enabledSources = result.enabledSources.map(enabledSource => oldDeviceIdToNew.has(enabledSource) ? oldDeviceIdToNew.get(enabledSource).toString() : enabledSource); - } - - return result; } async changeManagedObjectMigrationUpdateExisting(m: ManagedObjectMigration, existingId: string | undefined) { diff --git a/src/migrate/migration.service.ts b/src/migrate/migration.service.ts new file mode 100644 index 0000000..337ad63 --- /dev/null +++ b/src/migrate/migration.service.ts @@ -0,0 +1,392 @@ +import {DataClient} from "../DataClient"; +import {IApplication, IManagedObject} from "@c8y/client"; +import _ from "lodash"; +import objectScan from "object-scan"; +import {getIdPathsFromDashboard, getSimulatorId, isSimulatorDevice, setFromPath} from "../utils/utils"; +import {IExternalId} from "../c8y-interfaces/IExternalId"; +import {Subject} from "rxjs"; +import {ISmartRuleConfig} from "../c8y-interfaces/ISmartRuleConfig"; + +export interface ApplicationMigration { + newName: string, + newContextPath: string, + newAppKey: string, + application: IApplication & { binary:IManagedObject }, + updateExisting?: IApplication +} + +export interface ManagedObjectMigration { + newName: string, + managedObject: IManagedObject, + updateExisting?: IManagedObject +} + +export enum MigrationLogLevel { + Verbose, + Info, + Error +} + +export class MigrationLogEvent { + public timestamp: string; + constructor(public level: MigrationLogLevel, public description: string, timestamp?: string) { + if (timestamp !== undefined) { + this.timestamp = timestamp + } else { + this.timestamp = new Date().toISOString(); + } + } + + static verbose(description: string): MigrationLogEvent { + return new MigrationLogEvent(MigrationLogLevel.Verbose, description) + } + + static info(description: string): MigrationLogEvent { + return new MigrationLogEvent(MigrationLogLevel.Info, description) + } + + static error(description: string): MigrationLogEvent { + return new MigrationLogEvent(MigrationLogLevel.Error, description) + } +} + +export class Migration { + log$ = new Subject(); + + constructor(private sourceClient: DataClient, private destinationClient: DataClient) {} + + async migrate(deviceMigrations: ManagedObjectMigration[], simulatorMigrations: ManagedObjectMigration[], groupMigrations: ManagedObjectMigration[], otherMigrations: ManagedObjectMigration[], smartRuleMigrations: ManagedObjectMigration[], dashboardMigrations: ManagedObjectMigration[], binaryMigrations: ManagedObjectMigration[], appMigrations: ApplicationMigration[]) { + // Separate the simulated devices from the non-simulated devices + const [simulatorDeviceMigrations, nonSimulatorDeviceMigrations] = _.partition(deviceMigrations, deviceMigration => { + if (!isSimulatorDevice(deviceMigration.managedObject)) return false; + const simulatorId = getSimulatorId(deviceMigration.managedObject as (IManagedObject & {externalIds: IExternalId[]})); + return simulatorMigrations.some(simMigration => simMigration.managedObject.id.toString() === simulatorId); + }); + + const oldIdsToNewIds = new Map(); + + // Migrate the standard managedObjects + this.log$.next(MigrationLogEvent.info("Migrating Groups, Devices, and Other ManagedObjects...")); + const standardMoMigrations = [].concat(groupMigrations, nonSimulatorDeviceMigrations, otherMigrations); + for (let moMigration of standardMoMigrations) { + if (moMigration.updateExisting) { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${moMigration.managedObject.id} - Updating existing managed object: ${moMigration.updateExisting.id}.`)); + const mo = this.managedObjectMigrationToManagedObject(moMigration); + mo.id = moMigration.updateExisting.id; + oldIdsToNewIds.set(moMigration.managedObject.id.toString(), await this.destinationClient.updateManagedObject(mo)) + } else { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${moMigration.managedObject.id} - Creating new managed object.`)); + oldIdsToNewIds.set(moMigration.managedObject.id.toString(), await this.destinationClient.createManagedObject(this.managedObjectMigrationToManagedObject(moMigration))); + } + } + + // Migrate the simulators + this.log$.next(MigrationLogEvent.info("Migrating Simulators...")); + + // First find all of the devices linked to a simulator + const simulatorDeviceMigrationsBySimulatorId = new Map(); + for (let simulatorDeviceMigration of simulatorDeviceMigrations) { + const simulatorDevice = simulatorDeviceMigration.managedObject as (IManagedObject & {externalIds: IExternalId[]}); + const simulatorId = getSimulatorId(simulatorDevice); + if (simulatorId) { + if (!simulatorDeviceMigrationsBySimulatorId.has(simulatorId)) { + simulatorDeviceMigrationsBySimulatorId.set(simulatorId, []); + } + simulatorDeviceMigrationsBySimulatorId.get(simulatorId).push(simulatorDeviceMigration); + } else { + throw Error(`Unable to migrate simulator for device: ${simulatorDevice.id}, cannot find matching simulator definition`); + } + } + + // Then migrate all of those devices and the simulator + for (let [simulatorId, simDeviceMigrations] of simulatorDeviceMigrationsBySimulatorId) { + const simulatorMigration = simulatorMigrations.find(simMigration => simMigration.managedObject.id.toString() === simulatorId); + if (simulatorMigration) { + const simulatorConfig = _.omit(_.cloneDeep(simulatorMigration.managedObject.c8y_DeviceSimulator), 'id'); + simulatorConfig.name = simulatorMigration.newName; + + let newSimulatorId: string|number; + let deviceIds: (string|number)[]; + if (simulatorMigration.updateExisting) { + simulatorConfig.id = simulatorMigration.updateExisting.id; + simulatorConfig.instances = Math.max(simDeviceMigrations.length, simulatorConfig.instances); + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${simulatorMigration.managedObject.id} - Updating existing simulator: ${simulatorMigration.updateExisting.id}, with ${simulatorConfig.instances} instances.`)); + ({simulatorId: newSimulatorId, deviceIds: deviceIds} = await this.destinationClient.updateSimulator(simulatorConfig)); + } else { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${simulatorMigration.managedObject.id} - Creating new simulator, with ${simDeviceMigrations.length} instances.`)); + // Create the simulator (which creates the simulator devices + simulatorConfig.instances = simDeviceMigrations.length; + ({simulatorId: newSimulatorId, deviceIds: deviceIds} = await this.destinationClient.createSimulator(simulatorConfig)); + } + + // Update the simulator devices + for (let [simDeviceMigration, newDeviceId] of _.zip(simDeviceMigrations, deviceIds)) { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${simDeviceMigration.managedObject.id} - Updating simulated device: ${newDeviceId}.`)); + const update = this.managedObjectMigrationToManagedObject(simDeviceMigration); + update.id = newDeviceId.toString(); + await this.destinationClient.updateManagedObject(update); + oldIdsToNewIds.set(simDeviceMigration.managedObject.id.toString(), newDeviceId); + } + + oldIdsToNewIds.set(simulatorId.toString(), newSimulatorId); + } else { + this.log$.next(MigrationLogEvent.error('Unable to find simulator' + simulatorId)); + } + } + + // Migrate the SmartRules + this.log$.next(MigrationLogEvent.info("Migrating Smart Rules...")); + for (let srMigration of smartRuleMigrations) { + if (srMigration.updateExisting) { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${srMigration.managedObject.id} - Updating existing smart rule: ${srMigration.updateExisting.id}.`)); + const srConfig = _.omit(this.smartRuleMigrationToSmartRuleConfig(srMigration, oldIdsToNewIds),'type'); + srConfig.id = srMigration.updateExisting.id; + oldIdsToNewIds.set(srMigration.managedObject.id.toString(), await this.destinationClient.updateSmartRule(srConfig)); + } else { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${srMigration.managedObject.id} - Creating new smart rule.`)); + oldIdsToNewIds.set(srMigration.managedObject.id.toString(), await this.destinationClient.createSmartRule(this.smartRuleMigrationToSmartRuleConfig(srMigration, oldIdsToNewIds))); + } + } + + // Migrate the Binaries + this.log$.next(MigrationLogEvent.info("Migrating Binaries...")); + for (let bMigration of binaryMigrations) { + const blob = await this.sourceClient.getBinaryBlob(bMigration.managedObject); + if (bMigration.updateExisting) { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${bMigration.managedObject.id} - Updating existing binary: ${bMigration.updateExisting.id}.`)); + const mo = this.managedObjectMigrationToManagedObject(bMigration); + mo.id = bMigration.updateExisting.id; + oldIdsToNewIds.set(bMigration.managedObject.id.toString(), await this.destinationClient.updateBinary(mo, blob)); + } else { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${bMigration.managedObject.id} - Creating new binary.`)); + oldIdsToNewIds.set(bMigration.managedObject.id.toString(), await this.destinationClient.createBinary(this.managedObjectMigrationToManagedObject(bMigration), blob)); + } + } + + // Migrate the dashboards + this.log$.next(MigrationLogEvent.info("Migrating Dashboards...")); + for (let dashboardMigration of dashboardMigrations) { + if (dashboardMigration.updateExisting) { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${dashboardMigration.managedObject.id} - Updating existing dashboard: ${dashboardMigration.updateExisting.id}.`)); + const db = this.dashboardMigrationToManagedObject(dashboardMigration, new Map(oldIdsToNewIds)); + db.id = dashboardMigration.updateExisting.id; + oldIdsToNewIds.set(dashboardMigration.managedObject.id.toString(), await this.destinationClient.updateManagedObject(db)); + } else { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${dashboardMigration.managedObject.id} - Creating new dashboard.`)); + oldIdsToNewIds.set(dashboardMigration.managedObject.id.toString(), await this.destinationClient.createManagedObject(this.dashboardMigrationToManagedObject(dashboardMigration, oldIdsToNewIds))); + } + } + + // Create the parent child linkages + this.log$.next(MigrationLogEvent.info("Migrating Parent/Child linkages...")); + for (let moMigration of [...dashboardMigrations, ...groupMigrations, ...deviceMigrations, ...otherMigrations]) { + const newMoId = oldIdsToNewIds.get(moMigration.managedObject.id.toString()); + this.log$.next(MigrationLogEvent.verbose(`Creating Parent/Child linkages for: ${moMigration.managedObject.id} - Updating existing managed object: ${newMoId}.`)); + await this.destinationClient.createLinkages(newMoId.toString(), this.managedObjectMigrationToManagedObjectLinkages(moMigration, oldIdsToNewIds)) + } + + // Migrate the applications + this.log$.next(MigrationLogEvent.info("Migrating Applications...")); + for (let appMigration of appMigrations) { + const blob = await this.sourceClient.getApplicationBlob(appMigration.application); + if (appMigration.updateExisting) { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${appMigration.application.id} - Updating existing application: ${appMigration.updateExisting.id}.`)); + const app = Migration.appMigrationToApp(appMigration, oldIdsToNewIds); + app.id = appMigration.updateExisting.id; + await this.destinationClient.updateApplication(app, blob); + } else { + this.log$.next(MigrationLogEvent.verbose(`Migrating: ${appMigration.application.id} - Creating new application.`)); + await this.destinationClient.createApplication(Migration.appMigrationToApp(appMigration, oldIdsToNewIds), blob); + } + } + + this.log$.next(MigrationLogEvent.info("Done")); + } + + static appMigrationToApp(appMigration: ApplicationMigration, oldIdsToNewIds: Map): IApplication { + const result: IApplication & {applicationBuilder?: any} = {}; + + // Blacklist certain fields + function isBlacklistedKey(key) { + if (key.length) { + switch(key[0]) { + case 'downloading': // we added downloading so remove it + case 'binary': // we added the binary key so remove it + case 'activeVersionId': + return true; + } + + switch(key[key.length - 1]) { + case 'owner': + case 'self': + return true; + } + } + return false; + } + + const paths = objectScan(['**.*'], {useArraySelector: false, joined: false, breakFn: (key, value) => isBlacklistedKey(key), filterFn: (key, value) => { + if (isBlacklistedKey(key)) { + return false; + } + + // We want to copy the leaf nodes (and will create their paths) so we skip anything that isn't a leaf node + // In other words: only values (string or number) and empty objects or arrays + return _.isString(value) || _.isNumber(value) || _.isNull(value) || Object.keys(value).length === 0; + }})(appMigration.application); + + paths.forEach(path => { + _.set(result, path, _.get(appMigration.application, path)); + }); + + if (result.applicationBuilder) { + result.externalUrl = appMigration.application.externalUrl.split(appMigration.application.id.toString()).join('UNKNOWN-APP-ID'); + // Update application builder dashboard ids + if (result.applicationBuilder.dashboards) { + result.applicationBuilder.dashboards.forEach(dashboard => { + if (oldIdsToNewIds.has(dashboard.id.toString())) { + dashboard.id = oldIdsToNewIds.get(dashboard.id.toString()); + } else { + // TODO: add to warning + } + }) + } + } + + // Update the application with the user provided changes... + result.contextPath = appMigration.newContextPath; + result.name = appMigration.newName; + result.key = appMigration.newAppKey; + + return result; + } + + dashboardMigrationToManagedObject(dashboardMigration: ManagedObjectMigration, oldDeviceIdToNew: Map): IManagedObject { + let result = this.managedObjectMigrationToManagedObject(dashboardMigration); + + // Update all of the device/group ids to be the new ones + const idPaths = getIdPathsFromDashboard(result); + idPaths.forEach(path => { + if (_.has(result, path)) { + const oldId = _.get(result, path); + if (oldDeviceIdToNew.has(oldId.toString())) { + _.set(result, path, oldDeviceIdToNew.get(oldId.toString())); + } else { + // TODO: add to warning + } + } + }); + + // Update the c8y_Dashboard!device!1263673 property to point at the new id + const dashboardRegex = /^c8y_Dashboard!(group|device)!(\d+)$/; + const key = Object.keys(result).find(key => dashboardRegex.test(key)); + if (key) { + const match = key.match(dashboardRegex); + if (oldDeviceIdToNew.has(match[2])) { + const value = result[key]; + result = _.omit(result, key) as IManagedObject; + result[`c8y_Dashboard!${match[1]}!${oldDeviceIdToNew.get(match[2])}`] = value; + } else { + // TODO: add to warning + } + } else { + // TODO: warn? + } + + return result; + } + + managedObjectMigrationToManagedObject(managedObjectMigration: ManagedObjectMigration): IManagedObject { + const result: IManagedObject = {} as any; + + function isBlacklistedKey(key) { + if (key.length) { + switch(key[0]) { + case 'id': + case 'lastUpdated': + case 'additionParents': + case 'assetParents': + case 'childAdditions': + case 'childAssets': + case 'childDevices': + case 'c8y_Availability': + case 'c8y_Connection': + case 'c8y_ActiveAlarmsStatus': + case 'externalIds': // we added the externalIds key so remove it + return true; + } + + switch(key[key.length - 1]) { + case 'owner': + case 'self': + return true; + } + } + return false; + } + + const paths = objectScan(['**.*'], {useArraySelector: false, joined: false, breakFn: (key, value) => isBlacklistedKey(key), filterFn: (key, value) => { + if (isBlacklistedKey(key)) { + return false; + } + + // We want to copy the leaf nodes (and will create their paths) so we skip anything that isn't a leaf node + // In other words: only values (string or number) and empty objects or arrays + return _.isString(value) || _.isNumber(value) || _.isNull(value) || Object.keys(value).length === 0; + }})(managedObjectMigration.managedObject); + + paths.forEach(path => { + setFromPath(result, path, _.get(managedObjectMigration.managedObject, path)); + }); + + // Update the managedObject with the user provided changes... + if (result.name != null) { + result.name = managedObjectMigration.newName; + } + + return result; + } + + managedObjectMigrationToManagedObjectLinkages(managedObjectMigration: ManagedObjectMigration, oldDeviceIdToNew: Map): + { + additionParents: string[], + childAdditions: string[], + assetParents: string[], + childAssets: string[], + childDevices: string[], + deviceParents: string[] + } + { + return { + // TODO: add missing to warnings + additionParents: _.flatMap(_.get(managedObjectMigration.managedObject, 'additionParents.references', []), + reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), + childAdditions: _.flatMap(_.get(managedObjectMigration.managedObject, 'childAdditions.references', []), + reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), + assetParents: _.flatMap(_.get(managedObjectMigration.managedObject, 'assetParents.references', []), + reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), + childAssets: _.flatMap(_.get(managedObjectMigration.managedObject, 'childAssets.references', []), + reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), + childDevices: _.flatMap(_.get(managedObjectMigration.managedObject, 'childDevices.references', []), + reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []), + deviceParents: _.flatMap(_.get(managedObjectMigration.managedObject, 'deviceParents.references', []), + reference => oldDeviceIdToNew.has(reference.managedObject.id.toString()) ? oldDeviceIdToNew.get(reference.managedObject.id.toString()) : []) + } + } + + smartRuleMigrationToSmartRuleConfig(smartRuleMigration: ManagedObjectMigration, oldDeviceIdToNew: Map): ISmartRuleConfig { + const managedObject = this.managedObjectMigrationToManagedObject(smartRuleMigration); + + const result: ISmartRuleConfig = _.pick(managedObject, ['c8y_Context', 'config', 'enabled', 'enabledSources', 'name', 'ruleTemplateName', 'type']); + + if (result.c8y_Context && result.c8y_Context.id && oldDeviceIdToNew.has(result.c8y_Context.id)) { + result.c8y_Context.id = oldDeviceIdToNew.get(result.c8y_Context.id).toString(); + } + // TODO: warn about missing? + if (_.isArray(result.enabledSources)) { + result.enabledSources = result.enabledSources.map(enabledSource => oldDeviceIdToNew.has(enabledSource) ? oldDeviceIdToNew.get(enabledSource).toString() : enabledSource); + } + + return result; + } +} \ No newline at end of file