diff --git a/client/src/app/case/services/case.service.ts b/client/src/app/case/services/case.service.ts index 0f6b696542..96efd15f79 100644 --- a/client/src/app/case/services/case.service.ts +++ b/client/src/app/case/services/case.service.ts @@ -28,7 +28,7 @@ class CaseService { db:UserDatabase case:Case caseDefinition:CaseDefinition - + queryCaseEventDefinitionId: any queryEventFormDefinitionId: any queryFormId: any @@ -41,8 +41,8 @@ class CaseService { private deviceService:DeviceService, private userService:UserService, private http:HttpClient - ) { - + ) { + this.queryCaseEventDefinitionId = 'query-event'; this.queryEventFormDefinitionId = 'query-form-event'; this.queryFormId = 'query-form'; @@ -56,7 +56,7 @@ class CaseService { delete this.case._rev const tangyFormContainerEl:any = document.createElement('div') tangyFormContainerEl.innerHTML = await this.tangyFormService.getFormMarkup(this.caseDefinition.formId) - const tangyFormEl = tangyFormContainerEl.querySelector('tangy-form') + const tangyFormEl = tangyFormContainerEl.querySelector('tangy-form') tangyFormEl.style.display = 'none' document.body.appendChild(tangyFormContainerEl) try { @@ -86,7 +86,7 @@ class CaseService { this.case = caseInstance this.caseDefinition = (await this.caseDefinitionsService.load()) .find(caseDefinition => caseDefinition.id === this.case.caseDefinitionId) - + } async load(id:string) { @@ -109,7 +109,7 @@ class CaseService { const caseEventDefinition = this.caseDefinition .eventDefinitions .find(eventDefinition => eventDefinition.id === eventDefinitionId) - const caseEvent = { + const caseEvent = { id: UUID(), caseId: this.case._id, status: CASE_EVENT_STATUS_IN_PROGRESS, @@ -224,7 +224,7 @@ class CaseService { let numberOfUniqueCompleteCaseEvents = this.case .events .reduce((acc, instance) => instance.status === CASE_EVENT_STATUS_COMPLETED - ? Array.from(new Set([...acc, instance.caseEventDefinitionId])) + ? Array.from(new Set([...acc, instance.caseEventDefinitionId])) : acc , []) .length @@ -288,7 +288,7 @@ class CaseService { async getQueries (): Promise> { const userDbName = this.userService.getCurrentUser(); - const queryForms = await this.tangyFormService.getResponsesByFormId(this.queryFormId); + const queryForms = await this.tangyFormService.getResponsesByFormId(this.queryFormId); const queries = Array(); for (const queryForm of queryForms) { const query = Object.create(Query); @@ -395,11 +395,126 @@ class CaseService { ] }, []) for (let formResponseDocId of formResponseDocIds) { - docs.push(await this.tangyFormService.getResponse(formResponseDocId)) + if (formResponseDocId) { + const doc = await this.tangyFormService.getResponse(formResponseDocId) + if (doc) { + docs.push(await this.tangyFormService.getResponse(formResponseDocId)) + } else { + console.log('No response for ' + formResponseDocId); + } + } } return docs } + getRandomInt(min, max) { + return Math.floor(Math.random() * (max - min)) + min; + } + + async getDocCount() { + this.db.db.info().then(info => console.log(info.doc_count)) + } + + async generateCases(numberOfCases) { + let numberOfCasesCompleted = 0 + let firstnames = ['Mary', 'Jennifer', 'Lisa', 'Sandra ','Michelle', + 'Patricia', 'Maria','Nancy','Donna','Laura', 'Linda','Susan','Karen', + 'Carol','Sarah','Barbara','Margaret','Betty','Ruth','Kimberly','Elizabeth', + 'Dorothy','Helen','Sharon','Deborah'] + let surnames = ['Smith','Johnson','Williams','Jones','Brown','Davis','Miller', + 'Wilson','Moore','Taylor','Anderson','Thomas','Jackson','White','Harris', + 'Martin','Thompson','Garcia','Martinez','Robinson','Clark','Rodriguez','Lewis','Lee','Walker'] + + while (numberOfCasesCompleted < numberOfCases) { + const templateDocs = await this.export() + const caseDoc = templateDocs.find(doc => doc['type'] === 'case') + // Change the case's ID. + const caseId = UUID() + caseDoc._id = caseId + const participant_id = Math.round(Math.random() * 1000000) + // let firstname = "Helen" + Math.round(Math.random() * 100) + // let surname = "Smith" + Math.round(Math.random() * 100) + const firstname = firstnames[this.getRandomInt(0, firstnames.length + 1)] + const surname = surnames[this.getRandomInt(0, surnames.length + 1)] + let barcode_data = { "participant_id": participant_id, "treatment_assignment": "Experiment", "bin-mother": "A", "bin-infant": "B", "sub-studies": { "S1": true, "S2": false, "S3": false, "S4": true } } + let tangerineModifiedOn = new Date(); + // tangerineModifiedOn is set to numberOfCasesCompleted days before today, and its time is set based upon numberOfCasesCompleted. + tangerineModifiedOn.setDate( tangerineModifiedOn.getDate() - numberOfCasesCompleted ); + tangerineModifiedOn.setTime( tangerineModifiedOn.getTime() - ( numberOfCases - numberOfCasesCompleted ) ) + const day = String(tangerineModifiedOn.getDate()).padStart(2, '0'); + const month = String(tangerineModifiedOn.getMonth() + 1).padStart(2, '0'); + const year = tangerineModifiedOn.getFullYear(); + const screening_date = year + '-' + month + '-' + day; + const enrollment_date = screening_date; + let caseMother = { + _id: caseId, + tangerineModifiedOn: tangerineModifiedOn, + "participants": [{ + "id": participant_id, + "caseRoleId": "mother-role", + "data": { + "firstname": firstname, + "surname": surname, + "participant_id": participant_id + } + }], + } + const doc = Object.assign({}, caseDoc, caseMother); + caseDoc.items[0].inputs[1].value = participant_id; + caseDoc.items[0].inputs[2].value = enrollment_date; + caseDoc.items[0].inputs[8].value = firstname; + caseDoc.items[0].inputs[10].value = surname; + for (let caseEvent of caseDoc['events']) { + const caseEventId = UUID() + caseEvent.id = caseEventId + for (let eventForm of caseEvent.eventForms) { + eventForm.id = UUID() + eventForm.caseId = caseId + eventForm.caseEventId = caseEventId + // Some eventForms might not have a corresponding form response. + if (eventForm.formResponseId) { + const originalId = `${eventForm.formResponseId}` + const newId = UUID() + // Replace originalId with newId in both the reference to the FormResponse doc and the FormResponse doc itself. + eventForm.formResponseId = newId + const formResponse = templateDocs.find(doc => doc._id === originalId) + if (!formResponse) { + debugger + } + formResponse._id = newId + } + } + } + // modify the demographics form - s01a-participant-information-f254b9 + const demoDoc = templateDocs.find(doc => doc.form.id === 's01a-participant-information-f254b9') + demoDoc.items[0].inputs[4].value = screening_date; + // "id": "randomization", + demoDoc.items[10].inputs[1].value = barcode_data; + demoDoc.items[10].inputs[2].value = participant_id; + demoDoc.items[10].inputs[7].value = enrollment_date; + // "id": "participant_information", + demoDoc.items[12].inputs[2].value = surname; + demoDoc.items[12].inputs[3].value = firstname; + + this.db = await this.userService.getUserDatabase(this.userService.getCurrentUser()) + + for (let doc of templateDocs) { + // @ts-ignore + // sometimes doc is false... + if (doc !== false) { + try { + delete doc._rev + await this.db.put(doc) + } catch (e) { + console.log('Error: ' + e) + } + } + } + numberOfCasesCompleted++ + console.log("motherId: " + caseId + " participantId: " + participant_id + " Completed " + numberOfCasesCompleted + " of " + numberOfCases); + } + } + } export { CaseService }; diff --git a/client/src/app/device/components/device-sync/device-sync.component.ts b/client/src/app/device/components/device-sync/device-sync.component.ts index c0db00b891..6b55e17fd4 100644 --- a/client/src/app/device/components/device-sync/device-sync.component.ts +++ b/client/src/app/device/components/device-sync/device-sync.component.ts @@ -25,7 +25,7 @@ export class DeviceSyncComponent implements OnInit { this.syncInProgress = true this.syncService.syncMessage$.subscribe({ next: (progress) => { - this.syncMessage = progress.docs_written + ' docs saved.' + this.syncMessage = progress.docs_written + ' docs saved; ' + progress.pending + ' pending' console.log('Sync Progress: ' + JSON.stringify(progress)) } }) diff --git a/client/src/app/device/services/device.service.ts b/client/src/app/device/services/device.service.ts index 893c0f00bc..e6da2a1039 100644 --- a/client/src/app/device/services/device.service.ts +++ b/client/src/app/device/services/device.service.ts @@ -5,6 +5,7 @@ import { Device } from './../classes/device.class'; import { AppConfigService } from './../../shared/_services/app-config.service'; import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; +import {AppConfig} from '../../shared/_classes/app-config.class'; const bcrypt = window['dcodeIO'].bcrypt export interface AppInfo { @@ -23,6 +24,8 @@ export class DeviceService { username:string password:string + rawBuildChannel:string + buildId:string constructor( private httpClient:HttpClient, @@ -122,7 +125,8 @@ export class DeviceService { async getBuildId() { try { - return await this.httpClient.get('./assets/tangerine-build-id', {responseType: 'text'}).toPromise() + this.buildId = this.buildId ? this.buildId : await this.httpClient.get('./assets/tangerine-build-id', {responseType: 'text'}).toPromise(); + return this.buildId } catch (e) { return 'N/A' } @@ -130,10 +134,10 @@ export class DeviceService { async getBuildChannel() { try { - const raw = await this.httpClient.get('./assets/tangerine-build-channel', {responseType: 'text'}).toPromise() - return raw.includes('prod') + this.rawBuildChannel = this.rawBuildChannel ? this.rawBuildChannel : await this.httpClient.get('./assets/tangerine-build-channel', {responseType: 'text'}).toPromise() + return this.rawBuildChannel.includes('prod') ? 'live' - : raw.includes('qa') + : this.rawBuildChannel.includes('qa') ? 'test' : 'unknown' } catch (e) { diff --git a/client/src/app/shared/_classes/app-config.class.ts b/client/src/app/shared/_classes/app-config.class.ts index ba563538ad..16cd80c959 100644 --- a/client/src/app/shared/_classes/app-config.class.ts +++ b/client/src/app/shared/_classes/app-config.class.ts @@ -23,5 +23,6 @@ export class AppConfig { p2pSync = 'false' passwordPolicy:string passwordRecipe:string + couchdbSync4All:boolean } diff --git a/client/src/app/sync/components/sync/sync.component.ts b/client/src/app/sync/components/sync/sync.component.ts index cc67e41184..966e64ce7c 100644 --- a/client/src/app/sync/components/sync/sync.component.ts +++ b/client/src/app/sync/components/sync/sync.component.ts @@ -29,7 +29,14 @@ export class SyncComponent implements OnInit { this.status = STATUS_IN_PROGRESS this.syncService.syncMessage$.subscribe({ next: (progress) => { - this.syncMessage = progress.docs_written + ' docs saved.' + let pendingMessage = '' + if (progress.pending) { + pendingMessage = progress.pending + ' pending;' + } + this.syncMessage = progress.docs_written + ' docs saved; ' + pendingMessage + if (progress.direction !== '') { + this.syncMessage = this.syncMessage + ' Direction: ' + progress.direction + } console.log('Sync Progress: ' + JSON.stringify(progress)) } }) diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts index 10ce4033e5..7d4bc10c49 100644 --- a/client/src/app/sync/sync-couchdb.service.ts +++ b/client/src/app/sync/sync-couchdb.service.ts @@ -7,6 +7,8 @@ import { UserDatabase } from './../shared/_classes/user-database.class'; import { Injectable } from '@angular/core'; import PouchDB from 'pouchdb' import {Subject} from 'rxjs'; +import {VariableService} from '../shared/_services/variable.service'; +import {AppConfigService} from '../shared/_services/app-config.service'; export interface LocationQuery { level:string @@ -31,7 +33,9 @@ export class SyncCouchdbService { public readonly syncMessage$: Subject = new Subject(); constructor( - private http:HttpClient + private http: HttpClient, + private variableService: VariableService, + private appConfigService: AppConfigService ) { } async uploadQueue(userDb:UserDatabase, syncDetails:SyncCouchdbDetails) { @@ -51,14 +55,57 @@ export class SyncCouchdbService { async sync(userDb:UserDatabase, syncDetails:SyncCouchdbDetails): Promise { const syncSessionUrl = await this.http.get(`${syncDetails.serverUrl}sync-session/start/${syncDetails.groupId}/${syncDetails.deviceId}/${syncDetails.deviceToken}`, {responseType:'text'}).toPromise() const remoteDb = new PouchDB(syncSessionUrl) - const pouchDbSyncOptions ={ - selector: { - "$or": syncDetails.formInfos.reduce(($or, formInfo) => { - if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) { - $or = [ - ...$or, - ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation - ? syncDetails.deviceSyncLocations.map(locationConfig => { + let pull_last_seq = await this.variableService.get('sync-pull-last_seq') + let push_last_seq = await this.variableService.get('sync-push-last_seq') + if (typeof pull_last_seq === 'undefined') { + pull_last_seq = 0; + } + if (typeof push_last_seq === 'undefined') { + push_last_seq = 0; + } + + const pouchOptions = { + "push": { + "since": push_last_seq, + "batch_size": 50, + "batches_limit": 5, + ...(await this.appConfigService.getAppConfig()).couchdbSync4All ? {} : { "selector": { + "$or" : syncDetails.formInfos.reduce(($or, formInfo) => { + if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.push) { + $or = [ + ...$or, + ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation + ? syncDetails.deviceSyncLocations.map(locationConfig => { + // Get last value, that's the focused sync point. + let location = locationConfig.value.slice(-1).pop() + return { + "form.id": formInfo.id, + [`location.${location.level}`]: location.value + } + }) + : [ + { + "form.id": formInfo.id + } + ] + ] + } + return $or + }, []) + } + } + }, + "pull": { + "since": pull_last_seq, + "batch_size": 50, + "batches_limit": 5, + "selector": { + "$or" : syncDetails.formInfos.reduce(($or, formInfo) => { + if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) { + $or = [ + ...$or, + ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation + ? syncDetails.deviceSyncLocations.map(locationConfig => { // Get last value, that's the focused sync point. let location = locationConfig.value.slice(-1).pop() return { @@ -66,38 +113,50 @@ export class SyncCouchdbService { [`location.${location.level}`]: location.value } }) - : [ + : [ { "form.id": formInfo.id } ] - ] - } - return $or - }, []) + ] + } + return $or + }, []) + } } } + const replicationStatus = await new Promise((resolve, reject) => { - userDb.sync(remoteDb, pouchDbSyncOptions).on('complete', async (info) => { + userDb.sync(remoteDb, pouchOptions).on('complete', async (info) => { + await this.variableService.set('sync-push-last_seq', info.push.last_seq) + await this.variableService.set('sync-pull-last_seq', info.pull.last_seq) const conflictsQuery = await userDb.query('sync-conflicts'); resolve({ pulled: info.pull.docs_written, pushed: info.push.docs_written, conflicts: conflictsQuery.rows.map(row => row.id) }) - }).on('change', (info) => { - const docs_read = info.docs_read - const docs_written = info.docs_written - const doc_write_failures = info.doc_write_failures - // const errors = JSON.stringify(info.errors) + }).on('change', async (info) => { + if (typeof info.direction !== 'undefined') { + if (info.direction === 'push') { + await this.variableService.set('sync-push-last_seq', info.change.last_seq) + } else { + await this.variableService.set('sync-pull-last_seq', info.change.last_seq) + } + } + let pending = info.change.pending + let direction = info.direction + if (typeof info.direction === 'undefined') { + direction = '' + } const progress = { 'docs_read': info.change.docs_read, 'docs_written': info.change.docs_written, - 'doc_write_failures': info.change.doc_write_failures + 'doc_write_failures': info.change.doc_write_failures, + 'pending': info.change.pending, + 'direction': direction }; this.syncMessage$.next(progress) - }).on('paused', function (err) { - console.log('Sync paused; error: ' + JSON.stringify(err)) }).on('error', function (errorMessage) { console.log('boo, something went wrong! error: ' + errorMessage) reject(errorMessage) diff --git a/client/src/app/sync/sync-custom.service.ts b/client/src/app/sync/sync-custom.service.ts index 89f4dc4ec2..cb69ada8f1 100644 --- a/client/src/app/sync/sync-custom.service.ts +++ b/client/src/app/sync/sync-custom.service.ts @@ -28,7 +28,9 @@ export class SyncCustomService { async sync(userDb:UserDatabase, syncDetails:SyncCustomDetails) { const uploadQueue = await this.uploadQueue(userDb, syncDetails.formInfos) - await this.push(userDb, syncDetails, uploadQueue) + if (uploadQueue.length > 0) { + await this.push(userDb, syncDetails, uploadQueue) + } // @TODO pull } @@ -74,7 +76,7 @@ export class SyncCustomService { } } return queryKeys - }, []) + }, []) const response = await userDb.query('sync-queue', { keys: queryKeys }) return response .rows diff --git a/client/src/app/sync/sync.service.ts b/client/src/app/sync/sync.service.ts index ece0d5ad82..053b1e0c2b 100644 --- a/client/src/app/sync/sync.service.ts +++ b/client/src/app/sync/sync.service.ts @@ -60,7 +60,8 @@ export class SyncService { deviceSyncLocations: device.syncLocations, formInfos }) - // console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage)) + console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage)) + await this.syncCustomService.sync(userDb, { appConfig: appConfig, serverUrl: appConfig.serverUrl, diff --git a/client/src/app/tangy-forms/classes/form-info.class.ts b/client/src/app/tangy-forms/classes/form-info.class.ts index 9cc4823c3f..3843449c6b 100644 --- a/client/src/app/tangy-forms/classes/form-info.class.ts +++ b/client/src/app/tangy-forms/classes/form-info.class.ts @@ -26,6 +26,8 @@ export class FormInfo { export interface CouchdbSyncSettings { enabled: boolean + push: boolean + pull: boolean filterByLocation:boolean } diff --git a/config.defaults.sh b/config.defaults.sh index 6c0da27b82..432a59de02 100755 --- a/config.defaults.sh +++ b/config.defaults.sh @@ -94,3 +94,6 @@ T_HIDE_SKIP_IF="true" # In CSV output, set cell value to this when something is skipped. Set to "ORIGINAL_VALUE" if you want the actual value stored. T_REPORTING_MARK_SKIPPED_WITH="SKIPPED" +# Set to true if you want Tangerine to ignore any settings in Sync Configuration and use Couchdb Sync for replication. +T_COUCHDB_SYNC_4_ALL="false" + diff --git a/server/src/scripts/generate-cases/bin.js b/server/src/scripts/generate-cases/bin.js index e1a61da6bf..5d4748b397 100755 --- a/server/src/scripts/generate-cases/bin.js +++ b/server/src/scripts/generate-cases/bin.js @@ -28,7 +28,39 @@ async function go() { const caseDoc = templateDocs.find(doc => doc.type === 'case') // Change the case's ID. const caseId = uuidv1() - caseDoc._id = caseId + caseDoc._id = caseId + const participant_id = Math.round(Math.random() * 1000000) + let firstname = random_name({ first: true, gender: "female" }) + let surname = random_name({ last: true }) + let barcode_data = { "participant_id": participant_id, "treatment_assignment": "Experiment", "bin-mother": "A", "bin-infant": "B", "sub-studies": { "S1": true, "S2": false, "S3": false, "S4": true } } + let tangerineModifiedOn = new Date(); + // tangerineModifiedOn is set to numberOfCasesCompleted days before today, and its time is set based upon numberOfCasesCompleted. + tangerineModifiedOn.setDate( tangerineModifiedOn.getDate() - numberOfCasesCompleted ); + tangerineModifiedOn.setTime( tangerineModifiedOn.getTime() - ( numberOfCases - numberOfCasesCompleted ) ) + const day = String(tangerineModifiedOn.getDate()).padStart(2, '0'); + const month = String(tangerineModifiedOn.getMonth() + 1).padStart(2, '0'); + const year = tangerineModifiedOn.getFullYear(); + const screening_date = year + '-' + month + '-' + day; + const enrollment_date = screening_date; + let caseMother = { + _id: caseId, + tangerineModifiedOn: tangerineModifiedOn, + "participants": [{ + "id": participant_id, + "caseRoleId": "mother-role", + "data": { + "firstname": firstname, + "surname": surname, + "participant_id": participant_id + } + }], + } + console.log("motherId: " + caseId + " participantId: " + participant_id); + doc = Object.assign({}, caseDoc, caseMother); + caseDoc.items[0].inputs[1].value = participant_id; + caseDoc.items[0].inputs[2].value = enrollment_date; + caseDoc.items[0].inputs[8].value = firstname; + caseDoc.items[0].inputs[10].value = surname; for (let caseEvent of caseDoc.events) { const caseEventId = uuidv1() caseEvent.id = caseEventId @@ -50,6 +82,17 @@ async function go() { } } } + // modify the demographics form - s01a-participant-information-f254b9 + const demoDoc = templateDocs.find(doc => doc.form.id === 's01a-participant-information-f254b9') + demoDoc.items[0].inputs[4].value = screening_date; + // "id": "randomization", + demoDoc.items[10].inputs[1].value = barcode_data; + demoDoc.items[10].inputs[2].value = participant_id; + demoDoc.items[10].inputs[7].value = enrollment_date; + // "id": "participant_information", + demoDoc.items[12].inputs[2].value = surname; + demoDoc.items[12].inputs[3].value = firstname; + // Upload the profiles first // now upload the others for (let doc of templateDocs) { diff --git a/server/src/shared/classes/tangerine-config.ts b/server/src/shared/classes/tangerine-config.ts index de255b42e2..158a6e3ea7 100644 --- a/server/src/shared/classes/tangerine-config.ts +++ b/server/src/shared/classes/tangerine-config.ts @@ -11,4 +11,5 @@ export interface TangerineConfig { uploadToken: string reportingDelay: number hideSkipIf: boolean -} \ No newline at end of file + couchdbSync4All: boolean +} diff --git a/server/src/shared/services/tangerine-config/tangerine-config.service.ts b/server/src/shared/services/tangerine-config/tangerine-config.service.ts index 55e06e7d7b..60feae5921 100644 --- a/server/src/shared/services/tangerine-config/tangerine-config.service.ts +++ b/server/src/shared/services/tangerine-config/tangerine-config.service.ts @@ -21,7 +21,8 @@ export class TangerineConfigService { syncUsername: process.env.T_SYNC_USERNAME, syncPassword: process.env.T_SYNC_PASSWORD, hideSkipIf: process.env.T_HIDE_SKIP_IF === 'true' ? true : false, - reportingDelay: parseInt(process.env.T_REPORTING_DELAY) + reportingDelay: parseInt(process.env.T_REPORTING_DELAY), + couchdbSync4All: process.env.T_COUCHDB_SYNC_4_ALL === 'true' ? true : false } } }