From 4efb6e748afccb6f6d5dab9914a797fb891fe4b4 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 23 Mar 2020 15:17:05 -0500
Subject: [PATCH 01/14] Simplified pouchDbSyncOptions

Added pending to sync progress output
---
 .../device-sync/device-sync.component.ts      |  2 +-
 .../sync/components/sync/sync.component.ts    |  2 +-
 client/src/app/sync/sync-couchdb.service.ts   | 58 ++++++++-----------
 3 files changed, 27 insertions(+), 35 deletions(-)

diff --git a/client/src/app/device/components/device-sync/device-sync.component.ts b/client/src/app/device/components/device-sync/device-sync.component.ts
index c0db00b891..6b55e17fd4 100644
--- a/client/src/app/device/components/device-sync/device-sync.component.ts
+++ b/client/src/app/device/components/device-sync/device-sync.component.ts
@@ -25,7 +25,7 @@ export class DeviceSyncComponent implements OnInit {
     this.syncInProgress = true
     this.syncService.syncMessage$.subscribe({
       next: (progress) => {
-        this.syncMessage =  progress.docs_written + ' docs saved.'
+        this.syncMessage =  progress.docs_written + ' docs saved; ' + progress.pending + ' pending'
         console.log('Sync Progress: ' + JSON.stringify(progress))
       }
     })
diff --git a/client/src/app/sync/components/sync/sync.component.ts b/client/src/app/sync/components/sync/sync.component.ts
index cc67e41184..f307a4b2d3 100644
--- a/client/src/app/sync/components/sync/sync.component.ts
+++ b/client/src/app/sync/components/sync/sync.component.ts
@@ -29,7 +29,7 @@ export class SyncComponent implements OnInit {
     this.status = STATUS_IN_PROGRESS
     this.syncService.syncMessage$.subscribe({
       next: (progress) => {
-        this.syncMessage =  progress.docs_written + ' docs saved.'
+        this.syncMessage =  progress.docs_written + ' docs saved; ' + progress.pending + ' pending'
         console.log('Sync Progress: ' + JSON.stringify(progress))
       }
     })
diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index 10ce4033e5..b6ec119e39 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -7,6 +7,7 @@ import { UserDatabase } from './../shared/_classes/user-database.class';
 import { Injectable } from '@angular/core';
 import PouchDB from 'pouchdb'
 import {Subject} from 'rxjs';
+import {VariableService} from '../shared/_services/variable.service';
 
 export interface LocationQuery {
   level:string
@@ -31,7 +32,8 @@ export class SyncCouchdbService {
   public readonly syncMessage$: Subject<any> = new Subject();
 
   constructor(
-    private http:HttpClient
+    private http: HttpClient,
+    private variableService: VariableService
   ) { }
 
   async uploadQueue(userDb:UserDatabase, syncDetails:SyncCouchdbDetails) {
@@ -51,53 +53,43 @@ export class SyncCouchdbService {
   async sync(userDb:UserDatabase, syncDetails:SyncCouchdbDetails): Promise<ReplicationStatus> {
     const syncSessionUrl = await this.http.get(`${syncDetails.serverUrl}sync-session/start/${syncDetails.groupId}/${syncDetails.deviceId}/${syncDetails.deviceToken}`, {responseType:'text'}).toPromise()
     const remoteDb = new PouchDB(syncSessionUrl)
-    const pouchDbSyncOptions ={
-      selector: {
-        "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
-          if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) {
-            $or = [
-              ...$or,
-              ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
-                ? syncDetails.deviceSyncLocations.map(locationConfig => {
-                    // Get last value, that's the focused sync point.
-                    let location = locationConfig.value.slice(-1).pop()
-                    return {
-                      "form.id": formInfo.id,
-                      [`location.${location.level}`]: location.value
-                    }
-                  })
-                : [
-                    {
-                      "form.id": formInfo.id
-                    }
-                  ]
-            ]
+    let last_seq = await this.variableService.get('sync-checkpoint')
+    if (typeof last_seq === 'undefined') {
+      last_seq = 0;
+    }
+    const pouchDbSyncOptions = {
+      "since": last_seq
+    }
+
+    if (syncDetails.deviceSyncLocations.length > 0) {
+      const locationConfig = syncDetails.deviceSyncLocations[0]
+      const location = locationConfig.value.slice(-1).pop()
+      const selector = {
+          [`location.${location.level}`]: {
+            '$eq' : location.value
           }
-          return $or
-        }, [])
-      }
+        }
+      pouchDbSyncOptions['selector'] = selector
     }
+
     const replicationStatus = <ReplicationStatus>await new Promise((resolve, reject) => {
       userDb.sync(remoteDb, pouchDbSyncOptions).on('complete', async  (info) => {
+        await this.variableService.set('sync-checkpoint', info.pull.last_seq)
         const conflictsQuery = await userDb.query('sync-conflicts');
         resolve(<ReplicationStatus>{
           pulled: info.pull.docs_written,
           pushed: info.push.docs_written,
           conflicts: conflictsQuery.rows.map(row => row.id)
         })
-      }).on('change', (info) => {
-        const docs_read = info.docs_read
-        const docs_written = info.docs_written
-        const doc_write_failures = info.doc_write_failures
-        // const errors = JSON.stringify(info.errors)
+      }).on('change', async (info) => {
+        await this.variableService.set('sync-checkpoint', info.change.last_seq)
         const progress = {
           'docs_read': info.change.docs_read,
           'docs_written': info.change.docs_written,
-          'doc_write_failures': info.change.doc_write_failures
+          'doc_write_failures': info.change.doc_write_failures,
+          'pending': info.change.pending
         };
         this.syncMessage$.next(progress)
-      }).on('paused', function (err) {
-        console.log('Sync paused; error: ' + JSON.stringify(err))
       }).on('error', function (errorMessage) {
         console.log('boo, something went wrong! error: ' + errorMessage)
         reject(errorMessage)

From ea9a64a793c50a49f02949f61a684873d6e98a7f Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 23 Mar 2020 23:36:05 -0500
Subject: [PATCH 02/14] Random id's and names for generate-cases

Different options for push and pull sync.
Hard coded localPouchOptions since to 0.
---
 client/src/app/sync/sync-couchdb.service.ts | 17 ++++++--
 client/src/app/sync/sync.service.ts         | 16 ++++----
 server/src/scripts/generate-cases/bin.js    | 45 ++++++++++++++++++++-
 3 files changed, 65 insertions(+), 13 deletions(-)

diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index b6ec119e39..4dbe8b2ca7 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -49,7 +49,7 @@ export class SyncCouchdbService {
       .map(row => row.id)
   }
 
-  // Note that if you run this with no forms configured to CouchDB sync, that will result in no filter query and everything will be synced. Use carefully.
+  // Note that this ignores the sync config settings.
   async sync(userDb:UserDatabase, syncDetails:SyncCouchdbDetails): Promise<ReplicationStatus> {
     const syncSessionUrl = await this.http.get(`${syncDetails.serverUrl}sync-session/start/${syncDetails.groupId}/${syncDetails.deviceId}/${syncDetails.deviceToken}`, {responseType:'text'}).toPromise()
     const remoteDb = new PouchDB(syncSessionUrl)
@@ -57,7 +57,7 @@ export class SyncCouchdbService {
     if (typeof last_seq === 'undefined') {
       last_seq = 0;
     }
-    const pouchDbSyncOptions = {
+    const remotePouchOptions = {
       "since": last_seq
     }
 
@@ -69,11 +69,20 @@ export class SyncCouchdbService {
             '$eq' : location.value
           }
         }
-      pouchDbSyncOptions['selector'] = selector
+      remotePouchOptions['selector'] = selector
+    }
+
+    const localPouchOptions = {
+      "since": 0
+    }
+
+    const pouchOptions = {
+      "push": localPouchOptions,
+      "pull": remotePouchOptions
     }
 
     const replicationStatus = <ReplicationStatus>await new Promise((resolve, reject) => {
-      userDb.sync(remoteDb, pouchDbSyncOptions).on('complete', async  (info) => {
+      userDb.sync(remoteDb, pouchOptions).on('complete', async  (info) => {
         await this.variableService.set('sync-checkpoint', info.pull.last_seq)
         const conflictsQuery = await userDb.query('sync-conflicts');
         resolve(<ReplicationStatus>{
diff --git a/client/src/app/sync/sync.service.ts b/client/src/app/sync/sync.service.ts
index ece0d5ad82..da230531da 100644
--- a/client/src/app/sync/sync.service.ts
+++ b/client/src/app/sync/sync.service.ts
@@ -61,14 +61,14 @@ export class SyncService {
       formInfos
     })
     // console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage))
-    await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
-      appConfig: appConfig,
-      serverUrl: appConfig.serverUrl,
-      groupId: appConfig.groupId,
-      deviceId: device._id,
-      deviceToken: device.token,
-      formInfos
-    })
+    // await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
+    //   appConfig: appConfig,
+    //   serverUrl: appConfig.serverUrl,
+    //   groupId: appConfig.groupId,
+    //   deviceId: device._id,
+    //   deviceToken: device.token,
+    //   formInfos
+    // })
     await this.deviceService.didSync()
   }
 
diff --git a/server/src/scripts/generate-cases/bin.js b/server/src/scripts/generate-cases/bin.js
index e1a61da6bf..5d4748b397 100755
--- a/server/src/scripts/generate-cases/bin.js
+++ b/server/src/scripts/generate-cases/bin.js
@@ -28,7 +28,39 @@ async function go() {
     const caseDoc = templateDocs.find(doc => doc.type === 'case')
     // Change the case's ID.
     const caseId = uuidv1()
-    caseDoc._id = caseId 
+    caseDoc._id = caseId
+    const participant_id = Math.round(Math.random() * 1000000)
+    let firstname = random_name({ first: true, gender: "female" })
+    let surname = random_name({ last: true })
+    let barcode_data =  { "participant_id": participant_id, "treatment_assignment": "Experiment", "bin-mother": "A", "bin-infant": "B", "sub-studies": { "S1": true, "S2": false, "S3": false, "S4": true } }
+    let tangerineModifiedOn = new Date();
+    // tangerineModifiedOn is set to numberOfCasesCompleted days before today, and its time is set based upon numberOfCasesCompleted.
+    tangerineModifiedOn.setDate( tangerineModifiedOn.getDate() - numberOfCasesCompleted );
+    tangerineModifiedOn.setTime( tangerineModifiedOn.getTime() - ( numberOfCases - numberOfCasesCompleted ) )
+    const day = String(tangerineModifiedOn.getDate()).padStart(2, '0');
+    const month = String(tangerineModifiedOn.getMonth() + 1).padStart(2, '0');
+    const year = tangerineModifiedOn.getFullYear();
+    const screening_date = year + '-' + month + '-' + day;
+    const enrollment_date = screening_date;
+    let caseMother = {
+      _id: caseId,
+      tangerineModifiedOn: tangerineModifiedOn,
+      "participants": [{
+        "id": participant_id,
+        "caseRoleId": "mother-role",
+        "data": {
+          "firstname": firstname,
+          "surname": surname,
+          "participant_id": participant_id
+        }
+      }],
+    }
+    console.log("motherId: " + caseId + " participantId: " + participant_id);
+    doc = Object.assign({}, caseDoc, caseMother);
+    caseDoc.items[0].inputs[1].value = participant_id;
+    caseDoc.items[0].inputs[2].value = enrollment_date;
+    caseDoc.items[0].inputs[8].value = firstname;
+    caseDoc.items[0].inputs[10].value = surname;
     for (let caseEvent of caseDoc.events) {
       const caseEventId = uuidv1()
       caseEvent.id = caseEventId
@@ -50,6 +82,17 @@ async function go() {
         }
       }
     }
+    // modify the demographics form - s01a-participant-information-f254b9
+    const demoDoc = templateDocs.find(doc => doc.form.id === 's01a-participant-information-f254b9')
+    demoDoc.items[0].inputs[4].value = screening_date;
+    // "id": "randomization",
+    demoDoc.items[10].inputs[1].value = barcode_data;
+    demoDoc.items[10].inputs[2].value = participant_id;
+    demoDoc.items[10].inputs[7].value = enrollment_date;
+    // "id": "participant_information",
+    demoDoc.items[12].inputs[2].value = surname;
+    demoDoc.items[12].inputs[3].value = firstname;
+
     // Upload the profiles first
     // now upload the others
     for (let doc of templateDocs) {

From 2d97a20fd71083e08dc1e68424cea84d35a66630 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Tue, 24 Mar 2020 19:17:10 -0500
Subject: [PATCH 03/14] Added T_COUCHDB_SYNC_4_ALL variable

guardrail for sync-protocol-2 deployments.
Storing different last_seq for push/pull in sync.
Re-instated the option for using $or and a big list of forms.
---
 .../app/shared/_classes/app-config.class.ts   |  1 +
 client/src/app/sync/sync-couchdb.service.ts   | 81 +++++++++++++++----
 config.defaults.sh                            |  3 +
 server/src/shared/classes/tangerine-config.ts |  3 +-
 .../tangerine-config.service.ts               |  3 +-
 5 files changed, 75 insertions(+), 16 deletions(-)

diff --git a/client/src/app/shared/_classes/app-config.class.ts b/client/src/app/shared/_classes/app-config.class.ts
index ba563538ad..16cd80c959 100644
--- a/client/src/app/shared/_classes/app-config.class.ts
+++ b/client/src/app/shared/_classes/app-config.class.ts
@@ -23,5 +23,6 @@ export class AppConfig {
   p2pSync = 'false'
   passwordPolicy:string
   passwordRecipe:string
+  couchdbSync4All:boolean
 }
 
diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index 4dbe8b2ca7..04c8e1dc4a 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -8,6 +8,7 @@ import { Injectable } from '@angular/core';
 import PouchDB from 'pouchdb'
 import {Subject} from 'rxjs';
 import {VariableService} from '../shared/_services/variable.service';
+import {AppConfigService} from '../shared/_services/app-config.service';
 
 export interface LocationQuery {
   level:string
@@ -33,7 +34,8 @@ export class SyncCouchdbService {
 
   constructor(
     private http: HttpClient,
-    private variableService: VariableService
+    private variableService: VariableService,
+    private appConfigService: AppConfigService
   ) { }
 
   async uploadQueue(userDb:UserDatabase, syncDetails:SyncCouchdbDetails) {
@@ -53,16 +55,25 @@ export class SyncCouchdbService {
   async sync(userDb:UserDatabase, syncDetails:SyncCouchdbDetails): Promise<ReplicationStatus> {
     const syncSessionUrl = await this.http.get(`${syncDetails.serverUrl}sync-session/start/${syncDetails.groupId}/${syncDetails.deviceId}/${syncDetails.deviceToken}`, {responseType:'text'}).toPromise()
     const remoteDb = new PouchDB(syncSessionUrl)
-    let last_seq = await this.variableService.get('sync-checkpoint')
-    if (typeof last_seq === 'undefined') {
-      last_seq = 0;
+    let pull_last_seq = await this.variableService.get('sync-pull-last_seq')
+    let push_last_seq = await this.variableService.get('sync-push-last_seq')
+    if (typeof pull_last_seq === 'undefined') {
+      pull_last_seq = 0;
+    }
+    if (typeof push_last_seq === 'undefined') {
+      push_last_seq = 0;
     }
     const remotePouchOptions = {
-      "since": last_seq
+      "since": pull_last_seq
+    }
+
+    const localPouchOptions = {
+      "since": push_last_seq
     }
 
     if (syncDetails.deviceSyncLocations.length > 0) {
       const locationConfig = syncDetails.deviceSyncLocations[0]
+      // Get last value, that's the focused sync point.
       const location = locationConfig.value.slice(-1).pop()
       const selector = {
           [`location.${location.level}`]: {
@@ -72,18 +83,50 @@ export class SyncCouchdbService {
       remotePouchOptions['selector'] = selector
     }
 
-    const localPouchOptions = {
-      "since": 0
-    }
+    let pouchOptions;
+
+    const appConfig = await this.appConfigService.getAppConfig();
 
-    const pouchOptions = {
-      "push": localPouchOptions,
-      "pull": remotePouchOptions
+    if (appConfig.couchdbSync4All) {
+      // Passing false prevents the changes feed from keeping all the documents in memory
+      pouchOptions = {
+        "return_docs": false,
+        "push": localPouchOptions,
+        "pull": remotePouchOptions
+      }
+    } else {
+      pouchOptions = {
+        selector: {
+          "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
+            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) {
+              $or = [
+                ...$or,
+                ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
+                  ? syncDetails.deviceSyncLocations.map(locationConfig => {
+                    // Get last value, that's the focused sync point.
+                    let location = locationConfig.value.slice(-1).pop()
+                    return {
+                      "form.id": formInfo.id,
+                      [`location.${location.level}`]: location.value
+                    }
+                  })
+                  : [
+                    {
+                      "form.id": formInfo.id
+                    }
+                  ]
+              ]
+            }
+            return $or
+          }, [])
+        }
+      }
     }
 
     const replicationStatus = <ReplicationStatus>await new Promise((resolve, reject) => {
       userDb.sync(remoteDb, pouchOptions).on('complete', async  (info) => {
-        await this.variableService.set('sync-checkpoint', info.pull.last_seq)
+        await this.variableService.set('sync-push-last_seq', info.push.last_seq)
+        await this.variableService.set('sync-pull-last_seq', info.pull.last_seq)
         const conflictsQuery = await userDb.query('sync-conflicts');
         resolve(<ReplicationStatus>{
           pulled: info.pull.docs_written,
@@ -91,12 +134,22 @@ export class SyncCouchdbService {
           conflicts: conflictsQuery.rows.map(row => row.id)
         })
       }).on('change', async (info) => {
-        await this.variableService.set('sync-checkpoint', info.change.last_seq)
+        if (typeof info.direction !== 'undefined') {
+          if (info.direction === 'push') {
+            await this.variableService.set('sync-push-last_seq', info.change.last_seq)
+          } else {
+            await this.variableService.set('sync-pull-last_seq', info.change.last_seq)
+          }
+        }
+        let pending = info.change.pending
+        if (typeof info.change.pending === 'undefined') {
+          pending = 0;
+        }
         const progress = {
           'docs_read': info.change.docs_read,
           'docs_written': info.change.docs_written,
           'doc_write_failures': info.change.doc_write_failures,
-          'pending': info.change.pending
+          'pending': pending
         };
         this.syncMessage$.next(progress)
       }).on('error', function (errorMessage) {
diff --git a/config.defaults.sh b/config.defaults.sh
index 6c0da27b82..432a59de02 100755
--- a/config.defaults.sh
+++ b/config.defaults.sh
@@ -94,3 +94,6 @@ T_HIDE_SKIP_IF="true"
 # In CSV output, set cell value to this when something is skipped. Set to "ORIGINAL_VALUE" if you want the actual value stored.
 T_REPORTING_MARK_SKIPPED_WITH="SKIPPED"
 
+# Set to true if you want Tangerine to ignore any settings in Sync Configuration and use Couchdb Sync for replication.
+T_COUCHDB_SYNC_4_ALL="false"
+
diff --git a/server/src/shared/classes/tangerine-config.ts b/server/src/shared/classes/tangerine-config.ts
index de255b42e2..158a6e3ea7 100644
--- a/server/src/shared/classes/tangerine-config.ts
+++ b/server/src/shared/classes/tangerine-config.ts
@@ -11,4 +11,5 @@ export interface TangerineConfig {
   uploadToken: string
   reportingDelay: number
   hideSkipIf: boolean
-}
\ No newline at end of file
+  couchdbSync4All: boolean
+}
diff --git a/server/src/shared/services/tangerine-config/tangerine-config.service.ts b/server/src/shared/services/tangerine-config/tangerine-config.service.ts
index 55e06e7d7b..60feae5921 100644
--- a/server/src/shared/services/tangerine-config/tangerine-config.service.ts
+++ b/server/src/shared/services/tangerine-config/tangerine-config.service.ts
@@ -21,7 +21,8 @@ export class TangerineConfigService {
       syncUsername: process.env.T_SYNC_USERNAME,
       syncPassword: process.env.T_SYNC_PASSWORD,
       hideSkipIf: process.env.T_HIDE_SKIP_IF === 'true' ? true : false,
-      reportingDelay: parseInt(process.env.T_REPORTING_DELAY)
+      reportingDelay: parseInt(process.env.T_REPORTING_DELAY),
+      couchdbSync4All: process.env.T_COUCHDB_SYNC_4_ALL === 'true' ? true : false
     }
   }
 }

From 61740c72174374a89fc3ea657de71fed5c01e326 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Wed, 25 Mar 2020 13:16:11 -0500
Subject: [PATCH 04/14] Added couchdbSync4All check

---
 client/src/app/sync/sync.service.ts | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/client/src/app/sync/sync.service.ts b/client/src/app/sync/sync.service.ts
index da230531da..060c7740a7 100644
--- a/client/src/app/sync/sync.service.ts
+++ b/client/src/app/sync/sync.service.ts
@@ -60,15 +60,18 @@ export class SyncService {
       deviceSyncLocations: device.syncLocations,
       formInfos
     })
-    // console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage))
-    // await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
-    //   appConfig: appConfig,
-    //   serverUrl: appConfig.serverUrl,
-    //   groupId: appConfig.groupId,
-    //   deviceId: device._id,
-    //   deviceToken: device.token,
-    //   formInfos
-    // })
+    console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage))
+
+    if (!appConfig.couchdbSync4All) {
+      await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
+        appConfig: appConfig,
+        serverUrl: appConfig.serverUrl,
+        groupId: appConfig.groupId,
+        deviceId: device._id,
+        deviceToken: device.token,
+        formInfos
+      })
+    }
     await this.deviceService.didSync()
   }
 

From 6a4c1fcc424e45e6a48c6b0a120e016ca017f8ca Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 30 Mar 2020 08:35:50 -0500
Subject: [PATCH 05/14] Implemented generateCases in CaseService

Added sync direction
Separate push/pull pouchdb sync options
---
 client/src/app/case/services/case.service.ts  | 135 ++++++++++++++++--
 .../sync/components/sync/sync.component.ts    |   5 +-
 client/src/app/sync/sync-couchdb.service.ts   |  60 ++++++--
 .../tangy-forms/classes/form-info.class.ts    |   1 +
 4 files changed, 181 insertions(+), 20 deletions(-)

diff --git a/client/src/app/case/services/case.service.ts b/client/src/app/case/services/case.service.ts
index 0f6b696542..45cc3e2e87 100644
--- a/client/src/app/case/services/case.service.ts
+++ b/client/src/app/case/services/case.service.ts
@@ -28,7 +28,7 @@ class CaseService {
   db:UserDatabase
   case:Case
   caseDefinition:CaseDefinition
-  
+
   queryCaseEventDefinitionId: any
   queryEventFormDefinitionId: any
   queryFormId: any
@@ -41,8 +41,8 @@ class CaseService {
     private deviceService:DeviceService,
     private userService:UserService,
     private http:HttpClient
-  ) { 
-    
+  ) {
+
     this.queryCaseEventDefinitionId = 'query-event';
     this.queryEventFormDefinitionId = 'query-form-event';
     this.queryFormId = 'query-form';
@@ -56,7 +56,7 @@ class CaseService {
     delete this.case._rev
     const tangyFormContainerEl:any = document.createElement('div')
     tangyFormContainerEl.innerHTML = await this.tangyFormService.getFormMarkup(this.caseDefinition.formId)
-    const tangyFormEl = tangyFormContainerEl.querySelector('tangy-form') 
+    const tangyFormEl = tangyFormContainerEl.querySelector('tangy-form')
     tangyFormEl.style.display = 'none'
     document.body.appendChild(tangyFormContainerEl)
     try {
@@ -86,7 +86,7 @@ class CaseService {
     this.case = caseInstance
     this.caseDefinition = (await this.caseDefinitionsService.load())
       .find(caseDefinition => caseDefinition.id === this.case.caseDefinitionId)
- 
+
   }
 
   async load(id:string) {
@@ -109,7 +109,7 @@ class CaseService {
     const caseEventDefinition = this.caseDefinition
       .eventDefinitions
       .find(eventDefinition => eventDefinition.id === eventDefinitionId)
-    const caseEvent = <CaseEvent>{ 
+    const caseEvent = <CaseEvent>{
       id: UUID(),
       caseId: this.case._id,
       status: CASE_EVENT_STATUS_IN_PROGRESS,
@@ -224,7 +224,7 @@ class CaseService {
     let numberOfUniqueCompleteCaseEvents = this.case
       .events
       .reduce((acc, instance) => instance.status === CASE_EVENT_STATUS_COMPLETED
-          ? Array.from(new Set([...acc, instance.caseEventDefinitionId])) 
+          ? Array.from(new Set([...acc, instance.caseEventDefinitionId]))
           : acc
         , [])
         .length
@@ -288,7 +288,7 @@ class CaseService {
 
   async getQueries (): Promise<Array<Query>> {
     const userDbName = this.userService.getCurrentUser();
-    const queryForms = await this.tangyFormService.getResponsesByFormId(this.queryFormId);  
+    const queryForms = await this.tangyFormService.getResponsesByFormId(this.queryFormId);
     const queries = Array<Query>();
     for (const queryForm of queryForms) {
       const query = Object.create(Query);
@@ -395,11 +395,128 @@ class CaseService {
       ]
     }, [])
     for (let formResponseDocId of formResponseDocIds) {
-      docs.push(await this.tangyFormService.getResponse(formResponseDocId))
+      if (formResponseDocId) {
+        const doc = await this.tangyFormService.getResponse(formResponseDocId)
+        if (doc) {
+          docs.push(await this.tangyFormService.getResponse(formResponseDocId))
+        } else {
+          console.log('No response for ' + formResponseDocId);
+        }
+      }
     }
     return docs
   }
 
+  getRandomInt(min, max) {
+    return Math.floor(Math.random() * (max - min)) + min;
+  }
+
+  async getDocCount() {
+    this.db.db.info().then(info => console.log(info.doc_count))
+  }
+
+  async generateCases(numberOfCases) {
+    let numberOfCasesCompleted = 0
+    let firstnames = ['Mary', 'Jennifer', 'Lisa', 'Sandra	','Michelle',
+    'Patricia', 'Maria','Nancy','Donna','Laura', 'Linda','Susan','Karen',
+      'Carol','Sarah','Barbara','Margaret','Betty','Ruth','Kimberly','Elizabeth',
+      'Dorothy','Helen','Sharon','Deborah']
+    let surnames = ['Smith','Johnson','Williams','Jones','Brown','Davis','Miller',
+      'Wilson','Moore','Taylor','Anderson','Thomas','Jackson','White','Harris',
+      'Martin','Thompson','Garcia','Martinez','Robinson','Clark','Rodriguez','Lewis','Lee','Walker']
+
+    while (numberOfCasesCompleted < numberOfCases) {
+      const templateDocs = await this.export()
+      const caseDoc = templateDocs.find(doc => doc['type'] === 'case')
+      // Change the case's ID.
+      const caseId = UUID()
+      caseDoc._id = caseId
+      const participant_id = Math.round(Math.random() * 1000000)
+      // let firstname = "Helen" + Math.round(Math.random() * 100)
+      // let surname = "Smith" + Math.round(Math.random() * 100)
+      const firstname = firstnames[this.getRandomInt(0, firstnames.length + 1)]
+      const surname = surnames[this.getRandomInt(0, surnames.length + 1)]
+      let barcode_data =  { "participant_id": participant_id, "treatment_assignment": "Experiment", "bin-mother": "A", "bin-infant": "B", "sub-studies": { "S1": true, "S2": false, "S3": false, "S4": true } }
+      let tangerineModifiedOn = new Date();
+      // tangerineModifiedOn is set to numberOfCasesCompleted days before today, and its time is set based upon numberOfCasesCompleted.
+      tangerineModifiedOn.setDate( tangerineModifiedOn.getDate() - numberOfCasesCompleted );
+      tangerineModifiedOn.setTime( tangerineModifiedOn.getTime() - ( numberOfCases - numberOfCasesCompleted ) )
+      const day = String(tangerineModifiedOn.getDate()).padStart(2, '0');
+      const month = String(tangerineModifiedOn.getMonth() + 1).padStart(2, '0');
+      const year = tangerineModifiedOn.getFullYear();
+      const screening_date = year + '-' + month + '-' + day;
+      const enrollment_date = screening_date;
+      let caseMother = {
+        _id: caseId,
+        tangerineModifiedOn: tangerineModifiedOn,
+        "participants": [{
+          "id": participant_id,
+          "caseRoleId": "mother-role",
+          "data": {
+            "firstname": firstname,
+            "surname": surname,
+            "participant_id": participant_id
+          }
+        }],
+      }
+      console.log("motherId: " + caseId + " participantId: " + participant_id);
+      const doc = Object.assign({}, caseDoc, caseMother);
+      caseDoc.items[0].inputs[1].value = participant_id;
+      caseDoc.items[0].inputs[2].value = enrollment_date;
+      caseDoc.items[0].inputs[8].value = firstname;
+      caseDoc.items[0].inputs[10].value = surname;
+      for (let caseEvent of caseDoc['events']) {
+        const caseEventId = UUID()
+        caseEvent.id = caseEventId
+        for (let eventForm of caseEvent.eventForms) {
+          eventForm.id = UUID()
+          eventForm.caseId = caseId
+          eventForm.caseEventId = caseEventId
+          // Some eventForms might not have a corresponding form response.
+          if (eventForm.formResponseId) {
+            const originalId = `${eventForm.formResponseId}`
+            const newId = UUID()
+            // Replace originalId with newId in both the reference to the FormResponse doc and the FormResponse doc itself.
+            eventForm.formResponseId = newId
+            const formResponse = templateDocs.find(doc => doc._id === originalId)
+            if (!formResponse) {
+              debugger
+            }
+            formResponse._id = newId
+          }
+        }
+      }
+      // modify the demographics form - s01a-participant-information-f254b9
+      const demoDoc = templateDocs.find(doc => doc.form.id === 's01a-participant-information-f254b9')
+      demoDoc.items[0].inputs[4].value = screening_date;
+      // "id": "randomization",
+      demoDoc.items[10].inputs[1].value = barcode_data;
+      demoDoc.items[10].inputs[2].value = participant_id;
+      demoDoc.items[10].inputs[7].value = enrollment_date;
+      // "id": "participant_information",
+      demoDoc.items[12].inputs[2].value = surname;
+      demoDoc.items[12].inputs[3].value = firstname;
+
+      this.db = await this.userService.getUserDatabase(this.userService.getCurrentUser())
+
+      // Upload the profiles first
+      // now upload the others
+      for (let doc of templateDocs) {
+        // @ts-ignore
+        // sometimes doc is false...
+        if (doc !== false) {
+          try {
+            delete doc._rev
+            await this.db.put(doc)
+          } catch (e) {
+            console.log('Error: ' + e)
+          }
+        }
+      }
+      numberOfCasesCompleted++
+    }
+  }
+
 }
 
 export { CaseService };
diff --git a/client/src/app/sync/components/sync/sync.component.ts b/client/src/app/sync/components/sync/sync.component.ts
index f307a4b2d3..1499c5af32 100644
--- a/client/src/app/sync/components/sync/sync.component.ts
+++ b/client/src/app/sync/components/sync/sync.component.ts
@@ -29,7 +29,10 @@ export class SyncComponent implements OnInit {
     this.status = STATUS_IN_PROGRESS
     this.syncService.syncMessage$.subscribe({
       next: (progress) => {
-        this.syncMessage =  progress.docs_written + ' docs saved; ' + progress.pending + ' pending'
+        this.syncMessage =  progress.docs_written + ' docs saved; ' + progress.pending + ' pending;'
+        if (progress.direction !== '') {
+          this.syncMessage = this.syncMessage + '; Direction: ' + progress.direction
+        }
         console.log('Sync Progress: ' + JSON.stringify(progress))
       }
     })
diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index 04c8e1dc4a..2b5d61af81 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -67,10 +67,6 @@ export class SyncCouchdbService {
       "since": pull_last_seq
     }
 
-    const localPouchOptions = {
-      "since": push_last_seq
-    }
-
     if (syncDetails.deviceSyncLocations.length > 0) {
       const locationConfig = syncDetails.deviceSyncLocations[0]
       // Get last value, that's the focused sync point.
@@ -84,18 +80,57 @@ export class SyncCouchdbService {
     }
 
     let pouchOptions;
-
     const appConfig = await this.appConfigService.getAppConfig();
-
+    // TODO: replace with test for sync-protocol-2 instead ?
+    // actually, don't even need it - we are already sp2!
     if (appConfig.couchdbSync4All) {
       // Passing false prevents the changes feed from keeping all the documents in memory
+      // pouchOptions = {
+      //   "return_docs": false,
+      //   "push": {
+      //     "since": push_last_seq
+      //   },
+      //   "pull": remotePouchOptions
+      // }
       pouchOptions = {
-        "return_docs": false,
-        "push": localPouchOptions,
-        "pull": remotePouchOptions
+
+        "push": {
+          "return_docs": false,
+          "since": push_last_seq
+        },
+        "pull": {
+          "return_docs": false,
+          "since": pull_last_seq,
+          selector: {
+            "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
+              if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {
+                $or = [
+                  ...$or,
+                  ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
+                    ? syncDetails.deviceSyncLocations.map(locationConfig => {
+                      // Get last value, that's the focused sync point.
+                      let location = locationConfig.value.slice(-1).pop()
+                      return {
+                        "form.id": formInfo.id,
+                        [`location.${location.level}`]: location.value
+                      }
+                    })
+                    : [
+                      {
+                        "form.id": formInfo.id
+                      }
+                    ]
+                ]
+              }
+              return $or
+            }, [])
+          }
+        }
       }
     } else {
       pouchOptions = {
+        "since": pull_last_seq,
+        "return_docs": false,
         selector: {
           "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
             if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) {
@@ -142,14 +177,19 @@ export class SyncCouchdbService {
           }
         }
         let pending = info.change.pending
+        let direction = info.direction
         if (typeof info.change.pending === 'undefined') {
           pending = 0;
         }
+        if (typeof info.direction === 'undefined') {
+          direction = ''
+        }
         const progress = {
           'docs_read': info.change.docs_read,
           'docs_written': info.change.docs_written,
           'doc_write_failures': info.change.doc_write_failures,
-          'pending': pending
+          'pending': pending,
+          'direction': direction
         };
         this.syncMessage$.next(progress)
       }).on('error', function (errorMessage) {
diff --git a/client/src/app/tangy-forms/classes/form-info.class.ts b/client/src/app/tangy-forms/classes/form-info.class.ts
index 9cc4823c3f..6d7cdcea2b 100644
--- a/client/src/app/tangy-forms/classes/form-info.class.ts
+++ b/client/src/app/tangy-forms/classes/form-info.class.ts
@@ -26,6 +26,7 @@ export class FormInfo {
 
 export interface CouchdbSyncSettings {
   enabled: boolean
+  pull: boolean
   filterByLocation:boolean
 }
 

From f16b5745207822d18503ab7abbd357926a48a249 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 30 Mar 2020 08:38:30 -0500
Subject: [PATCH 06/14] Code cleanup

---
 client/src/app/sync/sync-couchdb.service.ts | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index 2b5d61af81..160bd5c414 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -51,7 +51,7 @@ export class SyncCouchdbService {
       .map(row => row.id)
   }
 
-  // Note that this ignores the sync config settings.
+  // Note that if you run this with no forms configured to CouchDB sync, that will result in no filter query and everything will be synced. Use carefully.
   async sync(userDb:UserDatabase, syncDetails:SyncCouchdbDetails): Promise<ReplicationStatus> {
     const syncSessionUrl = await this.http.get(`${syncDetails.serverUrl}sync-session/start/${syncDetails.groupId}/${syncDetails.deviceId}/${syncDetails.deviceToken}`, {responseType:'text'}).toPromise()
     const remoteDb = new PouchDB(syncSessionUrl)
@@ -81,19 +81,8 @@ export class SyncCouchdbService {
 
     let pouchOptions;
     const appConfig = await this.appConfigService.getAppConfig();
-    // TODO: replace with test for sync-protocol-2 instead ?
-    // actually, don't even need it - we are already sp2!
     if (appConfig.couchdbSync4All) {
-      // Passing false prevents the changes feed from keeping all the documents in memory
-      // pouchOptions = {
-      //   "return_docs": false,
-      //   "push": {
-      //     "since": push_last_seq
-      //   },
-      //   "pull": remotePouchOptions
-      // }
       pouchOptions = {
-
         "push": {
           "return_docs": false,
           "since": push_last_seq

From 92a2c44fe0580bf44438554dc1569fe42e2f4fec Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 30 Mar 2020 09:17:41 -0500
Subject: [PATCH 07/14] added description for couchdbSync4All setting

---
 client/src/app/sync/sync-couchdb.service.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index 160bd5c414..eaf8d7279f 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -81,6 +81,8 @@ export class SyncCouchdbService {
 
     let pouchOptions;
     const appConfig = await this.appConfigService.getAppConfig();
+    // couchdbSync4All = true:  creates separate push and pull options for sync.
+    // couchdbSync4All = false: it uses only the pull settings, and does custom sync for push.
     if (appConfig.couchdbSync4All) {
       pouchOptions = {
         "push": {

From 42306ee0d970ff36f2f4e97e78c787649ce62d60 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 30 Mar 2020 10:23:42 -0500
Subject: [PATCH 08/14] Implementing push and pull settings for
 couchdbSyncSettings

Not using appConfig.couchdbSync4All.
---
 client/src/app/sync/sync-couchdb.service.ts | 73 +++++++++------------
 client/src/app/sync/sync.service.ts         |  2 -
 2 files changed, 31 insertions(+), 44 deletions(-)

diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index eaf8d7279f..b6d4aaf144 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -79,52 +79,41 @@ export class SyncCouchdbService {
       remotePouchOptions['selector'] = selector
     }
 
-    let pouchOptions;
-    const appConfig = await this.appConfigService.getAppConfig();
-    // couchdbSync4All = true:  creates separate push and pull options for sync.
-    // couchdbSync4All = false: it uses only the pull settings, and does custom sync for push.
-    if (appConfig.couchdbSync4All) {
-      pouchOptions = {
-        "push": {
-          "return_docs": false,
-          "since": push_last_seq
-        },
-        "pull": {
-          "return_docs": false,
-          "since": pull_last_seq,
-          selector: {
-            "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
-              if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {
-                $or = [
-                  ...$or,
-                  ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
-                    ? syncDetails.deviceSyncLocations.map(locationConfig => {
-                      // Get last value, that's the focused sync point.
-                      let location = locationConfig.value.slice(-1).pop()
-                      return {
-                        "form.id": formInfo.id,
-                        [`location.${location.level}`]: location.value
-                      }
-                    })
-                    : [
-                      {
-                        "form.id": formInfo.id
-                      }
-                    ]
-                ]
-              }
-              return $or
-            }, [])
-          }
+    let pouchOptions = {
+      "push": {
+        "return_docs": false,
+        "since": push_last_seq,
+        selector: {
+          "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
+            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.push) {
+              $or = [
+                ...$or,
+                ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
+                  ? syncDetails.deviceSyncLocations.map(locationConfig => {
+                    // Get last value, that's the focused sync point.
+                    let location = locationConfig.value.slice(-1).pop()
+                    return {
+                      "form.id": formInfo.id,
+                      [`location.${location.level}`]: location.value
+                    }
+                  })
+                  : [
+                    {
+                      "form.id": formInfo.id
+                    }
+                  ]
+              ]
+            }
+            return $or
+          }, [])
         }
-      }
-    } else {
-      pouchOptions = {
-        "since": pull_last_seq,
+      },
+      "pull": {
         "return_docs": false,
+        "since": pull_last_seq,
         selector: {
           "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
-            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) {
+            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {
               $or = [
                 ...$or,
                 ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
diff --git a/client/src/app/sync/sync.service.ts b/client/src/app/sync/sync.service.ts
index 060c7740a7..f062272bf2 100644
--- a/client/src/app/sync/sync.service.ts
+++ b/client/src/app/sync/sync.service.ts
@@ -62,7 +62,6 @@ export class SyncService {
     })
     console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage))
 
-    if (!appConfig.couchdbSync4All) {
       await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
         appConfig: appConfig,
         serverUrl: appConfig.serverUrl,
@@ -71,7 +70,6 @@ export class SyncService {
         deviceToken: device.token,
         formInfos
       })
-    }
     await this.deviceService.didSync()
   }
 

From 9ad5f3650b693176de0f9f5fd6cae604c3ab31c3 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 30 Mar 2020 11:06:07 -0500
Subject: [PATCH 09/14] Added push field to CouchdbSyncSettings interface

---
 client/src/app/sync/sync.service.ts              | 16 ++++++++--------
 .../app/tangy-forms/classes/form-info.class.ts   |  1 +
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/client/src/app/sync/sync.service.ts b/client/src/app/sync/sync.service.ts
index f062272bf2..053b1e0c2b 100644
--- a/client/src/app/sync/sync.service.ts
+++ b/client/src/app/sync/sync.service.ts
@@ -62,14 +62,14 @@ export class SyncService {
     })
     console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage))
 
-      await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
-        appConfig: appConfig,
-        serverUrl: appConfig.serverUrl,
-        groupId: appConfig.groupId,
-        deviceId: device._id,
-        deviceToken: device.token,
-        formInfos
-      })
+    await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
+      appConfig: appConfig,
+      serverUrl: appConfig.serverUrl,
+      groupId: appConfig.groupId,
+      deviceId: device._id,
+      deviceToken: device.token,
+      formInfos
+    })
     await this.deviceService.didSync()
   }
 
diff --git a/client/src/app/tangy-forms/classes/form-info.class.ts b/client/src/app/tangy-forms/classes/form-info.class.ts
index 6d7cdcea2b..3843449c6b 100644
--- a/client/src/app/tangy-forms/classes/form-info.class.ts
+++ b/client/src/app/tangy-forms/classes/form-info.class.ts
@@ -26,6 +26,7 @@ export class FormInfo {
 
 export interface CouchdbSyncSettings {
   enabled: boolean
+  push: boolean
   pull: boolean
   filterByLocation:boolean
 }

From 8cbaaac29235d60732ca193304448a1204bc460c Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Mon, 30 Mar 2020 16:35:49 -0500
Subject: [PATCH 10/14] minor tweaks

---
 client/src/app/case/services/case.service.ts  |  2 --
 .../sync/components/sync/sync.component.ts    |  8 +++++--
 client/src/app/sync/sync-couchdb.service.ts   | 22 +------------------
 client/src/app/sync/sync-custom.service.ts    |  6 +++--
 4 files changed, 11 insertions(+), 27 deletions(-)

diff --git a/client/src/app/case/services/case.service.ts b/client/src/app/case/services/case.service.ts
index 45cc3e2e87..6b07e6f1e3 100644
--- a/client/src/app/case/services/case.service.ts
+++ b/client/src/app/case/services/case.service.ts
@@ -499,8 +499,6 @@ class CaseService {
 
       this.db = await this.userService.getUserDatabase(this.userService.getCurrentUser())
 
-      // Upload the profiles first
-      // now upload the others
       for (let doc of templateDocs) {
         // @ts-ignore
         // sometimes doc is false...
diff --git a/client/src/app/sync/components/sync/sync.component.ts b/client/src/app/sync/components/sync/sync.component.ts
index 1499c5af32..966e64ce7c 100644
--- a/client/src/app/sync/components/sync/sync.component.ts
+++ b/client/src/app/sync/components/sync/sync.component.ts
@@ -29,9 +29,13 @@ export class SyncComponent implements OnInit {
     this.status = STATUS_IN_PROGRESS
     this.syncService.syncMessage$.subscribe({
       next: (progress) => {
-        this.syncMessage =  progress.docs_written + ' docs saved; ' + progress.pending + ' pending;'
+        let pendingMessage = ''
+        if (progress.pending) {
+          pendingMessage = progress.pending + ' pending;'
+        }
+        this.syncMessage =  progress.docs_written + ' docs saved; ' + pendingMessage
         if (progress.direction !== '') {
-          this.syncMessage = this.syncMessage + '; Direction: ' + progress.direction
+          this.syncMessage = this.syncMessage + ' Direction: ' + progress.direction
         }
         console.log('Sync Progress: ' + JSON.stringify(progress))
       }
diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index b6d4aaf144..987af21ea8 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -63,25 +63,9 @@ export class SyncCouchdbService {
     if (typeof push_last_seq === 'undefined') {
       push_last_seq = 0;
     }
-    const remotePouchOptions = {
-      "since": pull_last_seq
-    }
-
-    if (syncDetails.deviceSyncLocations.length > 0) {
-      const locationConfig = syncDetails.deviceSyncLocations[0]
-      // Get last value, that's the focused sync point.
-      const location = locationConfig.value.slice(-1).pop()
-      const selector = {
-          [`location.${location.level}`]: {
-            '$eq' : location.value
-          }
-        }
-      remotePouchOptions['selector'] = selector
-    }
 
     let pouchOptions = {
       "push": {
-        "return_docs": false,
         "since": push_last_seq,
         selector: {
           "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
@@ -109,7 +93,6 @@ export class SyncCouchdbService {
         }
       },
       "pull": {
-        "return_docs": false,
         "since": pull_last_seq,
         selector: {
           "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
@@ -158,9 +141,6 @@ export class SyncCouchdbService {
         }
         let pending = info.change.pending
         let direction = info.direction
-        if (typeof info.change.pending === 'undefined') {
-          pending = 0;
-        }
         if (typeof info.direction === 'undefined') {
           direction = ''
         }
@@ -168,7 +148,7 @@ export class SyncCouchdbService {
           'docs_read': info.change.docs_read,
           'docs_written': info.change.docs_written,
           'doc_write_failures': info.change.doc_write_failures,
-          'pending': pending,
+          'pending': info.change.pending,
           'direction': direction
         };
         this.syncMessage$.next(progress)
diff --git a/client/src/app/sync/sync-custom.service.ts b/client/src/app/sync/sync-custom.service.ts
index 89f4dc4ec2..cb69ada8f1 100644
--- a/client/src/app/sync/sync-custom.service.ts
+++ b/client/src/app/sync/sync-custom.service.ts
@@ -28,7 +28,9 @@ export class SyncCustomService {
 
   async sync(userDb:UserDatabase, syncDetails:SyncCustomDetails) {
     const uploadQueue = await this.uploadQueue(userDb, syncDetails.formInfos)
-    await this.push(userDb, syncDetails, uploadQueue)
+    if (uploadQueue.length > 0) {
+      await this.push(userDb, syncDetails, uploadQueue)
+    }
     // @TODO pull
   }
 
@@ -74,7 +76,7 @@ export class SyncCustomService {
         }
       }
       return queryKeys
-    }, []) 
+    }, [])
     const response = await userDb.query('sync-queue', { keys: queryKeys })
     return response
       .rows

From a205fc2869fed3ae511bd789a271a42d731f3637 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Tue, 31 Mar 2020 10:02:07 -0500
Subject: [PATCH 11/14] Removed selector for push sync

Implemented caching for tangerine-build-id and tangerine-build-channel
---
 client/src/app/case/services/case.service.ts  |  2 +-
 .../src/app/device/services/device.service.ts | 12 +++--
 client/src/app/sync/sync-couchdb.service.ts   | 50 ++++++++++---------
 3 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/client/src/app/case/services/case.service.ts b/client/src/app/case/services/case.service.ts
index 6b07e6f1e3..96efd15f79 100644
--- a/client/src/app/case/services/case.service.ts
+++ b/client/src/app/case/services/case.service.ts
@@ -459,7 +459,6 @@ class CaseService {
           }
         }],
       }
-      console.log("motherId: " + caseId + " participantId: " + participant_id);
       const doc = Object.assign({}, caseDoc, caseMother);
       caseDoc.items[0].inputs[1].value = participant_id;
       caseDoc.items[0].inputs[2].value = enrollment_date;
@@ -512,6 +511,7 @@ class CaseService {
         }
       }
       numberOfCasesCompleted++
+      console.log("motherId: " + caseId + " participantId: " + participant_id + " Completed " + numberOfCasesCompleted + " of " + numberOfCases);
     }
   }
 
diff --git a/client/src/app/device/services/device.service.ts b/client/src/app/device/services/device.service.ts
index 893c0f00bc..e6da2a1039 100644
--- a/client/src/app/device/services/device.service.ts
+++ b/client/src/app/device/services/device.service.ts
@@ -5,6 +5,7 @@ import { Device } from './../classes/device.class';
 import { AppConfigService } from './../../shared/_services/app-config.service';
 import { HttpClient } from '@angular/common/http';
 import { Injectable } from '@angular/core';
+import {AppConfig} from '../../shared/_classes/app-config.class';
 const bcrypt = window['dcodeIO'].bcrypt
 
 export interface AppInfo {
@@ -23,6 +24,8 @@ export class DeviceService {
 
   username:string
   password:string
+  rawBuildChannel:string
+  buildId:string
 
   constructor(
     private httpClient:HttpClient,
@@ -122,7 +125,8 @@ export class DeviceService {
 
   async getBuildId() {
     try {
-      return await this.httpClient.get('./assets/tangerine-build-id', {responseType: 'text'}).toPromise()
+      this.buildId = this.buildId ? this.buildId : await this.httpClient.get('./assets/tangerine-build-id', {responseType: 'text'}).toPromise();
+      return this.buildId
     } catch (e) {
       return 'N/A'
     }
@@ -130,10 +134,10 @@ export class DeviceService {
 
   async getBuildChannel() {
     try {
-      const raw = await this.httpClient.get('./assets/tangerine-build-channel', {responseType: 'text'}).toPromise()
-      return raw.includes('prod')
+      this.rawBuildChannel = this.rawBuildChannel ? this.rawBuildChannel : await this.httpClient.get('./assets/tangerine-build-channel', {responseType: 'text'}).toPromise()
+      return this.rawBuildChannel.includes('prod')
         ? 'live'
-        : raw.includes('qa')
+        : this.rawBuildChannel.includes('qa')
           ? 'test'
           : 'unknown'
     } catch (e) {
diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index 987af21ea8..f0871a8528 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -67,33 +67,35 @@ export class SyncCouchdbService {
     let pouchOptions = {
       "push": {
         "since": push_last_seq,
-        selector: {
-          "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
-            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.push) {
-              $or = [
-                ...$or,
-                ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
-                  ? syncDetails.deviceSyncLocations.map(locationConfig => {
-                    // Get last value, that's the focused sync point.
-                    let location = locationConfig.value.slice(-1).pop()
-                    return {
-                      "form.id": formInfo.id,
-                      [`location.${location.level}`]: location.value
-                    }
-                  })
-                  : [
-                    {
-                      "form.id": formInfo.id
-                    }
-                  ]
-              ]
-            }
-            return $or
-          }, [])
-        }
+        "return_docs": false
+        // selector: {
+        //   "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
+        //     if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.push) {
+        //       $or = [
+        //         ...$or,
+        //         ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
+        //           ? syncDetails.deviceSyncLocations.map(locationConfig => {
+        //             // Get last value, that's the focused sync point.
+        //             let location = locationConfig.value.slice(-1).pop()
+        //             return {
+        //               "form.id": formInfo.id,
+        //               [`location.${location.level}`]: location.value
+        //             }
+        //           })
+        //           : [
+        //             {
+        //               "form.id": formInfo.id
+        //             }
+        //           ]
+        //       ]
+        //     }
+        //     return $or
+        //   }, [])
+        // }
       },
       "pull": {
         "since": pull_last_seq,
+        "return_docs": false,
         selector: {
           "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
             if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {

From 8e438108a6be24a12800cc51503d43a87ac8c997 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Tue, 31 Mar 2020 17:39:06 -0500
Subject: [PATCH 12/14] If no couchdbSync4All setting, use location to filter
 sync.

---
 client/src/app/sync/sync-couchdb.service.ts | 77 +++++++--------------
 1 file changed, 26 insertions(+), 51 deletions(-)

diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index f0871a8528..d0bed832af 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -64,61 +64,36 @@ export class SyncCouchdbService {
       push_last_seq = 0;
     }
 
-    let pouchOptions = {
+    const pouchOptions = {
       "push": {
-        "since": push_last_seq,
-        "return_docs": false
-        // selector: {
-        //   "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
-        //     if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.push) {
-        //       $or = [
-        //         ...$or,
-        //         ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
-        //           ? syncDetails.deviceSyncLocations.map(locationConfig => {
-        //             // Get last value, that's the focused sync point.
-        //             let location = locationConfig.value.slice(-1).pop()
-        //             return {
-        //               "form.id": formInfo.id,
-        //               [`location.${location.level}`]: location.value
-        //             }
-        //           })
-        //           : [
-        //             {
-        //               "form.id": formInfo.id
-        //             }
-        //           ]
-        //       ]
-        //     }
-        //     return $or
-        //   }, [])
-        // }
+        "since": push_last_seq
       },
       "pull": {
         "since": pull_last_seq,
-        "return_docs": false,
-        selector: {
-          "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
-            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {
-              $or = [
-                ...$or,
-                ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
-                  ? syncDetails.deviceSyncLocations.map(locationConfig => {
-                    // Get last value, that's the focused sync point.
-                    let location = locationConfig.value.slice(-1).pop()
-                    return {
-                      "form.id": formInfo.id,
-                      [`location.${location.level}`]: location.value
-                    }
-                  })
-                  : [
-                    {
-                      "form.id": formInfo.id
-                    }
-                  ]
-              ]
-            }
-            return $or
-          }, [])
+        ...(await this.appConfigService.getAppConfig()).couchdbSync4All ? {} : { "selector": {
+            "$or" : syncDetails.formInfos.reduce(($or, formInfo) => {
+              if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {
+                $or = [
+                  ...$or,
+                  ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
+                    ? syncDetails.deviceSyncLocations.map(locationConfig => {
+                      // Get last value, that's the focused sync point.
+                      let location = locationConfig.value.slice(-1).pop()
+                      return {
+                        "form.id": formInfo.id,
+                        [`location.${location.level}`]: location.value
+                      }
+                    })
+                    : [
+                      {
+                        "form.id": formInfo.id
+                      }
+                    ]
+                ]
+              }
+              return $or
+            }, [])
+          }
         }
       }
     }

From 9e0fe5ed4a6ccf1654d3a85548a0c99315da1cf4 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Tue, 31 Mar 2020 17:56:00 -0500
Subject: [PATCH 13/14] fixed push settings for sync

---
 client/src/app/sync/sync-couchdb.service.ts | 34 ++++++++++++++++++---
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index d0bed832af..7c30c7159c 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -66,13 +66,10 @@ export class SyncCouchdbService {
 
     const pouchOptions = {
       "push": {
-        "since": push_last_seq
-      },
-      "pull": {
-        "since": pull_last_seq,
+        "since": push_last_seq,
         ...(await this.appConfigService.getAppConfig()).couchdbSync4All ? {} : { "selector": {
             "$or" : syncDetails.formInfos.reduce(($or, formInfo) => {
-              if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {
+              if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.push) {
                 $or = [
                   ...$or,
                   ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
@@ -95,6 +92,33 @@ export class SyncCouchdbService {
             }, [])
           }
         }
+      },
+      "pull": {
+        "since": pull_last_seq,
+        "selector": {
+          "$or" : syncDetails.formInfos.reduce(($or, formInfo) => {
+            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {
+              $or = [
+                ...$or,
+                ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
+                  ? syncDetails.deviceSyncLocations.map(locationConfig => {
+                    // Get last value, that's the focused sync point.
+                    let location = locationConfig.value.slice(-1).pop()
+                    return {
+                      "form.id": formInfo.id,
+                      [`location.${location.level}`]: location.value
+                    }
+                  })
+                  : [
+                    {
+                      "form.id": formInfo.id
+                    }
+                  ]
+              ]
+            }
+            return $or
+          }, [])
+        }
       }
     }
 

From 9ea9f10a6ab44296a49c8baf2365d6c91edc7754 Mon Sep 17 00:00:00 2001
From: "Chris E. Kelley" <ckelley@rti.org>
Date: Tue, 31 Mar 2020 18:45:33 -0500
Subject: [PATCH 14/14] reduced batch-size and batches_limit

better for mobile devices w/ less memory.
---
 client/src/app/sync/sync-couchdb.service.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts
index 7c30c7159c..7d4bc10c49 100644
--- a/client/src/app/sync/sync-couchdb.service.ts
+++ b/client/src/app/sync/sync-couchdb.service.ts
@@ -67,6 +67,8 @@ export class SyncCouchdbService {
     const pouchOptions = {
       "push": {
         "since": push_last_seq,
+        "batch_size": 50,
+        "batches_limit": 5,
         ...(await this.appConfigService.getAppConfig()).couchdbSync4All ? {} : { "selector": {
             "$or" : syncDetails.formInfos.reduce(($or, formInfo) => {
               if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.push) {
@@ -95,6 +97,8 @@ export class SyncCouchdbService {
       },
       "pull": {
         "since": pull_last_seq,
+        "batch_size": 50,
+        "batches_limit": 5,
         "selector": {
           "$or" : syncDetails.formInfos.reduce(($or, formInfo) => {
             if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled && formInfo.couchdbSyncSettings.pull) {