From 99039385bfc66ad3c783d72df1f90101fabdd5b1 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Wed, 27 Mar 2019 09:28:30 +0100
Subject: [PATCH 01/26] Add consistency query to stage 009.

The first step in ensuring consistency between Oracle and ES is to obtain
a very basic set of task data - id and timestamp - from Oracle. Add a query
for doing so.
---
 Utils/Dataflow/009_oracleConnector/README     |  2 ++
 .../009_oracleConnector/query/consistency.sql | 30 +++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 Utils/Dataflow/009_oracleConnector/query/consistency.sql

diff --git a/Utils/Dataflow/009_oracleConnector/README b/Utils/Dataflow/009_oracleConnector/README
index 933dc5d5a..fb0036e8c 100644
--- a/Utils/Dataflow/009_oracleConnector/README
+++ b/Utils/Dataflow/009_oracleConnector/README
@@ -10,6 +10,8 @@ them as NDJSON.
 Currently works with specific set of queries only:
  * prodsys2ES + datasets
  * prodsys2ES
+ * consistency : simplified query that only obtains taskid and task_timestamp
+   for each task
 
 The goal is to make it work with any number and combination of queries.
 
diff --git a/Utils/Dataflow/009_oracleConnector/query/consistency.sql b/Utils/Dataflow/009_oracleConnector/query/consistency.sql
new file mode 100644
index 000000000..6b19c2df3
--- /dev/null
+++ b/Utils/Dataflow/009_oracleConnector/query/consistency.sql
@@ -0,0 +1,30 @@
+-- Select all tasks for specified period of time
+-- Query tables:
+------------------------------------------------
+-- ATLAS_DEFT.t_production_task
+-- ATLAS_DEFT.t_production_step
+-- ATLAS_DEFT.t_step_template
+-- ATLAS_DEFT.t_ht_to_task
+-- ATLAS_DEFT.t_hashtag
+-- ATLAS_PANDA.jedi_datasets
+--
+-- All fields:
+-- architecture, campaign, cloud, conditions_tags, core_count, description, end_time,
+-- energy_gev, evgen_job_opts, geometry_version, hashtag_list, job_config, physics_list, processed_events,
+-- phys_group, project, pr_id, requested_events, run_number, site, start_time, step_name, status, subcampaign,
+-- taskid, taskname, task_timestamp,  ticket_id, trans_home, trans_path, trans_uses, trigger_config, user_name, vo,
+-- n_files_per_job, n_events_per_job, n_files_to_be_used,
+
+-- RESTRICTIONS:
+-- 1. taskID must be more than 4 000 000 OR from the date > 12-03-2014
+-- 2. we collecting only PRODUCTION tasks OR only ANALYSIS tasks
+--    ('pr_id > 300' or 'pr_id = 300')
+  SELECT DISTINCT
+    t.taskid,
+    TO_CHAR(t.timestamp, 'dd-mm-yyyy hh24:mi:ss')           AS task_timestamp
+  FROM
+    ATLAS_DEFT.t_production_task t
+  WHERE
+    t.timestamp > :start_date AND
+    t.timestamp <= :end_date AND
+    t.pr_id %(production_or_analysis_cond)s 300

From 42d55b372e1972781e5d1cf8c91129a759b03631 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Fri, 29 Mar 2019 15:06:09 +0100
Subject: [PATCH 02/26] Add script for consistency check to stage 069.

---
 Utils/Dataflow/069_upload2es/consistency.py | 174 ++++++++++++++++++++
 1 file changed, 174 insertions(+)
 create mode 100755 Utils/Dataflow/069_upload2es/consistency.py

diff --git a/Utils/Dataflow/069_upload2es/consistency.py b/Utils/Dataflow/069_upload2es/consistency.py
new file mode 100755
index 000000000..2820842d8
--- /dev/null
+++ b/Utils/Dataflow/069_upload2es/consistency.py
@@ -0,0 +1,174 @@
+#!/bin/env python
+'''
+Script for consistency checking.
+
+Authors:
+  Vasilii Aulov (vasilii.aulov@cern.ch)
+'''
+import os
+import sys
+import traceback
+
+from datetime import datetime
+
+import elasticsearch
+
+
+def log(msg, prefix='DEBUG'):
+    ''' ??? '''
+    prefix = '(%s)' % (prefix)
+    prefix = prefix.ljust(8)
+    sys.stderr.write('%s%s %s\n' % (prefix, datetime.now().isoformat(), msg))
+
+
+try:
+    base_dir = os.path.dirname(__file__)
+    dkb_dir = os.path.join(base_dir, os.pardir)
+    sys.path.append(dkb_dir)
+    import pyDKB
+    from pyDKB.dataflow.stage import JSONProcessorStage
+    from pyDKB.dataflow.messages import JSONMessage
+    from pyDKB.dataflow.exceptions import DataflowException
+except Exception, err:
+    log('Failed to import pyDKB library: %s' % err, 'ERROR')
+    sys.exit(1)
+
+
+es = None
+
+
+INDEX = 'tasks_chain_data'
+
+
+def load_config(fname):
+    ''' ??? '''
+    cfg = {
+        'ES_HOST': 'localhost',
+        'ES_PORT': '9200',
+        'ES_USER': '',
+        'ES_PASSWORD': '',
+        'ES_INDEX': ''
+    }
+    with open(fname) as f:
+        lines = f.readlines()
+    for l in lines:
+        if l.startswith('ES'):
+            key = False
+            value = False
+            try:
+                (key, value) = l.split()[0].split('=')
+            except ValueError:
+                pass
+            if key in cfg:
+                cfg[key] = value
+    return cfg
+
+
+def es_connect(cfg):
+    ''' Establish a connection to elasticsearch.
+
+    TODO: take parameters from es config.
+    '''
+    global es
+    if cfg['ES_USER'] and cfg['ES_PASSWORD']:
+        s = 'http://%s:%s@%s:%s/' % (cfg['ES_USER'],
+                                     cfg['ES_PASSWORD'],
+                                     cfg['ES_HOST'],
+                                     cfg['ES_PORT'])
+    else:
+        s = '%s:%s' % (cfg['ES_HOST'], cfg['ES_PORT'])
+    es = elasticsearch.Elasticsearch([s])
+
+
+def get_field(index, taskid, field):
+    ''' Get field value by given taskid.
+
+    :param es: elasticsearch client
+    :type es: elasticsearch.client.Elasticsearch
+    :param index: index containing tasks
+    :type index: str
+    :param taskid: taskid of the task to look for
+    :type taskid: int or str
+    :param index: field name
+    :type index: str
+
+    :return: field value, or False if the task was not found
+    :rtype: int or bool
+    '''
+    try:
+        results = es.get(index=index, doc_type='_all', id=taskid,
+                         _source=[field])
+    except elasticsearch.exceptions.NotFoundError:
+        return False
+    return results['_source'].get(field)
+
+
+def process(stage, message):
+    ''' Process a message.
+
+    Implementation of :py:meth:`.AbstractProcessorStage.process` for hooking
+    the stage into DKB workflow.
+    '''
+    data = message.content()
+    if type(data) is not dict:
+        log('Incorrect data:' + str(data), 'INPUT')
+        return False
+    taskid = data.get('taskid')
+    if taskid is None:
+        log('No taskid in data:' + str(data), 'INPUT')
+        return False
+    timestamp = data.get('task_timestamp')
+    if timestamp is None:
+        log('No timestamp supplied for taskid ' + str(taskid), 'INPUT')
+        return False
+
+    es_timestamp = get_field(INDEX, taskid, 'task_timestamp')
+    if es_timestamp is None:
+        log('No timestamp in ES for taskid ' + str(taskid), 'DIFF')
+    elif not es_timestamp:
+        log('Taskid %d not found in ES' % taskid, 'DIFF')
+    elif es_timestamp != timestamp:
+        log('Taskid %d has timestamp %s in ES, %s in Oracle' % (taskid,
+                                                                es_timestamp,
+                                                                timestamp),
+            'DIFF')
+    else:
+        log('Taskid %d is up to date in ES' % taskid, 'INFO')
+
+    return True
+
+
+def main(args):
+    ''' ??? '''
+
+    stage = JSONProcessorStage()
+    stage.add_argument('--conf', help='elasticsearch config', required=True)
+
+    exit_code = 0
+    exc_info = None
+    try:
+        stage.parse_args(args)
+        cfg = load_config(stage.ARGS.conf)
+        stage.process = process
+        es_connect(cfg)
+        stage.run()
+    except (DataflowException, RuntimeError), err:
+        if str(err):
+            log(err, 'ERROR')
+        exit_code = 2
+    except Exception:
+        exc_info = sys.exc_info()
+        exit_code = 3
+    finally:
+        stage.stop()
+
+    if exc_info:
+        trace = traceback.format_exception(*exc_info)
+        for line in trace:
+            log(line, 'ERROR')
+
+    exit(exit_code)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

From f27f2781593d94b6279a7b23704e913aafcf843d Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Tue, 2 Apr 2019 11:44:30 +0200
Subject: [PATCH 03/26] Get index name from config rather than code.

---
 Utils/Dataflow/069_upload2es/consistency.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Utils/Dataflow/069_upload2es/consistency.py b/Utils/Dataflow/069_upload2es/consistency.py
index 2820842d8..e90aaec08 100755
--- a/Utils/Dataflow/069_upload2es/consistency.py
+++ b/Utils/Dataflow/069_upload2es/consistency.py
@@ -37,7 +37,7 @@ def log(msg, prefix='DEBUG'):
 es = None
 
 
-INDEX = 'tasks_chain_data'
+INDEX = None
 
 
 def load_config(fname):
@@ -61,6 +61,8 @@ def load_config(fname):
                 pass
             if key in cfg:
                 cfg[key] = value
+    global INDEX
+    INDEX = cfg['ES_INDEX']
     return cfg
 
 

From af9f212e787c32acc538ec43705094b574e9ad3d Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Tue, 2 Apr 2019 12:07:32 +0200
Subject: [PATCH 04/26] Check for index' existence before working.

es.get() raises NotFoundError in both cases - when index does not exist and
when document does not exist. Also, it's more reasonable to check index once
since it's the same for all messages.
---
 Utils/Dataflow/069_upload2es/consistency.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/Utils/Dataflow/069_upload2es/consistency.py b/Utils/Dataflow/069_upload2es/consistency.py
index e90aaec08..d70c30ef2 100755
--- a/Utils/Dataflow/069_upload2es/consistency.py
+++ b/Utils/Dataflow/069_upload2es/consistency.py
@@ -153,7 +153,11 @@ def main(args):
         cfg = load_config(stage.ARGS.conf)
         stage.process = process
         es_connect(cfg)
-        stage.run()
+        if not es.indices.exists(INDEX):
+            log('No such index: %s' % INDEX, 'ERROR')
+            exit_code = 4
+        else:
+            stage.run()
     except (DataflowException, RuntimeError), err:
         if str(err):
             log(err, 'ERROR')

From 4c560a6c588f57b99592b658023bdfaa60ee427b Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Tue, 2 Apr 2019 13:09:19 +0200
Subject: [PATCH 05/26] Update documentation.

- Add/update functions and their parameters' descriptions.
- Update the script's description.
- Add consistency' description into README.
---
 Utils/Dataflow/069_upload2es/README         | 20 ++++++++++++-
 Utils/Dataflow/069_upload2es/consistency.py | 31 +++++++++++++++++----
 2 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/Utils/Dataflow/069_upload2es/README b/Utils/Dataflow/069_upload2es/README
index a258165a5..d1ea14686 100644
--- a/Utils/Dataflow/069_upload2es/README
+++ b/Utils/Dataflow/069_upload2es/README
@@ -4,7 +4,7 @@
 
 Description
 -----------
-Uploads prepared data to ElasticSearch.
+load_data.sh uploads prepared data to ElasticSearch.
 
 Input
 -----
@@ -18,6 +18,24 @@ JSON documents, one per line:
 ...
 }}}
 
+Consistency
+-----------
+consistency.py checks that the data is present in ElasticSearch instead of
+uploading it. Input comes from Stage 009(in consistency mode) and only needs 2
+fields for now:
+{{{
+{taskid, task_timestamp}
+...
+}}}
+
+Consistency check can be run as following:
+
+  ./consistency.py --conf elasticsearch_config
+
+For more information about running the check and its arguments, use:
+
+  ./consistency.py -h
+
 TODO
 ----
 Make the stage aware of EOProcess/EOMessage markers
diff --git a/Utils/Dataflow/069_upload2es/consistency.py b/Utils/Dataflow/069_upload2es/consistency.py
index d70c30ef2..2215195c1 100755
--- a/Utils/Dataflow/069_upload2es/consistency.py
+++ b/Utils/Dataflow/069_upload2es/consistency.py
@@ -1,6 +1,9 @@
 #!/bin/env python
 '''
-Script for consistency checking.
+Script for checking the supplied task's presence in elasticsearch.
+
+Currently it performs the check by comparing the supplied timestamp
+with the one in elasticsearch.
 
 Authors:
   Vasilii Aulov (vasilii.aulov@cern.ch)
@@ -15,7 +18,7 @@
 
 
 def log(msg, prefix='DEBUG'):
-    ''' ??? '''
+    ''' Add prefix and current time to message and write it to stderr. '''
     prefix = '(%s)' % (prefix)
     prefix = prefix.ljust(8)
     sys.stderr.write('%s%s %s\n' % (prefix, datetime.now().isoformat(), msg))
@@ -41,7 +44,13 @@ def log(msg, prefix='DEBUG'):
 
 
 def load_config(fname):
-    ''' ??? '''
+    ''' Open elasticsearch config and obtain parameters from it.
+
+    Setup INDEX as global variable.
+
+    :param fname: config file's name
+    :type fname: str
+    '''
     cfg = {
         'ES_HOST': 'localhost',
         'ES_PORT': '9200',
@@ -67,9 +76,10 @@ def load_config(fname):
 
 
 def es_connect(cfg):
-    ''' Establish a connection to elasticsearch.
+    ''' Establish a connection to elasticsearch, as a global variable.
 
-    TODO: take parameters from es config.
+    :param cfg: connection parameters
+    :type cfg: dict
     '''
     global es
     if cfg['ES_USER'] and cfg['ES_PASSWORD']:
@@ -110,6 +120,11 @@ def process(stage, message):
 
     Implementation of :py:meth:`.AbstractProcessorStage.process` for hooking
     the stage into DKB workflow.
+
+    :param stage: stage instance
+    :type stage: pyDKB.dataflow.stage.ProcessorStage
+    :param msg: input message with task info
+    :type msg: pyDKB.dataflow.Message
     '''
     data = message.content()
     if type(data) is not dict:
@@ -141,7 +156,11 @@ def process(stage, message):
 
 
 def main(args):
-    ''' ??? '''
+    ''' Parse command line arguments and run the stage.
+
+    :param argv: arguments
+    :type argv: list
+    '''
 
     stage = JSONProcessorStage()
     stage.add_argument('--conf', help='elasticsearch config', required=True)

From 2296f6d6a0b613535b25d602581f0020e8766306 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Tue, 2 Apr 2019 13:36:28 +0200
Subject: [PATCH 06/26] Add a very basic consistency check script.

---
 Utils/Dataflow/run/data4es-consistency-check | 37 ++++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100755 Utils/Dataflow/run/data4es-consistency-check

diff --git a/Utils/Dataflow/run/data4es-consistency-check b/Utils/Dataflow/run/data4es-consistency-check
new file mode 100755
index 000000000..da8b8f524
--- /dev/null
+++ b/Utils/Dataflow/run/data4es-consistency-check
@@ -0,0 +1,37 @@
+#!/bin/bash -l
+
+DEBUG=
+
+base_dir=$( cd "$(dirname "$(readlink -f "$0")")"; pwd)
+
+# Directories with configuration files
+[ -n "$DATA4ES_CONSISTENCY_CONFIG_PATH" ] && \
+    CONFIG_PATH="$DATA4ES_CONSISTENCY_CONFIG_PATH" || \
+    CONFIG_PATH="${base_dir}/../config:${base_dir}/../../Elasticsearch/config"
+
+# Find configuration file in $CONFIG_PATH
+get_config() {
+  [ -z "$1" ] && echo "get_config(): no arguments passed." && return 1
+  dirs=$CONFIG_PATH
+  while [ -n "$dirs" ]; do
+    dir=${dirs%%:*}
+    [ "$dirs" = "$dir" ] && \
+        dirs='' || \
+        dirs="${dirs#*:}"
+    [ -f "${dir}/${1}" ] && readlink -f "${dir}/${1}" && return 0
+  done
+}
+
+# Oracle
+cfg009=`get_config "consistency009.cfg"`
+cmd_009="$base_dir/../009_oracleConnector/Oracle2JSON.py --config $cfg009"
+
+# ES
+cfg_es=`get_config "es"`
+cmd_069="$base_dir/../069_upload2es/consistency.py -m s --conf $cfg_es"
+
+[ -n "$DEBUG" ] &&
+  err009="009.err" ||
+    err009=/dev/null
+
+$cmd_009 2>"$err009" | $cmd_069

From b8a2ab11d83e80c419d3fa3a775ca0f277dca7e9 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 4 Apr 2019 13:43:52 +0200
Subject: [PATCH 07/26] Generalize 069-consistency.

Check that all fields supplied in input data are present in ES and their
values are matching the input data, instead of working only with tasks and
their timestamps. This will allow checking tasks' other fields as well as
different types of documents such as datasets.

Add stage 016 into consistency chain because it adds the fields required
for getting documents of given type from ES.
---
 Utils/Dataflow/069_upload2es/consistency.py  | 66 +++++++++++---------
 Utils/Dataflow/run/data4es-consistency-check | 20 ++++--
 2 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/Utils/Dataflow/069_upload2es/consistency.py b/Utils/Dataflow/069_upload2es/consistency.py
index 2215195c1..5f1a7e6d5 100755
--- a/Utils/Dataflow/069_upload2es/consistency.py
+++ b/Utils/Dataflow/069_upload2es/consistency.py
@@ -92,27 +92,29 @@ def es_connect(cfg):
     es = elasticsearch.Elasticsearch([s])
 
 
-def get_field(index, taskid, field):
-    ''' Get field value by given taskid.
+def get_fields(index, _id, _type, fields):
+    ''' Get fields value by given _id and _type.
 
     :param es: elasticsearch client
     :type es: elasticsearch.client.Elasticsearch
-    :param index: index containing tasks
+    :param index: index to search in
     :type index: str
-    :param taskid: taskid of the task to look for
-    :type taskid: int or str
-    :param index: field name
-    :type index: str
-
-    :return: field value, or False if the task was not found
-    :rtype: int or bool
+    :param _id: id of the document to look for
+    :type _id: int or str
+    :param _type: type of the document to look for
+    :type _type: str
+    :param fields: field names
+    :type fields: list
+
+    :return: field values, or False if the document was not found
+    :rtype: dict or bool
     '''
     try:
-        results = es.get(index=index, doc_type='_all', id=taskid,
-                         _source=[field])
+        results = es.get(index=index, doc_type=_type, id=_id,
+                         _source=fields)
     except elasticsearch.exceptions.NotFoundError:
         return False
-    return results['_source'].get(field)
+    return results['_source']
 
 
 def process(stage, message):
@@ -123,34 +125,36 @@ def process(stage, message):
 
     :param stage: stage instance
     :type stage: pyDKB.dataflow.stage.ProcessorStage
-    :param msg: input message with task info
+    :param msg: input message with document info
     :type msg: pyDKB.dataflow.Message
     '''
     data = message.content()
     if type(data) is not dict:
         log('Incorrect data:' + str(data), 'INPUT')
         return False
-    taskid = data.get('taskid')
-    if taskid is None:
-        log('No taskid in data:' + str(data), 'INPUT')
+    _id = data.pop('_id')
+    _type = data.pop('_type')
+    if _id is None or _type is None:
+        log('Insufficient ES info in data:' + str(data), 'INPUT')
         return False
-    timestamp = data.get('task_timestamp')
-    if timestamp is None:
-        log('No timestamp supplied for taskid ' + str(taskid), 'INPUT')
+
+    # Crutch. Remove unwanted (for now) field added by Stage 016.
+    if 'phys_category' in data:
+        del data['phys_category']
+
+    # Do not check empty documents with valid _id and _type.
+    # It's unlikely that such documents will be produced in DKB. In general,
+    # such documents should be checked by es.exists(), and not es.get().
+    if not data:
+        log('Nothing to check for document (%s, %d)' % (_type, _id), 'INPUT')
         return False
 
-    es_timestamp = get_field(INDEX, taskid, 'task_timestamp')
-    if es_timestamp is None:
-        log('No timestamp in ES for taskid ' + str(taskid), 'DIFF')
-    elif not es_timestamp:
-        log('Taskid %d not found in ES' % taskid, 'DIFF')
-    elif es_timestamp != timestamp:
-        log('Taskid %d has timestamp %s in ES, %s in Oracle' % (taskid,
-                                                                es_timestamp,
-                                                                timestamp),
-            'DIFF')
+    es_data = get_fields(INDEX, _id, _type, data.keys())
+    if data != es_data:
+        log('Document (%s, %d) differs between Oracle and ES: Oracle:%s ES:%s'
+            % (_type, _id, data, es_data), 'DIFF')
     else:
-        log('Taskid %d is up to date in ES' % taskid, 'INFO')
+        log('Document (%s, %d) is up to date in ES' % (_type, _id), 'INFO')
 
     return True
 
diff --git a/Utils/Dataflow/run/data4es-consistency-check b/Utils/Dataflow/run/data4es-consistency-check
index da8b8f524..74b7a5533 100755
--- a/Utils/Dataflow/run/data4es-consistency-check
+++ b/Utils/Dataflow/run/data4es-consistency-check
@@ -22,16 +22,26 @@ get_config() {
   done
 }
 
+# EOP filter (required due to the unconfigurable EOP marker in pyDKB)
+eop_filter() {
+  sed -e"s/\\x00//g"
+}
+
 # Oracle
 cfg009=`get_config "consistency009.cfg"`
 cmd_009="$base_dir/../009_oracleConnector/Oracle2JSON.py --config $cfg009"
+[ -n "$DEBUG" ] &&
+  err009="009.err" ||
+    err009=/dev/null
+
+# Formatting
+cmd_016="$base_dir/../016_task2es/task2es.py -m s"
+[ -n "$DEBUG" ] &&
+  err016="016.err" ||
+    err016=/dev/null
 
 # ES
 cfg_es=`get_config "es"`
 cmd_069="$base_dir/../069_upload2es/consistency.py -m s --conf $cfg_es"
 
-[ -n "$DEBUG" ] &&
-  err009="009.err" ||
-    err009=/dev/null
-
-$cmd_009 2>"$err009" | $cmd_069
+$cmd_009 2>"$err009" | $cmd_016 2>"$err016" | eop_filter | $cmd_069 >/dev/null

From acfe45a968b2967c0abe3aa32c62f2280b5d1333 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Fri, 5 Apr 2019 12:08:52 +0200
Subject: [PATCH 08/26] Save and display the info about different tasks.

Prepare the script for further development, where incosistent tasks will be
automatically reloaded into ES.
---
 Utils/Dataflow/069_upload2es/consistency.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/Utils/Dataflow/069_upload2es/consistency.py b/Utils/Dataflow/069_upload2es/consistency.py
index 5f1a7e6d5..a2557581e 100755
--- a/Utils/Dataflow/069_upload2es/consistency.py
+++ b/Utils/Dataflow/069_upload2es/consistency.py
@@ -41,6 +41,7 @@ def log(msg, prefix='DEBUG'):
 
 
 INDEX = None
+FOUND_DIFF = []
 
 
 def load_config(fname):
@@ -153,6 +154,8 @@ def process(stage, message):
     if data != es_data:
         log('Document (%s, %d) differs between Oracle and ES: Oracle:%s ES:%s'
             % (_type, _id, data, es_data), 'DIFF')
+        global FOUND_DIFF
+        FOUND_DIFF.append((_type, _id))
     else:
         log('Document (%s, %d) is up to date in ES' % (_type, _id), 'INFO')
 
@@ -196,6 +199,11 @@ def main(args):
         for line in trace:
             log(line, 'ERROR')
 
+    if exit_code == 0 and FOUND_DIFF:
+        exit_code = 1
+
+    print FOUND_DIFF
+
     exit(exit_code)
 
 

From 8a71791447ba40ce9257c643c94b74d91774848e Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Fri, 5 Apr 2019 14:41:53 +0200
Subject: [PATCH 09/26] Move certain shell functions to library.

These functions are either used by several scripts or will be in the future.
Move them to library to uphold DRY principle.
---
 Utils/Dataflow/run/data4es-consistency-check | 20 +++------------
 Utils/Dataflow/run/data4es-start             | 27 +++-----------------
 Utils/Dataflow/shell_lib/eop_filter          |  6 +++++
 Utils/Dataflow/shell_lib/get_config          | 18 +++++++++++++
 Utils/Dataflow/shell_lib/log                 |  9 +++++++
 5 files changed, 40 insertions(+), 40 deletions(-)
 create mode 100755 Utils/Dataflow/shell_lib/eop_filter
 create mode 100755 Utils/Dataflow/shell_lib/get_config
 create mode 100755 Utils/Dataflow/shell_lib/log

diff --git a/Utils/Dataflow/run/data4es-consistency-check b/Utils/Dataflow/run/data4es-consistency-check
index 74b7a5533..16e8b3a91 100755
--- a/Utils/Dataflow/run/data4es-consistency-check
+++ b/Utils/Dataflow/run/data4es-consistency-check
@@ -3,29 +3,15 @@
 DEBUG=
 
 base_dir=$( cd "$(dirname "$(readlink -f "$0")")"; pwd)
+lib="$base_dir/../shell_lib"
 
 # Directories with configuration files
 [ -n "$DATA4ES_CONSISTENCY_CONFIG_PATH" ] && \
     CONFIG_PATH="$DATA4ES_CONSISTENCY_CONFIG_PATH" || \
     CONFIG_PATH="${base_dir}/../config:${base_dir}/../../Elasticsearch/config"
 
-# Find configuration file in $CONFIG_PATH
-get_config() {
-  [ -z "$1" ] && echo "get_config(): no arguments passed." && return 1
-  dirs=$CONFIG_PATH
-  while [ -n "$dirs" ]; do
-    dir=${dirs%%:*}
-    [ "$dirs" = "$dir" ] && \
-        dirs='' || \
-        dirs="${dirs#*:}"
-    [ -f "${dir}/${1}" ] && readlink -f "${dir}/${1}" && return 0
-  done
-}
-
-# EOP filter (required due to the unconfigurable EOP marker in pyDKB)
-eop_filter() {
-  sed -e"s/\\x00//g"
-}
+source $lib/get_config
+source $lib/eop_filter
 
 # Oracle
 cfg009=`get_config "consistency009.cfg"`
diff --git a/Utils/Dataflow/run/data4es-start b/Utils/Dataflow/run/data4es-start
index 559b94fbc..0ae42dcf0 100755
--- a/Utils/Dataflow/run/data4es-start
+++ b/Utils/Dataflow/run/data4es-start
@@ -6,14 +6,9 @@ BATCH_SIZE=100
 DEBUG=
 
 base_dir=$( cd "$(dirname "$(readlink -f "$0")")"; pwd)
+lib="$base_dir/../shell_lib"
 
-log() {
-  level=INFO
-  [ $# -eq 2 ] && level="$1" && shift
-  [ -z "$SCRIPT_NAME" ] && SCRIPT_NAME="$(basename "$0")"
-  msg="$1"
-  echo "($level) `date +'%d-%m-%Y %T'` ($SCRIPT_NAME) $msg" >&2
-}
+source $lib/log
 
 # Directories with configuration files
 [ -n "$DATA4ES_CONFIG_PATH" ] && \
@@ -49,18 +44,7 @@ pidfile="$HOME_DIR/pid"
 # Define commands to be used as dataflow nodes
 # ---
 
-# Find configuration file in $CONFIG_PATH
-get_config() {
-  [ -z "$1" ] && log ERROR "get_config(): no arguments passed." && return 1
-  dirs=$CONFIG_PATH
-  while [ -n "$dirs" ]; do
-    dir=${dirs%%:*}
-    [ "$dirs" = "$dir" ] && \
-        dirs='' || \
-        dirs="${dirs#*:}"
-    [ -f "${dir}/${1}" ] && readlink -f "${dir}/${1}" && return 0
-  done
-}
+source $lib/get_config
 
 # Oracle Connector
 cmd_09="${base_dir}/../009_oracleConnector/Oracle2JSON.py"
@@ -101,10 +85,7 @@ cmd_95="${base_dir}/../095_datasetInfoAMI/amiDatasets.py -m s --userkey $AUTH_KE
 
 # Service (glue) functions
 # ---
-# EOP filter (required due to the unconfigurable EOP marker in pyDKB)
-eop_filter() {
-  sed -e"s/\\x00//g"
-}
+source $lib/eop_filter
 
 # Buffer file name
 get_buffer() {
diff --git a/Utils/Dataflow/shell_lib/eop_filter b/Utils/Dataflow/shell_lib/eop_filter
new file mode 100755
index 000000000..61f770255
--- /dev/null
+++ b/Utils/Dataflow/shell_lib/eop_filter
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# EOP filter (required due to the unconfigurable EOP marker in pyDKB)
+eop_filter() {
+  sed -e"s/\\x00//g"
+}
diff --git a/Utils/Dataflow/shell_lib/get_config b/Utils/Dataflow/shell_lib/get_config
new file mode 100755
index 000000000..9e6543464
--- /dev/null
+++ b/Utils/Dataflow/shell_lib/get_config
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+get_config_lib=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
+source $get_config_lib/log
+
+# Find configuration file in $CONFIG_PATH
+get_config() {
+  [ -z "$1" ] && log ERROR "get_config(): no arguments passed." && return 1
+  [ -z "$CONFIG_PATH" ] && CONFIG_PATH=`pwd`
+  dirs=$CONFIG_PATH
+  while [ -n "$dirs" ]; do
+    dir=${dirs%%:*}
+    [ "$dirs" = "$dir" ] && \
+        dirs='' || \
+        dirs="${dirs#*:}"
+    [ -f "${dir}/${1}" ] && readlink -f "${dir}/${1}" && return 0
+  done
+}
diff --git a/Utils/Dataflow/shell_lib/log b/Utils/Dataflow/shell_lib/log
new file mode 100755
index 000000000..ac9d1871c
--- /dev/null
+++ b/Utils/Dataflow/shell_lib/log
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+log() {
+  level=INFO
+  [ $# -eq 2 ] && level="$1" && shift
+  [ -z "$SCRIPT_NAME" ] && SCRIPT_NAME="$(basename "$0")"
+  msg="$1"
+  echo "($level) `date +'%d-%m-%Y %T'` ($SCRIPT_NAME) $msg" >&2
+}

From 90380a92bb91fcf0aa60d5b0c837cb4ccdc1aa8f Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Wed, 17 Apr 2019 15:24:29 +0200
Subject: [PATCH 10/26] Remove DEBUG mode.

DEBUG mode in data4es-start exists to check the workflow without uploading
anything to ES. Consistency check writes nothing, so DEBUG is unnecessary
here.
Do not redirect the stages' stderrs, leave them as-is.
---
 Utils/Dataflow/run/data4es-consistency-check | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/Utils/Dataflow/run/data4es-consistency-check b/Utils/Dataflow/run/data4es-consistency-check
index 16e8b3a91..54d23a7de 100755
--- a/Utils/Dataflow/run/data4es-consistency-check
+++ b/Utils/Dataflow/run/data4es-consistency-check
@@ -1,7 +1,5 @@
 #!/bin/bash -l
 
-DEBUG=
-
 base_dir=$( cd "$(dirname "$(readlink -f "$0")")"; pwd)
 lib="$base_dir/../shell_lib"
 
@@ -16,18 +14,12 @@ source $lib/eop_filter
 # Oracle
 cfg009=`get_config "consistency009.cfg"`
 cmd_009="$base_dir/../009_oracleConnector/Oracle2JSON.py --config $cfg009"
-[ -n "$DEBUG" ] &&
-  err009="009.err" ||
-    err009=/dev/null
 
 # Formatting
 cmd_016="$base_dir/../016_task2es/task2es.py -m s"
-[ -n "$DEBUG" ] &&
-  err016="016.err" ||
-    err016=/dev/null
 
 # ES
 cfg_es=`get_config "es"`
 cmd_069="$base_dir/../069_upload2es/consistency.py -m s --conf $cfg_es"
 
-$cmd_009 2>"$err009" | $cmd_016 2>"$err016" | eop_filter | $cmd_069 >/dev/null
+$cmd_009 | $cmd_016 | eop_filter | $cmd_069 >/dev/null

From 7bac20248ac886184739687e5c4db05e73b1838b Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Wed, 17 Apr 2019 16:16:06 +0200
Subject: [PATCH 11/26] Move ES consistency script into a separate stage.

While the script is the stage 069's counterpart in data4es-consistency-check,
they share no functionality.
---
 Utils/Dataflow/069_upload2es/README           | 20 +------------
 Utils/Dataflow/071_esConsistency/README       | 28 +++++++++++++++++++
 .../consistency.py                            |  0
 Utils/Dataflow/run/data4es-consistency-check  |  4 +--
 4 files changed, 31 insertions(+), 21 deletions(-)
 create mode 100644 Utils/Dataflow/071_esConsistency/README
 rename Utils/Dataflow/{069_upload2es => 071_esConsistency}/consistency.py (100%)

diff --git a/Utils/Dataflow/069_upload2es/README b/Utils/Dataflow/069_upload2es/README
index d1ea14686..a258165a5 100644
--- a/Utils/Dataflow/069_upload2es/README
+++ b/Utils/Dataflow/069_upload2es/README
@@ -4,7 +4,7 @@
 
 Description
 -----------
-load_data.sh uploads prepared data to ElasticSearch.
+Uploads prepared data to ElasticSearch.
 
 Input
 -----
@@ -18,24 +18,6 @@ JSON documents, one per line:
 ...
 }}}
 
-Consistency
------------
-consistency.py checks that the data is present in ElasticSearch instead of
-uploading it. Input comes from Stage 009(in consistency mode) and only needs 2
-fields for now:
-{{{
-{taskid, task_timestamp}
-...
-}}}
-
-Consistency check can be run as following:
-
-  ./consistency.py --conf elasticsearch_config
-
-For more information about running the check and its arguments, use:
-
-  ./consistency.py -h
-
 TODO
 ----
 Make the stage aware of EOProcess/EOMessage markers
diff --git a/Utils/Dataflow/071_esConsistency/README b/Utils/Dataflow/071_esConsistency/README
new file mode 100644
index 000000000..d4fd61cea
--- /dev/null
+++ b/Utils/Dataflow/071_esConsistency/README
@@ -0,0 +1,28 @@
+=============
+* Stage 071 *
+=============
+
+1. Description
+--------------
+Checks that the given data is present in ElasticSearch.
+
+Input must contain at least 2 fields:
+{{{
+{"_type": ..., "_id": ..., ...}
+...
+}}}
+
+_type and _id are required to retrieve the document from ES. All the other
+fields are compared with the document's corresponding ones. Results of the
+comparison are written to stderr.
+
+2. Running the stage
+--------------------
+The stage can be run as following:
+
+  ./consistency.py --conf elasticsearch_config
+
+For more information about running the stage and its arguments, use:
+
+  ./consistency.py -h
+
diff --git a/Utils/Dataflow/069_upload2es/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
similarity index 100%
rename from Utils/Dataflow/069_upload2es/consistency.py
rename to Utils/Dataflow/071_esConsistency/consistency.py
diff --git a/Utils/Dataflow/run/data4es-consistency-check b/Utils/Dataflow/run/data4es-consistency-check
index 54d23a7de..5733826e4 100755
--- a/Utils/Dataflow/run/data4es-consistency-check
+++ b/Utils/Dataflow/run/data4es-consistency-check
@@ -20,6 +20,6 @@ cmd_016="$base_dir/../016_task2es/task2es.py -m s"
 
 # ES
 cfg_es=`get_config "es"`
-cmd_069="$base_dir/../069_upload2es/consistency.py -m s --conf $cfg_es"
+cmd_071="$base_dir/../071_esConsistency/consistency.py -m s --conf $cfg_es"
 
-$cmd_009 | $cmd_016 | eop_filter | $cmd_069 >/dev/null
+$cmd_009 | $cmd_016 | eop_filter | $cmd_071 >/dev/null

From 12dd86e62842b30a5de00720fa406ffbd72ea91c Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 18 Apr 2019 11:55:46 +0200
Subject: [PATCH 12/26] Update a query description.

- State what is retrieved by the query.
- Remove unnecessary information.
---
 .../009_oracleConnector/query/consistency.sql | 22 +------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/Utils/Dataflow/009_oracleConnector/query/consistency.sql b/Utils/Dataflow/009_oracleConnector/query/consistency.sql
index 6b19c2df3..c0b386f9d 100644
--- a/Utils/Dataflow/009_oracleConnector/query/consistency.sql
+++ b/Utils/Dataflow/009_oracleConnector/query/consistency.sql
@@ -1,24 +1,4 @@
--- Select all tasks for specified period of time
--- Query tables:
-------------------------------------------------
--- ATLAS_DEFT.t_production_task
--- ATLAS_DEFT.t_production_step
--- ATLAS_DEFT.t_step_template
--- ATLAS_DEFT.t_ht_to_task
--- ATLAS_DEFT.t_hashtag
--- ATLAS_PANDA.jedi_datasets
---
--- All fields:
--- architecture, campaign, cloud, conditions_tags, core_count, description, end_time,
--- energy_gev, evgen_job_opts, geometry_version, hashtag_list, job_config, physics_list, processed_events,
--- phys_group, project, pr_id, requested_events, run_number, site, start_time, step_name, status, subcampaign,
--- taskid, taskname, task_timestamp,  ticket_id, trans_home, trans_path, trans_uses, trigger_config, user_name, vo,
--- n_files_per_job, n_events_per_job, n_files_to_be_used,
-
--- RESTRICTIONS:
--- 1. taskID must be more than 4 000 000 OR from the date > 12-03-2014
--- 2. we collecting only PRODUCTION tasks OR only ANALYSIS tasks
---    ('pr_id > 300' or 'pr_id = 300')
+-- Select taskid and timestamp of all tasks for specified period of time
   SELECT DISTINCT
     t.taskid,
     TO_CHAR(t.timestamp, 'dd-mm-yyyy hh24:mi:ss')           AS task_timestamp

From 944b5a2042aab59152bcc0bfa13f24173d1104eb Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 18 Apr 2019 12:06:54 +0200
Subject: [PATCH 13/26] Update and explain a magic number.

---
 Utils/Dataflow/071_esConsistency/consistency.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index a2557581e..58c91c427 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -20,7 +20,8 @@
 def log(msg, prefix='DEBUG'):
     ''' Add prefix and current time to message and write it to stderr. '''
     prefix = '(%s)' % (prefix)
-    prefix = prefix.ljust(8)
+    # 11 = len("(CRITICAL) "), where CRITICAL is the longest log level name.
+    prefix = prefix.ljust(11)
     sys.stderr.write('%s%s %s\n' % (prefix, datetime.now().isoformat(), msg))
 
 

From 26a1dfed7ce50e554793ffa92dfe18adf85f5751 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 18 Apr 2019 12:17:07 +0200
Subject: [PATCH 14/26] Reword es_connect() description.

---
 Utils/Dataflow/071_esConsistency/consistency.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index 58c91c427..76e97deff 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -78,7 +78,9 @@ def load_config(fname):
 
 
 def es_connect(cfg):
-    ''' Establish a connection to elasticsearch, as a global variable.
+    ''' Establish a connection to elasticsearch.
+
+    Initialize the global variable es with the resulting client object.
 
     :param cfg: connection parameters
     :type cfg: dict

From 20875b1fc0daefa030f8091e9df7ed3453f6a5d1 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 18 Apr 2019 14:21:39 +0200
Subject: [PATCH 15/26] Change log prefixes to standard ones.

---
 Utils/Dataflow/071_esConsistency/consistency.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index 76e97deff..facadd94c 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -134,12 +134,12 @@ def process(stage, message):
     '''
     data = message.content()
     if type(data) is not dict:
-        log('Incorrect data:' + str(data), 'INPUT')
+        log('Incorrect data:' + str(data), 'WARN')
         return False
     _id = data.pop('_id')
     _type = data.pop('_type')
     if _id is None or _type is None:
-        log('Insufficient ES info in data:' + str(data), 'INPUT')
+        log('Insufficient ES info in data:' + str(data), 'WARN')
         return False
 
     # Crutch. Remove unwanted (for now) field added by Stage 016.
@@ -150,13 +150,13 @@ def process(stage, message):
     # It's unlikely that such documents will be produced in DKB. In general,
     # such documents should be checked by es.exists(), and not es.get().
     if not data:
-        log('Nothing to check for document (%s, %d)' % (_type, _id), 'INPUT')
+        log('Nothing to check for document (%s, %d)' % (_type, _id), 'WARN')
         return False
 
     es_data = get_fields(INDEX, _id, _type, data.keys())
     if data != es_data:
         log('Document (%s, %d) differs between Oracle and ES: Oracle:%s ES:%s'
-            % (_type, _id, data, es_data), 'DIFF')
+            % (_type, _id, data, es_data), 'WARN')
         global FOUND_DIFF
         FOUND_DIFF.append((_type, _id))
     else:

From 46cf0af5e760659c8c0da5befe1c7e92f6261925 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 18 Apr 2019 14:34:10 +0200
Subject: [PATCH 16/26] Fix pop() results handling.

---
 Utils/Dataflow/071_esConsistency/consistency.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index facadd94c..686a351e1 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -136,9 +136,10 @@ def process(stage, message):
     if type(data) is not dict:
         log('Incorrect data:' + str(data), 'WARN')
         return False
-    _id = data.pop('_id')
-    _type = data.pop('_type')
-    if _id is None or _type is None:
+    try:
+        _id = data.pop('_id')
+        _type = data.pop('_type')
+    except KeyError:
         log('Insufficient ES info in data:' + str(data), 'WARN')
         return False
 

From 72d85a932c0fa59f69692010fea83faea06a8875 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 18 Apr 2019 15:10:46 +0200
Subject: [PATCH 17/26] Update ES parameters handling.

- Show an error message and exit if no host, port, or index is specified.
- Remove default values of the parameters.
---
 .../Dataflow/071_esConsistency/consistency.py | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index 686a351e1..641b581e6 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -54,8 +54,8 @@ def load_config(fname):
     :type fname: str
     '''
     cfg = {
-        'ES_HOST': 'localhost',
-        'ES_PORT': '9200',
+        'ES_HOST': '',
+        'ES_PORT': '',
         'ES_USER': '',
         'ES_PASSWORD': '',
         'ES_INDEX': ''
@@ -85,6 +85,13 @@ def es_connect(cfg):
     :param cfg: connection parameters
     :type cfg: dict
     '''
+    if not cfg['ES_HOST']:
+        log('No ES host specified', 'ERROR')
+        return False
+    if not cfg['ES_PORT']:
+        log('No ES port specified', 'ERROR')
+        return False
+
     global es
     if cfg['ES_USER'] and cfg['ES_PASSWORD']:
         s = 'http://%s:%s@%s:%s/' % (cfg['ES_USER'],
@@ -94,6 +101,7 @@ def es_connect(cfg):
     else:
         s = '%s:%s' % (cfg['ES_HOST'], cfg['ES_PORT'])
     es = elasticsearch.Elasticsearch([s])
+    return True
 
 
 def get_fields(index, _id, _type, fields):
@@ -182,10 +190,14 @@ def main(args):
         stage.parse_args(args)
         cfg = load_config(stage.ARGS.conf)
         stage.process = process
-        es_connect(cfg)
-        if not es.indices.exists(INDEX):
-            log('No such index: %s' % INDEX, 'ERROR')
+        if not es_connect(cfg):
             exit_code = 4
+        elif not INDEX:
+            log('No ES index specified', 'ERROR')
+            exit_code = 5
+        elif not es.indices.exists(INDEX):
+            log('No such index: %s' % INDEX, 'ERROR')
+            exit_code = 6
         else:
             stage.run()
     except (DataflowException, RuntimeError), err:

From cacba118e0d1e4514fdae9880f67d98f7a8574a8 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Thu, 18 Apr 2019 15:27:02 +0200
Subject: [PATCH 18/26] Remove batching of inconsistent records.

Printing all discovered inconsistent records to stdout as a batch contradicts
with various things, such as pyDKB's file mode and the possibility of
controlling the workflow with Apache Kafka.

Create an output message with _id and _type for each inconsistent record.
Still exit with code 1 if at least one inconsistent record was found,
0 otherwise.
---
 Utils/Dataflow/071_esConsistency/consistency.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index 641b581e6..e7f8214f1 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -42,7 +42,7 @@ def log(msg, prefix='DEBUG'):
 
 
 INDEX = None
-FOUND_DIFF = []
+FOUND_DIFF = False
 
 
 def load_config(fname):
@@ -166,8 +166,10 @@ def process(stage, message):
     if data != es_data:
         log('Document (%s, %d) differs between Oracle and ES: Oracle:%s ES:%s'
             % (_type, _id, data, es_data), 'WARN')
+        out_message = JSONMessage({'_type': _type, '_id': _id})
+        stage.output(out_message)
         global FOUND_DIFF
-        FOUND_DIFF.append((_type, _id))
+        FOUND_DIFF = True
     else:
         log('Document (%s, %d) is up to date in ES' % (_type, _id), 'INFO')
 
@@ -218,8 +220,6 @@ def main(args):
     if exit_code == 0 and FOUND_DIFF:
         exit_code = 1
 
-    print FOUND_DIFF
-
     exit(exit_code)
 
 

From 181fb14b6e8be54fdead990ae66ab1caf0cc56a8 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Fri, 19 Apr 2019 11:45:59 +0200
Subject: [PATCH 19/26] Add consistency data samples.

- Add output samples for stages 009 and 016. Stage 071 has no sample because
  all the tasks in 016's sample are up to date in ES.
- Update READMEs.
- Add symbolic link for stage 071's input.
---
 Utils/Dataflow/009_oracleConnector/README     |   4 +
 .../output/consistency.ndjson                 | 623 ++++++++++++++++++
 Utils/Dataflow/016_task2es/README             |   6 +-
 .../016_task2es/output/consistency.ndjson     | 623 ++++++++++++++++++
 Utils/Dataflow/071_esConsistency/input        |   1 +
 5 files changed, 1256 insertions(+), 1 deletion(-)
 create mode 100644 Utils/Dataflow/009_oracleConnector/output/consistency.ndjson
 create mode 100644 Utils/Dataflow/016_task2es/output/consistency.ndjson
 create mode 120000 Utils/Dataflow/071_esConsistency/input

diff --git a/Utils/Dataflow/009_oracleConnector/README b/Utils/Dataflow/009_oracleConnector/README
index c16c5bc45..814810a9a 100644
--- a/Utils/Dataflow/009_oracleConnector/README
+++ b/Utils/Dataflow/009_oracleConnector/README
@@ -32,3 +32,7 @@ The 'output' directory contains two samples of production data:
 
   sample2016.ndjson - collected with timestamps [09-05-2016 12:12:00, 09-05-2016 13:32:30]
   sample2018.ndjson - collected with timestamps [15-06-2018 12:00:00, 15-06-2018 13:00:00]
+
+Also, it contains a consistency checking sample:
+
+  consistency.ndjson - collected with timestamps [21-07-2018 00:00:00, 22-07-2018 00:00:00]
diff --git a/Utils/Dataflow/009_oracleConnector/output/consistency.ndjson b/Utils/Dataflow/009_oracleConnector/output/consistency.ndjson
new file mode 100644
index 000000000..d3e4ecd9a
--- /dev/null
+++ b/Utils/Dataflow/009_oracleConnector/output/consistency.ndjson
@@ -0,0 +1,623 @@
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14682166}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14682168}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14682064}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14682082}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14682104}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14682110}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14682113}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14682127}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14682152}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14681076}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690191}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690193}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690196}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690201}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690203}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690207}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690214}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690219}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690221}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14668910}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14668917}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14668945}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690224}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690229}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690247}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690258}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690260}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690262}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690269}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690277}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690288}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690298}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14682298}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14682301}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14682311}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14682313}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690312}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690319}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690326}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690329}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14682319}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690331}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690333}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690335}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690342}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690348}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690355}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690372}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690374}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690381}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690408}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690412}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690441}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690448}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690450}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690460}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690468}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690472}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690480}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14691091}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14691095}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14690486}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690511}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690517}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14690523}
+{"task_timestamp": "21-07-2018 14:37:15", "taskid": 14690834}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690839}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14683266}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14683313}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690528}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690537}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690545}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690549}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690560}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14690562}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14685891}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14683333}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14683340}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14683344}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14683433}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14683436}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14683440}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14683468}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14683473}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14683484}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14683503}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14683507}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14683515}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14683518}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14690080}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690097}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14690101}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690107}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690116}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690136}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690139}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690150}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690901}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690904}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690907}
+{"task_timestamp": "21-07-2018 02:17:32", "taskid": 14690911}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690914}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690917}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690920}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690923}
+{"task_timestamp": "21-07-2018 14:37:15", "taskid": 14690927}
+{"task_timestamp": "21-07-2018 14:37:15", "taskid": 14690931}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690934}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690938}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14690941}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690944}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690947}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690950}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14661566}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14683601}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690160}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690162}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14690165}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690173}
+{"task_timestamp": "21-07-2018 02:17:30", "taskid": 14690178}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690181}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14668830}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14686855}
+{"task_timestamp": "21-07-2018 17:36:26", "taskid": 14688263}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14678901}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14679112}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14658726}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14658730}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14658778}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14677745}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14677747}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14680896}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14689458}
+{"task_timestamp": "21-07-2018 07:16:09", "taskid": 14689467}
+{"task_timestamp": "21-07-2018 22:37:43", "taskid": 14695666}
+{"task_timestamp": "21-07-2018 02:17:31", "taskid": 14690660}
+{"task_timestamp": "21-07-2018 02:17:31", "taskid": 14690669}
+{"task_timestamp": "21-07-2018 02:17:31", "taskid": 14690673}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14690676}
+{"task_timestamp": "21-07-2018 22:37:40", "taskid": 14693665}
+{"task_timestamp": "21-07-2018 17:36:27", "taskid": 14693667}
+{"task_timestamp": "21-07-2018 17:36:27", "taskid": 14693669}
+{"task_timestamp": "21-07-2018 17:36:27", "taskid": 14693671}
+{"task_timestamp": "21-07-2018 17:36:27", "taskid": 14693794}
+{"task_timestamp": "21-07-2018 22:37:40", "taskid": 14693797}
+{"task_timestamp": "21-07-2018 17:36:27", "taskid": 14693801}
+{"task_timestamp": "21-07-2018 02:17:28", "taskid": 14686356}
+{"task_timestamp": "21-07-2018 20:16:45", "taskid": 14695067}
+{"task_timestamp": "21-07-2018 22:37:42", "taskid": 14695071}
+{"task_timestamp": "21-07-2018 22:37:42", "taskid": 14695080}
+{"task_timestamp": "21-07-2018 22:37:42", "taskid": 14695085}
+{"task_timestamp": "21-07-2018 22:37:42", "taskid": 14695090}
+{"task_timestamp": "21-07-2018 09:35:54", "taskid": 14692167}
+{"task_timestamp": "21-07-2018 12:16:36", "taskid": 14692169}
+{"task_timestamp": "21-07-2018 12:16:36", "taskid": 14692188}
+{"task_timestamp": "21-07-2018 09:35:54", "taskid": 14692191}
+{"task_timestamp": "21-07-2018 09:35:54", "taskid": 14692193}
+{"task_timestamp": "21-07-2018 12:16:36", "taskid": 14692195}
+{"task_timestamp": "21-07-2018 09:35:54", "taskid": 14692197}
+{"task_timestamp": "21-07-2018 09:35:54", "taskid": 14692199}
+{"task_timestamp": "21-07-2018 14:37:16", "taskid": 14692942}
+{"task_timestamp": "21-07-2018 14:37:16", "taskid": 14692948}
+{"task_timestamp": "21-07-2018 14:37:16", "taskid": 14692952}
+{"task_timestamp": "21-07-2018 12:16:37", "taskid": 14692961}
+{"task_timestamp": "21-07-2018 14:37:16", "taskid": 14692965}
+{"task_timestamp": "21-07-2018 17:36:27", "taskid": 14692969}
+{"task_timestamp": "21-07-2018 12:16:37", "taskid": 14692975}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14668306}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14659040}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14659059}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14659116}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14659125}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14659198}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14659252}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14658962}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14166563}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14166565}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14163173}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14163178}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14162739}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14162784}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14162790}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14162794}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14162838}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14162840}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14162844}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14162857}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14162860}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14162959}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14162961}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14163268}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14163272}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14163277}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14163281}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14162987}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14162989}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14162992}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14163004}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14163007}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14163011}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14163022}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14163057}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14163061}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14163064}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14163095}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14163098}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14163108}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14163110}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14163112}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14163148}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14163150}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14163568}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14163573}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14163655}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14163377}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14163380}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14163821}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14163823}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14163826}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14163485}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14163489}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14163491}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14163900}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14163903}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14230557}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14230567}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14267077}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14267083}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14267088}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14267096}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14267107}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14267113}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14267153}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14267156}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14219620}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14219626}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14273977}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14273982}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14217908}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14217915}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14217920}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14218995}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14219000}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14219005}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14217039}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14217046}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14219573}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14219577}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14227667}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14227675}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14227683}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14227693}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14228825}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14228828}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14228833}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14419976}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14419979}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14419981}
+{"task_timestamp": "21-07-2018 02:17:25", "taskid": 14420618}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14420623}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14420627}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14420632}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14429128}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14429133}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14429139}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14429143}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14481155}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14487397}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14589361}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14590092}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14648468}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14648472}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14648578}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14653174}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14653183}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14648691}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14648698}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14649825}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14650822}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14650828}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14650833}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14650947}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14650952}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14595057}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14595088}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14595121}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14595134}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14595184}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14649184}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14649190}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14651208}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14592156}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14592200}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14651236}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14651242}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14649471}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14649477}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14649483}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14649517}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14649523}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14649529}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14651541}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14620479}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14620558}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14620033}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14620052}
+{"task_timestamp": "21-07-2018 22:37:37", "taskid": 14620102}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14681851}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14673197}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14673207}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14673236}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14673245}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14673254}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14673261}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14678683}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14678685}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14678688}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14678707}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14678735}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14678738}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14678740}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14678744}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14673432}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14673444}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14673450}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14678746}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14678749}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14678781}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14673559}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14673612}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14673620}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14678805}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14678809}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14673628}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14673637}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14673652}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14673654}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14673658}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14673663}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14673669}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14673713}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14678912}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14678915}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14678919}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14678931}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14678933}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14678936}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14673834}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14673841}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14678939}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14673846}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14673851}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14673857}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14673861}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14673865}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14673878}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14673882}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14673887}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14673891}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14683604}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14683609}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14672389}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14672400}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14679059}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14679065}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14679073}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14679077}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14679166}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14679170}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14672857}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14672866}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14656704}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14656709}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14686627}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14686629}
+{"task_timestamp": "21-07-2018 17:36:26", "taskid": 14686631}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14686633}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14686635}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14686639}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14686642}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14658552}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14658582}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14692709}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14692712}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14692714}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14692717}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14692730}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14692732}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14692734}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14692736}
+{"task_timestamp": "21-07-2018 02:17:28", "taskid": 14686606}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14686609}
+{"task_timestamp": "21-07-2018 09:35:52", "taskid": 14686625}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14671148}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14671176}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14671182}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14671266}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14671282}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14671291}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14671297}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14681855}
+{"task_timestamp": "21-07-2018 02:17:32", "taskid": 14691001}
+{"task_timestamp": "21-07-2018 17:36:26", "taskid": 14691004}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14691018}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14668803}
+{"task_timestamp": "21-07-2018 02:17:32", "taskid": 14691030}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14691032}
+{"task_timestamp": "21-07-2018 02:17:32", "taskid": 14691042}
+{"task_timestamp": "21-07-2018 17:36:26", "taskid": 14691044}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14669673}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14669748}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14691054}
+{"task_timestamp": "21-07-2018 17:36:26", "taskid": 14691056}
+{"task_timestamp": "21-07-2018 07:16:10", "taskid": 14691067}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14669755}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14669928}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14669936}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14669943}
+{"task_timestamp": "21-07-2018 04:38:02", "taskid": 14691079}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14691081}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14691110}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14691112}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14691120}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14646145}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14670036}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14670043}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14670061}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14670067}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14670074}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14670085}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14691122}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14646635}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14670159}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14670184}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14670191}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14670207}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14670216}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14670223}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14670232}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14670326}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14670335}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14670342}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14670348}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14670362}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14670369}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14670374}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14670381}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14688923}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14634255}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14670722}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14670734}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14670811}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14670818}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14670827}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14670889}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14671003}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14671015}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14671042}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14671052}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14671062}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14671140}
+{"task_timestamp": "21-07-2018 09:35:53", "taskid": 14691106}
+{"task_timestamp": "21-07-2018 09:35:53", "taskid": 14691108}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14682705}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14682802}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657735}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657757}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657784}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657801}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657872}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657877}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657880}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14657918}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657923}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14657928}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14642082}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14656303}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14656352}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14656362}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14656373}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14656411}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14656419}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14656431}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14656433}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14656448}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14656482}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14656486}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14656502}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14656517}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14656548}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14656553}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657449}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14657457}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657503}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14657511}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657515}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657517}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14657523}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657525}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657528}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14657530}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657536}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657548}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657552}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14657556}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14657564}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657568}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657580}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657586}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657588}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657592}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657594}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657604}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657607}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657611}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657626}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14657631}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14657640}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657647}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14657656}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657658}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657665}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657692}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657703}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657720}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657722}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657738}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14657773}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14657839}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14657892}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14658086}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14657968}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14658001}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14658134}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14658145}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14658151}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14658023}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14658052}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14658064}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14658207}
+{"task_timestamp": "21-07-2018 07:16:08", "taskid": 14658231}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14658008}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14658272}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14661560}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14661562}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14661565}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14661568}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14661570}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14657047}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657058}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657063}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657081}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657098}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14657101}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657106}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14657111}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657141}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657149}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14657179}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14657199}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657208}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14657214}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14657223}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657227}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14657231}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14657243}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657246}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14657250}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657256}
+{"task_timestamp": "21-07-2018 02:17:26", "taskid": 14657261}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657264}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657273}
+{"task_timestamp": "21-07-2018 12:16:33", "taskid": 14657277}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14657281}
+{"task_timestamp": "21-07-2018 17:36:24", "taskid": 14657300}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657304}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657322}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657326}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14657328}
+{"task_timestamp": "21-07-2018 07:16:07", "taskid": 14657330}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14657340}
+{"task_timestamp": "21-07-2018 20:16:40", "taskid": 14657342}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657345}
+{"task_timestamp": "21-07-2018 09:35:50", "taskid": 14657212}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14657153}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14661905}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14661911}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14661917}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14661921}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14661925}
+{"task_timestamp": "21-07-2018 04:37:59", "taskid": 14650433}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14677098}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14661232}
+{"task_timestamp": "21-07-2018 14:37:15", "taskid": 14691222}
+{"task_timestamp": "21-07-2018 20:16:42", "taskid": 14692898}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14693003}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14686836}
+{"task_timestamp": "21-07-2018 02:17:28", "taskid": 14686839}
+{"task_timestamp": "21-07-2018 12:16:35", "taskid": 14686843}
+{"task_timestamp": "21-07-2018 04:38:00", "taskid": 14667828}
+{"task_timestamp": "21-07-2018 14:37:12", "taskid": 14638656}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14667956}
+{"task_timestamp": "21-07-2018 14:37:13", "taskid": 14667999}
+{"task_timestamp": "21-07-2018 12:16:34", "taskid": 14681611}
+{"task_timestamp": "21-07-2018 02:17:28", "taskid": 14687576}
+{"task_timestamp": "21-07-2018 07:16:09", "taskid": 14687580}
+{"task_timestamp": "21-07-2018 22:37:39", "taskid": 14687618}
+{"task_timestamp": "21-07-2018 22:37:38", "taskid": 14680910}
+{"task_timestamp": "21-07-2018 14:37:14", "taskid": 14684195}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14684204}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14684211}
+{"task_timestamp": "21-07-2018 17:36:25", "taskid": 14684216}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14684220}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14684224}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14687667}
+{"task_timestamp": "21-07-2018 02:17:28", "taskid": 14684262}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14684265}
+{"task_timestamp": "21-07-2018 09:35:51", "taskid": 14678665}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14682626}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14682630}
+{"task_timestamp": "21-07-2018 20:16:41", "taskid": 14682656}
+{"task_timestamp": "21-07-2018 04:38:01", "taskid": 14682679}
+{"task_timestamp": "21-07-2018 02:17:27", "taskid": 14682700}
diff --git a/Utils/Dataflow/016_task2es/README b/Utils/Dataflow/016_task2es/README
index c148578ad..4e23d9616 100644
--- a/Utils/Dataflow/016_task2es/README
+++ b/Utils/Dataflow/016_task2es/README
@@ -21,7 +21,11 @@ Goes to 019 esFormat.
 
 4. Samples
 ----------
-The 'output' directory contains 2 samples:
+The 'output' directory contains 2 samples of production data:
 
 sample2016.ndjson - 2016 data (from 025's 2016 sample)
 sample2018.ndjson - 2018 data (from 025's 2018 sample)
+
+Also, it contains a consistency checking sample:
+
+consistency.ndjson (from 009's consistency sample)
diff --git a/Utils/Dataflow/016_task2es/output/consistency.ndjson b/Utils/Dataflow/016_task2es/output/consistency.ndjson
new file mode 100644
index 000000000..a8f5fd46a
--- /dev/null
+++ b/Utils/Dataflow/016_task2es/output/consistency.ndjson
@@ -0,0 +1,623 @@
+{"_type": "task", "_id": 14682166, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14682166}
+{"_type": "task", "_id": 14682168, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14682168}
+{"_type": "task", "_id": 14682064, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14682064}
+{"_type": "task", "_id": 14682082, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14682082}
+{"_type": "task", "_id": 14682104, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14682104}
+{"_type": "task", "_id": 14682110, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14682110}
+{"_type": "task", "_id": 14682113, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14682113}
+{"_type": "task", "_id": 14682127, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14682127}
+{"_type": "task", "_id": 14682152, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14682152}
+{"_type": "task", "_id": 14681076, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14681076}
+{"_type": "task", "_id": 14690191, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690191}
+{"_type": "task", "_id": 14690193, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690193}
+{"_type": "task", "_id": 14690196, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690196}
+{"_type": "task", "_id": 14690201, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690201}
+{"_type": "task", "_id": 14690203, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690203}
+{"_type": "task", "_id": 14690207, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690207}
+{"_type": "task", "_id": 14690214, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690214}
+{"_type": "task", "_id": 14690219, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690219}
+{"_type": "task", "_id": 14690221, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690221}
+{"_type": "task", "_id": 14668910, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14668910}
+{"_type": "task", "_id": 14668917, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14668917}
+{"_type": "task", "_id": 14668945, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14668945}
+{"_type": "task", "_id": 14690224, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690224}
+{"_type": "task", "_id": 14690229, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690229}
+{"_type": "task", "_id": 14690247, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690247}
+{"_type": "task", "_id": 14690258, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690258}
+{"_type": "task", "_id": 14690260, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690260}
+{"_type": "task", "_id": 14690262, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690262}
+{"_type": "task", "_id": 14690269, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690269}
+{"_type": "task", "_id": 14690277, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690277}
+{"_type": "task", "_id": 14690288, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690288}
+{"_type": "task", "_id": 14690298, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690298}
+{"_type": "task", "_id": 14682298, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14682298}
+{"_type": "task", "_id": 14682301, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14682301}
+{"_type": "task", "_id": 14682311, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14682311}
+{"_type": "task", "_id": 14682313, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14682313}
+{"_type": "task", "_id": 14690312, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690312}
+{"_type": "task", "_id": 14690319, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690319}
+{"_type": "task", "_id": 14690326, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690326}
+{"_type": "task", "_id": 14690329, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690329}
+{"_type": "task", "_id": 14682319, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14682319}
+{"_type": "task", "_id": 14690331, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690331}
+{"_type": "task", "_id": 14690333, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690333}
+{"_type": "task", "_id": 14690335, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690335}
+{"_type": "task", "_id": 14690342, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690342}
+{"_type": "task", "_id": 14690348, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690348}
+{"_type": "task", "_id": 14690355, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690355}
+{"_type": "task", "_id": 14690372, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690372}
+{"_type": "task", "_id": 14690374, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690374}
+{"_type": "task", "_id": 14690381, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690381}
+{"_type": "task", "_id": 14690408, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690408}
+{"_type": "task", "_id": 14690412, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690412}
+{"_type": "task", "_id": 14690441, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690441}
+{"_type": "task", "_id": 14690448, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690448}
+{"_type": "task", "_id": 14690450, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690450}
+{"_type": "task", "_id": 14690460, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690460}
+{"_type": "task", "_id": 14690468, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690468}
+{"_type": "task", "_id": 14690472, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690472}
+{"_type": "task", "_id": 14690480, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690480}
+{"_type": "task", "_id": 14691091, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14691091}
+{"_type": "task", "_id": 14691095, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14691095}
+{"_type": "task", "_id": 14690486, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14690486}
+{"_type": "task", "_id": 14690511, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690511}
+{"_type": "task", "_id": 14690517, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690517}
+{"_type": "task", "_id": 14690523, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14690523}
+{"_type": "task", "_id": 14690834, "task_timestamp": "21-07-2018 14:37:15", "phys_category": ["Uncategorized"], "taskid": 14690834}
+{"_type": "task", "_id": 14690839, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690839}
+{"_type": "task", "_id": 14683266, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14683266}
+{"_type": "task", "_id": 14683313, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14683313}
+{"_type": "task", "_id": 14690528, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690528}
+{"_type": "task", "_id": 14690537, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690537}
+{"_type": "task", "_id": 14690545, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690545}
+{"_type": "task", "_id": 14690549, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690549}
+{"_type": "task", "_id": 14690560, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690560}
+{"_type": "task", "_id": 14690562, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14690562}
+{"_type": "task", "_id": 14685891, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14685891}
+{"_type": "task", "_id": 14683333, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14683333}
+{"_type": "task", "_id": 14683340, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14683340}
+{"_type": "task", "_id": 14683344, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14683344}
+{"_type": "task", "_id": 14683433, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14683433}
+{"_type": "task", "_id": 14683436, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14683436}
+{"_type": "task", "_id": 14683440, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14683440}
+{"_type": "task", "_id": 14683468, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14683468}
+{"_type": "task", "_id": 14683473, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14683473}
+{"_type": "task", "_id": 14683484, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14683484}
+{"_type": "task", "_id": 14683503, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14683503}
+{"_type": "task", "_id": 14683507, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14683507}
+{"_type": "task", "_id": 14683515, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14683515}
+{"_type": "task", "_id": 14683518, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14683518}
+{"_type": "task", "_id": 14690080, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14690080}
+{"_type": "task", "_id": 14690097, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690097}
+{"_type": "task", "_id": 14690101, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14690101}
+{"_type": "task", "_id": 14690107, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690107}
+{"_type": "task", "_id": 14690116, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690116}
+{"_type": "task", "_id": 14690136, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690136}
+{"_type": "task", "_id": 14690139, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690139}
+{"_type": "task", "_id": 14690150, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690150}
+{"_type": "task", "_id": 14690901, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690901}
+{"_type": "task", "_id": 14690904, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690904}
+{"_type": "task", "_id": 14690907, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690907}
+{"_type": "task", "_id": 14690911, "task_timestamp": "21-07-2018 02:17:32", "phys_category": ["Uncategorized"], "taskid": 14690911}
+{"_type": "task", "_id": 14690914, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690914}
+{"_type": "task", "_id": 14690917, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690917}
+{"_type": "task", "_id": 14690920, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690920}
+{"_type": "task", "_id": 14690923, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690923}
+{"_type": "task", "_id": 14690927, "task_timestamp": "21-07-2018 14:37:15", "phys_category": ["Uncategorized"], "taskid": 14690927}
+{"_type": "task", "_id": 14690931, "task_timestamp": "21-07-2018 14:37:15", "phys_category": ["Uncategorized"], "taskid": 14690931}
+{"_type": "task", "_id": 14690934, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690934}
+{"_type": "task", "_id": 14690938, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690938}
+{"_type": "task", "_id": 14690941, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14690941}
+{"_type": "task", "_id": 14690944, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690944}
+{"_type": "task", "_id": 14690947, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690947}
+{"_type": "task", "_id": 14690950, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690950}
+{"_type": "task", "_id": 14661566, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14661566}
+{"_type": "task", "_id": 14683601, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14683601}
+{"_type": "task", "_id": 14690160, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690160}
+{"_type": "task", "_id": 14690162, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690162}
+{"_type": "task", "_id": 14690165, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14690165}
+{"_type": "task", "_id": 14690173, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690173}
+{"_type": "task", "_id": 14690178, "task_timestamp": "21-07-2018 02:17:30", "phys_category": ["Uncategorized"], "taskid": 14690178}
+{"_type": "task", "_id": 14690181, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690181}
+{"_type": "task", "_id": 14668830, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14668830}
+{"_type": "task", "_id": 14686855, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14686855}
+{"_type": "task", "_id": 14688263, "task_timestamp": "21-07-2018 17:36:26", "phys_category": ["Uncategorized"], "taskid": 14688263}
+{"_type": "task", "_id": 14678901, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14678901}
+{"_type": "task", "_id": 14679112, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14679112}
+{"_type": "task", "_id": 14658726, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14658726}
+{"_type": "task", "_id": 14658730, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14658730}
+{"_type": "task", "_id": 14658778, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14658778}
+{"_type": "task", "_id": 14677745, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14677745}
+{"_type": "task", "_id": 14677747, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14677747}
+{"_type": "task", "_id": 14680896, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14680896}
+{"_type": "task", "_id": 14689458, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14689458}
+{"_type": "task", "_id": 14689467, "task_timestamp": "21-07-2018 07:16:09", "phys_category": ["Uncategorized"], "taskid": 14689467}
+{"_type": "task", "_id": 14695666, "task_timestamp": "21-07-2018 22:37:43", "phys_category": ["Uncategorized"], "taskid": 14695666}
+{"_type": "task", "_id": 14690660, "task_timestamp": "21-07-2018 02:17:31", "phys_category": ["Uncategorized"], "taskid": 14690660}
+{"_type": "task", "_id": 14690669, "task_timestamp": "21-07-2018 02:17:31", "phys_category": ["Uncategorized"], "taskid": 14690669}
+{"_type": "task", "_id": 14690673, "task_timestamp": "21-07-2018 02:17:31", "phys_category": ["Uncategorized"], "taskid": 14690673}
+{"_type": "task", "_id": 14690676, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14690676}
+{"_type": "task", "_id": 14693665, "task_timestamp": "21-07-2018 22:37:40", "phys_category": ["Uncategorized"], "taskid": 14693665}
+{"_type": "task", "_id": 14693667, "task_timestamp": "21-07-2018 17:36:27", "phys_category": ["Uncategorized"], "taskid": 14693667}
+{"_type": "task", "_id": 14693669, "task_timestamp": "21-07-2018 17:36:27", "phys_category": ["Uncategorized"], "taskid": 14693669}
+{"_type": "task", "_id": 14693671, "task_timestamp": "21-07-2018 17:36:27", "phys_category": ["Uncategorized"], "taskid": 14693671}
+{"_type": "task", "_id": 14693794, "task_timestamp": "21-07-2018 17:36:27", "phys_category": ["Uncategorized"], "taskid": 14693794}
+{"_type": "task", "_id": 14693797, "task_timestamp": "21-07-2018 22:37:40", "phys_category": ["Uncategorized"], "taskid": 14693797}
+{"_type": "task", "_id": 14693801, "task_timestamp": "21-07-2018 17:36:27", "phys_category": ["Uncategorized"], "taskid": 14693801}
+{"_type": "task", "_id": 14686356, "task_timestamp": "21-07-2018 02:17:28", "phys_category": ["Uncategorized"], "taskid": 14686356}
+{"_type": "task", "_id": 14695067, "task_timestamp": "21-07-2018 20:16:45", "phys_category": ["Uncategorized"], "taskid": 14695067}
+{"_type": "task", "_id": 14695071, "task_timestamp": "21-07-2018 22:37:42", "phys_category": ["Uncategorized"], "taskid": 14695071}
+{"_type": "task", "_id": 14695080, "task_timestamp": "21-07-2018 22:37:42", "phys_category": ["Uncategorized"], "taskid": 14695080}
+{"_type": "task", "_id": 14695085, "task_timestamp": "21-07-2018 22:37:42", "phys_category": ["Uncategorized"], "taskid": 14695085}
+{"_type": "task", "_id": 14695090, "task_timestamp": "21-07-2018 22:37:42", "phys_category": ["Uncategorized"], "taskid": 14695090}
+{"_type": "task", "_id": 14692167, "task_timestamp": "21-07-2018 09:35:54", "phys_category": ["Uncategorized"], "taskid": 14692167}
+{"_type": "task", "_id": 14692169, "task_timestamp": "21-07-2018 12:16:36", "phys_category": ["Uncategorized"], "taskid": 14692169}
+{"_type": "task", "_id": 14692188, "task_timestamp": "21-07-2018 12:16:36", "phys_category": ["Uncategorized"], "taskid": 14692188}
+{"_type": "task", "_id": 14692191, "task_timestamp": "21-07-2018 09:35:54", "phys_category": ["Uncategorized"], "taskid": 14692191}
+{"_type": "task", "_id": 14692193, "task_timestamp": "21-07-2018 09:35:54", "phys_category": ["Uncategorized"], "taskid": 14692193}
+{"_type": "task", "_id": 14692195, "task_timestamp": "21-07-2018 12:16:36", "phys_category": ["Uncategorized"], "taskid": 14692195}
+{"_type": "task", "_id": 14692197, "task_timestamp": "21-07-2018 09:35:54", "phys_category": ["Uncategorized"], "taskid": 14692197}
+{"_type": "task", "_id": 14692199, "task_timestamp": "21-07-2018 09:35:54", "phys_category": ["Uncategorized"], "taskid": 14692199}
+{"_type": "task", "_id": 14692942, "task_timestamp": "21-07-2018 14:37:16", "phys_category": ["Uncategorized"], "taskid": 14692942}
+{"_type": "task", "_id": 14692948, "task_timestamp": "21-07-2018 14:37:16", "phys_category": ["Uncategorized"], "taskid": 14692948}
+{"_type": "task", "_id": 14692952, "task_timestamp": "21-07-2018 14:37:16", "phys_category": ["Uncategorized"], "taskid": 14692952}
+{"_type": "task", "_id": 14692961, "task_timestamp": "21-07-2018 12:16:37", "phys_category": ["Uncategorized"], "taskid": 14692961}
+{"_type": "task", "_id": 14692965, "task_timestamp": "21-07-2018 14:37:16", "phys_category": ["Uncategorized"], "taskid": 14692965}
+{"_type": "task", "_id": 14692969, "task_timestamp": "21-07-2018 17:36:27", "phys_category": ["Uncategorized"], "taskid": 14692969}
+{"_type": "task", "_id": 14692975, "task_timestamp": "21-07-2018 12:16:37", "phys_category": ["Uncategorized"], "taskid": 14692975}
+{"_type": "task", "_id": 14668306, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14668306}
+{"_type": "task", "_id": 14659040, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14659040}
+{"_type": "task", "_id": 14659059, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14659059}
+{"_type": "task", "_id": 14659116, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14659116}
+{"_type": "task", "_id": 14659125, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14659125}
+{"_type": "task", "_id": 14659198, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14659198}
+{"_type": "task", "_id": 14659252, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14659252}
+{"_type": "task", "_id": 14658962, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14658962}
+{"_type": "task", "_id": 14166563, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14166563}
+{"_type": "task", "_id": 14166565, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14166565}
+{"_type": "task", "_id": 14163173, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14163173}
+{"_type": "task", "_id": 14163178, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14163178}
+{"_type": "task", "_id": 14162739, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14162739}
+{"_type": "task", "_id": 14162784, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14162784}
+{"_type": "task", "_id": 14162790, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14162790}
+{"_type": "task", "_id": 14162794, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14162794}
+{"_type": "task", "_id": 14162838, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14162838}
+{"_type": "task", "_id": 14162840, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14162840}
+{"_type": "task", "_id": 14162844, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14162844}
+{"_type": "task", "_id": 14162857, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14162857}
+{"_type": "task", "_id": 14162860, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14162860}
+{"_type": "task", "_id": 14162959, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14162959}
+{"_type": "task", "_id": 14162961, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14162961}
+{"_type": "task", "_id": 14163268, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14163268}
+{"_type": "task", "_id": 14163272, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14163272}
+{"_type": "task", "_id": 14163277, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14163277}
+{"_type": "task", "_id": 14163281, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14163281}
+{"_type": "task", "_id": 14162987, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14162987}
+{"_type": "task", "_id": 14162989, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14162989}
+{"_type": "task", "_id": 14162992, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14162992}
+{"_type": "task", "_id": 14163004, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14163004}
+{"_type": "task", "_id": 14163007, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14163007}
+{"_type": "task", "_id": 14163011, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14163011}
+{"_type": "task", "_id": 14163022, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14163022}
+{"_type": "task", "_id": 14163057, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14163057}
+{"_type": "task", "_id": 14163061, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14163061}
+{"_type": "task", "_id": 14163064, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14163064}
+{"_type": "task", "_id": 14163095, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14163095}
+{"_type": "task", "_id": 14163098, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14163098}
+{"_type": "task", "_id": 14163108, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14163108}
+{"_type": "task", "_id": 14163110, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14163110}
+{"_type": "task", "_id": 14163112, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14163112}
+{"_type": "task", "_id": 14163148, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14163148}
+{"_type": "task", "_id": 14163150, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14163150}
+{"_type": "task", "_id": 14163568, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14163568}
+{"_type": "task", "_id": 14163573, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14163573}
+{"_type": "task", "_id": 14163655, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14163655}
+{"_type": "task", "_id": 14163377, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14163377}
+{"_type": "task", "_id": 14163380, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14163380}
+{"_type": "task", "_id": 14163821, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14163821}
+{"_type": "task", "_id": 14163823, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14163823}
+{"_type": "task", "_id": 14163826, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14163826}
+{"_type": "task", "_id": 14163485, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14163485}
+{"_type": "task", "_id": 14163489, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14163489}
+{"_type": "task", "_id": 14163491, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14163491}
+{"_type": "task", "_id": 14163900, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14163900}
+{"_type": "task", "_id": 14163903, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14163903}
+{"_type": "task", "_id": 14230557, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14230557}
+{"_type": "task", "_id": 14230567, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14230567}
+{"_type": "task", "_id": 14267077, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14267077}
+{"_type": "task", "_id": 14267083, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14267083}
+{"_type": "task", "_id": 14267088, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14267088}
+{"_type": "task", "_id": 14267096, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14267096}
+{"_type": "task", "_id": 14267107, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14267107}
+{"_type": "task", "_id": 14267113, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14267113}
+{"_type": "task", "_id": 14267153, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14267153}
+{"_type": "task", "_id": 14267156, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14267156}
+{"_type": "task", "_id": 14219620, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14219620}
+{"_type": "task", "_id": 14219626, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14219626}
+{"_type": "task", "_id": 14273977, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14273977}
+{"_type": "task", "_id": 14273982, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14273982}
+{"_type": "task", "_id": 14217908, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14217908}
+{"_type": "task", "_id": 14217915, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14217915}
+{"_type": "task", "_id": 14217920, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14217920}
+{"_type": "task", "_id": 14218995, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14218995}
+{"_type": "task", "_id": 14219000, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14219000}
+{"_type": "task", "_id": 14219005, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14219005}
+{"_type": "task", "_id": 14217039, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14217039}
+{"_type": "task", "_id": 14217046, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14217046}
+{"_type": "task", "_id": 14219573, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14219573}
+{"_type": "task", "_id": 14219577, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14219577}
+{"_type": "task", "_id": 14227667, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14227667}
+{"_type": "task", "_id": 14227675, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14227675}
+{"_type": "task", "_id": 14227683, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14227683}
+{"_type": "task", "_id": 14227693, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14227693}
+{"_type": "task", "_id": 14228825, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14228825}
+{"_type": "task", "_id": 14228828, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14228828}
+{"_type": "task", "_id": 14228833, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14228833}
+{"_type": "task", "_id": 14419976, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14419976}
+{"_type": "task", "_id": 14419979, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14419979}
+{"_type": "task", "_id": 14419981, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14419981}
+{"_type": "task", "_id": 14420618, "task_timestamp": "21-07-2018 02:17:25", "phys_category": ["Uncategorized"], "taskid": 14420618}
+{"_type": "task", "_id": 14420623, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14420623}
+{"_type": "task", "_id": 14420627, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14420627}
+{"_type": "task", "_id": 14420632, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14420632}
+{"_type": "task", "_id": 14429128, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14429128}
+{"_type": "task", "_id": 14429133, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14429133}
+{"_type": "task", "_id": 14429139, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14429139}
+{"_type": "task", "_id": 14429143, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14429143}
+{"_type": "task", "_id": 14481155, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14481155}
+{"_type": "task", "_id": 14487397, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14487397}
+{"_type": "task", "_id": 14589361, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14589361}
+{"_type": "task", "_id": 14590092, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14590092}
+{"_type": "task", "_id": 14648468, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14648468}
+{"_type": "task", "_id": 14648472, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14648472}
+{"_type": "task", "_id": 14648578, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14648578}
+{"_type": "task", "_id": 14653174, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14653174}
+{"_type": "task", "_id": 14653183, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14653183}
+{"_type": "task", "_id": 14648691, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14648691}
+{"_type": "task", "_id": 14648698, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14648698}
+{"_type": "task", "_id": 14649825, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14649825}
+{"_type": "task", "_id": 14650822, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14650822}
+{"_type": "task", "_id": 14650828, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14650828}
+{"_type": "task", "_id": 14650833, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14650833}
+{"_type": "task", "_id": 14650947, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14650947}
+{"_type": "task", "_id": 14650952, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14650952}
+{"_type": "task", "_id": 14595057, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14595057}
+{"_type": "task", "_id": 14595088, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14595088}
+{"_type": "task", "_id": 14595121, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14595121}
+{"_type": "task", "_id": 14595134, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14595134}
+{"_type": "task", "_id": 14595184, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14595184}
+{"_type": "task", "_id": 14649184, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14649184}
+{"_type": "task", "_id": 14649190, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14649190}
+{"_type": "task", "_id": 14651208, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14651208}
+{"_type": "task", "_id": 14592156, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14592156}
+{"_type": "task", "_id": 14592200, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14592200}
+{"_type": "task", "_id": 14651236, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14651236}
+{"_type": "task", "_id": 14651242, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14651242}
+{"_type": "task", "_id": 14649471, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14649471}
+{"_type": "task", "_id": 14649477, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14649477}
+{"_type": "task", "_id": 14649483, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14649483}
+{"_type": "task", "_id": 14649517, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14649517}
+{"_type": "task", "_id": 14649523, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14649523}
+{"_type": "task", "_id": 14649529, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14649529}
+{"_type": "task", "_id": 14651541, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14651541}
+{"_type": "task", "_id": 14620479, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14620479}
+{"_type": "task", "_id": 14620558, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14620558}
+{"_type": "task", "_id": 14620033, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14620033}
+{"_type": "task", "_id": 14620052, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14620052}
+{"_type": "task", "_id": 14620102, "task_timestamp": "21-07-2018 22:37:37", "phys_category": ["Uncategorized"], "taskid": 14620102}
+{"_type": "task", "_id": 14681851, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14681851}
+{"_type": "task", "_id": 14673197, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14673197}
+{"_type": "task", "_id": 14673207, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14673207}
+{"_type": "task", "_id": 14673236, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14673236}
+{"_type": "task", "_id": 14673245, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14673245}
+{"_type": "task", "_id": 14673254, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14673254}
+{"_type": "task", "_id": 14673261, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14673261}
+{"_type": "task", "_id": 14678683, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14678683}
+{"_type": "task", "_id": 14678685, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14678685}
+{"_type": "task", "_id": 14678688, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14678688}
+{"_type": "task", "_id": 14678707, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14678707}
+{"_type": "task", "_id": 14678735, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14678735}
+{"_type": "task", "_id": 14678738, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14678738}
+{"_type": "task", "_id": 14678740, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14678740}
+{"_type": "task", "_id": 14678744, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14678744}
+{"_type": "task", "_id": 14673432, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14673432}
+{"_type": "task", "_id": 14673444, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14673444}
+{"_type": "task", "_id": 14673450, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14673450}
+{"_type": "task", "_id": 14678746, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14678746}
+{"_type": "task", "_id": 14678749, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14678749}
+{"_type": "task", "_id": 14678781, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14678781}
+{"_type": "task", "_id": 14673559, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14673559}
+{"_type": "task", "_id": 14673612, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14673612}
+{"_type": "task", "_id": 14673620, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14673620}
+{"_type": "task", "_id": 14678805, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14678805}
+{"_type": "task", "_id": 14678809, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14678809}
+{"_type": "task", "_id": 14673628, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14673628}
+{"_type": "task", "_id": 14673637, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14673637}
+{"_type": "task", "_id": 14673652, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14673652}
+{"_type": "task", "_id": 14673654, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14673654}
+{"_type": "task", "_id": 14673658, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14673658}
+{"_type": "task", "_id": 14673663, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14673663}
+{"_type": "task", "_id": 14673669, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14673669}
+{"_type": "task", "_id": 14673713, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14673713}
+{"_type": "task", "_id": 14678912, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14678912}
+{"_type": "task", "_id": 14678915, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14678915}
+{"_type": "task", "_id": 14678919, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14678919}
+{"_type": "task", "_id": 14678931, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14678931}
+{"_type": "task", "_id": 14678933, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14678933}
+{"_type": "task", "_id": 14678936, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14678936}
+{"_type": "task", "_id": 14673834, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14673834}
+{"_type": "task", "_id": 14673841, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14673841}
+{"_type": "task", "_id": 14678939, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14678939}
+{"_type": "task", "_id": 14673846, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14673846}
+{"_type": "task", "_id": 14673851, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14673851}
+{"_type": "task", "_id": 14673857, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14673857}
+{"_type": "task", "_id": 14673861, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14673861}
+{"_type": "task", "_id": 14673865, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14673865}
+{"_type": "task", "_id": 14673878, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14673878}
+{"_type": "task", "_id": 14673882, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14673882}
+{"_type": "task", "_id": 14673887, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14673887}
+{"_type": "task", "_id": 14673891, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14673891}
+{"_type": "task", "_id": 14683604, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14683604}
+{"_type": "task", "_id": 14683609, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14683609}
+{"_type": "task", "_id": 14672389, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14672389}
+{"_type": "task", "_id": 14672400, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14672400}
+{"_type": "task", "_id": 14679059, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14679059}
+{"_type": "task", "_id": 14679065, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14679065}
+{"_type": "task", "_id": 14679073, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14679073}
+{"_type": "task", "_id": 14679077, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14679077}
+{"_type": "task", "_id": 14679166, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14679166}
+{"_type": "task", "_id": 14679170, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14679170}
+{"_type": "task", "_id": 14672857, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14672857}
+{"_type": "task", "_id": 14672866, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14672866}
+{"_type": "task", "_id": 14656704, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14656704}
+{"_type": "task", "_id": 14656709, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14656709}
+{"_type": "task", "_id": 14686627, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14686627}
+{"_type": "task", "_id": 14686629, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14686629}
+{"_type": "task", "_id": 14686631, "task_timestamp": "21-07-2018 17:36:26", "phys_category": ["Uncategorized"], "taskid": 14686631}
+{"_type": "task", "_id": 14686633, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14686633}
+{"_type": "task", "_id": 14686635, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14686635}
+{"_type": "task", "_id": 14686639, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14686639}
+{"_type": "task", "_id": 14686642, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14686642}
+{"_type": "task", "_id": 14658552, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14658552}
+{"_type": "task", "_id": 14658582, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14658582}
+{"_type": "task", "_id": 14692709, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14692709}
+{"_type": "task", "_id": 14692712, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14692712}
+{"_type": "task", "_id": 14692714, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14692714}
+{"_type": "task", "_id": 14692717, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14692717}
+{"_type": "task", "_id": 14692730, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14692730}
+{"_type": "task", "_id": 14692732, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14692732}
+{"_type": "task", "_id": 14692734, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14692734}
+{"_type": "task", "_id": 14692736, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14692736}
+{"_type": "task", "_id": 14686606, "task_timestamp": "21-07-2018 02:17:28", "phys_category": ["Uncategorized"], "taskid": 14686606}
+{"_type": "task", "_id": 14686609, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14686609}
+{"_type": "task", "_id": 14686625, "task_timestamp": "21-07-2018 09:35:52", "phys_category": ["Uncategorized"], "taskid": 14686625}
+{"_type": "task", "_id": 14671148, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14671148}
+{"_type": "task", "_id": 14671176, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14671176}
+{"_type": "task", "_id": 14671182, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14671182}
+{"_type": "task", "_id": 14671266, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14671266}
+{"_type": "task", "_id": 14671282, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14671282}
+{"_type": "task", "_id": 14671291, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14671291}
+{"_type": "task", "_id": 14671297, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14671297}
+{"_type": "task", "_id": 14681855, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14681855}
+{"_type": "task", "_id": 14691001, "task_timestamp": "21-07-2018 02:17:32", "phys_category": ["Uncategorized"], "taskid": 14691001}
+{"_type": "task", "_id": 14691004, "task_timestamp": "21-07-2018 17:36:26", "phys_category": ["Uncategorized"], "taskid": 14691004}
+{"_type": "task", "_id": 14691018, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14691018}
+{"_type": "task", "_id": 14668803, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14668803}
+{"_type": "task", "_id": 14691030, "task_timestamp": "21-07-2018 02:17:32", "phys_category": ["Uncategorized"], "taskid": 14691030}
+{"_type": "task", "_id": 14691032, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14691032}
+{"_type": "task", "_id": 14691042, "task_timestamp": "21-07-2018 02:17:32", "phys_category": ["Uncategorized"], "taskid": 14691042}
+{"_type": "task", "_id": 14691044, "task_timestamp": "21-07-2018 17:36:26", "phys_category": ["Uncategorized"], "taskid": 14691044}
+{"_type": "task", "_id": 14669673, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14669673}
+{"_type": "task", "_id": 14669748, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14669748}
+{"_type": "task", "_id": 14691054, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14691054}
+{"_type": "task", "_id": 14691056, "task_timestamp": "21-07-2018 17:36:26", "phys_category": ["Uncategorized"], "taskid": 14691056}
+{"_type": "task", "_id": 14691067, "task_timestamp": "21-07-2018 07:16:10", "phys_category": ["Uncategorized"], "taskid": 14691067}
+{"_type": "task", "_id": 14669755, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14669755}
+{"_type": "task", "_id": 14669928, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14669928}
+{"_type": "task", "_id": 14669936, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14669936}
+{"_type": "task", "_id": 14669943, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14669943}
+{"_type": "task", "_id": 14691079, "task_timestamp": "21-07-2018 04:38:02", "phys_category": ["Uncategorized"], "taskid": 14691079}
+{"_type": "task", "_id": 14691081, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14691081}
+{"_type": "task", "_id": 14691110, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14691110}
+{"_type": "task", "_id": 14691112, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14691112}
+{"_type": "task", "_id": 14691120, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14691120}
+{"_type": "task", "_id": 14646145, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14646145}
+{"_type": "task", "_id": 14670036, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14670036}
+{"_type": "task", "_id": 14670043, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14670043}
+{"_type": "task", "_id": 14670061, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14670061}
+{"_type": "task", "_id": 14670067, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14670067}
+{"_type": "task", "_id": 14670074, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14670074}
+{"_type": "task", "_id": 14670085, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14670085}
+{"_type": "task", "_id": 14691122, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14691122}
+{"_type": "task", "_id": 14646635, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14646635}
+{"_type": "task", "_id": 14670159, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14670159}
+{"_type": "task", "_id": 14670184, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14670184}
+{"_type": "task", "_id": 14670191, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14670191}
+{"_type": "task", "_id": 14670207, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14670207}
+{"_type": "task", "_id": 14670216, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14670216}
+{"_type": "task", "_id": 14670223, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14670223}
+{"_type": "task", "_id": 14670232, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14670232}
+{"_type": "task", "_id": 14670326, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14670326}
+{"_type": "task", "_id": 14670335, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14670335}
+{"_type": "task", "_id": 14670342, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14670342}
+{"_type": "task", "_id": 14670348, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14670348}
+{"_type": "task", "_id": 14670362, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14670362}
+{"_type": "task", "_id": 14670369, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14670369}
+{"_type": "task", "_id": 14670374, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14670374}
+{"_type": "task", "_id": 14670381, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14670381}
+{"_type": "task", "_id": 14688923, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14688923}
+{"_type": "task", "_id": 14634255, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14634255}
+{"_type": "task", "_id": 14670722, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14670722}
+{"_type": "task", "_id": 14670734, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14670734}
+{"_type": "task", "_id": 14670811, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14670811}
+{"_type": "task", "_id": 14670818, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14670818}
+{"_type": "task", "_id": 14670827, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14670827}
+{"_type": "task", "_id": 14670889, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14670889}
+{"_type": "task", "_id": 14671003, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14671003}
+{"_type": "task", "_id": 14671015, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14671015}
+{"_type": "task", "_id": 14671042, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14671042}
+{"_type": "task", "_id": 14671052, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14671052}
+{"_type": "task", "_id": 14671062, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14671062}
+{"_type": "task", "_id": 14671140, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14671140}
+{"_type": "task", "_id": 14691106, "task_timestamp": "21-07-2018 09:35:53", "phys_category": ["Uncategorized"], "taskid": 14691106}
+{"_type": "task", "_id": 14691108, "task_timestamp": "21-07-2018 09:35:53", "phys_category": ["Uncategorized"], "taskid": 14691108}
+{"_type": "task", "_id": 14682705, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14682705}
+{"_type": "task", "_id": 14682802, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14682802}
+{"_type": "task", "_id": 14657735, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657735}
+{"_type": "task", "_id": 14657757, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657757}
+{"_type": "task", "_id": 14657784, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657784}
+{"_type": "task", "_id": 14657801, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657801}
+{"_type": "task", "_id": 14657872, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657872}
+{"_type": "task", "_id": 14657877, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657877}
+{"_type": "task", "_id": 14657880, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657880}
+{"_type": "task", "_id": 14657918, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14657918}
+{"_type": "task", "_id": 14657923, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657923}
+{"_type": "task", "_id": 14657928, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14657928}
+{"_type": "task", "_id": 14642082, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14642082}
+{"_type": "task", "_id": 14656303, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14656303}
+{"_type": "task", "_id": 14656352, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14656352}
+{"_type": "task", "_id": 14656362, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14656362}
+{"_type": "task", "_id": 14656373, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14656373}
+{"_type": "task", "_id": 14656411, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14656411}
+{"_type": "task", "_id": 14656419, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14656419}
+{"_type": "task", "_id": 14656431, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14656431}
+{"_type": "task", "_id": 14656433, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14656433}
+{"_type": "task", "_id": 14656448, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14656448}
+{"_type": "task", "_id": 14656482, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14656482}
+{"_type": "task", "_id": 14656486, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14656486}
+{"_type": "task", "_id": 14656502, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14656502}
+{"_type": "task", "_id": 14656517, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14656517}
+{"_type": "task", "_id": 14656548, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14656548}
+{"_type": "task", "_id": 14656553, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14656553}
+{"_type": "task", "_id": 14657449, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657449}
+{"_type": "task", "_id": 14657457, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14657457}
+{"_type": "task", "_id": 14657503, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657503}
+{"_type": "task", "_id": 14657511, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14657511}
+{"_type": "task", "_id": 14657515, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657515}
+{"_type": "task", "_id": 14657517, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657517}
+{"_type": "task", "_id": 14657523, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14657523}
+{"_type": "task", "_id": 14657525, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657525}
+{"_type": "task", "_id": 14657528, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657528}
+{"_type": "task", "_id": 14657530, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14657530}
+{"_type": "task", "_id": 14657536, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657536}
+{"_type": "task", "_id": 14657548, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657548}
+{"_type": "task", "_id": 14657552, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657552}
+{"_type": "task", "_id": 14657556, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14657556}
+{"_type": "task", "_id": 14657564, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14657564}
+{"_type": "task", "_id": 14657568, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657568}
+{"_type": "task", "_id": 14657580, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657580}
+{"_type": "task", "_id": 14657586, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657586}
+{"_type": "task", "_id": 14657588, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657588}
+{"_type": "task", "_id": 14657592, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657592}
+{"_type": "task", "_id": 14657594, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657594}
+{"_type": "task", "_id": 14657604, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657604}
+{"_type": "task", "_id": 14657607, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657607}
+{"_type": "task", "_id": 14657611, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657611}
+{"_type": "task", "_id": 14657626, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657626}
+{"_type": "task", "_id": 14657631, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14657631}
+{"_type": "task", "_id": 14657640, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14657640}
+{"_type": "task", "_id": 14657647, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657647}
+{"_type": "task", "_id": 14657656, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14657656}
+{"_type": "task", "_id": 14657658, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657658}
+{"_type": "task", "_id": 14657665, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657665}
+{"_type": "task", "_id": 14657692, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657692}
+{"_type": "task", "_id": 14657703, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657703}
+{"_type": "task", "_id": 14657720, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657720}
+{"_type": "task", "_id": 14657722, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657722}
+{"_type": "task", "_id": 14657738, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657738}
+{"_type": "task", "_id": 14657773, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14657773}
+{"_type": "task", "_id": 14657839, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14657839}
+{"_type": "task", "_id": 14657892, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14657892}
+{"_type": "task", "_id": 14658086, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14658086}
+{"_type": "task", "_id": 14657968, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14657968}
+{"_type": "task", "_id": 14658001, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14658001}
+{"_type": "task", "_id": 14658134, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14658134}
+{"_type": "task", "_id": 14658145, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14658145}
+{"_type": "task", "_id": 14658151, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14658151}
+{"_type": "task", "_id": 14658023, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14658023}
+{"_type": "task", "_id": 14658052, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14658052}
+{"_type": "task", "_id": 14658064, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14658064}
+{"_type": "task", "_id": 14658207, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14658207}
+{"_type": "task", "_id": 14658231, "task_timestamp": "21-07-2018 07:16:08", "phys_category": ["Uncategorized"], "taskid": 14658231}
+{"_type": "task", "_id": 14658008, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14658008}
+{"_type": "task", "_id": 14658272, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14658272}
+{"_type": "task", "_id": 14661560, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14661560}
+{"_type": "task", "_id": 14661562, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14661562}
+{"_type": "task", "_id": 14661565, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14661565}
+{"_type": "task", "_id": 14661568, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14661568}
+{"_type": "task", "_id": 14661570, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14661570}
+{"_type": "task", "_id": 14657047, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14657047}
+{"_type": "task", "_id": 14657058, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657058}
+{"_type": "task", "_id": 14657063, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657063}
+{"_type": "task", "_id": 14657081, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657081}
+{"_type": "task", "_id": 14657098, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657098}
+{"_type": "task", "_id": 14657101, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14657101}
+{"_type": "task", "_id": 14657106, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657106}
+{"_type": "task", "_id": 14657111, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14657111}
+{"_type": "task", "_id": 14657141, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657141}
+{"_type": "task", "_id": 14657149, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657149}
+{"_type": "task", "_id": 14657179, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14657179}
+{"_type": "task", "_id": 14657199, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14657199}
+{"_type": "task", "_id": 14657208, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657208}
+{"_type": "task", "_id": 14657214, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14657214}
+{"_type": "task", "_id": 14657223, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14657223}
+{"_type": "task", "_id": 14657227, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657227}
+{"_type": "task", "_id": 14657231, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14657231}
+{"_type": "task", "_id": 14657243, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14657243}
+{"_type": "task", "_id": 14657246, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657246}
+{"_type": "task", "_id": 14657250, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14657250}
+{"_type": "task", "_id": 14657256, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657256}
+{"_type": "task", "_id": 14657261, "task_timestamp": "21-07-2018 02:17:26", "phys_category": ["Uncategorized"], "taskid": 14657261}
+{"_type": "task", "_id": 14657264, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657264}
+{"_type": "task", "_id": 14657273, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657273}
+{"_type": "task", "_id": 14657277, "task_timestamp": "21-07-2018 12:16:33", "phys_category": ["Uncategorized"], "taskid": 14657277}
+{"_type": "task", "_id": 14657281, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14657281}
+{"_type": "task", "_id": 14657300, "task_timestamp": "21-07-2018 17:36:24", "phys_category": ["Uncategorized"], "taskid": 14657300}
+{"_type": "task", "_id": 14657304, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657304}
+{"_type": "task", "_id": 14657322, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657322}
+{"_type": "task", "_id": 14657326, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657326}
+{"_type": "task", "_id": 14657328, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14657328}
+{"_type": "task", "_id": 14657330, "task_timestamp": "21-07-2018 07:16:07", "phys_category": ["Uncategorized"], "taskid": 14657330}
+{"_type": "task", "_id": 14657340, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14657340}
+{"_type": "task", "_id": 14657342, "task_timestamp": "21-07-2018 20:16:40", "phys_category": ["Uncategorized"], "taskid": 14657342}
+{"_type": "task", "_id": 14657345, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657345}
+{"_type": "task", "_id": 14657212, "task_timestamp": "21-07-2018 09:35:50", "phys_category": ["Uncategorized"], "taskid": 14657212}
+{"_type": "task", "_id": 14657153, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14657153}
+{"_type": "task", "_id": 14661905, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14661905}
+{"_type": "task", "_id": 14661911, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14661911}
+{"_type": "task", "_id": 14661917, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14661917}
+{"_type": "task", "_id": 14661921, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14661921}
+{"_type": "task", "_id": 14661925, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14661925}
+{"_type": "task", "_id": 14650433, "task_timestamp": "21-07-2018 04:37:59", "phys_category": ["Uncategorized"], "taskid": 14650433}
+{"_type": "task", "_id": 14677098, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14677098}
+{"_type": "task", "_id": 14661232, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14661232}
+{"_type": "task", "_id": 14691222, "task_timestamp": "21-07-2018 14:37:15", "phys_category": ["Uncategorized"], "taskid": 14691222}
+{"_type": "task", "_id": 14692898, "task_timestamp": "21-07-2018 20:16:42", "phys_category": ["Uncategorized"], "taskid": 14692898}
+{"_type": "task", "_id": 14693003, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14693003}
+{"_type": "task", "_id": 14686836, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14686836}
+{"_type": "task", "_id": 14686839, "task_timestamp": "21-07-2018 02:17:28", "phys_category": ["Uncategorized"], "taskid": 14686839}
+{"_type": "task", "_id": 14686843, "task_timestamp": "21-07-2018 12:16:35", "phys_category": ["Uncategorized"], "taskid": 14686843}
+{"_type": "task", "_id": 14667828, "task_timestamp": "21-07-2018 04:38:00", "phys_category": ["Uncategorized"], "taskid": 14667828}
+{"_type": "task", "_id": 14638656, "task_timestamp": "21-07-2018 14:37:12", "phys_category": ["Uncategorized"], "taskid": 14638656}
+{"_type": "task", "_id": 14667956, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14667956}
+{"_type": "task", "_id": 14667999, "task_timestamp": "21-07-2018 14:37:13", "phys_category": ["Uncategorized"], "taskid": 14667999}
+{"_type": "task", "_id": 14681611, "task_timestamp": "21-07-2018 12:16:34", "phys_category": ["Uncategorized"], "taskid": 14681611}
+{"_type": "task", "_id": 14687576, "task_timestamp": "21-07-2018 02:17:28", "phys_category": ["Uncategorized"], "taskid": 14687576}
+{"_type": "task", "_id": 14687580, "task_timestamp": "21-07-2018 07:16:09", "phys_category": ["Uncategorized"], "taskid": 14687580}
+{"_type": "task", "_id": 14687618, "task_timestamp": "21-07-2018 22:37:39", "phys_category": ["Uncategorized"], "taskid": 14687618}
+{"_type": "task", "_id": 14680910, "task_timestamp": "21-07-2018 22:37:38", "phys_category": ["Uncategorized"], "taskid": 14680910}
+{"_type": "task", "_id": 14684195, "task_timestamp": "21-07-2018 14:37:14", "phys_category": ["Uncategorized"], "taskid": 14684195}
+{"_type": "task", "_id": 14684204, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14684204}
+{"_type": "task", "_id": 14684211, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14684211}
+{"_type": "task", "_id": 14684216, "task_timestamp": "21-07-2018 17:36:25", "phys_category": ["Uncategorized"], "taskid": 14684216}
+{"_type": "task", "_id": 14684220, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14684220}
+{"_type": "task", "_id": 14684224, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14684224}
+{"_type": "task", "_id": 14687667, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14687667}
+{"_type": "task", "_id": 14684262, "task_timestamp": "21-07-2018 02:17:28", "phys_category": ["Uncategorized"], "taskid": 14684262}
+{"_type": "task", "_id": 14684265, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14684265}
+{"_type": "task", "_id": 14678665, "task_timestamp": "21-07-2018 09:35:51", "phys_category": ["Uncategorized"], "taskid": 14678665}
+{"_type": "task", "_id": 14682626, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14682626}
+{"_type": "task", "_id": 14682630, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14682630}
+{"_type": "task", "_id": 14682656, "task_timestamp": "21-07-2018 20:16:41", "phys_category": ["Uncategorized"], "taskid": 14682656}
+{"_type": "task", "_id": 14682679, "task_timestamp": "21-07-2018 04:38:01", "phys_category": ["Uncategorized"], "taskid": 14682679}
+{"_type": "task", "_id": 14682700, "task_timestamp": "21-07-2018 02:17:27", "phys_category": ["Uncategorized"], "taskid": 14682700}
diff --git a/Utils/Dataflow/071_esConsistency/input b/Utils/Dataflow/071_esConsistency/input
new file mode 120000
index 000000000..322251d34
--- /dev/null
+++ b/Utils/Dataflow/071_esConsistency/input
@@ -0,0 +1 @@
+../016_task2es/output
\ No newline at end of file

From 711724291c9ed9387310a29c831fc8673fbfa9c4 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Fri, 19 Apr 2019 11:59:55 +0200
Subject: [PATCH 20/26] Update the dataflow README.

Add information about the new stage and dataflow to the README.
---
 Utils/Dataflow/README | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/Utils/Dataflow/README b/Utils/Dataflow/README
index 84b7f9c94..cb61c723a 100644
--- a/Utils/Dataflow/README
+++ b/Utils/Dataflow/README
@@ -43,6 +43,7 @@
 1.9     019_oracle2esFormat            Prepare data for bulk upload to ES
 2.5     025_chicagoES                  Get additional metadata from Chicago ES
 6.9     069_upload2es                  Upload data to ES
+7.1     071_esConsistency              Check data in ES
 9.1     091_datasetsRucio              Get dataset metadata from Rucio
 9.3     093_datasetsFormat             Update dataset metadata: add
                                          "data_format" field
@@ -79,6 +80,16 @@
                                           |
     ** ----> 019 -> 069                   | Prepare and upload metadata to ES
 
+3. Tasks metadata consistency check
+---
+
+009 ---> 016 ---> 071
+
+This is a simplified and slightly changed version of the previous dataflow,
+intended for making sure that information is consistent between ProdSys2 and
+ES. It gets a very basic set of metadata from ProdSys2, adds ES-related
+fields, and checks that it is present in ES rather than uploading it.
+
 ==============
 * REFERENCES *
 ==============

From 165c5d27ef967b5c99d202d4418e4e17242e7b9a Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Tue, 21 May 2019 16:18:51 +0200
Subject: [PATCH 21/26] Ignore two additional fields.

---
 Utils/Dataflow/071_esConsistency/consistency.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index e7f8214f1..084a62064 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -151,9 +151,10 @@ def process(stage, message):
         log('Insufficient ES info in data:' + str(data), 'WARN')
         return False
 
-    # Crutch. Remove unwanted (for now) field added by Stage 016.
-    if 'phys_category' in data:
-        del data['phys_category']
+    # Crutch. Remove unwanted (for now) fields added by Stage 016.
+    for field in ['phys_category', 'chain_data', 'chain_id']:
+        if field in data:
+            del data[field]
 
     # Do not check empty documents with valid _id and _type.
     # It's unlikely that such documents will be produced in DKB. In general,

From 8f84d223de81c270b76e4f5cf044bffb530ef850 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Wed, 22 May 2019 12:40:18 +0200
Subject: [PATCH 22/26] Change messages formatting.

Type of _id is unknown - it can be str or int for task, and str for dataset.
---
 Utils/Dataflow/071_esConsistency/consistency.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index 084a62064..4d97a630a 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -160,19 +160,19 @@ def process(stage, message):
     # It's unlikely that such documents will be produced in DKB. In general,
     # such documents should be checked by es.exists(), and not es.get().
     if not data:
-        log('Nothing to check for document (%s, %d)' % (_type, _id), 'WARN')
+        log('Nothing to check for document (%s, %r)' % (_type, _id), 'WARN')
         return False
 
     es_data = get_fields(INDEX, _id, _type, data.keys())
     if data != es_data:
-        log('Document (%s, %d) differs between Oracle and ES: Oracle:%s ES:%s'
+        log('Document (%s, %r) differs between Oracle and ES: Oracle:%s ES:%s'
             % (_type, _id, data, es_data), 'WARN')
         out_message = JSONMessage({'_type': _type, '_id': _id})
         stage.output(out_message)
         global FOUND_DIFF
         FOUND_DIFF = True
     else:
-        log('Document (%s, %d) is up to date in ES' % (_type, _id), 'INFO')
+        log('Document (%s, %r) is up to date in ES' % (_type, _id), 'INFO')
 
     return True
 

From d19565027aa827c73c90d4cd33e10c16e46eefc2 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Wed, 22 May 2019 12:47:06 +0200
Subject: [PATCH 23/26] Add _parent field handling.

The field is required to get child documents such as output datasets.
---
 Utils/Dataflow/071_esConsistency/consistency.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index 4d97a630a..ea72b7484 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -104,7 +104,7 @@ def es_connect(cfg):
     return True
 
 
-def get_fields(index, _id, _type, fields):
+def get_fields(index, _id, _type, fields, _parent):
     ''' Get fields value by given _id and _type.
 
     :param es: elasticsearch client
@@ -123,7 +123,7 @@ def get_fields(index, _id, _type, fields):
     '''
     try:
         results = es.get(index=index, doc_type=_type, id=_id,
-                         _source=fields)
+                         _source=fields, parent=_parent)
     except elasticsearch.exceptions.NotFoundError:
         return False
     return results['_source']
@@ -151,6 +151,10 @@ def process(stage, message):
         log('Insufficient ES info in data:' + str(data), 'WARN')
         return False
 
+    _parent = None
+    if '_parent' in data:
+        _parent = data.pop('_parent')
+
     # Crutch. Remove unwanted (for now) fields added by Stage 016.
     for field in ['phys_category', 'chain_data', 'chain_id']:
         if field in data:
@@ -163,7 +167,7 @@ def process(stage, message):
         log('Nothing to check for document (%s, %r)' % (_type, _id), 'WARN')
         return False
 
-    es_data = get_fields(INDEX, _id, _type, data.keys())
+    es_data = get_fields(INDEX, _id, _type, data.keys(), _parent)
     if data != es_data:
         log('Document (%s, %r) differs between Oracle and ES: Oracle:%s ES:%s'
             % (_type, _id, data, es_data), 'WARN')

From 612bf52f4093f177c2e4ec41d091329a4a97d978 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Wed, 22 May 2019 13:02:05 +0200
Subject: [PATCH 24/26] Remove service fields before checking.

Service fields are different from data fields and shouldn't be checked.
---
 Utils/Dataflow/071_esConsistency/consistency.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index ea72b7484..db2f6aed4 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -160,6 +160,11 @@ def process(stage, message):
         if field in data:
             del data[field]
 
+    # Fields starting with an underscore are service fields. Some of them are
+    # treated in special way (see _id above). Service fields should not be
+    # checked, so they are removed.
+    data = {field: data[field] for field in data if field[0] != '_'}
+
     # Do not check empty documents with valid _id and _type.
     # It's unlikely that such documents will be produced in DKB. In general,
     # such documents should be checked by es.exists(), and not es.get().

From 01ae258a1186fda73ff9b378a212017bf6fd55d4 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Wed, 22 May 2019 13:12:04 +0200
Subject: [PATCH 25/26] Remove interpreter directives from lib files.

These are unnecessary because library files are not supposed to be executed.
---
 Utils/Dataflow/shell_lib/eop_filter | 2 --
 Utils/Dataflow/shell_lib/get_config | 2 --
 Utils/Dataflow/shell_lib/log        | 2 --
 3 files changed, 6 deletions(-)

diff --git a/Utils/Dataflow/shell_lib/eop_filter b/Utils/Dataflow/shell_lib/eop_filter
index 61f770255..30f00d2f7 100755
--- a/Utils/Dataflow/shell_lib/eop_filter
+++ b/Utils/Dataflow/shell_lib/eop_filter
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
 # EOP filter (required due to the unconfigurable EOP marker in pyDKB)
 eop_filter() {
   sed -e"s/\\x00//g"
diff --git a/Utils/Dataflow/shell_lib/get_config b/Utils/Dataflow/shell_lib/get_config
index 9e6543464..3f1986987 100755
--- a/Utils/Dataflow/shell_lib/get_config
+++ b/Utils/Dataflow/shell_lib/get_config
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
 get_config_lib=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
 source $get_config_lib/log
 
diff --git a/Utils/Dataflow/shell_lib/log b/Utils/Dataflow/shell_lib/log
index ac9d1871c..cf17d721d 100755
--- a/Utils/Dataflow/shell_lib/log
+++ b/Utils/Dataflow/shell_lib/log
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
 log() {
   level=INFO
   [ $# -eq 2 ] && level="$1" && shift

From 8f86dddb0e847fe89981594ef55fdfebf8712a59 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Tue, 28 May 2019 11:16:56 +0200
Subject: [PATCH 26/26] Simplify a field retrieval.

---
 Utils/Dataflow/071_esConsistency/consistency.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/Utils/Dataflow/071_esConsistency/consistency.py b/Utils/Dataflow/071_esConsistency/consistency.py
index db2f6aed4..df12b1c30 100755
--- a/Utils/Dataflow/071_esConsistency/consistency.py
+++ b/Utils/Dataflow/071_esConsistency/consistency.py
@@ -151,9 +151,7 @@ def process(stage, message):
         log('Insufficient ES info in data:' + str(data), 'WARN')
         return False
 
-    _parent = None
-    if '_parent' in data:
-        _parent = data.pop('_parent')
+    _parent = data.pop('_parent', None)
 
     # Crutch. Remove unwanted (for now) fields added by Stage 016.
     for field in ['phys_category', 'chain_data', 'chain_id']: