Advertise on replag (#22)

Bug: T60841

Dockerfile

@@ -1,5 +1,5 @@
 # Use official python base image, small and debian edition
-FROM amd64/python:3.7.3-slim
+FROM amd64/python:3.7.16-slim
 
 ARG purpose=dev
 

docker-replica/replica.sql

@@ -1197,3 +1197,14 @@ UNLOCK TABLES;
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
+
+CREATE DATABASE /*!32312 IF NOT EXISTS*/ `heartbeat_p` /*!40100 DEFAULT CHARACTER SET utf8mb4 */;
+USE `heartbeat_p`;
+GRANT SELECT ON heartbeat_p.* TO 'repl'@'%';
+
+CREATE TABLE `heartbeat` (
+  `shard` varbinary(10) NOT NULL,
+  `last_updated` varbinary(26) NOT NULL,
+  `lag` decimal(25, 4)
+) ENGINE=InnoDB DEFAULT CHARSET=binary;
+INSERT INTO heartbeat_p.heartbeat (shard, last_updated, `lag`) VALUES (0x7331, 0x323032332D30352D32355430353A33393A33382E303030393930, 278631.9990);

quarry/web/app.py

@@ -44,7 +44,8 @@ def setup_context():
 
 
 def kill_context(exception=None):
-    g.conn.close_all()
+    if g.conn:
+        g.conn.close_all()
     del g.replica.connection
 
 

quarry/web/static/templates/compiled.js

@@ -156,6 +156,13 @@ output += "\n";
 ;
 }
 output += "\n";
+if(runtime.memberLookup((runtime.contextOrFrameLookup(context, frame, "extra")),"replag")) {
+output += "\n<div class=\"alert alert-warning\">\n  <h4 class=\"alert-heading\">Replication lag</h4>\n  <p class=\"mb-0\">The database on which this query was executed has a <a href=\"https://replag.toolforge.org/\">synchronization delay</a> with the wiki.\n      This can be caused by maintenance or incident on database, and should be resolved soon.<br>\n      The modifications that was made in last <b>";
+output += runtime.suppressValue(runtime.memberLookup((runtime.contextOrFrameLookup(context, frame, "extra")),"replag"), env.opts.autoescape);
+output += "</b> on the wiki are not taken into account in results bellow.<br>\n</div>\n";
+;
+}
+output += "\n";
 if(parentTemplate) {
 parentTemplate.rootRenderFunc(env, context, frame, runtime, cb);
 } else {

quarry/web/static/templates/query-status.html

@@ -13,3 +13,11 @@
 <!--<button id="show-explain" type="button" class="btn btn-default btn-xs">Explain</button>-->
 {% endif %}
 {% endif %}
+{% if extra.replag %}
+<div class="alert alert-warning">
+  <h4 class="alert-heading">Replication lag</h4>
+  <p class="mb-0">The database on which this query was executed has a <a href="https://replag.toolforge.org/">synchronization delay</a> with the wiki.
+      This can be caused by maintenance or incident on database, and should be resolved soon.<br>
+      The modifications that was made in last <b>{{ extra.replag }}</b> on the wiki are not taken into account in results bellow.<br>
+</div>
+{% endif %}

quarry/web/webhelpers.py

@@ -4,18 +4,14 @@
 templatehelpers = Blueprint("templatehelpers", __name__)
 
 
-@templatehelpers.add_app_template_filter
-def timesince(dt, default="just now"):
+def get_pretty_delay(diff, suffix="", default="just now"):
     """
     Returns string representing "time since" e.g.
     3 days ago, 5 hours ago etc.
 
     From http://flask.pocoo.org/snippets/33/
     """
 
-    now = datetime.utcnow()
-    diff = now - dt
-
     periods = (
         (diff.days // 365, "year", "years"),
         (diff.days // 30, "month", "months"),
@@ -27,8 +23,18 @@ def timesince(dt, default="just now"):
     )
 
     for period, singular, plural in periods:
-
         if period:
-            return "%d %s ago" % (period, singular if period == 1 else plural)
+            return "%d %s %s" % (
+                period,
+                singular if period == 1 else plural,
+                suffix,
+            )
 
     return default
+
+
+@templatehelpers.add_app_template_filter
+def timesince(dt, default="just now"):
+    now = datetime.utcnow()
+    diff = now - dt
+    return get_pretty_delay(diff, suffix="ago", default=default)

quarry/web/worker.py

@@ -1,6 +1,7 @@
 import json
 import os
 import timeit
+from datetime import timedelta
 
 from celery import Celery
 from celery.signals import worker_process_init, worker_process_shutdown
@@ -13,6 +14,7 @@
 from .results import SQLiteResultWriter
 from .replica import Replica
 from .utils import monkey as _unused  # noqa: F401
+from .webhelpers import get_pretty_delay
 
 
 __dir__ = os.path.dirname(__file__)
@@ -34,6 +36,11 @@
 conn = None
 
 
+def get_replag(cur):
+    cur.execute("SELECT lag FROM heartbeat_p.heartbeat;")
+    return int(cur.fetchall()[0][0])
+
+
 @worker_process_init.connect
 def init(*args, **kwargs):
     global conn
@@ -107,12 +114,16 @@ def run_query(query_run_id):
         output.close()
         stoptime = timeit.default_timer()
         qrun.status = QueryRun.STATUS_COMPLETE
-        qrun.extra_info = json.dumps(
-            {
-                "resultsets": output.get_resultsets(),
-                "runningtime": "%.2f" % (stoptime - starttime),
-            }
-        )
+        extra_info = {
+            "resultsets": output.get_resultsets(),
+            "runningtime": "%.2f" % (stoptime - starttime),
+        }
+        # Add replica lag if it's above threshold, to be displayed to user
+        replag = get_replag(cur)
+        if replag > 180:  # 3 minutes
+            extra_info["replag"] = get_pretty_delay(timedelta(seconds=replag))
+        qrun.extra_info = json.dumps(extra_info)
+
         celery_log.info("Completed run for qrun:%s successfully", qrun.id)
         conn.session.add(qrun)
         conn.session.commit()

tests/test_worker.py

@@ -19,7 +19,7 @@ def __init__(self):
         self.nextsetcalled = False
 
     def fetchall(self):
-        return [["a", "b", "c"], ["1", "2", "3"]]
+        return [[1, "b", "c"], ["1", "2", "3"]]
 
     def fetchmany(self, _count):
         if not self.called: