Merge pull request #1 from codinggeeks06/main

mercury compatible with jupyterhub

Dockerfile/README.md

@@ -0,0 +1,3 @@
+This dockerfile is for standalone mercury pod. The thought process here is to have basic setup of Python3.9 and then install mercury with its dependencies. and run `mercury run 0.0.0.0:9000` on start. This approach worked for me. 
+
+We can customise this docker file by adding all the extra libraries which was used by notebooks.

howItIsWorking.md

@@ -0,0 +1,45 @@
+# This is a version of mercury that supports mercury inside the jupyterhub pod
+Here this version that works with jupyter server proxy.
+
+prerequsites:
+- working setup of jupyterhub with any kind of spawner. This version is tested and working with `Kubespawner`. You can contribute by checking if this works with other spawner.
+
+### Here this version works with jupyter server proxy.
+
+Before starting you need to set environment varibales `MERCURY_APP_PREFIX` and `MECURY_SERVER_URL`. 
+
+`MERCURY_APP_PREFIX` is a subpath where the requests will be made and frontend will be served.
+
+for example for jupyterhub user with name  'username' `MERCURY_APP_PREFIX` will be '/user/username/mercury'.
+
+and `MECURY_SERVER_URL` will be '127.0.0.1:8080'.
+
+
+### Setup required to run this version on jupyterhub:
+- jupyter-server-proxy installed and activated with this command:
+```
+jupyter serverextension enable --sys-prefix jupyter_server_proxy
+```
+- setting up jupyter_server_config.py at on of `jupyter --paths`. You can check documentation of `jupyter-server-proxy` [here](https://jupyter-server-proxy.readthedocs.io/en/latest/)  
+
+    the `jupyter_server_config.py` will look like this:
+```
+c = get_config() #noqa
+
+c.ServerProxy.servers = {
+    'mercury': {
+        'command': ['mercury', 'run', '0.0.0.0:8080', '--verbose'],
+        'timeout': 2 * 60,
+        'absolute_url': False,
+        'port': '8080',
+        'new_browser_window': False,
+        'launcher_entry':{
+            'enabled': True,
+            'title': 'mercury server'
+        }
+    }
+}
+```
+
+- in `jupyterhub_config.py` you need to set environement variables `MERCURY_APP_PREFIX` and `MECURY_SERVER_URL`,
+- you need to make custom front end with subpath '/user/username/mercury' for each user as per shown here[https://runmercury.com/docs/docker-compose/#deploy-on-subpath] by changing `package.json` and `Routes.tsx`. create wheel using this [script](scripts/build_wheel_custom_frontend.sh) and this use user specific wheel in each user environment.

jupyter_config/jupyter_server_config.py

@@ -0,0 +1,15 @@
+c = get_config() #noqa
+
+c.ServerProxy.servers = {
+    'mercury': {
+        'command': ['mercury', 'run', '0.0.0.0:8080', '--verbose'],
+        'timeout': 2 * 60,
+        'absolute_url': False,
+        'port': '10000'
+        'new_browser_window': False,
+        'launcher_entry':{
+            'enabled': True,
+            'title': 'mercury server'
+        }
+    }
+}

mercury/apps/nbworker/__main__.py

@@ -39,6 +39,8 @@
         "wss://", "https://"
     )
 
+# ks
+server_url = server_url.replace("http://", "ws://").replace("https://", "wss://")
 
 def signal_handler(signal, frame):
     global stop_event

mercury/apps/storage/utils.py

@@ -18,7 +18,8 @@
 MERCURY_DATA_DIR = Path(os.getenv('MERCURY_DATA_DIR', BASE_DIR))
 
 MEDIA_ROOT = str(MERCURY_DATA_DIR / "media")
-MEDIA_URL = "/media/"
+# ks
+MEDIA_URL = os.environ.get("MERCURY_APP_PREFIX", "") + "/media/"
 
 
 def get_bucket_key(site, user, filename):

mercury/apps/ws/client.py

@@ -16,6 +16,7 @@
 from apps.ws.tasks import task_start_websocket_worker
 from apps.ws.utils import client_group, worker_group
 from apps.storage.s3utils import clean_worker_files
+import os
 
 log = logging.getLogger(__name__)
 
@@ -123,6 +124,8 @@ def need_worker(self):
         #         "payload": {"purpose": "worker-state", "state": "UsageLimitReached"},
         #     },
         # )
+        # ks
+        self.server_address = os.environ.get("MERCURY_SERVER_URL", "http://127.0.0.1:8080")
 
         with transaction.atomic():
             log.debug("Create worker in db")

mercury/server/settings.py

@@ -245,7 +245,8 @@
     except Exception as e:
         raise Exception(f"Cannot create directory {MEDIA_ROOT}. {str(e)}")
 
-MEDIA_URL = "/media/"
+# ks
+MEDIA_URL = os.environ.get("MERCURY_APP_PREFIX", "") + "/media/"
 
 # Default primary key field type
 # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field

proxy/package.json

@@ -0,0 +1,21 @@
+{
+    "name": "proxy",
+    "version": "2.3.7",
+    "private": true,
+    "dependencies": {
+        "http-proxy": "^1.18.1"
+    },
+    "browserslist": {
+      "production": [
+        ">0.2%",
+        "not dead",
+        "not op_mini all"
+      ],
+      "development": [
+        "last 1 chrome version",
+        "last 1 firefox version",
+        "last 1 safari version"
+      ]
+    }
+  }
+

proxy/proxy.js

@@ -0,0 +1,34 @@
+var httpProxy = require('http-proxy')
+
+let server_ip='127.0.0.1'
+let server_port=8080, proxy_port=10000, 
+var proxy = httpProxy.createProxyServer({target: "http://" +`${server_ip}:${server_port}`, ws:true}).listen(proxy_port);
+var username = process.env.JHUB_USER
+proxy.on('proxyReq', (proxyReq, req, res, options) => {
+    let path = proxyReq.path;
+    let tokens = path.split('/')
+    if(path.includes("/static/media")){
+        if(tokens[1] == 'user' && tokens[3] == 'mercury'){
+            tokens.splice(1,3)
+            path = tokens.join('/')
+        }
+    }else if(path.includes('/media/')){
+        path = '/user/' + username + '/mercury' + path
+    }else if(tokens[1]  == 'user' && tokens[3] == 'mercury'){
+        tokens.splice(1,3)
+        path = tokens.join('/')
+    }
+
+    if(path === ''){
+        path = '/'
+    }
+    proxyReq = path
+});
+
+proxy.on('error', (err, req, res) => {
+    if(err.errno == -111){
+        res.end("waiting for mercury to get up")
+    }else{
+        res.end("Error occures" + err)
+    }
+});

scripts/build_wheel_custom_frontend.sh

@@ -0,0 +1,8 @@
+# have node14 environment for creating custom frontend
+which node
+cd ../frontend
+rm -rf build && yarn build
+rm -rf ../mercury/frontend-dist
+cd ../ 
+rm -rf build/* dist/*
+python3.9 setup.py bdist_wheel