3.bidaf-mir.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deploy NCD model to Triton Inference Server on MIR\n",
    "\n",
    "description: (preview) deploy a bi-directional attention flow (bidaf) Q&A model to V100s on AKS via Triton"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please note that this Public Preview release is subject to the [Supplemental Terms of Use for Microsoft Azure Previews](https://azure.microsoft.com/support/legal/preview-supplemental-terms/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from azureml.core import Workspace\n",
    "\n",
    "subscription_id = os.getenv(\"SUBSCRIPTION_ID\", default=\"<subscription_id>\")\n",
    "resource_group = os.getenv(\"RESOURCE_GROUP\", default=\"<resource_group>\")\n",
    "workspace_name = os.getenv(\"WORKSPACE_NAME\", default=\"<workspace_name>\")\n",
    "\n",
    "ws = Workspace.get(\n",
    "    subscription_id = subscription_id, \n",
    "    resource_group = resource_group, \n",
    "    name = workspace_name)\n",
    "\n",
    "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "from src.model_utils import download_triton_models, delete_triton_models\n",
    "\n",
    "prefix = Path(\".\")\n",
    "download_triton_models(prefix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Register model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from azureml.core.model import Model\n",
    "\n",
    "model = Model.register(\n",
    "    model_path=\"./models\",\n",
    "    model_name=\"bidaf-9\",\n",
    "    tags={\"area\": \"Natural language processing\", \"type\": \"Question-answering\"},\n",
    "    description=\"Question answering from ONNX model zoo\",\n",
    "    workspace=ws,\n",
    "    model_framework=Model.Framework.MULTI,\n",
    ")\n",
    "\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploy webservice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from azureml.contrib.mir.webservice import MirWebservice\n",
    "from azureml.exceptions import WebserviceException\n",
    "from azureml.core import Webservice\n",
    "\n",
    "mir_config = MirWebservice.deploy_configuration(cpu_cores = 1, memory_gb = 4, sku = \"Standard_NC24s_v3\", \n",
    "    num_replicas = 1, gpu_cores = 1, tags = {\"mir.enableMount\": \"true\"})\n",
    "\n",
    "service_name = \"triton-ncd-bidaf-gpus-1\"\n",
    "\n",
    "mir_service = Model.deploy(ws, service_name , [model], deployment_config = mir_config)\n",
    "mir_service.wait_for_deployment(show_output = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(mir_service.state)\n",
    "print(mir_service.scoring_uri)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the webservice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "access_token = mir_service.get_access_token().access_token\n",
    "print(access_token)\n",
    "\n",
    "service_endpoint = mir_service.scoring_uri.replace(\"/score\", \"\")\n",
    "print(service_endpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install Triton client libraries @https://github.com/triton-inference-server/server/releases\n",
    "\n",
    "Open cmd window and run the following commands\n",
    "\n",
    "! cd /tmp && \\\n",
    "  wget https://github.com/NVIDIA/triton-inference-server/releases/download/v2.6.0/v2.6.0_ubuntu2004.clients.tar.gz && \\\n",
    "  tar xzf v2.6.0_ubuntu2004.clients.tar.gz && \\\n",
    "  pip install python/tritonclient-2.6.0-py3-none-any.whl numpy pillow\n",
    "\n",
    "Note this depends on the local OS (in this case Ubuntu) version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tritonclient.http as tritonhttpclient\n",
    "from tritonclient.utils import triton_to_np_dtype\n",
    "\n",
    "headers = {}\n",
    "headers[\"Authorization\"] = f\"Bearer {access_token}\"\n",
    "\n",
    "uri = service_endpoint[8:]\n",
    "print(uri)\n",
    "\n",
    "triton_client = tritonhttpclient.InferenceServerClient(uri, ssl=True)\n",
    "\n",
    "context = \"A quick brown fox jumped over the lazy dog.\"\n",
    "query = \"Which animal was lower?\"\n",
    "\n",
    "model_name = \"bidaf-9\"\n",
    "\n",
    "model_metadata = triton_client.get_model_metadata(\n",
    "    model_name=model_name, headers=headers\n",
    ")\n",
    "\n",
    "input_meta = model_metadata[\"inputs\"]\n",
    "output_meta = model_metadata[\"outputs\"]\n",
    "\n",
    "# We use the np.object data type for string data\n",
    "np_dtype = triton_to_np_dtype(input_meta[0][\"datatype\"])\n",
    "cw, cc = preprocess(context, np_dtype)\n",
    "qw, qc = preprocess(query, np_dtype)\n",
    "\n",
    "input_mapping = {\n",
    "    \"query_word\": qw,\n",
    "    \"query_char\": qc,\n",
    "    \"context_word\": cw,\n",
    "    \"context_char\": cc,\n",
    "}\n",
    "\n",
    "inputs = []\n",
    "outputs = []\n",
    "\n",
    "# Populate the inputs array\n",
    "for in_meta in input_meta:\n",
    "    input_name = in_meta[\"name\"]\n",
    "    data = input_mapping[input_name]\n",
    "\n",
    "    input = tritonhttpclient.InferInput(input_name, data.shape, in_meta[\"datatype\"])\n",
    "\n",
    "    input.set_data_from_numpy(data, binary_data=False)\n",
    "    inputs.append(input)\n",
    "\n",
    "# Populate the outputs array\n",
    "for out_meta in output_meta:\n",
    "    output_name = out_meta[\"name\"]\n",
    "    output = tritonhttpclient.InferRequestedOutput(output_name, binary_data=False)\n",
    "    outputs.append(output)\n",
    "\n",
    "# Run inference\n",
    "res = triton_client.infer(\n",
    "    model_name,\n",
    "    inputs,\n",
    "    request_id=\"0\",\n",
    "    outputs=outputs,\n",
    "    model_version=\"1\",\n",
    "    headers=headers,\n",
    ")\n",
    "\n",
    "result = postprocess(context_words=cw, answer=res)\n",
    "\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete the webservice and the downloaded model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "mir_service.delete()\n",
    "model.delete()\n",
    "delete_triton_models(prefix)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "name": "deploy-bidaf-aks",
  "task": "Use the high-performance Triton Inference Server with Azure Machine Learning"
 },
 "nbformat": 4,
 "nbformat_minor": 4
}