From fe14a313ee96d4a170c036bfb63ec36793938b99 Mon Sep 17 00:00:00 2001 From: cs128781 Date: Tue, 10 Dec 2024 15:33:43 -0800 Subject: [PATCH 1/6] add init api key injection and scheduler version (again) --- .../templates/ai-unlimited/ai-unlimited-with-nlb.yaml | 6 ++++-- .../ai-unlimited/ai-unlimited-without-lb.yaml | 10 +++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deployments/aws/templates/ai-unlimited/ai-unlimited-with-nlb.yaml b/deployments/aws/templates/ai-unlimited/ai-unlimited-with-nlb.yaml index 1e19a32..4b3ed67 100644 --- a/deployments/aws/templates/ai-unlimited/ai-unlimited-with-nlb.yaml +++ b/deployments/aws/templates/ai-unlimited/ai-unlimited-with-nlb.yaml @@ -611,7 +611,8 @@ Resources: RestartSec=2 ExecStartPre=-/usr/bin/docker stop %n || true ExecStartPre=-/usr/bin/docker rm %n || true - ExecStartPre=/usr/bin/docker pull teradata/ai-unlimited-scheduler:latest + EnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt + ExecStartPre=/usr/bin/docker pull teradata/ai-unlimited-scheduler:${ AiUnlimitedSchedulerVersion } ExecStart=/usr/bin/docker run \ --network ai_unlimited \ -p ${ AiUnlimitedSchedulerGrpcPort }:50051 \ @@ -621,7 +622,8 @@ Resources: -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \ -e TD_WSSCHED_POL_INTERVAL=60 \ -e TD_WS_CONTAINER_NAME=ai-unlimited.service \ - --rm --name %n teradata/ai-unlimited-scheduler:latest workspace-event-scheduler serve + -e TD_VCD_INIT_API_KEY \ + --rm --name %n teradata/ai-unlimited-scheduler:${ AiUnlimitedSchedulerVersion } workspace-event-scheduler serve [Install] WantedBy=multi-user.target group: root diff --git a/deployments/aws/templates/ai-unlimited/ai-unlimited-without-lb.yaml b/deployments/aws/templates/ai-unlimited/ai-unlimited-without-lb.yaml index 4ae712d..ee9fbf6 100644 --- a/deployments/aws/templates/ai-unlimited/ai-unlimited-without-lb.yaml +++ b/deployments/aws/templates/ai-unlimited/ai-unlimited-without-lb.yaml @@ -559,11 +559,11 @@ Resources: StartLimitBurst=10 [Service] + EnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt TimeoutStartSec=0 Restart=always RestartSec=2 ExecStartPre=-/usr/bin/mkdir -p /etc/td/ai-unlimited - EnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt ExecStartPre=-/usr/bin/docker stop %n || true ExecStartPre=-/usr/bin/docker rm %n || true ExecStartPre=/usr/bin/docker pull teradata/ai-unlimited-workspaces:${ AiUnlimitedVersion } @@ -604,9 +604,10 @@ Resources: TimeoutStartSec=0 Restart=always RestartSec=2 + EnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt ExecStartPre=-/usr/bin/docker exec %n stop || true ExecStartPre=-/usr/bin/docker rm %n || true - ExecStartPre=/usr/bin/docker pull teradata/ai-unlimited-scheduler:latest + ExecStartPre=/usr/bin/docker pull teradata/ai-unlimited-scheduler:${ AiUnlimitedSchedulerVersion } ExecStart=/usr/bin/docker run \ --network ai_unlimited \ -p ${ AiUnlimitedSchedulerGrpcPort }:50051 \ @@ -615,8 +616,11 @@ Resources: -e TD_WSSCHED_LOG_PATH=/etc/td/workspaces/scheduler_logs \ -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \ -e TD_WSSCHED_POL_INTERVAL=60 \ + -e TD_WSSCHED_DEV_LOC_OVR=ignore \ -e TD_WS_CONTAINER_NAME=ai-unlimited.service \ - --rm --name %n teradata/ai-unlimited-scheduler:latest workspace-event-scheduler serve + -e TD_WS_SERVER_PORT=${ AiUnlimitedGrpcPort } \ + -e TD_VCD_INIT_API_KEY \ + --rm --name %n teradata/ai-unlimited-scheduler:${ AiUnlimitedSchedulerVersion } workspace-event-scheduler serve [Install] WantedBy=multi-user.target From 533acc2ed51aaabde72cb8b095b4cdeeb4b9ceab Mon Sep 17 00:00:00 2001 From: cs128781 Date: Thu, 12 Dec 2024 15:39:52 -0800 Subject: [PATCH 2/6] change scheduler service for tls --- .../scripts/ai-unlimited-scheduler.service | 6 ++- .../ai-unlimited/ai-unlimited-without-lb.json | 50 ++++++++++++------- .../ai-unlimited-without-lb.bicep | 23 ++++++--- 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/deployments/azure/scripts/ai-unlimited-scheduler.service b/deployments/azure/scripts/ai-unlimited-scheduler.service index 9a315b9..9f545e0 100644 --- a/deployments/azure/scripts/ai-unlimited-scheduler.service +++ b/deployments/azure/scripts/ai-unlimited-scheduler.service @@ -9,6 +9,7 @@ StartLimitBurst=10 TimeoutStartSec=0 Restart=always RestartSec=2 +EnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt ExecStartPre=-/usr/bin/docker stop %n ExecStartPre=-/usr/bin/docker rm %n ExecStartPre=/usr/bin/docker pull {0}/{1}:{2} @@ -18,8 +19,11 @@ ExecStart=/usr/bin/docker run \ -v /etc/td/ai-unlimited:/etc/td \ -e TD_WSSCHED_LOG_PATH=/etc/td/workspaces/scheduler_logs \ -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \ - -e TD_WSSCHED_POL_INTERVAL=2 \ + -e TD_WSSCHED_POL_INTERVAL=60 \ -e TD_WS_CONTAINER_NAME=ai-unlimited.service \ + -e TD_WSSCHED_DEV_LOC_OVR=ignore \ + -e TD_WS_SERVER_PORT={4} \ + -e TD_VCD_INIT_API_KEY \ --rm --name %n {0}/{1}:{2} workspace-event-scheduler serve [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-without-lb.json b/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-without-lb.json index 84936a5..2aca3da 100644 --- a/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-without-lb.json +++ b/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-without-lb.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "7904991390945018605" + "version": "0.32.4.45862", + "templateHash": "14038604803180625097" } }, "parameters": { @@ -83,6 +83,13 @@ "description": "port to access the AI Unlimited service api." } }, + "AiUnlimitedSchedulerHttpPort": { + "type": "int", + "defaultValue": 50061, + "metadata": { + "description": "port to access the AI Unlimited scheduler api." + } + }, "SourceAppSecGroups": { "type": "array", "defaultValue": [], @@ -153,6 +160,13 @@ "description": "Container Version of the AI Unlimited UI service" } }, + "AiUnlimitedSchedulerVersion": { + "type": "string", + "defaultValue": "latest", + "metadata": { + "description": "Container Version of the AI Unlimited scheduler service" + } + }, "Tags": { "type": "object", "defaultValue": {}, @@ -164,19 +178,17 @@ "variables": { "$fxv#0": "#cloud-config\nwrite_files:\n- encoding: b64\n content: \"{0}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited.service\n permissions: '0640'\n- encoding: b64\n content: \"{1}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited-scheduler.service\n permissions: '0640'\n- encoding: b64\n content: \"{2}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited-ui.service\n permissions: '0640'\n\nruncmd:\n- mkdir -p /etc/td\n- |\n export PERMDISK=$(lsscsi 1:0:0:0 -b | awk '{{print $2}}');\n if [ -n \"${{PERMDISK}}\" ]; then blkid --match-token TYPE=ext4 ${{PERMDISK}} || (mkfs.ext4 -m0 ${{PERMDISK}} && e2label ${{PERMDISK}} WORKSPACES); fi\n /usr/bin/echo \"LABEL=WORKSPACES /etc/td ext4 defaults 0 2\" >> /etc/fstab\n /usr/bin/mount -a\n- while [ $(systemctl status docker | grep \"active (running)\" | wc -l) -lt 1 ]; do sleep 5; done\n- mkdir -p /etc/td/ai-unlimited\n- echo \"TD_VCD_INIT_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 /etc/td/ai-unlimited/init_api_key.txt\n- sleep 60\n- systemctl enable ai-unlimited.service\n- systemctl start ai-unlimited.service\n- systemctl enable ai-unlimited-scheduler.service\n- systemctl start ai-unlimited-scheduler.service\n- systemctl enable ai-unlimited-ui.service\n- systemctl start ai-unlimited-ui.service\n", "$fxv#1": "[Unit]\nDescription=AI Unlimited\nAfter=docker.service\nRequires=docker.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker volume create ssl_certs\nExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited\nExecStartPre=-/usr/bin/mkdir -p /etc/td/ai-unlimited\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e accept_license=Y \\\n -e PLATFORM=azure \\\n -e ARM_USE_MSI=true \\\n -e ARM_SUBSCRIPTION_ID={5} \\\n -e ARM_TENANT_ID={6} \\\n -e TD_VCD_INIT_API_KEY \\\n -p {3}:3000 \\\n -p {4}:3282 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -v ssl_certs:/etc/td/ssl \\\n --network ai_unlimited {7} \\\n --rm --name %n {0}/{1}:{2} workspaces serve -v\n\n[Install]\nWantedBy=multi-user.target", - "$fxv#2": "[Unit]\nDescription=AI Unlimited Scheduler\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\nExecStart=/usr/bin/docker run \\\n --network ai_unlimited \\\n -p {3}:50061 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -e TD_WSSCHED_LOG_PATH=/etc/td/workspaces/scheduler_logs \\\n -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \\\n -e TD_WSSCHED_POL_INTERVAL=2 \\\n -e TD_WS_CONTAINER_NAME=ai-unlimited.service \\\n --rm --name %n {0}/{1}:{2} workspace-event-scheduler serve\n[Install]\nWantedBy=multi-user.target", + "$fxv#2": "[Unit]\nDescription=AI Unlimited Scheduler\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\nExecStart=/usr/bin/docker run \\\n --network ai_unlimited \\\n -p {3}:50061 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -e TD_WSSCHED_LOG_PATH=/etc/td/workspaces/scheduler_logs \\\n -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \\\n -e TD_WSSCHED_POL_INTERVAL=60 \\\n -e TD_WS_CONTAINER_NAME=ai-unlimited.service \\\n -e TD_WSSCHED_DEV_LOC_OVR=ignore \\\n -e TD_WS_SERVER_PORT={4} \\\n -e TD_VCD_INIT_API_KEY \\\n --rm --name %n {0}/{1}:{2} workspace-event-scheduler serve\n[Install]\nWantedBy=multi-user.target", "$fxv#3": "[Unit]\nDescription=AI Unlimited UI\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e TD_VCD_USE_TLS=false \\\n -e TD_VCD_AUTH_PORT={4}\\\n -e TD_VCD_API_PORT={5}\\\n -e TD_VCD_INIT_API_KEY \\\n -p 80:80 \\\n -p 443:443 \\\n -v ssl_certs:/etc/ssl/td \\\n --network ai_unlimited {6} \\\n --rm --name %n {0}/{1}:{2} \n\n[Install]\nWantedBy=multi-user.target", - "AiUnlimitedSchedulerHttpPort": 50061, "AiUnlimitedUIHttpPort": 80, "AiUnlimitedUIHttpsPort": 443, - "AiUnlimitedSchedulerVersion": "latest", "roleAssignmentName": "[guid(subscription().id, parameters('AiUnlimitedName'), subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), parameters('RoleDefinitionId'))]", "dnsLabelPrefix": "[format('td{0}', uniqueString(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), deployment().name, parameters('AiUnlimitedName')))]", "registry": "teradata", "workspaceRepository": "ai-unlimited-workspaces", "workspaceSchedulerRepository": "ai-unlimited-scheduler", "workspaceUIRepository": "ai-unlimited-workspaces-ui", - "cloudInitData": "[base64(format(variables('$fxv#0'), base64(format(variables('$fxv#1'), variables('registry'), variables('workspaceRepository'), parameters('AiUnlimitedVersion'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), subscription().subscriptionId, subscription().tenantId, '--network-alias ai-unlimited')), base64(format(variables('$fxv#2'), variables('registry'), variables('workspaceSchedulerRepository'), variables('AiUnlimitedSchedulerVersion'), variables('AiUnlimitedSchedulerHttpPort'))), base64(format(variables('$fxv#3'), variables('registry'), variables('workspaceUIRepository'), parameters('AiUnlimitedUIVersion'), variables('AiUnlimitedUIHttpPort'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), '--network-alias ai-unlimited'))))]" + "cloudInitData": "[base64(format(variables('$fxv#0'), base64(format(variables('$fxv#1'), variables('registry'), variables('workspaceRepository'), parameters('AiUnlimitedVersion'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), subscription().subscriptionId, subscription().tenantId, '--network-alias ai-unlimited')), base64(format(variables('$fxv#2'), variables('registry'), variables('workspaceSchedulerRepository'), parameters('AiUnlimitedSchedulerVersion'), parameters('AiUnlimitedSchedulerHttpPort'), parameters('AiUnlimitedGrpcPort'))), base64(format(variables('$fxv#3'), variables('registry'), variables('workspaceUIRepository'), parameters('AiUnlimitedUIVersion'), variables('AiUnlimitedUIHttpPort'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), '--network-alias ai-unlimited'))))]" }, "resources": [ { @@ -222,8 +234,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "29287785012335710" + "version": "0.32.4.45862", + "templateHash": "18082039460890227567" } }, "parameters": { @@ -341,8 +353,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "16095084913002426133" + "version": "0.32.4.45862", + "templateHash": "4331975894596953088" } }, "parameters": { @@ -402,7 +414,7 @@ "value": "[parameters('AiUnlimitedGrpcPort')]" }, "aiUnlimitedSchedulerHttpPort": { - "value": "[variables('AiUnlimitedSchedulerHttpPort')]" + "value": "[parameters('AiUnlimitedSchedulerHttpPort')]" }, "aiUnlimitedUIHttpPort": { "value": "[variables('AiUnlimitedUIHttpPort')]" @@ -426,8 +438,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "392354314460473239" + "version": "0.32.4.45862", + "templateHash": "16013689345643124539" } }, "parameters": { @@ -838,8 +850,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "11687747621230861550" + "version": "0.32.4.45862", + "templateHash": "17662981702954878393" } }, "parameters": { @@ -1119,8 +1131,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "4325662974998231491" + "version": "0.32.4.45862", + "templateHash": "3737753991380666295" } }, "parameters": { @@ -1224,8 +1236,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "9690061458974637253" + "version": "0.32.4.45862", + "templateHash": "12388745557700980847" } }, "parameters": { diff --git a/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicep b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicep index 66544ee..437786c 100644 --- a/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicep +++ b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicep @@ -35,11 +35,14 @@ param AiUnlimitedAuthPort int = 3000 @description('port to access the AI Unlimited service api.') param AiUnlimitedGrpcPort int = 3282 +@description('port to access the AI Unlimited scheduler api.') +param AiUnlimitedSchedulerHttpPort int = 50061 + // @description('port to access the AI Unlimited scheduler service api.') // var AiUnlimitedSchedulerGrpcPort = 50051 // @description('port to access the AI Unlimited scheduler service api.') -var AiUnlimitedSchedulerHttpPort = 50061 +//var AiUnlimitedSchedulerHttpPort = 50061 // @description('port to access the AI Unlimited service UI http.') var AiUnlimitedUIHttpPort = 80 @@ -76,8 +79,8 @@ param AiUnlimitedVersion string = 'v0.3.0' @description('Container Version of the AI Unlimited UI service') param AiUnlimitedUIVersion string = 'v0.1.0' -// @description('Container Version of the AI Unlimited scheduler service') -var AiUnlimitedSchedulerVersion = 'latest' +@description('Container Version of the AI Unlimited scheduler service') +param AiUnlimitedSchedulerVersion string = 'latest' @description('Tags to apply to all newly created resources, in the form of {"key_one":"value_one","key_two":"value_two"}') param Tags object = {} @@ -109,8 +112,8 @@ var cloudInitData = base64(format( registry, workspaceSchedulerRepository, AiUnlimitedSchedulerVersion, - // AiUnlimitedSchedulerGrpcPort, - AiUnlimitedSchedulerHttpPort + AiUnlimitedSchedulerHttpPort, + AiUnlimitedGrpcPort )), base64(format( loadTextContent('../../../scripts/ai-unlimited-ui.service'), @@ -246,8 +249,14 @@ resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { output PublicIP string = aiUnlimited.outputs.PublicIP output PrivateIP string = aiUnlimited.outputs.PrivateIP -output AiUnlimitedPublicHttpAccess string = concat('http://${aiUnlimited.outputs.PublicIP}', (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '')) -output AiUnlimitedPrivateHttpAccess string = concat('http://${aiUnlimited.outputs.PrivateIP}', (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '')) +output AiUnlimitedPublicHttpAccess string = concat( + 'http://${aiUnlimited.outputs.PublicIP}', + (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '') +) +output AiUnlimitedPrivateHttpAccess string = concat( + 'http://${aiUnlimited.outputs.PrivateIP}', + (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '') +) output AiUnlimitedPublicGrpcAccess string = 'http://${aiUnlimited.outputs.PublicIP}:${AiUnlimitedGrpcPort}' output AiUnlimitedPrivateGrpcAccess string = 'http://${aiUnlimited.outputs.PrivateIP}:${AiUnlimitedGrpcPort}' output KeyVaultName string = (UseKeyVault == 'New') ? vault.outputs.name : '' From e47e6d2533e609e0019b506d7663a0984ab30cc1 Mon Sep 17 00:00:00 2001 From: cs128781 Date: Mon, 16 Dec 2024 13:31:22 -0800 Subject: [PATCH 3/6] scheduler changes for api key and latest features --- .../ai-unlimited/ai-unlimited-with-nlb.json | 66 +++++++++++-------- .../ai-unlimited/ai-unlimited-with-nlb.bicep | 23 ++++--- .../ai-unlimited-without-lb.bicepparam | 2 +- .../azure/templates/bicep/modules/nlb.bicep | 3 +- deployments/docker/ai-unlimited.yaml | 21 ++++++ 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-with-nlb.json b/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-with-nlb.json index c28afa3..eff331f 100644 --- a/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-with-nlb.json +++ b/deployments/azure/templates/arm/ai-unlimited/ai-unlimited-with-nlb.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "7009335605344942432" + "version": "0.32.4.45862", + "templateHash": "14115047365666761920" } }, "parameters": { @@ -83,6 +83,13 @@ "description": "port to access the AI Unlimited service api." } }, + "AiUnlimitedSchedulerHttpPort": { + "type": "int", + "defaultValue": 50061, + "metadata": { + "description": "port to access the AI Unlimited scheduler api." + } + }, "SourceAppSecGroups": { "type": "array", "defaultValue": [], @@ -153,6 +160,13 @@ "description": "Container Version of the AI Unlimited UI service" } }, + "AiUnlimitedSchedulerVersion": { + "type": "string", + "defaultValue": "latest", + "metadata": { + "description": "Container Version of the AI Unlimited scheduler service" + } + }, "Tags": { "type": "object", "defaultValue": {}, @@ -164,12 +178,10 @@ "variables": { "$fxv#0": "#cloud-config\nwrite_files:\n- encoding: b64\n content: \"{0}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited.service\n permissions: '0640'\n- encoding: b64\n content: \"{1}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited-scheduler.service\n permissions: '0640'\n- encoding: b64\n content: \"{2}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited-ui.service\n permissions: '0640'\n\nruncmd:\n- mkdir -p /etc/td\n- |\n export PERMDISK=$(lsscsi 1:0:0:0 -b | awk '{{print $2}}');\n if [ -n \"${{PERMDISK}}\" ]; then blkid --match-token TYPE=ext4 ${{PERMDISK}} || (mkfs.ext4 -m0 ${{PERMDISK}} && e2label ${{PERMDISK}} WORKSPACES); fi\n /usr/bin/echo \"LABEL=WORKSPACES /etc/td ext4 defaults 0 2\" >> /etc/fstab\n /usr/bin/mount -a\n- while [ $(systemctl status docker | grep \"active (running)\" | wc -l) -lt 1 ]; do sleep 5; done\n- mkdir -p /etc/td/ai-unlimited\n- echo \"TD_VCD_INIT_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 /etc/td/ai-unlimited/init_api_key.txt\n- sleep 60\n- systemctl enable ai-unlimited.service\n- systemctl start ai-unlimited.service\n- systemctl enable ai-unlimited-scheduler.service\n- systemctl start ai-unlimited-scheduler.service\n- systemctl enable ai-unlimited-ui.service\n- systemctl start ai-unlimited-ui.service\n", "$fxv#1": "[Unit]\nDescription=AI Unlimited\nAfter=docker.service\nRequires=docker.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker volume create ssl_certs\nExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited\nExecStartPre=-/usr/bin/mkdir -p /etc/td/ai-unlimited\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e accept_license=Y \\\n -e PLATFORM=azure \\\n -e ARM_USE_MSI=true \\\n -e ARM_SUBSCRIPTION_ID={5} \\\n -e ARM_TENANT_ID={6} \\\n -e TD_VCD_INIT_API_KEY \\\n -p {3}:3000 \\\n -p {4}:3282 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -v ssl_certs:/etc/td/ssl \\\n --network ai_unlimited {7} \\\n --rm --name %n {0}/{1}:{2} workspaces serve -v\n\n[Install]\nWantedBy=multi-user.target", - "$fxv#2": "[Unit]\nDescription=AI Unlimited Scheduler\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\nExecStart=/usr/bin/docker run \\\n --network ai_unlimited \\\n -p {3}:50061 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -e TD_WSSCHED_LOG_PATH=/etc/td/workspaces/scheduler_logs \\\n -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \\\n -e TD_WSSCHED_POL_INTERVAL=2 \\\n -e TD_WS_CONTAINER_NAME=ai-unlimited.service \\\n --rm --name %n {0}/{1}:{2} workspace-event-scheduler serve\n[Install]\nWantedBy=multi-user.target", + "$fxv#2": "[Unit]\nDescription=AI Unlimited Scheduler\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\nExecStart=/usr/bin/docker run \\\n --network ai_unlimited \\\n -p {3}:50061 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -e TD_WSSCHED_LOG_PATH=/etc/td/workspaces/scheduler_logs \\\n -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \\\n -e TD_WSSCHED_POL_INTERVAL=60 \\\n -e TD_WS_CONTAINER_NAME=ai-unlimited.service \\\n -e TD_WSSCHED_DEV_LOC_OVR=ignore \\\n -e TD_WS_SERVER_PORT={4} \\\n -e TD_VCD_INIT_API_KEY \\\n --rm --name %n {0}/{1}:{2} workspace-event-scheduler serve\n[Install]\nWantedBy=multi-user.target", "$fxv#3": "[Unit]\nDescription=AI Unlimited UI\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e TD_VCD_USE_TLS=false \\\n -e TD_VCD_AUTH_PORT={4}\\\n -e TD_VCD_API_PORT={5}\\\n -e TD_VCD_INIT_API_KEY \\\n -p 80:80 \\\n -p 443:443 \\\n -v ssl_certs:/etc/ssl/td \\\n --network ai_unlimited {6} \\\n --rm --name %n {0}/{1}:{2} \n\n[Install]\nWantedBy=multi-user.target", - "AiUnlimitedSchedulerHttpPort": 50061, "AiUnlimitedUIHttpPort": 80, "AiUnlimitedUIHttpsPort": 443, - "AiUnlimitedSchedulerVersion": "latest", "roleAssignmentName": "[guid(subscription().id, parameters('AiUnlimitedName'), subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), parameters('RoleDefinitionId'))]", "dnsId": "[uniqueString(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), deployment().name, parameters('AiUnlimitedName'))]", "dnsLabelPrefix": "[format('td{0}', variables('dnsId'))]", @@ -223,8 +235,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "29287785012335710" + "version": "0.32.4.45862", + "templateHash": "18082039460890227567" } }, "parameters": { @@ -342,8 +354,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "16095084913002426133" + "version": "0.32.4.45862", + "templateHash": "4331975894596953088" } }, "parameters": { @@ -400,7 +412,7 @@ "value": "[parameters('AiUnlimitedGrpcPort')]" }, "aiUnlimitedSchedulerHttpPort": { - "value": "[variables('AiUnlimitedSchedulerHttpPort')]" + "value": "[parameters('AiUnlimitedSchedulerHttpPort')]" }, "aiUnlimitedUIHttpPort": { "value": "[variables('AiUnlimitedUIHttpPort')]" @@ -427,8 +439,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "392354314460473239" + "version": "0.32.4.45862", + "templateHash": "16013689345643124539" } }, "parameters": { @@ -803,7 +815,7 @@ "value": "[parameters('AiUnlimitedGrpcPort')]" }, "aiUnlimitedSchedulerHttpPort": { - "value": "[variables('AiUnlimitedSchedulerHttpPort')]" + "value": "[parameters('AiUnlimitedSchedulerHttpPort')]" }, "aiUnlimitedUIHttpPort": { "value": "[variables('AiUnlimitedUIHttpPort')]" @@ -821,8 +833,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "370499549708602593" + "version": "0.32.4.45862", + "templateHash": "12614640311343730358" } }, "parameters": { @@ -901,7 +913,7 @@ } ], "loadBalancingRules": "[flatten(createArray(if(not(equals(parameters('aiUnlimitedAuthPort'), 0)), createArray(createObject('name', 'AiUnlimitedAuth', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedAuthPort'), 'backendPort', parameters('aiUnlimitedAuthPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}AuthLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedGrpcPort'), 0)), createArray(createObject('name', 'AiUnlimitedAPI', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedGrpcPort'), 'backendPort', parameters('aiUnlimitedGrpcPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}APILbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('jupyterHttpPort'), 0)), createArray(createObject('name', 'JupyterUI', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('jupyterHttpPort'), 'backendPort', parameters('jupyterHttpPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}JupyterLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedSchedulerHttpPort'), 0)), createArray(createObject('name', 'AiUnlimitedSchedulerHttp', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedSchedulerHttpPort'), 'backendPort', parameters('aiUnlimitedSchedulerHttpPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}SchedulerHttpLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpPort'), 0)), createArray(createObject('name', 'AiUnlimitedUIHttp', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedUIHttpPort'), 'backendPort', parameters('aiUnlimitedUIHttpPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}UIHttpLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpsPort'), 0)), createArray(createObject('name', 'AiUnlimitedUIHttps', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedUIHttpsPort'), 'backendPort', parameters('aiUnlimitedUIHttpsPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}UIHttpsLbProbe', parameters('name'))))))), createArray())))]", - "probes": "[flatten(createArray(if(not(equals(parameters('aiUnlimitedAuthPort'), 0)), createArray(createObject('name', format('{0}AuthLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedAuthPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedGrpcPort'), 0)), createArray(createObject('name', format('{0}APILbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedGrpcPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('jupyterHttpPort'), 0)), createArray(createObject('name', format('{0}JupyterLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('jupyterHttpPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedSchedulerHttpPort'), 0)), createArray(createObject('name', format('{0}SchedulerHttpLbProbe', parameters('name')), 'properties', createObject('protocol', 'Http', 'port', parameters('aiUnlimitedSchedulerHttpPort'), 'requestPath', '/healthcheck', 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpPort'), 0)), createArray(createObject('name', format('{0}UIHttpLbProbe', parameters('name')), 'properties', createObject('protocol', 'Http', 'port', parameters('aiUnlimitedUIHttpPort'), 'requestPath', '/', 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpsPort'), 0)), createArray(createObject('name', format('{0}UIHttpsLbProbe', parameters('name')), 'properties', createObject('protocol', 'Https', 'port', parameters('aiUnlimitedUIHttpsPort'), 'requestPath', '/', 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray())))]", + "probes": "[flatten(createArray(if(not(equals(parameters('aiUnlimitedAuthPort'), 0)), createArray(createObject('name', format('{0}AuthLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedAuthPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedGrpcPort'), 0)), createArray(createObject('name', format('{0}APILbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedGrpcPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('jupyterHttpPort'), 0)), createArray(createObject('name', format('{0}JupyterLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('jupyterHttpPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedSchedulerHttpPort'), 0)), createArray(createObject('name', format('{0}SchedulerHttpLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedSchedulerHttpPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpPort'), 0)), createArray(createObject('name', format('{0}UIHttpLbProbe', parameters('name')), 'properties', createObject('protocol', 'Http', 'port', parameters('aiUnlimitedUIHttpPort'), 'requestPath', '/', 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpsPort'), 0)), createArray(createObject('name', format('{0}UIHttpsLbProbe', parameters('name')), 'properties', createObject('protocol', 'Https', 'port', parameters('aiUnlimitedUIHttpsPort'), 'requestPath', '/', 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray())))]", "outboundRules": [ { "name": "myOutboundRule", @@ -957,8 +969,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "4325662974998231491" + "version": "0.32.4.45862", + "templateHash": "3737753991380666295" } }, "parameters": { @@ -1039,8 +1051,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "4325662974998231491" + "version": "0.32.4.45862", + "templateHash": "3737753991380666295" } }, "parameters": { @@ -1152,7 +1164,7 @@ "value": "[parameters('OSVersion')]" }, "cloudInitData": { - "value": "[base64(format(variables('$fxv#0'), base64(format(variables('$fxv#1'), variables('registry'), variables('workspaceRepository'), parameters('AiUnlimitedVersion'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), subscription().subscriptionId, subscription().tenantId, format('--network-alias {0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value))), base64(format(variables('$fxv#2'), variables('registry'), variables('workspaceSchedulerRepository'), variables('AiUnlimitedSchedulerVersion'), variables('AiUnlimitedSchedulerHttpPort'))), base64(format(variables('$fxv#3'), variables('registry'), variables('workspaceUIRepository'), parameters('AiUnlimitedUIVersion'), variables('AiUnlimitedUIHttpPort'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), format('--network-alias {0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value)))))]" + "value": "[base64(format(variables('$fxv#0'), base64(format(variables('$fxv#1'), variables('registry'), variables('workspaceRepository'), parameters('AiUnlimitedVersion'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), subscription().subscriptionId, subscription().tenantId, format('--network-alias {0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value))), base64(format(variables('$fxv#2'), variables('registry'), variables('workspaceSchedulerRepository'), parameters('AiUnlimitedSchedulerVersion'), parameters('AiUnlimitedSchedulerHttpPort'), parameters('AiUnlimitedGrpcPort'))), base64(format(variables('$fxv#3'), variables('registry'), variables('workspaceUIRepository'), parameters('AiUnlimitedUIVersion'), variables('AiUnlimitedUIHttpPort'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), format('--network-alias {0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value)))))]" }, "usePersistentVolume": { "value": "[parameters('UsePersistentVolume')]" @@ -1182,8 +1194,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "11687747621230861550" + "version": "0.32.4.45862", + "templateHash": "17662981702954878393" } }, "parameters": { @@ -1463,8 +1475,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "4325662974998231491" + "version": "0.32.4.45862", + "templateHash": "3737753991380666295" } }, "parameters": { @@ -1569,8 +1581,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.31.92.45157", - "templateHash": "9690061458974637253" + "version": "0.32.4.45862", + "templateHash": "12388745557700980847" } }, "parameters": { diff --git a/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.bicep b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.bicep index 9cfdb10..132f629 100644 --- a/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.bicep +++ b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.bicep @@ -38,8 +38,8 @@ param AiUnlimitedGrpcPort int = 3282 // @description('port to access the AI Unlimited scheduler service grpc api.') // var AiUnlimitedSchedulerGrpcPort = 50051 -// @description('port to access the AI Unlimited scheduler service grpc api.') -var AiUnlimitedSchedulerHttpPort = 50061 +@description('port to access the AI Unlimited scheduler api.') +param AiUnlimitedSchedulerHttpPort int = 50061 // @description('port to access the AI Unlimited service UI http.') var AiUnlimitedUIHttpPort = 80 @@ -76,8 +76,8 @@ param AiUnlimitedVersion string = 'v0.3.0' @description('Container Version of the AI Unlimited UI service') param AiUnlimitedUIVersion string = 'v0.1.0' -// @description('Container Version of the AI Unlimited scheduler service') -var AiUnlimitedSchedulerVersion = 'latest' +@description('Container Version of the AI Unlimited scheduler service') +param AiUnlimitedSchedulerVersion string = 'latest' @description('Tags to apply to all newly created resources, in the form of {"key_one":"value_one","key_two":"value_two"}') param Tags object = {} @@ -93,7 +93,6 @@ var workspaceRepository = 'ai-unlimited-workspaces' var workspaceSchedulerRepository = 'ai-unlimited-scheduler' var workspaceUIRepository = 'ai-unlimited-workspaces-ui' - var cloudInitData = base64(format( loadTextContent('../../../scripts/ai-unlimited.cloudinit.yaml'), base64(format( @@ -112,8 +111,8 @@ var cloudInitData = base64(format( registry, workspaceSchedulerRepository, AiUnlimitedSchedulerVersion, - // AiUnlimitedSchedulerGrpcPort, - AiUnlimitedSchedulerHttpPort + AiUnlimitedSchedulerHttpPort, + AiUnlimitedGrpcPort )), base64(format( loadTextContent('../../../scripts/ai-unlimited-ui.service'), @@ -268,8 +267,14 @@ resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { output PublicIP string = nlb.outputs.PublicIp output PrivateIP string = aiUnlimited.outputs.PrivateIP -output AiUnlimitedPublicHttpAccess string = concat('http://${nlb.outputs.PublicDns}', (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '')) -output AiUnlimitedPrivateHttpAccess string = concat('http://${aiUnlimited.outputs.PrivateIP}', (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '')) +output AiUnlimitedPublicHttpAccess string = concat( + 'http://${nlb.outputs.PublicDns}', + (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '') +) +output AiUnlimitedPrivateHttpAccess string = concat( + 'http://${aiUnlimited.outputs.PrivateIP}', + (AiUnlimitedUIHttpPort != 80 ? concat(':', string(AiUnlimitedUIHttpPort)) : '') +) output AiUnlimitedPublicGrpcAccess string = 'http://${nlb.outputs.PublicDns}:${AiUnlimitedGrpcPort}' output AiUnlimitedPrivateGrpcAccess string = 'http://${aiUnlimited.outputs.PrivateIP}:${AiUnlimitedGrpcPort}' output KeyVaultName string = (UseKeyVault == 'New') ? vault.outputs.name : '' diff --git a/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicepparam b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicepparam index 881bdac..57b0103 100644 --- a/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicepparam +++ b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-without-lb.bicepparam @@ -13,7 +13,7 @@ param AccessCIDRs = [ param AiUnlimitedAuthPort = 3000 param AiUnlimitedGrpcPort = 3282 // param AiUnlimitedSchedulerGrpcPort = 50051 -// param AiUnlimitedSchedulerHttpPort = 50061 +param AiUnlimitedSchedulerHttpPort = 50061 // param AiUnlimitedUIHttpPort = 80 // param AiUnlimitedUIHttpsPort = 443 param SourceAppSecGroups = [] diff --git a/deployments/azure/templates/bicep/modules/nlb.bicep b/deployments/azure/templates/bicep/modules/nlb.bicep index 8b42f17..e95d9be 100644 --- a/deployments/azure/templates/bicep/modules/nlb.bicep +++ b/deployments/azure/templates/bicep/modules/nlb.bicep @@ -320,9 +320,8 @@ resource lb 'Microsoft.Network/loadBalancers@2021-08-01' = { { name: '${name}SchedulerHttpLbProbe' properties: { - protocol: 'Http' + protocol: 'Tcp' port: aiUnlimitedSchedulerHttpPort - requestPath: '/healthcheck' intervalInSeconds: 5 numberOfProbes: 2 } diff --git a/deployments/docker/ai-unlimited.yaml b/deployments/docker/ai-unlimited.yaml index 77d8598..e79f570 100644 --- a/deployments/docker/ai-unlimited.yaml +++ b/deployments/docker/ai-unlimited.yaml @@ -42,6 +42,27 @@ services: networks: - ai-unlimited-network + scheduler: + deploy: + replicas: 1 + container_name: workspace-event-scheduler + image: ${AI_UNLIMITED_SCHEDULER_IMAGE_NAME:-teradata/ai-unlimited-scheduler}:${AI_UNLIMITED_SCHEDULER_IMAGE_TAG:-v0.01.65} + command: workspace-event-scheduler serve + restart: unless-stopped + ports: + - "${TD_WSSCHED_SCHED_SERVER_PORT:-50051}:50051/tcp" + - "${TD_WS_HTTP_SERVER_PORT:-50061}:50061/tcp" + depends_on: + - ai-unlimited + environment: + TD_WS_SERVER_PORT: "${TD_VCD_API_PORT:-3282}" + TD_VCD_INIT_API_KEY: "${AI_UNLIMITED_INIT_API_KEY}" + TD_WSSCHED_POL_INTERVAL: "${TD_WSSCHED_POL_INTERVAL:-60}" + TD_WS_CONTAINER_NAME: "${TD_WS_CONTAINER_NAME:-ai-unlimited-workspaces}" + volumes: + - ${AI_UNLIMITED_HOME:-./volumes/ai-unlimited-workspaces}:/etc/td + networks: + - ai-unlimited-network volumes: ssl_certs: # external: true From 375b20c47860040162be253aa8db9854eb55e56d Mon Sep 17 00:00:00 2001 From: cs128781 Date: Mon, 16 Dec 2024 13:33:52 -0800 Subject: [PATCH 4/6] scheduler changes for api key and latest features --- .../templates/ai-unlimited-scheduler.service | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 deployments/azure/templates/ai-unlimited-scheduler.service diff --git a/deployments/azure/templates/ai-unlimited-scheduler.service b/deployments/azure/templates/ai-unlimited-scheduler.service new file mode 100644 index 0000000..f7d100a --- /dev/null +++ b/deployments/azure/templates/ai-unlimited-scheduler.service @@ -0,0 +1,31 @@ +[Unit] +Description=AI Unlimited +After=docker.service +Requires=docker.service +StartLimitInterval=200 +StartLimitBurst=10 + +[Service] +TimeoutStartSec=0 +Restart=always +RestartSec=2 +ExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited +ExecStartPre=-/usr/bin/mkdir -p /etc/td/ai-unlimited +ExecStartPre=-/usr/bin/docker exec %n stop || true +ExecStartPre=-/usr/bin/docker rm %n || true +ExecStartPre=/usr/bin/docker pull {0}/{1}:{2} + +ExecStart=/usr/bin/docker run \ + -e accept_license=Y \ + -e PLATFORM=azure \ + -e ARM_USE_MSI=true \ + -e ARM_SUBSCRIPTION_ID={5} \ + -e ARM_TENANT_ID={6} \ + -v /etc/td/ai-unlimited:/etc/td \ + -p {3}:3000 \ + -p {4}:3282 \ + --network ai_unlimited {7} \ + --rm --name %n {0}/{1}:{2} workspaces serve -v + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 973f1829978129d6d53f2b4818e749abd04f3928 Mon Sep 17 00:00:00 2001 From: cs128781 Date: Mon, 16 Dec 2024 13:34:10 -0800 Subject: [PATCH 5/6] scheduler changes for api key and latest features --- .../ai-unlimited/ai-unlimited-with-nlb.json | 1721 +++++++++++++++++ 1 file changed, 1721 insertions(+) create mode 100644 deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.json diff --git a/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.json b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.json new file mode 100644 index 0000000..eff331f --- /dev/null +++ b/deployments/azure/templates/bicep/ai-unlimited/ai-unlimited-with-nlb.json @@ -0,0 +1,1721 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "14115047365666761920" + } + }, + "parameters": { + "ResourceGroupName": { + "type": "string", + "defaultValue": "ai-unlimited-workspace", + "metadata": { + "description": "name for the resource group." + } + }, + "AiUnlimitedName": { + "type": "string", + "metadata": { + "description": "Name for the AI Unlimited service's virtual machine." + } + }, + "OSVersion": { + "type": "string", + "defaultValue": "Ubuntu-2004", + "allowedValues": [ + "Ubuntu-1804", + "Ubuntu-2004", + "Ubuntu-2204" + ], + "metadata": { + "description": "The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version." + } + }, + "InstanceType": { + "type": "string", + "defaultValue": "Standard_D2s_v3", + "metadata": { + "description": "The AI Unlimited VM type" + } + }, + "Network": { + "type": "string", + "metadata": { + "description": "Name of the network to run the AI Unlimited service in" + } + }, + "Subnet": { + "type": "string", + "metadata": { + "description": "Name of the subnet to run the AI Unlimited service in" + } + }, + "SecurityGroup": { + "type": "string", + "defaultValue": "AiUnlimitedSecurityGroup", + "metadata": { + "description": "Name of the network security group" + } + }, + "AccessCIDRs": { + "type": "array", + "defaultValue": [ + "0.0.0.0/0" + ], + "metadata": { + "description": "The CIDR ranges that can be used to communicate with the AI Unlimited service instance." + } + }, + "AiUnlimitedAuthPort": { + "type": "int", + "defaultValue": 3000, + "metadata": { + "description": "port to access the AI Unlimited auth service." + } + }, + "AiUnlimitedGrpcPort": { + "type": "int", + "defaultValue": 3282, + "metadata": { + "description": "port to access the AI Unlimited service api." + } + }, + "AiUnlimitedSchedulerHttpPort": { + "type": "int", + "defaultValue": 50061, + "metadata": { + "description": "port to access the AI Unlimited scheduler api." + } + }, + "SourceAppSecGroups": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Source Application Security Groups to access the AI Unlimited service api." + } + }, + "detinationAppSecGroups": { + "type": "array", + "defaultValue": [], + "metadata": { + "description": "Destination Application Security Groups to give access to AI Unlimited service instance." + } + }, + "RoleDefinitionId": { + "type": "string", + "metadata": { + "description": "GUID of the AI Unlimited Role" + } + }, + "UseKeyVault": { + "type": "string", + "defaultValue": "New", + "allowedValues": [ + "New", + "None" + ], + "metadata": { + "description": "should we create a new Azure Key Vault for bootstrapping the AI Unlimited Engine nodes." + } + }, + "UsePersistentVolume": { + "type": "string", + "defaultValue": "New", + "allowedValues": [ + "New", + "Existing" + ], + "metadata": { + "description": "should we use a new or existing volume for persistent data on the AI Unlimited server." + } + }, + "PersistentVolumeSize": { + "type": "int", + "defaultValue": 100, + "metadata": { + "description": "size of the optional persistent disk to the AI Unlimited server." + } + }, + "ExistingPersistentVolume": { + "type": "string", + "defaultValue": "NONE", + "metadata": { + "description": "Name of the existing persistent volume to attach. Must be in the same region and resourcegroup zone as the AI Unlimited server." + } + }, + "AiUnlimitedVersion": { + "type": "string", + "defaultValue": "v0.3.0", + "metadata": { + "description": "Container Version of the AI Unlimited service" + } + }, + "AiUnlimitedUIVersion": { + "type": "string", + "defaultValue": "v0.1.0", + "metadata": { + "description": "Container Version of the AI Unlimited UI service" + } + }, + "AiUnlimitedSchedulerVersion": { + "type": "string", + "defaultValue": "latest", + "metadata": { + "description": "Container Version of the AI Unlimited scheduler service" + } + }, + "Tags": { + "type": "object", + "defaultValue": {}, + "metadata": { + "description": "Tags to apply to all newly created resources, in the form of {\"key_one\":\"value_one\",\"key_two\":\"value_two\"}" + } + } + }, + "variables": { + "$fxv#0": "#cloud-config\nwrite_files:\n- encoding: b64\n content: \"{0}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited.service\n permissions: '0640'\n- encoding: b64\n content: \"{1}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited-scheduler.service\n permissions: '0640'\n- encoding: b64\n content: \"{2}\"\n owner: root:root\n path: /usr/lib/systemd/system/ai-unlimited-ui.service\n permissions: '0640'\n\nruncmd:\n- mkdir -p /etc/td\n- |\n export PERMDISK=$(lsscsi 1:0:0:0 -b | awk '{{print $2}}');\n if [ -n \"${{PERMDISK}}\" ]; then blkid --match-token TYPE=ext4 ${{PERMDISK}} || (mkfs.ext4 -m0 ${{PERMDISK}} && e2label ${{PERMDISK}} WORKSPACES); fi\n /usr/bin/echo \"LABEL=WORKSPACES /etc/td ext4 defaults 0 2\" >> /etc/fstab\n /usr/bin/mount -a\n- while [ $(systemctl status docker | grep \"active (running)\" | wc -l) -lt 1 ]; do sleep 5; done\n- mkdir -p /etc/td/ai-unlimited\n- echo \"TD_VCD_INIT_API_KEY=$(LC_ALL=C tr -dc A-Za-z0-9 /etc/td/ai-unlimited/init_api_key.txt\n- sleep 60\n- systemctl enable ai-unlimited.service\n- systemctl start ai-unlimited.service\n- systemctl enable ai-unlimited-scheduler.service\n- systemctl start ai-unlimited-scheduler.service\n- systemctl enable ai-unlimited-ui.service\n- systemctl start ai-unlimited-ui.service\n", + "$fxv#1": "[Unit]\nDescription=AI Unlimited\nAfter=docker.service\nRequires=docker.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker volume create ssl_certs\nExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited\nExecStartPre=-/usr/bin/mkdir -p /etc/td/ai-unlimited\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e accept_license=Y \\\n -e PLATFORM=azure \\\n -e ARM_USE_MSI=true \\\n -e ARM_SUBSCRIPTION_ID={5} \\\n -e ARM_TENANT_ID={6} \\\n -e TD_VCD_INIT_API_KEY \\\n -p {3}:3000 \\\n -p {4}:3282 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -v ssl_certs:/etc/td/ssl \\\n --network ai_unlimited {7} \\\n --rm --name %n {0}/{1}:{2} workspaces serve -v\n\n[Install]\nWantedBy=multi-user.target", + "$fxv#2": "[Unit]\nDescription=AI Unlimited Scheduler\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\nExecStart=/usr/bin/docker run \\\n --network ai_unlimited \\\n -p {3}:50061 \\\n -v /etc/td/ai-unlimited:/etc/td \\\n -e TD_WSSCHED_LOG_PATH=/etc/td/workspaces/scheduler_logs \\\n -e TD_WSSCHED_TASK_LOG_PATH=/etc/td/workspaces/scheduler_logs/projects \\\n -e TD_WSSCHED_POL_INTERVAL=60 \\\n -e TD_WS_CONTAINER_NAME=ai-unlimited.service \\\n -e TD_WSSCHED_DEV_LOC_OVR=ignore \\\n -e TD_WS_SERVER_PORT={4} \\\n -e TD_VCD_INIT_API_KEY \\\n --rm --name %n {0}/{1}:{2} workspace-event-scheduler serve\n[Install]\nWantedBy=multi-user.target", + "$fxv#3": "[Unit]\nDescription=AI Unlimited UI\nAfter=ai-unlimited.service\nRequires=ai-unlimited.service\nStartLimitInterval=200\nStartLimitBurst=10\n\n[Service]\nTimeoutStartSec=0\nRestart=always\nRestartSec=2\nEnvironmentFile=/etc/td/ai-unlimited/init_api_key.txt\nExecStartPre=-/usr/bin/docker stop %n\nExecStartPre=-/usr/bin/docker rm %n\nExecStartPre=/usr/bin/docker pull {0}/{1}:{2}\n\nExecStart=/usr/bin/docker run \\\n -e TD_VCD_USE_TLS=false \\\n -e TD_VCD_AUTH_PORT={4}\\\n -e TD_VCD_API_PORT={5}\\\n -e TD_VCD_INIT_API_KEY \\\n -p 80:80 \\\n -p 443:443 \\\n -v ssl_certs:/etc/ssl/td \\\n --network ai_unlimited {6} \\\n --rm --name %n {0}/{1}:{2} \n\n[Install]\nWantedBy=multi-user.target", + "AiUnlimitedUIHttpPort": 80, + "AiUnlimitedUIHttpsPort": 443, + "roleAssignmentName": "[guid(subscription().id, parameters('AiUnlimitedName'), subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), parameters('RoleDefinitionId'))]", + "dnsId": "[uniqueString(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), deployment().name, parameters('AiUnlimitedName'))]", + "dnsLabelPrefix": "[format('td{0}', variables('dnsId'))]", + "nlbDnsLabelPrefix": "[format('td{0}-nlb', variables('dnsId'))]", + "registry": "teradata", + "workspaceRepository": "ai-unlimited-workspaces", + "workspaceSchedulerRepository": "ai-unlimited-scheduler", + "workspaceUIRepository": "ai-unlimited-workspaces-ui" + }, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('roleAssignmentName')]", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', parameters('RoleDefinitionId'))]", + "principalId": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'ai-unlimited'), '2022-09-01').outputs.PrincipleId.value]" + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'ai-unlimited')]" + ] + }, + { + "condition": "[equals(parameters('UseKeyVault'), 'New')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "vault", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "encryptVolumes": { + "value": true + }, + "keyVaultName": { + "value": "[parameters('AiUnlimitedName')]" + }, + "location": { + "value": "[reference(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), '2022-09-01', 'full').location]" + }, + "tags": { + "value": "[parameters('Tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "18082039460890227567" + } + }, + "parameters": { + "encryptVolumes": { + "type": "bool" + }, + "keyVaultName": { + "type": "string" + }, + "location": { + "type": "string" + }, + "tags": { + "type": "object", + "defaultValue": {} + }, + "uuid": { + "type": "string", + "defaultValue": "[newGuid()]" + } + }, + "variables": { + "nameCharLimit": 24, + "uniqueName": "[format('{0}-{1}', parameters('keyVaultName'), uniqueString(parameters('uuid')))]", + "uniqueKeyVaultName": "[substring(format('{0}', variables('uniqueName')), 0, if(less(length(variables('uniqueName')), variables('nameCharLimit')), length(variables('uniqueName')), variables('nameCharLimit')))]" + }, + "resources": [ + { + "type": "Microsoft.KeyVault/vaults", + "apiVersion": "2023-02-01", + "name": "[variables('uniqueKeyVaultName')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "sku": { + "family": "A", + "name": "standard" + }, + "tenantId": "[subscription().tenantId]", + "softDeleteRetentionInDays": 7, + "enableSoftDelete": true, + "enablePurgeProtection": "[if(parameters('encryptVolumes'), true(), null())]", + "enabledForDiskEncryption": "[parameters('encryptVolumes')]", + "accessPolicies": [] + } + } + ], + "outputs": { + "id": { + "type": "string", + "value": "[resourceId('Microsoft.KeyVault/vaults', variables('uniqueKeyVaultName'))]" + }, + "name": { + "type": "string", + "value": "[variables('uniqueKeyVaultName')]" + } + } + } + } + }, + { + "condition": "[equals(parameters('UseKeyVault'), 'New')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "vault-access-policy", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "vaultName": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'vault'), '2022-09-01').outputs.name.value]" + }, + "accessPolicy": { + "value": { + "tenantId": "[subscription().tenantId]", + "objectId": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'ai-unlimited'), '2022-09-01').outputs.PrincipleId.value]", + "permissions": { + "keys": [ + "Create", + "Delete", + "Get", + "List", + "Update", + "Purge", + "Recover", + "Decrypt", + "Encrypt", + "Sign", + "UnwrapKey", + "Verify", + "WrapKey", + "GetRotationPolicy", + "SetRotationPolicy" + ], + "secrets": [ + "Get", + "Set", + "Delete", + "List", + "Purge" + ], + "storage": [ + "Get" + ] + } + } + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "4331975894596953088" + } + }, + "parameters": { + "vaultName": { + "type": "string" + }, + "accessPolicy": { + "type": "object" + } + }, + "resources": [ + { + "type": "Microsoft.KeyVault/vaults/accessPolicies", + "apiVersion": "2022-07-01", + "name": "[format('{0}/add', parameters('vaultName'))]", + "properties": { + "accessPolicies": [ + "[parameters('accessPolicy')]" + ] + } + } + ] + } + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'ai-unlimited')]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'vault')]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "firewall", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[reference(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), '2022-09-01', 'full').location]" + }, + "name": { + "value": "[parameters('SecurityGroup')]" + }, + "accessCidrs": { + "value": "[parameters('AccessCIDRs')]" + }, + "aiUnlimitedAuthPort": { + "value": "[parameters('AiUnlimitedAuthPort')]" + }, + "aiUnlimitedGrpcPort": { + "value": "[parameters('AiUnlimitedGrpcPort')]" + }, + "aiUnlimitedSchedulerHttpPort": { + "value": "[parameters('AiUnlimitedSchedulerHttpPort')]" + }, + "aiUnlimitedUIHttpPort": { + "value": "[variables('AiUnlimitedUIHttpPort')]" + }, + "aiUnlimitedUIHttpsPort": { + "value": "[variables('AiUnlimitedUIHttpsPort')]" + }, + "sourceAppSecGroups": { + "value": "[parameters('SourceAppSecGroups')]" + }, + "detinationAppSecGroups": { + "value": "[parameters('detinationAppSecGroups')]" + }, + "sshAccess": { + "value": false + }, + "tags": { + "value": "[parameters('Tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "16013689345643124539" + } + }, + "parameters": { + "location": { + "type": "string" + }, + "name": { + "type": "string" + }, + "accessCidrs": { + "type": "array", + "defaultValue": [] + }, + "sourceAppSecGroups": { + "type": "array", + "defaultValue": [] + }, + "detinationAppSecGroups": { + "type": "array", + "defaultValue": [] + }, + "sshAccess": { + "type": "bool", + "defaultValue": false + }, + "aiUnlimitedAuthPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedGrpcPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedSchedulerHttpPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedUIHttpPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedUIHttpsPort": { + "type": "int", + "defaultValue": 0 + }, + "jupyterHttpPort": { + "type": "int", + "defaultValue": 0 + }, + "tags": { + "type": "object", + "defaultValue": {} + }, + "uuid": { + "type": "string", + "defaultValue": "[newGuid()]" + } + }, + "variables": { + "nameCharLimit": 60, + "uniqueName": "[format('{0}-{1}', parameters('name'), uniqueString(parameters('uuid')))]", + "uniqueSecurityGroupName": "[substring(format('{0}', variables('uniqueName')), 0, if(less(length(variables('uniqueName')), variables('nameCharLimit')), length(variables('uniqueName')), variables('nameCharLimit')))]" + }, + "resources": [ + { + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2022-11-01", + "name": "[variables('uniqueSecurityGroupName')]", + "tags": "[parameters('tags')]", + "location": "[parameters('location')]" + }, + { + "condition": "[parameters('sshAccess')]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', variables('uniqueSecurityGroupName'), format('{0}-ssh-allow', variables('uniqueSecurityGroupName')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow ssh to the workspace instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "direction": "Inbound", + "priority": 700, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + ] + }, + { + "condition": "[not(equals(parameters('aiUnlimitedAuthPort'), 0))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', variables('uniqueSecurityGroupName'), format('{0}-workspace-auth-allow', variables('uniqueSecurityGroupName')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow http to the workspace instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[string(parameters('aiUnlimitedAuthPort'))]", + "direction": "Inbound", + "priority": 701, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + ] + }, + { + "condition": "[not(equals(parameters('aiUnlimitedGrpcPort'), 0))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', variables('uniqueSecurityGroupName'), format('{0}-workspace-grpc-allow', variables('uniqueSecurityGroupName')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow grpc to the workspace instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[string(parameters('aiUnlimitedGrpcPort'))]", + "direction": "Inbound", + "priority": 702, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + ] + }, + { + "condition": "[not(equals(parameters('jupyterHttpPort'), 0))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', variables('uniqueSecurityGroupName'), format('{0}-juptyer-http-allow', variables('uniqueSecurityGroupName')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow http to the jupyter instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[string(parameters('jupyterHttpPort'))]", + "direction": "Inbound", + "priority": 703, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + ] + }, + { + "condition": "[not(equals(parameters('aiUnlimitedSchedulerHttpPort'), 0))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', variables('uniqueSecurityGroupName'), format('{0}-scheduler-http-allow', variables('uniqueSecurityGroupName')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow http to the scheduler instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[string(parameters('aiUnlimitedSchedulerHttpPort'))]", + "direction": "Inbound", + "priority": 704, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + ] + }, + { + "condition": "[not(equals(parameters('aiUnlimitedUIHttpPort'), 0))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', variables('uniqueSecurityGroupName'), format('{0}-workspace-ui-http-allow', parameters('name')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow http to the workspace ui instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[string(parameters('aiUnlimitedUIHttpPort'))]", + "direction": "Inbound", + "priority": 705, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + ] + }, + { + "condition": "[not(equals(parameters('aiUnlimitedUIHttpsPort'), 0))]", + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "apiVersion": "2023-04-01", + "name": "[format('{0}/{1}', variables('uniqueSecurityGroupName'), format('{0}-workspace-ui-https-allow', parameters('name')))]", + "properties": { + "copy": [ + { + "name": "destinationApplicationSecurityGroups", + "count": "[length(parameters('detinationAppSecGroups'))]", + "input": { + "id": "[parameters('detinationAppSecGroups')[copyIndex('destinationApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + }, + { + "name": "sourceApplicationSecurityGroups", + "count": "[length(parameters('sourceAppSecGroups'))]", + "input": { + "id": "[parameters('sourceAppSecGroups')[copyIndex('sourceApplicationSecurityGroups')]]", + "location": "[parameters('location')]" + } + } + ], + "access": "Allow", + "description": "allow https to the workspace ui instance", + "destinationAddressPrefix": "*", + "destinationPortRange": "[string(parameters('aiUnlimitedUIHttpsPort'))]", + "direction": "Inbound", + "priority": 706, + "protocol": "Tcp", + "sourceAddressPrefixes": "[parameters('accessCidrs')]", + "sourcePortRange": "*" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + ] + } + ], + "outputs": { + "Id": { + "type": "string", + "value": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('uniqueSecurityGroupName'))]" + } + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "loadbalancer", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "name": { + "value": "[parameters('AiUnlimitedName')]" + }, + "dnsPrefix": { + "value": "[variables('nlbDnsLabelPrefix')]" + }, + "location": { + "value": "[reference(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), '2022-09-01', 'full').location]" + }, + "aiUnlimitedAuthPort": { + "value": "[parameters('AiUnlimitedAuthPort')]" + }, + "aiUnlimitedGrpcPort": { + "value": "[parameters('AiUnlimitedGrpcPort')]" + }, + "aiUnlimitedSchedulerHttpPort": { + "value": "[parameters('AiUnlimitedSchedulerHttpPort')]" + }, + "aiUnlimitedUIHttpPort": { + "value": "[variables('AiUnlimitedUIHttpPort')]" + }, + "aiUnlimitedUIHttpsPort": { + "value": "[variables('AiUnlimitedUIHttpsPort')]" + }, + "tags": { + "value": "[parameters('Tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "12614640311343730358" + } + }, + "parameters": { + "name": { + "type": "string" + }, + "location": { + "type": "string" + }, + "dnsPrefix": { + "type": "string" + }, + "aiUnlimitedAuthPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedGrpcPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedSchedulerHttpPort": { + "type": "int", + "defaultValue": 0 + }, + "jupyterHttpPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedUIHttpPort": { + "type": "int", + "defaultValue": 0 + }, + "aiUnlimitedUIHttpsPort": { + "type": "int", + "defaultValue": 0 + }, + "tags": { + "type": "object", + "defaultValue": {} + } + }, + "resources": [ + { + "type": "Microsoft.Network/loadBalancers", + "apiVersion": "2021-08-01", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard" + }, + "properties": { + "frontendIPConfigurations": [ + { + "name": "[format('{0}Inbound', parameters('name'))]", + "properties": { + "publicIPAddress": { + "id": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-inbound', parameters('name'))), '2022-09-01').outputs.Id.value]" + } + } + }, + { + "name": "[format('{0}Outbound', parameters('name'))]", + "properties": { + "publicIPAddress": { + "id": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-outbound', parameters('name'))), '2022-09-01').outputs.Id.value]" + } + } + } + ], + "backendAddressPools": [ + { + "name": "[format('{0}InboundBackendPool', parameters('name'))]" + }, + { + "name": "[format('{0}OutboundBackendPool', parameters('name'))]" + } + ], + "loadBalancingRules": "[flatten(createArray(if(not(equals(parameters('aiUnlimitedAuthPort'), 0)), createArray(createObject('name', 'AiUnlimitedAuth', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedAuthPort'), 'backendPort', parameters('aiUnlimitedAuthPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}AuthLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedGrpcPort'), 0)), createArray(createObject('name', 'AiUnlimitedAPI', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedGrpcPort'), 'backendPort', parameters('aiUnlimitedGrpcPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}APILbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('jupyterHttpPort'), 0)), createArray(createObject('name', 'JupyterUI', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('jupyterHttpPort'), 'backendPort', parameters('jupyterHttpPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}JupyterLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedSchedulerHttpPort'), 0)), createArray(createObject('name', 'AiUnlimitedSchedulerHttp', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedSchedulerHttpPort'), 'backendPort', parameters('aiUnlimitedSchedulerHttpPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}SchedulerHttpLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpPort'), 0)), createArray(createObject('name', 'AiUnlimitedUIHttp', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedUIHttpPort'), 'backendPort', parameters('aiUnlimitedUIHttpPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}UIHttpLbProbe', parameters('name'))))))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpsPort'), 0)), createArray(createObject('name', 'AiUnlimitedUIHttps', 'properties', createObject('frontendIPConfiguration', createObject('id', resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Inbound', parameters('name')))), 'backendAddressPool', createObject('id', resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))), 'frontendPort', parameters('aiUnlimitedUIHttpsPort'), 'backendPort', parameters('aiUnlimitedUIHttpsPort'), 'enableFloatingIP', false(), 'idleTimeoutInMinutes', 15, 'protocol', 'Tcp', 'enableTcpReset', true(), 'loadDistribution', 'Default', 'disableOutboundSnat', true(), 'probe', createObject('id', resourceId('Microsoft.Network/loadBalancers/probes', parameters('name'), format('{0}UIHttpsLbProbe', parameters('name'))))))), createArray())))]", + "probes": "[flatten(createArray(if(not(equals(parameters('aiUnlimitedAuthPort'), 0)), createArray(createObject('name', format('{0}AuthLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedAuthPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedGrpcPort'), 0)), createArray(createObject('name', format('{0}APILbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedGrpcPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('jupyterHttpPort'), 0)), createArray(createObject('name', format('{0}JupyterLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('jupyterHttpPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedSchedulerHttpPort'), 0)), createArray(createObject('name', format('{0}SchedulerHttpLbProbe', parameters('name')), 'properties', createObject('protocol', 'Tcp', 'port', parameters('aiUnlimitedSchedulerHttpPort'), 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpPort'), 0)), createArray(createObject('name', format('{0}UIHttpLbProbe', parameters('name')), 'properties', createObject('protocol', 'Http', 'port', parameters('aiUnlimitedUIHttpPort'), 'requestPath', '/', 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray()), if(not(equals(parameters('aiUnlimitedUIHttpsPort'), 0)), createArray(createObject('name', format('{0}UIHttpsLbProbe', parameters('name')), 'properties', createObject('protocol', 'Https', 'port', parameters('aiUnlimitedUIHttpsPort'), 'requestPath', '/', 'intervalInSeconds', 5, 'numberOfProbes', 2))), createArray())))]", + "outboundRules": [ + { + "name": "myOutboundRule", + "properties": { + "allocatedOutboundPorts": 10000, + "protocol": "All", + "enableTcpReset": false, + "idleTimeoutInMinutes": 15, + "backendAddressPool": { + "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('name'), format('{0}OutboundBackendPool', parameters('name')))]" + }, + "frontendIPConfigurations": [ + { + "id": "[resourceId('Microsoft.Network/loadBalancers/frontendIPConfigurations', parameters('name'), format('{0}Outbound', parameters('name')))]" + } + ] + } + } + ] + }, + "tags": "[parameters('tags')]", + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', format('{0}-inbound', parameters('name')))]", + "[resourceId('Microsoft.Resources/deployments', format('{0}-outbound', parameters('name')))]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-inbound', parameters('name'))]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "name": { + "value": "[format('{0}-inbound', parameters('name'))]" + }, + "location": { + "value": "[parameters('location')]" + }, + "dnsPrefix": { + "value": "[parameters('dnsPrefix')]" + }, + "tags": { + "value": "[parameters('tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "3737753991380666295" + } + }, + "parameters": { + "name": { + "type": "string" + }, + "location": { + "type": "string" + }, + "dnsPrefix": { + "type": "string" + }, + "tags": { + "type": "object" + } + }, + "resources": [ + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2021-05-01", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard" + }, + "properties": { + "dnsSettings": "[if(not(equals(parameters('dnsPrefix'), '')), createObject('domainNameLabel', parameters('dnsPrefix')), null())]", + "publicIPAddressVersion": "IPv4", + "publicIPAllocationMethod": "Static", + "idleTimeoutInMinutes": 5 + }, + "tags": "[parameters('tags')]" + } + ], + "outputs": { + "Id": { + "type": "string", + "value": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('name'))]" + }, + "Ip": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('name')), '2021-05-01').ipAddress]" + }, + "Dns": { + "type": "string", + "value": "[if(not(equals(parameters('dnsPrefix'), '')), reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('name')), '2021-05-01').dnsSettings.fqdn, '')]" + } + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[format('{0}-outbound', parameters('name'))]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "name": { + "value": "[format('{0}-outbound', parameters('name'))]" + }, + "location": { + "value": "[parameters('location')]" + }, + "dnsPrefix": { + "value": "" + }, + "tags": { + "value": "[parameters('tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "3737753991380666295" + } + }, + "parameters": { + "name": { + "type": "string" + }, + "location": { + "type": "string" + }, + "dnsPrefix": { + "type": "string" + }, + "tags": { + "type": "object" + } + }, + "resources": [ + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2021-05-01", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard" + }, + "properties": { + "dnsSettings": "[if(not(equals(parameters('dnsPrefix'), '')), createObject('domainNameLabel', parameters('dnsPrefix')), null())]", + "publicIPAddressVersion": "IPv4", + "publicIPAllocationMethod": "Static", + "idleTimeoutInMinutes": 5 + }, + "tags": "[parameters('tags')]" + } + ], + "outputs": { + "Id": { + "type": "string", + "value": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('name'))]" + }, + "Ip": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('name')), '2021-05-01').ipAddress]" + }, + "Dns": { + "type": "string", + "value": "[if(not(equals(parameters('dnsPrefix'), '')), reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('name')), '2021-05-01').dnsSettings.fqdn, '')]" + } + } + } + } + } + ], + "outputs": { + "nlbPools": { + "type": "array", + "value": [ + "[format('{0}InboundBackendPool', parameters('name'))]", + "[format('{0}OutboundBackendPool', parameters('name'))]" + ] + }, + "PublicIp": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-inbound', parameters('name'))), '2022-09-01').outputs.Ip.value]" + }, + "PublicDns": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-inbound', parameters('name'))), '2022-09-01').outputs.Dns.value]" + } + } + } + } + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "ai-unlimited", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[reference(subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('ResourceGroupName')), '2022-09-01', 'full').location]" + }, + "name": { + "value": "[parameters('AiUnlimitedName')]" + }, + "adminUsername": { + "value": "azureuser" + }, + "sshPublicKey": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'Public-Key'), '2022-09-01').outputs.PublicKey.value]" + }, + "dnsLabelPrefix": { + "value": "[variables('dnsLabelPrefix')]" + }, + "vmSize": { + "value": "[parameters('InstanceType')]" + }, + "subnetId": { + "value": "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Network/virtualNetworks/subnets', parameters('Network'), parameters('Subnet'))]" + }, + "networkSecurityGroupID": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'firewall'), '2022-09-01').outputs.Id.value]" + }, + "osVersion": { + "value": "[parameters('OSVersion')]" + }, + "cloudInitData": { + "value": "[base64(format(variables('$fxv#0'), base64(format(variables('$fxv#1'), variables('registry'), variables('workspaceRepository'), parameters('AiUnlimitedVersion'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), subscription().subscriptionId, subscription().tenantId, format('--network-alias {0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value))), base64(format(variables('$fxv#2'), variables('registry'), variables('workspaceSchedulerRepository'), parameters('AiUnlimitedSchedulerVersion'), parameters('AiUnlimitedSchedulerHttpPort'), parameters('AiUnlimitedGrpcPort'))), base64(format(variables('$fxv#3'), variables('registry'), variables('workspaceUIRepository'), parameters('AiUnlimitedUIVersion'), variables('AiUnlimitedUIHttpPort'), parameters('AiUnlimitedAuthPort'), parameters('AiUnlimitedGrpcPort'), format('--network-alias {0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value)))))]" + }, + "usePersistentVolume": { + "value": "[parameters('UsePersistentVolume')]" + }, + "persistentVolumeSize": { + "value": "[parameters('PersistentVolumeSize')]" + }, + "existingPersistentVolume": { + "value": "[parameters('ExistingPersistentVolume')]" + }, + "nlbName": { + "value": "[parameters('AiUnlimitedName')]" + }, + "nlbPoolNames": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.nlbPools.value]" + }, + "usePublicIp": { + "value": false + }, + "tags": { + "value": "[parameters('Tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "17662981702954878393" + } + }, + "parameters": { + "location": { + "type": "string" + }, + "name": { + "type": "string" + }, + "adminUsername": { + "type": "string" + }, + "sshPublicKey": { + "type": "string" + }, + "vmSize": { + "type": "string" + }, + "subnetId": { + "type": "string" + }, + "networkSecurityGroupID": { + "type": "string" + }, + "osVersion": { + "type": "string" + }, + "usePersistentVolume": { + "type": "string" + }, + "persistentVolumeSize": { + "type": "int" + }, + "existingPersistentVolume": { + "type": "string" + }, + "cloudInitData": { + "type": "string" + }, + "usePublicIp": { + "type": "bool", + "defaultValue": false + }, + "nlbName": { + "type": "string", + "defaultValue": "" + }, + "albName": { + "type": "string", + "defaultValue": "" + }, + "nlbPoolNames": { + "type": "array", + "defaultValue": [] + }, + "tags": { + "type": "object", + "defaultValue": {} + }, + "dnsLabelPrefix": { + "type": "string", + "defaultValue": "" + } + }, + "variables": { + "copy": [ + { + "name": "resourcePools", + "count": "[length(parameters('nlbPoolNames'))]", + "input": { + "id": "[resourceId('Microsoft.Network/loadBalancers/backendAddressPools', parameters('nlbName'), parameters('nlbPoolNames')[copyIndex('resourcePools')])]" + } + } + ], + "imageReference": { + "Ubuntu-1804": { + "publisher": "Canonical", + "offer": "UbuntuServer", + "sku": "18_04-lts-gen2", + "version": "latest" + }, + "Ubuntu-2004": { + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-focal", + "sku": "20_04-lts-gen2", + "version": "latest" + }, + "Ubuntu-2204": { + "publisher": "Canonical", + "offer": "0001-com-ubuntu-server-jammy", + "sku": "22_04-lts-gen2", + "version": "latest" + } + }, + "publicIPAddressName": "[format('{0}PublicIP', parameters('name'))]", + "networkInterfaceName": "[format('{0}-nic', parameters('name'))]", + "osDiskType": "Standard_LRS", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[format('/home/{0}/.ssh/authorized_keys', parameters('adminUsername'))]", + "keyData": "[parameters('sshPublicKey')]" + } + ] + } + }, + "trustedExtensionName": "GuestAttestation", + "trustedExtensionPublisher": "Microsoft.Azure.Security.LinuxAttestation", + "trustedExtensionVersion": "1.0", + "trustedMaaTenantName": "GuestAttestation", + "trustedMaaEndpoint": "[substring('emptystring', 0, 0)]", + "dockerExtensionName": "DockerExtension", + "dockerExtensionPublisher": "Microsoft.Azure.Extensions", + "dockerExtensionVersion": "1.1" + }, + "resources": [ + { + "condition": "[equals(parameters('usePersistentVolume'), 'New')]", + "type": "Microsoft.Compute/disks", + "apiVersion": "2023-04-02", + "name": "[format('{0}-disk', parameters('name'))]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "creationData": { + "createOption": "Empty" + }, + "diskSizeGB": "[parameters('persistentVolumeSize')]", + "maxShares": 1, + "osType": "Linux" + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2022-11-01", + "name": "[variables('networkInterfaceName')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "properties": { + "ipConfigurations": [ + "[if(parameters('usePublicIp'), createObject('name', 'ipconfigpublic', 'properties', createObject('privateIPAllocationMethod', 'Dynamic', 'subnet', createObject('id', parameters('subnetId')), 'publicIPAddress', createObject('id', reference(resourceId('Microsoft.Resources/deployments', variables('publicIPAddressName')), '2022-09-01').outputs.Id.value), 'loadBalancerBackendAddressPools', variables('resourcePools'))), if(equals(parameters('albName'), ''), createObject('name', 'ipconfigprivate', 'properties', createObject('privateIPAllocationMethod', 'Dynamic', 'subnet', createObject('id', parameters('subnetId')), 'loadBalancerBackendAddressPools', variables('resourcePools'))), createObject('name', 'ipconfigprivate', 'properties', createObject('privateIPAllocationMethod', 'Dynamic', 'subnet', createObject('id', parameters('subnetId'))))))]" + ], + "networkSecurityGroup": { + "id": "[parameters('networkSecurityGroupID')]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', variables('publicIPAddressName'))]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2023-03-01", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]", + "identity": { + "type": "SystemAssigned" + }, + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('vmSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "[variables('osDiskType')]" + } + }, + "dataDisks": [ + { + "lun": 0, + "createOption": "Attach", + "managedDisk": { + "id": "[if(equals(parameters('usePersistentVolume'), 'New'), resourceId('Microsoft.Compute/disks', format('{0}-disk', parameters('name'))), resourceId('Microsoft.Compute/disks', parameters('existingPersistentVolume')))]" + } + } + ], + "imageReference": "[variables('imageReference')[parameters('osVersion')]]" + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + } + ] + }, + "osProfile": { + "computerName": "[parameters('name')]", + "adminUsername": "[parameters('adminUsername')]", + "linuxConfiguration": "[variables('linuxConfiguration')]" + }, + "securityProfile": { + "securityType": "TrustedLaunch", + "uefiSettings": { + "secureBootEnabled": true, + "vTpmEnabled": true + } + }, + "userData": "[parameters('cloudInitData')]" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]", + "[resourceId('Microsoft.Compute/disks', format('{0}-disk', parameters('name')))]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-03-01", + "name": "[format('{0}/{1}', parameters('name'), variables('trustedExtensionName'))]", + "tags": "[parameters('tags')]", + "location": "[parameters('location')]", + "properties": { + "publisher": "[variables('trustedExtensionPublisher')]", + "type": "[variables('trustedExtensionName')]", + "typeHandlerVersion": "[variables('trustedExtensionVersion')]", + "autoUpgradeMinorVersion": true, + "settings": { + "AttestationConfig": { + "MaaSettings": { + "maaEndpoint": "[variables('trustedMaaEndpoint')]", + "maaTenantName": "[variables('trustedMaaTenantName')]" + } + } + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('name'))]" + ] + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "apiVersion": "2023-03-01", + "name": "[format('{0}/{1}', parameters('name'), variables('dockerExtensionName'))]", + "tags": "[parameters('tags')]", + "location": "[parameters('location')]", + "properties": { + "publisher": "[variables('dockerExtensionPublisher')]", + "type": "[variables('dockerExtensionName')]", + "typeHandlerVersion": "[variables('dockerExtensionVersion')]", + "autoUpgradeMinorVersion": true + }, + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines', parameters('name'))]" + ] + }, + { + "condition": "[parameters('usePublicIp')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "[variables('publicIPAddressName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "name": { + "value": "[variables('publicIPAddressName')]" + }, + "location": { + "value": "[parameters('location')]" + }, + "dnsPrefix": { + "value": "[parameters('dnsLabelPrefix')]" + }, + "tags": { + "value": "[parameters('tags')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "3737753991380666295" + } + }, + "parameters": { + "name": { + "type": "string" + }, + "location": { + "type": "string" + }, + "dnsPrefix": { + "type": "string" + }, + "tags": { + "type": "object" + } + }, + "resources": [ + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2021-05-01", + "name": "[parameters('name')]", + "location": "[parameters('location')]", + "sku": { + "name": "Standard" + }, + "properties": { + "dnsSettings": "[if(not(equals(parameters('dnsPrefix'), '')), createObject('domainNameLabel', parameters('dnsPrefix')), null())]", + "publicIPAddressVersion": "IPv4", + "publicIPAllocationMethod": "Static", + "idleTimeoutInMinutes": 5 + }, + "tags": "[parameters('tags')]" + } + ], + "outputs": { + "Id": { + "type": "string", + "value": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('name'))]" + }, + "Ip": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('name')), '2021-05-01').ipAddress]" + }, + "Dns": { + "type": "string", + "value": "[if(not(equals(parameters('dnsPrefix'), '')), reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('name')), '2021-05-01').dnsSettings.fqdn, '')]" + } + } + } + } + } + ], + "outputs": { + "PublicIP": { + "type": "string", + "value": "[if(parameters('usePublicIp'), reference(resourceId('Microsoft.Resources/deployments', variables('publicIPAddressName')), '2022-09-01').outputs.Ip.value, '')]" + }, + "PrivateIP": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName')), '2022-11-01').ipConfigurations[0].properties.privateIPAddress]" + }, + "PrincipleId": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Compute/virtualMachines', parameters('name')), '2023-03-01', 'full').identity.principalId]" + } + } + } + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'firewall')]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer')]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'Public-Key')]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2022-09-01", + "name": "Public-Key", + "resourceGroup": "[parameters('ResourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "Name": { + "value": "[parameters('AiUnlimitedName')]" + }, + "Location": { + "value": "[deployment().location]" + }, + "VaultName": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'vault'), '2022-09-01').outputs.name.value]" + }, + "RoleID": { + "value": "[parameters('RoleDefinitionId')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.32.4.45862", + "templateHash": "12388745557700980847" + } + }, + "parameters": { + "Location": { + "type": "string" + }, + "Name": { + "type": "string" + }, + "VaultName": { + "type": "string" + }, + "RoleID": { + "type": "string" + }, + "Uuid": { + "type": "string", + "defaultValue": "[newGuid()]" + } + }, + "variables": { + "SecretName": "[format('{0}-PrivateKey', parameters('Name'))]", + "ScriptName": "[format('{0}-createKeys', parameters('Name'))]", + "IdentityName": "[format('{0}-scratch', parameters('Name'))]", + "RoleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', parameters('RoleID'))]", + "RoleDefinitionName": "[guid(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('IdentityName')), variables('RoleDefinitionId'), resourceGroup().id)]" + }, + "resources": [ + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "apiVersion": "2023-01-31", + "name": "[variables('IdentityName')]", + "location": "[parameters('Location')]" + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[variables('RoleDefinitionName')]", + "properties": { + "roleDefinitionId": "[variables('RoleDefinitionId')]", + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('IdentityName')), '2023-01-31').principalId]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('IdentityName'))]" + ] + }, + { + "type": "Microsoft.Resources/deploymentScripts", + "apiVersion": "2023-08-01", + "name": "[variables('ScriptName')]", + "location": "[parameters('Location')]", + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "[format('{0}', resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('IdentityName')))]": {} + } + }, + "kind": "AzureCLI", + "properties": { + "forceUpdateTag": "[parameters('Uuid')]", + "azCliVersion": "2.0.80", + "timeout": "PT30M", + "retentionInterval": "P1D", + "cleanupPreference": "OnSuccess", + "scriptContent": " #/bin/bash -e\n\n echo -e 'y' | ssh-keygen -f scratch\n\n privateKey=$(cat scratch)\n publicKey=$(cat 'scratch.pub')\n\n json=\"{\\\"keyinfo\\\":{\\\"privateKey\\\":\\\"$privateKey\\\",\\\"publicKey\\\":\\\"$publicKey\\\"}}\"\n\n echo \"$json\" > $AZ_SCRIPTS_OUTPUT_PATH\n " + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', variables('IdentityName'))]", + "[resourceId('Microsoft.Authorization/roleAssignments', variables('RoleDefinitionName'))]" + ] + }, + { + "type": "Microsoft.KeyVault/vaults/secrets", + "apiVersion": "2023-07-01", + "name": "[format('{0}/{1}', parameters('VaultName'), variables('SecretName'))]", + "properties": { + "value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', variables('ScriptName')), '2023-08-01').outputs.keyinfo.privateKey]" + }, + "dependsOn": [ + "[resourceId('Microsoft.Resources/deploymentScripts', variables('ScriptName'))]" + ] + } + ], + "outputs": { + "PublicKey": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', variables('ScriptName')), '2023-08-01').outputs.keyinfo.publicKey]" + }, + "Status": { + "type": "object", + "value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', variables('ScriptName')), '2023-08-01').status]" + } + } + } + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'vault')]" + ] + } + ], + "outputs": { + "PublicIP": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicIp.value]" + }, + "PrivateIP": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'ai-unlimited'), '2022-09-01').outputs.PrivateIP.value]" + }, + "AiUnlimitedPublicHttpAccess": { + "type": "string", + "value": "[concat(format('http://{0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value), if(not(equals(variables('AiUnlimitedUIHttpPort'), 80)), concat(':', string(variables('AiUnlimitedUIHttpPort'))), ''))]" + }, + "AiUnlimitedPrivateHttpAccess": { + "type": "string", + "value": "[concat(format('http://{0}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'ai-unlimited'), '2022-09-01').outputs.PrivateIP.value), if(not(equals(variables('AiUnlimitedUIHttpPort'), 80)), concat(':', string(variables('AiUnlimitedUIHttpPort'))), ''))]" + }, + "AiUnlimitedPublicGrpcAccess": { + "type": "string", + "value": "[format('http://{0}:{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'loadbalancer'), '2022-09-01').outputs.PublicDns.value, parameters('AiUnlimitedGrpcPort'))]" + }, + "AiUnlimitedPrivateGrpcAccess": { + "type": "string", + "value": "[format('http://{0}:{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'ai-unlimited'), '2022-09-01').outputs.PrivateIP.value, parameters('AiUnlimitedGrpcPort'))]" + }, + "KeyVaultName": { + "type": "string", + "value": "[if(equals(parameters('UseKeyVault'), 'New'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'vault'), '2022-09-01').outputs.name.value, '')]" + }, + "NetworkSecurityGroupId": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('ResourceGroupName')), 'Microsoft.Resources/deployments', 'firewall'), '2022-09-01').outputs.Id.value]" + } + } +} \ No newline at end of file From d72ffaa517ed77caa66b0f33cc514fde3b8addd2 Mon Sep 17 00:00:00 2001 From: cs128781 Date: Mon, 16 Dec 2024 13:45:43 -0800 Subject: [PATCH 6/6] remove extra service file --- .../templates/ai-unlimited-scheduler.service | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 deployments/azure/templates/ai-unlimited-scheduler.service diff --git a/deployments/azure/templates/ai-unlimited-scheduler.service b/deployments/azure/templates/ai-unlimited-scheduler.service deleted file mode 100644 index f7d100a..0000000 --- a/deployments/azure/templates/ai-unlimited-scheduler.service +++ /dev/null @@ -1,31 +0,0 @@ -[Unit] -Description=AI Unlimited -After=docker.service -Requires=docker.service -StartLimitInterval=200 -StartLimitBurst=10 - -[Service] -TimeoutStartSec=0 -Restart=always -RestartSec=2 -ExecStartPre=-/usr/bin/docker network create -d bridge ai_unlimited -ExecStartPre=-/usr/bin/mkdir -p /etc/td/ai-unlimited -ExecStartPre=-/usr/bin/docker exec %n stop || true -ExecStartPre=-/usr/bin/docker rm %n || true -ExecStartPre=/usr/bin/docker pull {0}/{1}:{2} - -ExecStart=/usr/bin/docker run \ - -e accept_license=Y \ - -e PLATFORM=azure \ - -e ARM_USE_MSI=true \ - -e ARM_SUBSCRIPTION_ID={5} \ - -e ARM_TENANT_ID={6} \ - -v /etc/td/ai-unlimited:/etc/td \ - -p {3}:3000 \ - -p {4}:3282 \ - --network ai_unlimited {7} \ - --rm --name %n {0}/{1}:{2} workspaces serve -v - -[Install] -WantedBy=multi-user.target \ No newline at end of file