CostExport init

CostExport/ARM/CostExport.json

@@ -0,0 +1,234 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "functionAppName": {
+      "type": "string",
+      "defaultValue": "[format('coralogixcostsexport-{0}', uniqueString(resourceGroup().id))]",
+      "metadata": {
+        "description": "The name of the Azure Function app."
+      }
+    },
+    "storageAccountType": {
+      "type": "string",
+      "defaultValue": "Standard_LRS",
+      "allowedValues": [
+        "Standard_LRS",
+        "Standard_GRS",
+        "Standard_RAGRS"
+      ],
+      "metadata": {
+        "description": "Storage Account type."
+      }
+    },
+    "coralogix_domain": {
+      "type": "string",
+      "defaultValue": "coralogix.com",
+      "allowedValues": [
+        "coralogix.com",
+        "eu2.coralogix.com",
+        "coralogix.us",
+        "cx498.coralogix.com",
+        "coralogix.in",
+        "coralogixsg.com"
+      ],
+      "metadata": {
+        "description": "Coralogix domain."
+      }
+    },
+    "coralogix_key": {
+      "type": "string",
+      "metadata": {
+        "description": "Coralogix private key."
+      }
+    },
+    "location": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]",
+      "metadata": {
+        "description": "Location for all resources."
+      }
+    },
+    "appInsightsLocation": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]",
+      "metadata": {
+        "description": "Location for Application Insights."
+      }
+    },
+    "functionWorkerRuntime": {
+      "type": "string",
+      "defaultValue": "python",
+      "allowedValues": [
+        "python"
+      ],
+      "metadata": {
+        "description": "The language worker runtime to load in the function app."
+      }
+    },
+    "linuxFxVersion": {
+      "type": "string",
+      "defaultValue": "Python|3.11",
+      "allowedValues": [
+        "Python|3.11"
+      ],
+      "metadata": {
+        "description": "Required for Linux app to represent runtime stack in the format of 'runtime|runtimeVersion'. For example: 'python|3.9'."
+      }
+    },
+    "packageUri": {
+      "type": "string",
+      "defaultValue": null,
+      "metadata": {
+        "description": "The zip content URL."
+      }
+    }
+  },
+  "variables": {
+    "hostingPlanName": "[parameters('functionAppName')]",
+    "applicationInsightsName": "[parameters('functionAppName')]",
+    "storageAccountName": "[format('{0}azfunctions', uniqueString(resourceGroup().id))]",
+    "roleAssignmentName": "[guid(resourceGroup().id, 'CostManagementContributor')]",
+    "costManagementContributorRoleDefinitionId": "/subscriptions/7d91656d-73e8-4f2e-a3df-4d17ad2c3ef7/providers/Microsoft.Authorization/roleDefinitions/434105ed-43f6-45c7-a02f-909b2ba83430"
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Storage/storageAccounts",
+      "apiVersion": "2022-05-01",
+      "name": "[variables('storageAccountName')]",
+      "location": "[parameters('location')]",
+      "sku": {
+        "name": "[parameters('storageAccountType')]"
+      },
+      "kind": "Storage"
+    },
+    {
+      "type": "Microsoft.Web/serverfarms",
+      "apiVersion": "2022-03-01",
+      "name": "[variables('hostingPlanName')]",
+      "location": "[parameters('location')]",
+      "sku": {
+        "name": "Y1",
+        "tier": "Dynamic",
+        "size": "Y1",
+        "family": "Y"
+      },
+      "properties": {
+        "reserved": true
+      }
+    },
+    {
+      "type": "Microsoft.Insights/components",
+      "apiVersion": "2020-02-02",
+      "name": "[variables('applicationInsightsName')]",
+      "location": "[parameters('appInsightsLocation')]",
+      "tags": {
+        "[format('hidden-link:{0}', resourceId('Microsoft.Web/sites', parameters('functionAppName')))]": "Resource"
+      },
+      "kind": "web",
+      "properties": {
+        "Application_Type": "web"
+      }
+    },
+    {
+      "type": "Microsoft.Web/sites",
+      "apiVersion": "2022-03-01",
+      "name": "[parameters('functionAppName')]",
+      "location": "[parameters('location')]",
+      "identity": {
+        "type": "SystemAssigned"
+      },
+      "dependsOn": [
+        "[resourceId('Microsoft.Insights/components', variables('applicationInsightsName'))]",
+        "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]",
+        "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName'))]"
+      ],
+      "kind": "functionapp,linux",
+      "properties": {
+        "reserved": true,
+        "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('hostingPlanName'))]",
+        "siteConfig": {
+          "linuxFxVersion": "[parameters('linuxFxVersion')]",
+          "appSettings": [
+            {
+              "name": "APPINSIGHTS_INSTRUMENTATIONKEY",
+              "value": "[reference(resourceId('Microsoft.Insights/components', parameters('functionAppName')), '2020-02-02').InstrumentationKey]"
+            },
+            {
+              "name": "SUBSCRIPTION_ID",
+              "value": "[subscription().subscriptionId]"
+            },
+            {
+              "name": "CORALOGIX_DOMAIN",
+              "value": "[parameters('coralogix_domain')]"
+            },
+            {
+              "name": "CORALOGIX_PRIVATE_KEY",
+              "value": "[parameters('coralogix_key')]"
+            },
+            {
+              "name": "AzureWebJobsStorage",
+              "value": "[format('DefaultEndpointsProtocol=https;AccountName={0};EndpointSuffix={1};AccountKey={2}', variables('storageAccountName'), environment().suffixes.storage, listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2022-05-01').keys[0].value)]"
+            },
+            {
+              "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING",
+              "value": "[format('DefaultEndpointsProtocol=https;AccountName={0};EndpointSuffix={1};AccountKey={2}', variables('storageAccountName'), environment().suffixes.storage, listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccountName')), '2022-05-01').keys[0].value)]"
+            },
+            {
+              "name": "WEBSITE_CONTENTSHARE",
+              "value": "[toLower(parameters('functionAppName'))]"
+            },
+            {
+              "name": "FUNCTIONS_EXTENSION_VERSION",
+              "value": "~4"
+            },
+            {
+              "name": "FUNCTIONS_WORKER_RUNTIME",
+              "value": "[parameters('functionWorkerRuntime')]"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "type": "Microsoft.Authorization/roleAssignments",
+      "apiVersion": "2021-04-01-preview",
+      "name": "[variables('roleAssignmentName')]",
+      "properties": {
+        "roleDefinitionId": "[variables('costManagementContributorRoleDefinitionId')]",
+        "principalId": "[reference(resourceId('Microsoft.Web/sites', parameters('functionAppName')), '2022-03-01', 'full').identity.principalId]",
+        "principalType": "ServicePrincipal",
+        "scope": "[resourceGroup().id]"
+      },
+      "dependsOn": [
+        "[resourceId('Microsoft.Web/sites', parameters('functionAppName'))]"
+      ]
+    },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2021-04-01",
+      "name": "zipDeploy",
+      "dependsOn": [
+        "[resourceId('Microsoft.Web/sites', parameters('functionAppName'))]",
+        "[variables('roleAssignmentName')]"
+      ],
+      "properties": {
+        "mode": "Incremental",
+        "template": {
+          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+          "contentVersion": "1.0.0.0",
+          "resources": [
+            {
+              "type": "Microsoft.Web/sites/extensions",
+              "apiVersion": "2022-03-01",
+              "name": "[concat(parameters('functionAppName'), '/zipdeploy')]",
+              "properties": {
+                "packageUri": "[parameters('packageUri')]"
+              }
+            }
+          ]
+        }
+      }
+    }
+  ]
+}

CostExport/CostExport/function_app.py

@@ -0,0 +1,144 @@
+import datetime
+import logging
+import os
+import requests
+from azure.identity import DefaultAzureCredential
+import azure.functions as func
+import pandas as pd
+from io import StringIO
+import json
+import math
+
+def parseCSV(url):
+    # Reads CSV file from URL and converts it to JSON
+    response = requests.get(url)
+    if response.status_code==200:
+        csv_content = response.content.decode('utf-8')
+        csv_df = pd.read_csv(StringIO(csv_content))
+
+        json_rows = csv_df.to_dict(orient='records')
+        return json_rows
+
+
+def clean_json(data):
+    # recursively cleans the data to ensure all values are JSON serializable
+    if isinstance(data, dict):
+        return {key: clean_json(value) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [clean_json(element) for element in data]
+    elif isinstance(data, float):
+        if math.isfinite(data):
+            return data
+        else:
+            return None
+    elif isinstance(data, (bool, int, str)) or data is None:
+        return data
+    else:
+        return str(data)
+
+def get_cost_data(subscription_id, token):
+
+    # Tells Azure to enerate a cost detail report of our subscription
+    url = f"https://management.azure.com/subscriptions/{subscription_id}/providers/Microsoft.CostManagement/generateCostDetailsReport?api-version=2023-11-01"
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+        "Accept": "application/json"
+    }
+
+    # Get from 00:00 of day before to 23:59 of same day
+    end_date = datetime.datetime.utcnow()
+    start_date = end_date - datetime.timedelta(days=1)
+    start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)
+    end_date = start_date + datetime.timedelta(days=1) - datetime.timedelta(seconds=1)
+
+    start_date_iso = start_date.strftime('%Y-%m-%dT%H:%M:%SZ')
+    end_date_iso = end_date.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+    body = {
+        "timePeriod": {
+            "start": start_date_iso,
+            "end": end_date_iso
+        },
+    }
+
+    response = requests.post(url, headers=headers, json=body)
+
+    # If the request was successful get the follow up URL from headers
+    if response.status_code == 202:
+        url=response.headers["Location"]
+        response = requests.get(url, headers=headers)
+
+        # Retrieve CSV file and return the parsed JSON data back to the main function
+        if response.status_code == 200:
+            json=response.json()
+            logging.info(json['manifest']['blobs'][0]['blobLink'])
+            return parseCSV(json['manifest']['blobs'][0]['blobLink'])
+            # if json.status == "Completed":
+            #     logging.info(json.blobs[0].blobLink)
+            #     return parseCSV(json.blobs[0].blobLink)
+        else:
+            logging.info(response.status_code)
+            logging.error("Failed to fetch AAAAAAAAAAAAA data: %s", response.text)
+    else:
+        logging.info(response.status_code)
+        logging.error("Failed to fetch cost data: %s", response.text)
+        return None
+
+
+def send_to_coralogix(log_data):
+    # Retrieves the correct coralogix domain and private key from Azure application settings
+    coralogix_domain = os.getenv('CORALOGIX_DOMAIN')
+    coralogix_key = os.getenv('CORALOGIX_PRIVATE_KEY')
+
+    headers = {
+        'Content-Type': 'application/json',
+        'Authorization': f"Bearer {coralogix_key}"
+    }
+
+    # Makes sure the JSON is serializable before sending it to Coralogix
+    cleaned_data = clean_json(log_data)
+
+    payload = {
+        "applicationName": "AzureFunctionApp",
+        "subsystemName": "CostManagement",
+        "timestamp": int(datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).timestamp() * 1000),
+        "severity": 3,
+        "text": cleaned_data
+    }
+
+    response = requests.post(f"https://ingress.{coralogix_domain}/logs/v1/singles", json=payload, headers=headers)
+
+    if response.status_code == 200:
+        logging.info("Successfully sent data to Coralogix")
+    else:
+        logging.error("Failed to send data to Coralogix: %s", response.text)
+
+
+
+app = func.FunctionApp()
+
+@app.schedule(schedule="0 0 * * *", arg_name="myTimer", run_on_startup=True,
+              use_monitor=False) 
+def timer_trigger(myTimer: func.TimerRequest) -> None:
+    if myTimer.past_due:
+        logging.info('The timer is past due!')
+
+    logging.info('Python timer trigger function executed.')
+    subscription_id = os.getenv('SUBSCRIPTION_ID')
+    if not subscription_id:
+        logging.error("Environment variable 'SUBSCRIPTION_ID' is not set.")
+        return
+
+    try:
+        credential = DefaultAzureCredential()
+        logging.info("Attempting to obtain token using DefaultAzureCredential.")
+        token = credential.get_token("https://management.azure.com/.default").token
+        logging.info("Successfully obtained token.")
+    except Exception as e:
+        logging.error(f"Failed to obtain token: {e}")
+        return
+
+    cost_data = get_cost_data(subscription_id, token)
+    for row in cost_data:
+        send_to_coralogix(row)

CostExport/CostExport/host.json

@@ -0,0 +1,15 @@
+{
+  "version": "2.0",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      }
+    }
+  },
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[4.*, 5.0.0)"
+  }
+}