✨ (analysis) add possibility to add redis energy to feedservice

loadtesting/data/01-caching/2024-07-31/16-18/output/run.log

@@ -1,20 +1,20 @@
-19:33:49 INFO - Starting
-19:33:51 INFO - Start: 2024-07-31 16:40:44 ; End: 2024-07-31 18:19:55
-19:33:51 INFO - Total energy consumed from 2024-07-31 16:40:44 to 2024-07-31 18:19:55 tracked by pdu metrics: 1.17822 MJ
-19:33:51 INFO - Total energy consumed from 2024-07-31 16:40:44 to 2024-07-31 18:19:55 tracked by kepler: 351.088 kJ
-19:33:51 INFO - Mean Abs Error of Keplers Trends compared to PDU (MAE over all data point diffs): 2019.93 J
-19:33:51 INFO - Stage: 100, From: 2024-07-31 16:40:44, To: 2024-07-31 16:59:55, Kepler: 20.309 kJ, PDU: 161.74 kJ
-19:33:51 INFO - Stage: 200, From: 2024-07-31 17:01:35, To: 2024-07-31 17:19:55, Kepler: 39.643 kJ, PDU: 180.64 kJ
-19:33:51 INFO - Stage: 300, From: 2024-07-31 17:21:36, To: 2024-07-31 17:39:55, Kepler: 76.277 kJ, PDU: 239.135 kJ
-19:33:51 INFO - Stage: 400, From: 2024-07-31 17:41:36, To: 2024-07-31 17:59:55, Kepler: 92.541 kJ, PDU: 253.89 kJ
-19:33:51 INFO - Stage: 500, From: 2024-07-31 18:01:35, To: 2024-07-31 18:19:55, Kepler: 95.15 kJ, PDU: 262.52 kJ
-19:33:52 INFO - Stage: 100, From: 2024-07-31 16:40:44, To: 2024-07-31 16:59:55, Successfull Requests: 392632.0 ,Kepler efficiency: 19332.906593136046, PDU efficiency: 2427.5503895140346
-19:33:52 INFO - Stage: 200, From: 2024-07-31 17:01:35, To: 2024-07-31 17:19:55, Successfull Requests: 715373.0 ,Kepler efficiency: 18045.38001664859, PDU efficiency: 3960.2136846767053
-19:33:52 INFO - Stage: 300, From: 2024-07-31 17:21:36, To: 2024-07-31 17:39:55, Successfull Requests: 912672.0 ,Kepler efficiency: 11965.23198342882, PDU efficiency: 3816.555502122232
-19:33:52 INFO - Stage: 400, From: 2024-07-31 17:41:36, To: 2024-07-31 17:59:55, Successfull Requests: 844999.0 ,Kepler efficiency: 9131.077036124529, PDU efficiency: 3328.2090669187446
-19:33:52 INFO - Stage: 500, From: 2024-07-31 18:01:35, To: 2024-07-31 18:19:55, Successfull Requests: 704956.0 ,Kepler efficiency: 7408.891224382553, PDU efficiency: 2685.342069175682
-19:33:52 INFO - Total Energy Efficiency (Kepler): 11023.190911336134 successful requests/kJ
-19:33:52 INFO - Total Energy Efficiency (PDU): 3252.1638545437986 successful requests/kJ
-19:33:52 INFO - Total Energy Efficiency (Kepler): 11.023190911336135 successful requests/J
-19:33:52 INFO - Total Energy Efficiency (PDU): 3.2521638545437987 successful requests/J
-19:33:53 INFO - Finished
+08:43:11 INFO - Starting
+08:43:11 INFO - Start: 2024-07-31 16:40:44 ; End: 2024-07-31 18:19:55
+08:43:11 INFO - Total energy consumed from 2024-07-31 16:40:44 to 2024-07-31 18:19:55 tracked by pdu metrics: 1.17822 MJ
+08:43:11 INFO - Total energy consumed from 2024-07-31 16:40:44 to 2024-07-31 18:19:55 tracked by kepler: 351.088 kJ
+08:43:11 INFO - Mean Abs Error of Keplers Trends compared to PDU (MAE over all data point diffs): 2019.93 J
+08:43:11 INFO - Stage: 100, From: 2024-07-31 16:40:44, To: 2024-07-31 16:59:55, Kepler: 20.309 kJ, PDU: 161.74 kJ
+08:43:11 INFO - Stage: 200, From: 2024-07-31 17:01:35, To: 2024-07-31 17:19:55, Kepler: 39.643 kJ, PDU: 180.64 kJ
+08:43:11 INFO - Stage: 300, From: 2024-07-31 17:21:36, To: 2024-07-31 17:39:55, Kepler: 76.277 kJ, PDU: 239.135 kJ
+08:43:11 INFO - Stage: 400, From: 2024-07-31 17:41:36, To: 2024-07-31 17:59:55, Kepler: 92.541 kJ, PDU: 253.89 kJ
+08:43:11 INFO - Stage: 500, From: 2024-07-31 18:01:35, To: 2024-07-31 18:19:55, Kepler: 95.15 kJ, PDU: 262.52 kJ
+08:43:11 INFO - Stage: 100, From: 2024-07-31 16:40:44, To: 2024-07-31 16:59:55, Successfull Requests: 392632.0 ,Kepler efficiency: 19332.906593136046, PDU efficiency: 2427.5503895140346
+08:43:11 INFO - Stage: 200, From: 2024-07-31 17:01:35, To: 2024-07-31 17:19:55, Successfull Requests: 715373.0 ,Kepler efficiency: 18045.38001664859, PDU efficiency: 3960.2136846767053
+08:43:11 INFO - Stage: 300, From: 2024-07-31 17:21:36, To: 2024-07-31 17:39:55, Successfull Requests: 912672.0 ,Kepler efficiency: 11965.23198342882, PDU efficiency: 3816.555502122232
+08:43:11 INFO - Stage: 400, From: 2024-07-31 17:41:36, To: 2024-07-31 17:59:55, Successfull Requests: 844999.0 ,Kepler efficiency: 9131.077036124529, PDU efficiency: 3328.2090669187446
+08:43:11 INFO - Stage: 500, From: 2024-07-31 18:01:35, To: 2024-07-31 18:19:55, Successfull Requests: 704956.0 ,Kepler efficiency: 7408.891224382553, PDU efficiency: 2685.342069175682
+08:43:11 INFO - Total Energy Efficiency (Kepler): 11023.190911336134 successful requests/kJ
+08:43:11 INFO - Total Energy Efficiency (PDU): 3252.1638545437986 successful requests/kJ
+08:43:11 INFO - Total Energy Efficiency (Kepler): 11.023190911336135 successful requests/J
+08:43:11 INFO - Total Energy Efficiency (PDU): 3.2521638545437987 successful requests/J
+08:43:12 INFO - Finished

loadtesting/data/01-caching/2024-07-31/16-18/output/service_efficiency_energy_requests_table.tex

@@ -2,23 +2,23 @@
 \toprule
  & efficiency & total\_energy & successful\_requests \\
 \midrule
+postservice & 32.883686 & 18794.000000 & 618016.000000 \\
 grafana & 0.000000 & 1011.000000 & 0.000000 \\
-servicediscovery & 0.000000 & 716.000000 & 0.000000 \\
-userservice-db & 0.000000 & 59789.000000 & 0.000000 \\
-redis & 0.000000 & 4743.000000 & 0.000000 \\
-postservice-db & 0.000000 & 30989.000000 & 0.000000 \\
-feedservice-db & 0.000000 & 5567.000000 & 0.000000 \\
-userservice & 94.453649 & 7001.000000 & 661270.000000 \\
 zipkin & 0.000000 & 679.000000 & 0.000000 \\
+userservice & 94.453649 & 7001.000000 & 661270.000000 \\
+redis & 0.000000 & 4743.000000 & 0.000000 \\
+gateway & 98.928577 & 13707.000000 & 1356014.000000 \\
+snmp-exporter & 0.000000 & 304.000000 & 0.000000 \\
+notificationservice-db & 0.000000 & 23493.000000 & 0.000000 \\
+feedservice & 12.213368 & 33918.000000 & 414253.000000 \\
+servicediscovery & 0.000000 & 716.000000 & 0.000000 \\
 statisticservice & 41.660183 & 9717.000000 & 404812.000000 \\
 prometheus & 0.000000 & 1860.000000 & 0.000000 \\
-statisticservice-db & 0.000000 & 133334.000000 & 0.000000 \\
+userservice-db & 0.000000 & 59789.000000 & 0.000000 \\
 rabbitmq & 0.000000 & 2438.000000 & 0.000000 \\
+statisticservice-db & 0.000000 & 133334.000000 & 0.000000 \\
+feedservice-db & 0.000000 & 5567.000000 & 0.000000 \\
 notificationservice & 129.934280 & 3028.000000 & 393441.000000 \\
-snmp-exporter & 0.000000 & 304.000000 & 0.000000 \\
-notificationservice-db & 0.000000 & 23493.000000 & 0.000000 \\
-gateway & 98.928577 & 13707.000000 & 1356014.000000 \\
-feedservice & 12.213368 & 33918.000000 & 414253.000000 \\
-postservice & 32.883686 & 18794.000000 & 618016.000000 \\
+postservice-db & 0.000000 & 30989.000000 & 0.000000 \\
 \bottomrule
 \end{tabular}

loadtesting/data/01-caching/2024-07-31/16-18/output/service_merged_with_database_efficiency_energy_requests_table.tex

@@ -2,18 +2,18 @@
 \toprule
  & efficiency & total\_energy & successful\_requests \\
 \midrule
+postservice & 12.414198 & 49783.000000 & 618016.000000 \\
 grafana & 0.000000 & 1011.000000 & 0.000000 \\
-servicediscovery & 0.000000 & 716.000000 & 0.000000 \\
-redis & 0.000000 & 4743.000000 & 0.000000 \\
-userservice & 9.900734 & 66790.000000 & 661270.000000 \\
 zipkin & 0.000000 & 679.000000 & 0.000000 \\
+userservice & 9.900734 & 66790.000000 & 661270.000000 \\
+redis & 0.000000 & 4743.000000 & 0.000000 \\
+gateway & 98.928577 & 13707.000000 & 1356014.000000 \\
+snmp-exporter & 0.000000 & 304.000000 & 0.000000 \\
+feedservice & 9.366306 & 44228.000000 & 414253.000000 \\
+servicediscovery & 0.000000 & 716.000000 & 0.000000 \\
 statisticservice & 2.829844 & 143051.000000 & 404812.000000 \\
 prometheus & 0.000000 & 1860.000000 & 0.000000 \\
 rabbitmq & 0.000000 & 2438.000000 & 0.000000 \\
 notificationservice & 14.835074 & 26521.000000 & 393441.000000 \\
-snmp-exporter & 0.000000 & 304.000000 & 0.000000 \\
-gateway & 98.928577 & 13707.000000 & 1356014.000000 \\
-feedservice & 10.491402 & 39485.000000 & 414253.000000 \\
-postservice & 12.414198 & 49783.000000 & 618016.000000 \\
 \bottomrule
 \end{tabular}

loadtesting/notebooks/energy_efficiency.ipynb

@@ -21,7 +21,9 @@
    "id": "3d6de811535875d3",
    "metadata": {},
    "source": [
-    "basedir = \"../data/01-caching/2024-07-30/16-18/\"\n",
+    "basedir = \"../data/01-caching/2024-07-31/16-18/\"\n",
+    "add_redis_to_feedservice = True # will add redis container energy to feedservice total energy\n",
+    "\n",
     "outputdir = basedir+\"output/\"\n",
     "os.makedirs(outputdir, exist_ok=True)\n",
     "locust_data_history_file = basedir + \"data_stats_history.csv\"\n",
@@ -674,13 +676,16 @@
    "metadata": {},
    "cell_type": "code",
    "source": [
-    "combined_metrics = service_metrics[~service_metrics.index.str.endswith('-db')].copy()\n",
+    "combined_metrics = service_metrics[\n",
+    "    (~service_metrics.index.str.endswith('-db')) | (service_metrics.index == 'redis')\n",
+    "].copy()\n",
     "\n",
     "# Add DB energy consumption to the respective services\n",
     "combined_metrics['total_energy'] += combined_metrics.apply(\n",
     "    lambda row: service_metrics.loc[f\"{row.name}-db\", 'total_energy'] if f\"{row.name}-db\" in service_metrics.index else 0, axis=1\n",
     ")\n",
-    "\n",
+    "if add_redis_to_feedservice:\n",
+    "    combined_metrics.loc['feedservice', 'total_energy'] += service_metrics.loc['redis', 'total_energy']\n",
     "# Calculate efficiency\n",
     "combined_metrics['efficiency'] = combined_metrics['successful_requests'] / combined_metrics['total_energy']\n",
     "combined_metrics.to_latex(outputdir+'service_merged_with_database_efficiency_energy_requests_table.tex', escape=True)\n",
@@ -721,7 +726,7 @@
    "cell_type": "code",
    "source": [
     "# Function to calculate energy consumption for each service during stages\n",
-    "def calculate_combined_service_metrics_by_stage(service_list, stages, locust_df, kepler_df, service_work_metrics_df):\n",
+    "def calculate_combined_service_metrics_by_stage(service_list, stages, locust_df, kepler_df, service_work_metrics_df, add_redis_to_feedservice=False):\n",
     "    metrics_results_by_stage = {stage['users']: {} for stage in stages}\n",
     "\n",
     "    for stage in stages:\n",
@@ -755,6 +760,9 @@
     "            if f\"{service}-db\" in service_list:\n",
     "                total_energy_service += calculate_service_energy_consumption(kepler_df, f\"{service}-db\", stage_start_time, stage_end_time)\n",
     "\n",
+    "            if add_redis_to_feedservice and service == 'feedservice' and 'redis' in service_list:\n",
+    "                total_energy_service += calculate_service_energy_consumption(kepler_df, 'redis', stage_start_time, stage_end_time)\n",
+    "\n",
     "            if service.upper() in set(map(str.upper, stage_work_metrics_df.columns)):\n",
     "                total_requests = stage_work_metrics_df[service.upper()].max() - stage_work_metrics_df[\n",
     "                    service.upper()].min()\n",
@@ -774,8 +782,7 @@
     "\n",
     "\n",
     "service_metrics_by_stage = calculate_combined_service_metrics_by_stage(service_names, stages, locust_df, kepler_df,\n",
-    "                                                                       service_work_metrics_df)\n",
-    "service_metrics_by_stage\n",
+    "                                                                       service_work_metrics_df, add_redis_to_feedservice)\n",
     "\n",
     "# Convert dictionary to DataFrame for easier manipulation\n",
     "metrics_df_list = []\n",