[Cgroups] Add cgroups stats

databricks · Oct 12, 2023 · 084d0ae · 084d0ae
1 parent 436f1ac
commit 084d0ae
Show file tree

Hide file tree

Showing 11 changed files with 154 additions and 11 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/test/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/analysis/test/TestStrategy.java
@@ -43,6 +43,7 @@
 import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
 import com.google.devtools.build.lib.server.FailureDetails.TestAction;
 import com.google.devtools.build.lib.server.FailureDetails.TestAction.Code;
+import com.google.devtools.build.lib.shell.TerminationStatus;
 import com.google.devtools.build.lib.util.Fingerprint;
 import com.google.devtools.build.lib.util.OS;
 import com.google.devtools.build.lib.util.io.OutErr;
@@ -323,6 +324,7 @@ protected TestCase parseTestResult(Path resultFile) {
   protected void processTestOutput(
       ActionExecutionContext actionExecutionContext,
       TestResultData testResultData,
+      TerminationStatus ts,
       String testName,
       Path testLog)
       throws IOException {
@@ -354,9 +356,10 @@ protected void processTestOutput(
               .getEventHandler()
               .handle(Event.of(EventKind.CANCELLED, null, testName));
         } else {
+          String message = String.format("%s (%s) (see %s)", testName, ts.toShortString(), testLog);
           actionExecutionContext
               .getEventHandler()
-              .handle(Event.of(EventKind.FAIL, null, testName + " (see " + testLog + ")"));
+              .handle(Event.of(EventKind.FAIL, null, message));
         }
       }
     }

diff --git a/src/main/java/com/google/devtools/build/lib/exec/BUILD b/src/main/java/com/google/devtools/build/lib/exec/BUILD
@@ -356,6 +356,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
         "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/shell",
         "//src/main/java/com/google/devtools/build/lib/skyframe:tree_artifact_value",
         "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/util/io",

diff --git a/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java
@@ -52,6 +52,7 @@
 import com.google.devtools.build.lib.server.FailureDetails.Execution.Code;
 import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
 import com.google.devtools.build.lib.server.FailureDetails.TestAction;
+import com.google.devtools.build.lib.shell.TerminationStatus;
 import com.google.devtools.build.lib.util.Pair;
 import com.google.devtools.build.lib.util.io.FileOutErr;
 import com.google.devtools.build.lib.vfs.FileStatus;
@@ -278,7 +279,11 @@ private StandaloneFailedAttemptResult processTestAttempt(
         .post(
             TestAttempt.forExecutedTestResult(
                 action, data, attemptId, testOutputs, result.executionInfo(), isLastAttempt));
-    processTestOutput(actionExecutionContext, data, action.getTestName(), renamedTestLog);
+    TerminationStatus ts = TerminationStatus.builder()
+        .setWaitResponse(result.spawnResults().get(0).exitCode())
+        .setTimedOut(data.getStatus() == BlazeTestStatus.TIMEOUT)
+        .build();
+    processTestOutput(actionExecutionContext, data, ts, action.getTestName(), renamedTestLog);
     return new StandaloneFailedAttemptResult(data);
   }
 

diff --git a/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java b/src/main/java/com/google/devtools/build/lib/profiler/Profiler.java
@@ -13,6 +13,8 @@
 // limitations under the License.
 package com.google.devtools.build.lib.profiler;
 
+import static com.google.devtools.build.lib.profiler.ProfilerTask.SANDBOX_CPU_INFO;
+import static com.google.devtools.build.lib.profiler.ProfilerTask.SANDBOX_MEMORY_INFO;
 import static com.google.devtools.build.lib.profiler.ProfilerTask.TASK_COUNT;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -970,12 +972,24 @@ private void writeTask(JsonWriter writer, TaskData data) throws IOException {
       writer.setIndent("  ");
       writer.beginObject();
       writer.setIndent("");
-      if (data.type == null) {
-        writer.setIndent("    ");
+      if (data.type == SANDBOX_CPU_INFO || data.type == SANDBOX_MEMORY_INFO) {
+        writer.name("cat").value("sandbox info");
+        writer.name("name").value(data.type.description);
+        writer.name("args");
+        writer.beginObject();
+        for (String stat : data.description.split("\n")) {
+          String [] pair = stat.split(" ", 2);
+          writer.name(pair[0]).value(pair[1]);
+        }
+        writer.endObject();
       } else {
-        writer.name("cat").value(data.type.description);
+        if (data.type == null) {
+          writer.setIndent("    ");
+        } else {
+          writer.name("cat").value(data.type.description);
+        }
+        writer.name("name").value(data.description);
       }
-      writer.name("name").value(data.description);
       writer.name("ph").value(eventType);
       writer
           .name("ts")

diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java
@@ -73,6 +73,8 @@ public enum ProfilerTask {
   SYSTEM_NETWORK_UP_USAGE("Network Up usage (total)"),
   SYSTEM_NETWORK_DOWN_USAGE("Network Down usage (total)"),
   WORKERS_MEMORY_USAGE("Workers memory usage"),
+  SANDBOX_CPU_INFO("CPU stats (Sandbox)"),
+  SANDBOX_MEMORY_INFO("Memory stats (Sandbox)"),
   SYSTEM_LOAD_AVERAGE("System load average"),
   STARLARK_PARSER("Starlark Parser", Threshold.FIFTY_MILLIS),
   STARLARK_USER_FN("Starlark user function call", Threshold.FIFTY_MILLIS),

diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java
@@ -37,6 +37,7 @@
 import com.google.devtools.build.lib.exec.local.LocalExecutionOptions;
 import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider;
 import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
 import com.google.devtools.build.lib.profiler.SilentCloseable;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
 import com.google.devtools.build.lib.sandbox.SandboxHelpers.SandboxInputs;
@@ -59,6 +60,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.SortedMap;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nullable;
 
@@ -70,6 +72,7 @@ final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner {
   private static final AtomicBoolean warnedAboutNonHermeticTmp = new AtomicBoolean();
 
   private static final AtomicBoolean warnedAboutUnsupportedModificationCheck = new AtomicBoolean();
+  private ConcurrentHashMap<Integer, Optional<VirtualCGroup>> cgroups;
 
   /**
    * Returns whether the linux sandbox is supported on the local machine by running a small command
@@ -172,12 +175,17 @@ private static boolean computeIsSupported(CommandEnvironment cmdEnv, Path linuxS
     this.localEnvProvider = new PosixLocalEnvProvider(cmdEnv.getClientEnv());
     this.treeDeleter = treeDeleter;
     this.reporter = cmdEnv.getReporter();
+    this.cgroups = new ConcurrentHashMap<>();
   }
 
-  private VirtualCGroup getCgroup(Spawn spawn, SpawnExecutionContext context) throws ExecException, IOException {
+  private Optional<VirtualCGroup> getCgroup(Spawn spawn, SpawnExecutionContext context) throws ExecException, IOException {
     if (spawn.getExecutionInfo().get(ExecutionRequirements.NO_SUPPORTS_CGROUPS) != null) {
       return null;
     }
+    if (cgroups.containsKey(context.getId())) {
+      return cgroups.get(context.getId());
+    }
+
     SandboxOptions sandboxOptions = getSandboxOptions();
 
     VirtualCGroup cgroup = null;
@@ -252,7 +260,9 @@ private VirtualCGroup getCgroup(Spawn spawn, SpawnExecutionContext context) thro
       cgroup.cpu().setCpus(cpuLimit);
     }
 
-    return cgroup;
+    cgroups.put(context.getId(), Optional.ofNullable(cgroup));
+
+    return cgroups.get(context.getId());
   }
 
   @Override
@@ -326,10 +336,10 @@ protected SandboxedSpawn prepareSpawn(Spawn spawn, SpawnExecutionContext context
             .setKillDelay(timeoutKillDelay);
 
 
-    VirtualCGroup cgroup = getCgroup(spawn, context);
-    if (cgroup != null) {
+    Optional<VirtualCGroup> cgroup = getCgroup(spawn, context);
+    if (cgroup.isPresent()) {
       commandLineBuilder.setCgroupsDirs(
-          cgroup.paths().stream()
+          cgroup.get().paths().stream()
             .map(p -> fileSystem.getPath(p.toString()))
             .collect(ImmutableSet.toImmutableSet()));
     }
@@ -528,6 +538,27 @@ public void verifyPostCondition(
     if (getSandboxOptions().useHermetic) {
       checkForConcurrentModifications(context);
     }
+    Optional<VirtualCGroup> cgroup = cgroups.remove(context.getId());
+    if (cgroup == null || cgroup.isEmpty()) {
+      return;
+    }
+    long now = System.nanoTime();
+    if (cgroup.get().cpu() != null) {
+      String stats = cgroup.get().cpu().getStats();
+      Profiler.instance().logEventAtTime(now, ProfilerTask.SANDBOX_CPU_INFO, stats);
+    }
+    if (cgroup.get().memory() != null) {
+      Long kills = cgroup.get().memory().oomKills();
+      Long limit = cgroup.get().memory().getMaxBytes();
+      Long usage = cgroup.get().memory().maxUsage();
+
+      StringBuilder stats = new StringBuilder(cgroup.get().memory().getStats());
+      if (usage > 0) stats.append("max_usage_in_bytes").append(" ").append(usage).append("\n");
+      if (limit > 0) stats.append("limit_in_bytes").append(" ").append(limit).append("\n");
+      if (kills > 0) stats.append("oom_kills").append(" ").append(kills).append("\n");
+
+      Profiler.instance().logEventAtTime(now, ProfilerTask.SANDBOX_MEMORY_INFO, stats.toString());
+    }
   }
 
   private void checkForConcurrentModifications(SpawnExecutionContext context)

diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Controller.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/Controller.java
@@ -8,7 +8,10 @@
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public interface Controller {
     default boolean isLegacy() throws IOException {
@@ -33,9 +36,30 @@ protected FailureDetails.FailureDetail getFailureDetail(String message) {
 
     Path getPath() throws IOException;
 
+    Path statFile() throws IOException;
+
+    default String getStats() throws IOException {
+        if (statFile() != null && statFile().toFile().exists()) {
+            return Files.readString(statFile());
+        }
+        return "";
+    }
+
     interface Memory extends Controller {
         void setMaxBytes(long bytes) throws IOException;
         long getMaxBytes() throws IOException;
+        long oomKills() throws IOException;
+
+        long maxUsage() throws IOException;
+
+        default String getStats() throws IOException {
+            String stats = Controller.super.getStats();
+            long maxUsage = this.maxUsage();
+            if (maxUsage > 0) {
+                return stats.strip() + "\n" + "max_usage_in_bytes " + maxUsage;
+            }
+            return stats;
+        }
     }
     interface Cpu extends Controller {
         void setCpus(float cpus) throws IOException;

diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyCpu.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyCpu.java
@@ -5,6 +5,8 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class LegacyCpu implements Controller.Cpu {
     private final Path path;
@@ -20,6 +22,11 @@ public Path getPath() {
         return path;
     }
 
+    @Override
+    public Path statFile() throws IOException {
+        return path.resolve("cpu.stat");
+    }
+
     @Override
     public void setCpus(float cpus) throws IOException {
         int quota = Math.round(cpus * period);

diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyMemory.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v1/LegacyMemory.java
@@ -5,6 +5,9 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class LegacyMemory implements Controller.Memory {
     private final Path path;
@@ -14,6 +17,11 @@ public Path getPath() {
         return path;
     }
 
+    @Override
+    public Path statFile() throws IOException {
+        return path.resolve("memory.stat");
+    }
+
     public LegacyMemory(Path path) {
         this.path = path;
     }
@@ -27,4 +35,19 @@ public void setMaxBytes(long bytes) throws IOException {
     public long getMaxBytes() throws IOException {
         return Long.parseLong(Files.readString(path.resolve("memory.limit_in_bytes")).trim());
     }
+
+    @Override
+    public long oomKills() throws IOException {
+        for (String line: Files.readAllLines(getPath().resolve("memory.oom_control"))) {
+            if (line.startsWith("oom_kill ")) {
+                return Long.parseLong(line.substring(line.indexOf(" ") + 1));
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public long maxUsage() throws IOException {
+        return Long.parseLong(Files.readString(path.resolve("memory.max_usage_in_bytes")).trim());
+    }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedCpu.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedCpu.java
@@ -17,6 +17,11 @@ public Path getPath() {
         return path;
     }
 
+    @Override
+    public Path statFile() throws IOException {
+        return path.resolve("cpu.stat");
+    }
+
     @Override
     public void setCpus(float cpus) throws IOException {
         int period = 1000_000;

diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedMemory.java b/src/main/java/com/google/devtools/build/lib/sandbox/cgroups/v2/UnifiedMemory.java
@@ -5,6 +5,8 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 public class UnifiedMemory implements Controller.Memory {
     private final Path path;
@@ -17,6 +19,11 @@ public Path getPath() {
         return path;
     }
 
+    @Override
+    public Path statFile() throws IOException {
+        return path.resolve("memory.stat");
+    }
+
     @Override
     public void setMaxBytes(long bytes) throws IOException {
         Files.writeString(path.resolve("memory.max"), Long.toString(bytes));
@@ -26,4 +33,25 @@ public void setMaxBytes(long bytes) throws IOException {
     public long getMaxBytes() throws IOException {
         return Long.parseLong(Files.readString(path.resolve("memory.max")).trim());
     }
+
+    @Override
+    public long oomKills() throws IOException {
+        for (String line: Files.readAllLines(getPath().resolve("memory.events"))) {
+            if (line.startsWith("oom_kill ")) {
+                return Long.parseLong(line.substring(line.indexOf(" ") + 1));
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public long maxUsage() throws IOException {
+        // This file has been added relatively recently, so it might not exist.
+        // Return -1 in that case, to signal its absence
+        // Ref. https://github.com/torvalds/linux/commit/8e20d4b332660a32e842e20c34cfc3b3456bc6dc
+        if (path.resolve("memory.peak").toFile().exists()) {
+            return Long.parseLong(Files.readString(path.resolve("memory.max")).trim());
+        }
+        return -1;
+    }
 }