From 8198f7347d5d8c66428ea5a1ed8c33f81921b646 Mon Sep 17 00:00:00 2001 From: Peter Palaga Date: Mon, 29 Apr 2024 23:37:09 +0200 Subject: [PATCH] Support podman https://github.com/quarkiverse/quarkus-langchain4j/issues/493 --- deployment/pom.xml | 14 - .../antora/deployment/AntoraProcessor.java | 256 +------------- .../antora/deployment/LinuxIDUtil.java | 61 ++++ .../deployment/NativeImageBuildRunner.java | 333 ++++++++++++++++++ docs/modules/ROOT/pages/index.adoc | 1 - 5 files changed, 408 insertions(+), 257 deletions(-) create mode 100644 deployment/src/main/java/io/quarkiverse/antora/deployment/LinuxIDUtil.java create mode 100644 deployment/src/main/java/io/quarkiverse/antora/deployment/NativeImageBuildRunner.java diff --git a/deployment/pom.xml b/deployment/pom.xml index 95315ba..45f92aa 100644 --- a/deployment/pom.xml +++ b/deployment/pom.xml @@ -26,20 +26,6 @@ org.yaml snakeyaml - - org.testcontainers - testcontainers - - - junit - junit - - - - - io.quarkus - quarkus-junit4-mock - diff --git a/deployment/src/main/java/io/quarkiverse/antora/deployment/AntoraProcessor.java b/deployment/src/main/java/io/quarkiverse/antora/deployment/AntoraProcessor.java index aa5ee35..bfb550b 100644 --- a/deployment/src/main/java/io/quarkiverse/antora/deployment/AntoraProcessor.java +++ b/deployment/src/main/java/io/quarkiverse/antora/deployment/AntoraProcessor.java @@ -9,26 +9,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.jboss.logging.Logger; -import org.junit.jupiter.api.Assertions; -import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.output.BaseConsumer; -import org.testcontainers.containers.output.OutputFrame; -import org.testcontainers.containers.output.OutputFrame.OutputType; -import org.testcontainers.containers.output.WaitingConsumer; import org.yaml.snakeyaml.Yaml; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - import io.quarkus.deployment.IsDevelopment; import io.quarkus.deployment.annotations.BuildProducer; import io.quarkus.deployment.annotations.BuildStep; @@ -38,6 +23,7 @@ import io.quarkus.deployment.util.FileUtil; public class AntoraProcessor { + private static final String FEATURE = "antora"; private static final int INVALID_UID = -1; @@ -79,66 +65,20 @@ void buildAntoraSite( } final Path baseDir = targetDir.getParent(); - /* - * We need the current user's uid so that Antora container generates the files as that user - * so that they can be later deleted with mvn clean - */ - int uid = INVALID_UID; - try { - uid = (Integer) Files.getAttribute(targetDir, "unix:uid"); - log.info("Detected unix:uid " + uid); - } catch (Exception e) { - if (System.getProperty("os.name").toLowerCase().indexOf("win") < 0) { - /* Warn on non-Windows, ignore otherwise */ - log.warn("Could not read unix:uid of " + targetDir + " directory", e); - } - } - final int finalUid = uid; - - WaitingConsumer logConsumer = new WaitingConsumer().withRemoveAnsiCodes(true); - - AntoraFrameConsumer antoraFrameConsumer = new AntoraFrameConsumer(); - final Path gitRepoRoot = gitRepoRoot(baseDir); final PlaybookInfo pbInfo = augmentAntoraPlaybook(gitRepoRoot, baseDir, targetDir); final Path absAntoraPlaybookPath = pbInfo.playbookPath; final Path antoraPlaybookPath = gitRepoRoot.relativize(absAntoraPlaybookPath); if (Files.isDirectory(pbInfo.outDir)) { - final Path movedOutDir = targetDir - .resolve("antora-site-" + UUID.randomUUID().toString()); - try { - Files.move(pbInfo.outDir, movedOutDir); - } catch (IOException e) { - throw new RuntimeException("Could not move " + pbInfo.outDir + " -> " + movedOutDir); - } try { - FileUtil.deleteDirectory(movedOutDir); + FileUtil.deleteDirectory(pbInfo.outDir); } catch (IOException e) { - throw new RuntimeException("Could not remove " + movedOutDir); + throw new RuntimeException("Could not remove " + pbInfo.outDir); } } - try (GenericContainer antoraContainer = new GenericContainer<>("antora/antora:3.0.1")) { - if (finalUid >= 0) { - antoraContainer - .withCreateContainerCmdModifier(cmd -> { - cmd.withUser(String.valueOf(finalUid)); - }); - } - antoraContainer - .withFileSystemBind(gitRepoRoot.toString(), "/antora", BindMode.READ_WRITE) - .withCommand("--cache-dir=./antora-cache", antoraPlaybookPath.toString()) - .withLogConsumer(antoraFrameConsumer) - .withLogConsumer(logConsumer); - antoraContainer.start(); - try { - logConsumer.waitUntilEnd(30, TimeUnit.SECONDS); - } catch (TimeoutException e) { - throw new RuntimeException("Timeout waiting for Antora log consumer", e); - } - } - antoraFrameConsumer.assertNoErrors(); + buildWithContainer(gitRepoRoot, antoraPlaybookPath); try (Stream files = Files.walk(pbInfo.outDir)) { files.forEach(absP -> { @@ -185,6 +125,16 @@ void buildAntoraSite( } + private void buildWithContainer(final Path gitRepoRoot, final Path antoraPlaybookPath) { + try { + new NativeImageBuildRunner().build(gitRepoRoot, antoraPlaybookPath); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + throw new RuntimeException("Failed to build native image", e); + } + } + private Path gitRepoRoot(Path startDir) { Path gitRepoRoot = startDir.toAbsolutePath().normalize(); while (!Files.exists(gitRepoRoot.resolve(".git"))) { @@ -287,182 +237,4 @@ static Map mutableMap(Object... entries) { private record PlaybookInfo(Path playbookPath, Path outDir) { } - - static class AntoraFrame { - String level; - long time; - String name; - AntoraFile file; - AntoraSource source; - String msg; - String hint; - List stack; - - public String getLevel() { - return level; - } - - public long getTime() { - return time; - } - - public String getName() { - return name; - } - - public AntoraFile getFile() { - return file; - } - - public AntoraSource getSource() { - return source; - } - - public String getMsg() { - return msg; - } - - public List getStack() { - return stack; - } - - public String getHint() { - return hint; - } - - @Override - public String toString() { - return file + ": " + msg - + (hint != null ? (" " + hint) : "") - + (stack != null && !stack.isEmpty() - ? ("\n at " + stack.stream() - .map(AntoraStackFrame::toString) - .collect(Collectors.joining("\n at "))) - : ""); - } - - static class AntoraSource { - String url; - String worktree; - String refname; - String startPath; - - public String getUrl() { - return url; - } - - public String getWorktree() { - return worktree; - } - - public String getRefname() { - return refname; - } - - public String getStartPath() { - return startPath; - } - } - - static class AntoraFile { - String path; - int line; - - @Override - public String toString() { - return path + ":" + line; - } - - public String getPath() { - return path; - } - - public int getLine() { - return line; - } - } - - static class AntoraStackFrame { - AntoraFile file; - AntoraSource source; - - public AntoraFile getFile() { - return file; - } - - public AntoraSource getSource() { - return source; - } - - @Override - public String toString() { - return file != null ? file.toString() : ""; - } - } - } - - private static class AntoraFrameConsumer extends BaseConsumer { - - private final ObjectMapper mapper; - private final List frames = new ArrayList<>(); - private final Map exceptions = new LinkedHashMap<>(); - - public AntoraFrameConsumer() { - mapper = new ObjectMapper(); - } - - @Override - public void accept(OutputFrame t) { - if (t.getType() == OutputType.END) { - return; - } - final String rawFrame = t.getUtf8String(); - try { - final AntoraFrame frame = mapper.readValue(rawFrame, AntoraFrame.class); - synchronized (frames) { - frames.add(frame); - } - switch (frame.getLevel()) { - case "info": - log.info(frame.toString()); - break; - case "warn": - log.warn(frame.toString()); - break; - case "error": - log.error(frame.toString()); - break; - case "fatal": - log.fatal(frame.toString()); - break; - default: - throw new IllegalStateException("Unexpected AntoraFrame.level " + frame.getLevel()); - } - } catch (JsonProcessingException e) { - synchronized (exceptions) { - exceptions.put(rawFrame, e); - } - } - } - - public void assertNoErrors() { - synchronized (exceptions) { - if (!exceptions.isEmpty()) { - Entry e = exceptions.entrySet().iterator().next(); - throw new RuntimeException("Could not parse AntoraFrame " + e.getKey(), e.getValue()); - } - } - - synchronized (frames) { - String errors = frames.stream() - .filter(f -> f.getLevel().equals("error") || f.getLevel().equals("fatal")) - .map(AntoraFrame::toString) - .collect(Collectors.joining("\n")); - if (errors != null && !errors.isEmpty()) { - Assertions.fail(errors); - } - } - } - - } } diff --git a/deployment/src/main/java/io/quarkiverse/antora/deployment/LinuxIDUtil.java b/deployment/src/main/java/io/quarkiverse/antora/deployment/LinuxIDUtil.java new file mode 100644 index 0000000..07948c5 --- /dev/null +++ b/deployment/src/main/java/io/quarkiverse/antora/deployment/LinuxIDUtil.java @@ -0,0 +1,61 @@ +package io.quarkiverse.antora.deployment; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +final class LinuxIDUtil { + + private LinuxIDUtil() { + } + + static String getLinuxID(String option) { + Process process; + + try { + StringBuilder responseBuilder = new StringBuilder(); + String line; + + ProcessBuilder idPB = new ProcessBuilder().command("id", option); + idPB.redirectError(new File("/dev/null")); + idPB.redirectInput(new File("/dev/null")); + + process = idPB.start(); + try (InputStream inputStream = process.getInputStream()) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + while ((line = reader.readLine()) != null) { + responseBuilder.append(line); + } + safeWaitFor(process); + return responseBuilder.toString(); + } + } catch (Throwable t) { + safeWaitFor(process); + throw t; + } + } catch (IOException e) { //from process.start() + //swallow and return null id + return null; + } + } + + private static void safeWaitFor(Process process) { + boolean intr = false; + try { + for (;;) + try { + process.waitFor(); + return; + } catch (InterruptedException ex) { + intr = true; + } + } finally { + if (intr) { + Thread.currentThread().interrupt(); + } + } + } +} diff --git a/deployment/src/main/java/io/quarkiverse/antora/deployment/NativeImageBuildRunner.java b/deployment/src/main/java/io/quarkiverse/antora/deployment/NativeImageBuildRunner.java new file mode 100644 index 0000000..772ef79 --- /dev/null +++ b/deployment/src/main/java/io/quarkiverse/antora/deployment/NativeImageBuildRunner.java @@ -0,0 +1,333 @@ +package io.quarkiverse.antora.deployment; + +import static io.quarkiverse.antora.deployment.LinuxIDUtil.getLinuxID; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.SystemUtils; +import org.jboss.logging.Logger; +import org.junit.jupiter.api.Assertions; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.quarkus.deployment.util.ContainerRuntimeUtil; +import io.quarkus.deployment.util.FileUtil; + +public class NativeImageBuildRunner { + + private static final Logger log = Logger.getLogger(NativeImageBuildRunner.class); + + private final ContainerRuntimeUtil.ContainerRuntime containerRuntime; + + private final String containerName; + + public NativeImageBuildRunner() { + containerRuntime = ContainerRuntimeUtil.detectContainerRuntime(); + containerName = "antora-" + RandomStringUtils.random(5, true, false); + } + + public void build(Path outputDir, Path antoraPlaybookPath) + throws InterruptedException, IOException { + + final List cmd = new ArrayList<>(); + cmd.add(containerRuntime.getExecutableName()); + cmd.add("run"); + + cmd.add("--rm"); + + if (SystemUtils.IS_OS_LINUX || SystemUtils.IS_OS_MAC) { + if (containerRuntime.isInWindowsWSL()) { + cmd.add("--interactive"); + } + if (containerRuntime.isDocker() && containerRuntime.isRootless()) { + Collections.addAll(cmd, "--user", String.valueOf(0)); + } else { + String uid = getLinuxID("-ur"); + String gid = getLinuxID("-gr"); + if (uid != null && gid != null && !uid.isEmpty() && !gid.isEmpty()) { + Collections.addAll(cmd, "--user", uid + ":" + gid); + if (containerRuntime.isPodman() && containerRuntime.isRootless()) { + // Needed to avoid AccessDeniedExceptions + cmd.add("--userns=keep-id"); + } + } + } + } + + String volumeOutputPath = outputDir.toAbsolutePath().toString(); + if (SystemUtils.IS_OS_WINDOWS) { + volumeOutputPath = FileUtil.translateToVolumePath(volumeOutputPath); + } + + final String selinuxBindOption; + if (SystemUtils.IS_OS_MAC && containerRuntime.isPodman()) { + selinuxBindOption = ""; + } else { + selinuxBindOption = ":z"; + } + cmd.add("-v"); + cmd.add(volumeOutputPath + ":/antora" + selinuxBindOption); + + cmd.add("--name"); + cmd.add(containerName); + + cmd.add("antora/antora:3.0.1"); + + cmd.add("--cache-dir=./antora-cache"); + cmd.add(antoraPlaybookPath.toString()); + + final String[] buildCommand = cmd.toArray(new String[0]); + + log.infof("Running Antora with %s:", containerRuntime.getExecutableName()); + log.info(String.join(" ", buildCommand).replace("$", "\\$")); + final Process process = new ProcessBuilder(buildCommand) + .directory(outputDir.toFile()) + .redirectErrorStream(true) + .start(); + addShutdownHook(process); + + final OutputSlurper output = new OutputSlurper(containerName, process.getInputStream(), System.out); + final int exitCode = process.waitFor(); + + output.assertNoErrors(1000); + + if (exitCode != 0) { + throw new IllegalStateException("Antora exited with " + exitCode); + } + } + + void addShutdownHook(Process process) { + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + if (process.isAlive()) { + try { + Process removeProcess = new ProcessBuilder( + List.of(containerRuntime.getExecutableName(), "rm", "-f", containerName)) + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .redirectError(ProcessBuilder.Redirect.DISCARD) + .start(); + removeProcess.waitFor(2, TimeUnit.SECONDS); + } catch (IOException | InterruptedException e) { + log.debug("Unable to stop running container", e); + } + } + })); + } + + private static final class OutputSlurper { + + private final AntoraFrameConsumer frameConsumer = new AntoraFrameConsumer(); + private final CountDownLatch finished = new CountDownLatch(1); + + private OutputSlurper(String containerName, final InputStream processStream, final PrintStream consumer) { + final Thread t = new Thread(() -> { + try (final BufferedReader reader = new BufferedReader( + new InputStreamReader(processStream, StandardCharsets.UTF_8))) { + String line = null; + while ((line = reader.readLine()) != null) { + frameConsumer.accept(line); + } + } catch (IOException e) { + throw new RuntimeException("Could not read stdout of container " + containerName, e); + } + finished.countDown(); + }); + t.setName(containerName + ":stdout"); + t.start(); + } + + public void assertNoErrors(long timeout) throws InterruptedException { + finished.await(timeout, TimeUnit.MILLISECONDS); + frameConsumer.assertNoErrors(); + } + } + + private static class AntoraFrameConsumer { + + private final ObjectMapper mapper; + private final List frames = new ArrayList<>(); + private final Map exceptions = new LinkedHashMap<>(); + + public AntoraFrameConsumer() { + mapper = new ObjectMapper(); + } + + public void accept(String rawFrame) { + try { + final AntoraFrame frame = mapper.readValue(rawFrame, AntoraFrame.class); + synchronized (frames) { + frames.add(frame); + } + switch (frame.getLevel()) { + case "info": + log.info(frame.toString()); + break; + case "warn": + log.warn(frame.toString()); + break; + case "error": + log.error(frame.toString()); + break; + case "fatal": + log.fatal(frame.toString()); + break; + default: + throw new IllegalStateException("Unexpected AntoraFrame.level " + frame.getLevel()); + } + } catch (JsonProcessingException e) { + synchronized (exceptions) { + exceptions.put(rawFrame, e); + } + } + } + + public void assertNoErrors() { + synchronized (exceptions) { + if (!exceptions.isEmpty()) { + Entry e = exceptions.entrySet().iterator().next(); + throw new RuntimeException("Could not parse AntoraFrame " + e.getKey(), e.getValue()); + } + } + + synchronized (frames) { + String errors = frames.stream() + .filter(f -> f.getLevel().equals("error") || f.getLevel().equals("fatal")) + .map(AntoraFrame::toString) + .collect(Collectors.joining("\n")); + if (errors != null && !errors.isEmpty()) { + Assertions.fail(errors); + } + } + } + + } + + static class AntoraFrame { + String level; + long time; + String name; + AntoraFile file; + AntoraSource source; + String msg; + String hint; + List stack; + + public String getLevel() { + return level; + } + + public long getTime() { + return time; + } + + public String getName() { + return name; + } + + public AntoraFile getFile() { + return file; + } + + public AntoraSource getSource() { + return source; + } + + public String getMsg() { + return msg; + } + + public List getStack() { + return stack; + } + + public String getHint() { + return hint; + } + + @Override + public String toString() { + return file + ": " + msg + + (hint != null ? (" " + hint) : "") + + (stack != null && !stack.isEmpty() + ? ("\n at " + stack.stream() + .map(AntoraStackFrame::toString) + .collect(Collectors.joining("\n at "))) + : ""); + } + + static class AntoraSource { + String url; + String worktree; + String refname; + String startPath; + + public String getUrl() { + return url; + } + + public String getWorktree() { + return worktree; + } + + public String getRefname() { + return refname; + } + + public String getStartPath() { + return startPath; + } + } + + static class AntoraFile { + String path; + int line; + + @Override + public String toString() { + return path + ":" + line; + } + + public String getPath() { + return path; + } + + public int getLine() { + return line; + } + } + + static class AntoraStackFrame { + AntoraFile file; + AntoraSource source; + + public AntoraFile getFile() { + return file; + } + + public AntoraSource getSource() { + return source; + } + + @Override + public String toString() { + return file != null ? file.toString() : ""; + } + } + } +} diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc index c76dc08..707124f 100644 --- a/docs/modules/ROOT/pages/index.adoc +++ b/docs/modules/ROOT/pages/index.adoc @@ -11,7 +11,6 @@ include::./includes/attributes.adoc[] The extension uses the Antora container to build the site. You will thus need a working container runtime, such as Docker or Podman. -If you can run tests with Testcontainers then this extension should work flawlessly for you. == Installation