From 6b6e8d6f785ccb9f61fd4e8599dcdc855dd5965f Mon Sep 17 00:00:00 2001 From: David Handermann Date: Mon, 26 Aug 2024 13:22:04 -0500 Subject: [PATCH] NIFI-13665 Refactored Bootstrap and Runtime Process Handling (#9192) - Added BootstrapProcess main class to nifi-bootstrap - Added HTTP Management Server to nifi-runtime for status - Added HTTP client to nifi-bootstrap for status - Removed TCP socket listeners from nifi-bootstrap and nifi-runtime - Removed dump and env commands - Removed java property from bootstrap.conf - Removed nifi.bootstrap.listen.port from bootstrap.conf --- .../bootstrap/BootstrapRequestReader.java | 1 - .../bootstrap}/LimitingInputStream.java | 2 +- nifi-bootstrap/pom.xml | 7 +- .../apache/nifi/bootstrap/BootstrapCodec.java | 106 -- .../nifi/bootstrap/BootstrapProcess.java | 53 + .../apache/nifi/bootstrap/NiFiListener.java | 141 -- .../org/apache/nifi/bootstrap/RunNiFi.java | 1500 ----------------- .../apache/nifi/bootstrap/ShutdownHook.java | 98 -- ...licationProcessStatusBootstrapCommand.java | 63 + .../bootstrap/command/BootstrapCommand.java | 29 + .../command/BootstrapCommandProvider.java | 30 + .../nifi/bootstrap/command/CommandStatus.java | 49 + .../GetRunCommandBootstrapCommand.java | 91 + .../ManagementServerBootstrapCommand.java | 178 ++ .../command/RunBootstrapCommand.java | 122 ++ .../command/SequenceBootstrapCommand.java | 51 + .../StandardBootstrapCommandProvider.java | 238 +++ .../command/StartBootstrapCommand.java | 94 ++ .../command/StopBootstrapCommand.java | 142 ++ .../command/UnknownBootstrapCommand.java | 33 + .../command/io/BootstrapArgument.java | 38 + .../command/io/BootstrapArgumentParser.java | 32 + .../command/io/FileResponseStreamHandler.java | 49 + .../command/io/HttpRequestMethod.java | 12 +- .../io/LoggerResponseStreamHandler.java | 45 + .../command/io/ResponseStreamHandler.java | 31 + .../io/StandardBootstrapArgumentParser.java | 57 + .../ManagementServerAddressProvider.java | 31 + .../process/ProcessBuilderProvider.java | 29 + ...HandleManagementServerAddressProvider.java | 75 + .../process/ProcessHandleProvider.java | 38 + ...andardManagementServerAddressProvider.java | 86 + .../StandardProcessBuilderProvider.java | 118 ++ .../StandardProcessHandleProvider.java | 83 + .../configuration/ApplicationClassName.java | 36 + .../configuration/BootstrapProperty.java | 44 + .../configuration/ConfigurationProvider.java | 91 + .../configuration/EnvironmentVariable.java | 24 + .../ManagementServerPath.java} | 28 +- .../StandardConfigurationProvider.java | 290 ++++ .../configuration/SystemProperty.java | 44 + .../bootstrap/util/DumpFileValidator.java | 59 - .../bootstrap/util/LimitingInputStream.java | 107 -- ...tionProcessStatusBootstrapCommandTest.java | 76 + .../GetRunCommandBootstrapCommandTest.java | 82 + .../command/RunBootstrapCommandTest.java | 54 + .../StandardBootstrapCommandProviderTest.java | 40 + .../command/StartBootstrapCommandTest.java | 68 + .../command/StopBootstrapCommandTest.java | 87 + .../io/FileResponseStreamHandlerTest.java | 47 + .../StandardBootstrapArgumentParserTest.java | 51 + ...leManagementServerAddressProviderTest.java | 86 + ...rdManagementServerAddressProviderTest.java | 48 + .../StandardProcessBuilderProviderTest.java | 78 + .../StandardProcessHandleProviderTest.java | 55 + .../StandardConfigurationProviderTest.java | 139 ++ .../main/asciidoc/administration-guide.adoc | 1 - .../src/main/resources/bin/nifi.cmd | 9 +- .../src/main/resources/bin/nifi.sh | 86 +- .../src/main/resources/conf/bootstrap.conf | 5 - .../src/main/resources/conf/logback.xml | 2 - .../org/apache/nifi/BootstrapListener.java | 406 ----- .../src/main/java/org/apache/nifi/NiFi.java | 110 +- .../runtime/HealthClusterHttpHandler.java | 106 ++ .../runtime/HealthDiagnosticsHttpHandler.java | 88 + .../nifi/runtime/HealthHttpHandler.java | 58 + .../HealthStatusHistoryHttpHandler.java | 101 ++ .../apache/nifi/runtime/ManagementServer.java | 32 + .../runtime/StandardManagementServer.java | 97 ++ .../runtime/StandardManagementServerTest.java | 121 ++ .../SpawnedStandaloneNiFiInstanceFactory.java | 62 +- .../conf/clustered/node1/logback.xml | 2 - .../conf/clustered/node2/logback.xml | 2 - .../test/resources/conf/default/logback.xml | 2 - .../test/resources/conf/pythonic/logback.xml | 2 - pom.xml | 2 +- 76 files changed, 4100 insertions(+), 2580 deletions(-) rename {nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/util => minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap}/LimitingInputStream.java (98%) delete mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapProcess.java delete mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java delete mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java delete mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommandProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/CommandStatus.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ManagementServerBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/RunBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/SequenceBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StartBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StopBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/UnknownBootstrapCommand.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgument.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgumentParser.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandler.java rename nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFiEntryPoint.java => nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/HttpRequestMethod.java (81%) create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/LoggerResponseStreamHandler.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/ResponseStreamHandler.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParser.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ManagementServerAddressProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessBuilderProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ApplicationClassName.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/BootstrapProperty.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ConfigurationProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/EnvironmentVariable.java rename nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/{InvalidCommandException.java => configuration/ManagementServerPath.java} (63%) create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProvider.java create mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/SystemProperty.java delete mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DumpFileValidator.java delete mode 100644 nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommandTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommandTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/RunBootstrapCommandTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProviderTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StartBootstrapCommandTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StopBootstrapCommandTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandlerTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParserTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProviderTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProviderTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProviderTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProviderTest.java create mode 100644 nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProviderTest.java delete mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/BootstrapListener.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthClusterHttpHandler.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthDiagnosticsHttpHandler.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthHttpHandler.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthStatusHistoryHttpHandler.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/ManagementServer.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/StandardManagementServer.java create mode 100644 nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/java/org/apache/nifi/runtime/StandardManagementServerTest.java diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapRequestReader.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapRequestReader.java index 89816d6c2853..b2cf4d6c186a 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapRequestReader.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap/BootstrapRequestReader.java @@ -22,7 +22,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.Arrays; -import org.apache.nifi.util.LimitingInputStream; public class BootstrapRequestReader { private final String secretKey; diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/util/LimitingInputStream.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap/LimitingInputStream.java similarity index 98% rename from nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/util/LimitingInputStream.java rename to minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap/LimitingInputStream.java index ce3a6dbff066..d762d0d8d25f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/util/LimitingInputStream.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/bootstrap/LimitingInputStream.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.util; +package org.apache.nifi.minifi.bootstrap; import java.io.IOException; import java.io.InputStream; diff --git a/nifi-bootstrap/pom.xml b/nifi-bootstrap/pom.xml index 2fee97fa2640..378962aa2e19 100644 --- a/nifi-bootstrap/pom.xml +++ b/nifi-bootstrap/pom.xml @@ -24,20 +24,17 @@ language governing permissions and limitations under the License. --> org.slf4j slf4j-api - - org.apache.nifi - nifi-utils - 2.0.0-SNAPSHOT - org.apache.nifi nifi-security-cert-builder 2.0.0-SNAPSHOT + org.apache.nifi nifi-properties-loader 2.0.0-SNAPSHOT + runtime diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java deleted file mode 100644 index 5d93a80a9593..000000000000 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.bootstrap; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.util.Arrays; - -public class BootstrapCodec { - - private final RunNiFi runner; - private final BufferedReader reader; - private final BufferedWriter writer; - - public BootstrapCodec(final RunNiFi runner, final InputStream in, final OutputStream out) { - this.runner = runner; - this.reader = new BufferedReader(new InputStreamReader(in)); - this.writer = new BufferedWriter(new OutputStreamWriter(out)); - } - - public void communicate() throws IOException { - final String line = reader.readLine(); - final String[] splits = line.split(" "); - if (splits.length < 0) { - throw new IOException("Received invalid command from NiFi: " + line); - } - - final String cmd = splits[0]; - final String[] args; - if (splits.length == 1) { - args = new String[0]; - } else { - args = Arrays.copyOfRange(splits, 1, splits.length); - } - - try { - processRequest(cmd, args); - } catch (final InvalidCommandException ice) { - throw new IOException("Received invalid command from NiFi: " + line + (ice.getMessage() == null ? "" : " - Details: " + ice.toString())); - } - } - - private void processRequest(final String cmd, final String[] args) throws InvalidCommandException, IOException { - switch (cmd) { - case "PORT": { - if (args.length != 2) { - throw new InvalidCommandException(); - } - - final int port; - try { - port = Integer.parseInt(args[0]); - } catch (final NumberFormatException nfe) { - throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); - } - - if (port < 1 || port > 65535) { - throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); - } - - final String secretKey = args[1]; - - runner.setNiFiCommandControlPort(port, secretKey); - writer.write("OK"); - writer.newLine(); - writer.flush(); - } - break; - case "STARTED": { - if (args.length != 1) { - throw new InvalidCommandException("STARTED command must contain a status argument"); - } - - if (!"true".equals(args[0]) && !"false".equals(args[0])) { - throw new InvalidCommandException("Invalid status for STARTED command; should be true or false, but was '" + args[0] + "'"); - } - - final boolean started = Boolean.parseBoolean(args[0]); - runner.setNiFiStarted(started); - writer.write("OK"); - writer.newLine(); - writer.flush(); - } - break; - } - } -} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapProcess.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapProcess.java new file mode 100644 index 000000000000..57dd7ebb9f99 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapProcess.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap; + +import org.apache.nifi.bootstrap.command.BootstrapCommand; +import org.apache.nifi.bootstrap.command.BootstrapCommandProvider; +import org.apache.nifi.bootstrap.command.CommandStatus; +import org.apache.nifi.bootstrap.command.StandardBootstrapCommandProvider; +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Bootstrap Process responsible for reading configuration and maintaining application status + */ +public class BootstrapProcess { + /** + * Start Application + * + * @param arguments Array of arguments + */ + public static void main(final String[] arguments) { + final BootstrapCommandProvider bootstrapCommandProvider = new StandardBootstrapCommandProvider(); + final BootstrapCommand bootstrapCommand = bootstrapCommandProvider.getBootstrapCommand(arguments); + run(bootstrapCommand); + } + + private static void run(final BootstrapCommand bootstrapCommand) { + bootstrapCommand.run(); + final CommandStatus commandStatus = bootstrapCommand.getCommandStatus(); + if (CommandStatus.RUNNING == commandStatus) { + final Logger logger = LoggerFactory.getLogger(ApplicationClassName.BOOTSTRAP_COMMAND.getName()); + logger.info("Bootstrap Process Running"); + } else { + final int status = commandStatus.getStatus(); + System.exit(status); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java deleted file mode 100644 index 16fe1955cef8..000000000000 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.bootstrap; - -import java.io.IOException; -import java.io.InputStream; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import org.apache.nifi.bootstrap.util.LimitingInputStream; - -public class NiFiListener { - - private ServerSocket serverSocket; - private volatile Listener listener; - - int start(final RunNiFi runner, final int listenPort) throws IOException { - serverSocket = new ServerSocket(); - serverSocket.bind(new InetSocketAddress("localhost", listenPort)); - - final int localPort = serverSocket.getLocalPort(); - listener = new Listener(serverSocket, runner); - final Thread listenThread = new Thread(listener); - listenThread.setName("Listen to NiFi"); - listenThread.setDaemon(true); - listenThread.start(); - return localPort; - } - - public void stop() throws IOException { - final Listener listener = this.listener; - if (listener == null) { - return; - } - - listener.stop(); - } - - private class Listener implements Runnable { - - private final ServerSocket serverSocket; - private final ExecutorService executor; - private final RunNiFi runner; - private volatile boolean stopped = false; - - public Listener(final ServerSocket serverSocket, final RunNiFi runner) { - this.serverSocket = serverSocket; - this.executor = Executors.newFixedThreadPool(2, new ThreadFactory() { - @Override - public Thread newThread(final Runnable runnable) { - final Thread t = Executors.defaultThreadFactory().newThread(runnable); - t.setDaemon(true); - t.setName("NiFi Bootstrap Command Listener"); - return t; - } - }); - - this.runner = runner; - } - - public void stop() throws IOException { - stopped = true; - - executor.shutdown(); - try { - executor.awaitTermination(3, TimeUnit.SECONDS); - } catch (final InterruptedException ie) { - } - - serverSocket.close(); - } - - @Override - public void run() { - while (!serverSocket.isClosed()) { - try { - if (stopped) { - return; - } - - final Socket socket; - try { - socket = serverSocket.accept(); - } catch (final IOException ioe) { - if (stopped) { - return; - } - - throw ioe; - } - - executor.submit(new Runnable() { - @Override - public void run() { - try { - // we want to ensure that we don't try to read data from an InputStream directly - // by a BufferedReader because any user on the system could open a socket and send - // a multi-gigabyte file without any new lines in order to crash the Bootstrap, - // which in turn may cause the Shutdown Hook to shutdown NiFi. - // So we will limit the amount of data to read to 4 KB - final InputStream limitingIn = new LimitingInputStream(socket.getInputStream(), 4096); - final BootstrapCodec codec = new BootstrapCodec(runner, limitingIn, socket.getOutputStream()); - codec.communicate(); - } catch (final Throwable t) { - System.out.println("Failed to communicate with NiFi due to " + t); - t.printStackTrace(); - } finally { - try { - socket.close(); - } catch (final IOException ioe) { - } - } - } - }); - } catch (final Throwable t) { - System.err.println("Failed to receive information from NiFi due to " + t); - t.printStackTrace(); - } - } - } - } -} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java deleted file mode 100644 index 89d0693e1082..000000000000 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java +++ /dev/null @@ -1,1500 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.bootstrap; - -import org.apache.nifi.bootstrap.process.RuntimeValidatorExecutor; -import org.apache.nifi.bootstrap.property.ApplicationPropertyHandler; -import org.apache.nifi.bootstrap.property.SecurityApplicationPropertyHandler; -import org.apache.nifi.bootstrap.util.DumpFileValidator; -import org.apache.nifi.util.file.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.Reader; -import java.lang.management.ManagementFactory; -import java.lang.reflect.Method; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.PosixFilePermission; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import javax.management.MBeanServer; -import javax.management.ObjectName; - -/** - *

- * The class which bootstraps Apache NiFi. This class looks for the - * bootstrap.conf file by looking in the following places (in order):

- *
    - *
  1. Java System Property named - * {@code org.apache.nifi.bootstrap.config.file}
    2. - *
  2. ${NIFI_HOME}/./conf/bootstrap.conf, where ${NIFI_HOME} references an - * environment variable {@code NIFI_HOME}
    3. - *
  3. ./conf/bootstrap.conf, where {@code ./} represents the working - * directory.
    4. - *
- *

- * If the {@code bootstrap.conf} file cannot be found, throws a {@code FileNotFoundException}. - */ -public class RunNiFi { - - public static final String DEFAULT_CONFIG_FILE = "./conf/bootstrap.conf"; - public static final String DEFAULT_JAVA_CMD = "java"; - public static final String DEFAULT_PID_DIR = "bin"; - public static final String DEFAULT_LOG_DIR = "./logs"; - public static final String DEFAULT_STATUS_HISTORY_DAYS = "1"; - - public static final String GRACEFUL_SHUTDOWN_PROP = "graceful.shutdown.seconds"; - public static final String DEFAULT_GRACEFUL_SHUTDOWN_VALUE = "20"; - - public static final String NIFI_PID_DIR_PROP = "org.apache.nifi.bootstrap.config.pid.dir"; - - public static final String NIFI_PID_FILE_NAME = "nifi.pid"; - public static final String NIFI_STATUS_FILE_NAME = "nifi.status"; - public static final String NIFI_LOCK_FILE_NAME = "nifi.lock"; - - public static final String NIFI_BOOTSTRAP_LISTEN_PORT_PROP = "nifi.bootstrap.listen.port"; - - public static final String PID_KEY = "pid"; - - public static final int STARTUP_WAIT_SECONDS = 60; - public static final long GRACEFUL_SHUTDOWN_RETRY_MILLIS = 2000L; - - public static final String SHUTDOWN_CMD = "SHUTDOWN"; - public static final String DECOMMISSION_CMD = "DECOMMISSION"; - public static final String PING_CMD = "PING"; - public static final String DUMP_CMD = "DUMP"; - public static final String DIAGNOSTICS_CMD = "DIAGNOSTICS"; - public static final String CLUSTER_STATUS_CMD = "CLUSTER_STATUS"; - public static final String IS_LOADED_CMD = "IS_LOADED"; - public static final String STATUS_HISTORY_CMD = "STATUS_HISTORY"; - - private static final int UNINITIALIZED_CC_PORT = -1; - - private static final int INVALID_CMD_ARGUMENT = -1; - - private volatile boolean autoRestartNiFi = true; - private volatile int ccPort = UNINITIALIZED_CC_PORT; - private volatile long nifiPid = -1L; - private volatile String secretKey; - private volatile ShutdownHook shutdownHook; - private volatile boolean nifiStarted; - - private final Lock startedLock = new ReentrantLock(); - private final Lock lock = new ReentrantLock(); - private final Condition startupCondition = lock.newCondition(); - - private final File bootstrapConfigFile; - - // used for logging initial info; these will be logged to console by default when the app is started - private final Logger cmdLogger = LoggerFactory.getLogger("org.apache.nifi.bootstrap.Command"); - // used for logging all info. These by default will be written to the log file - private final Logger defaultLogger = LoggerFactory.getLogger(RunNiFi.class); - - private final ExecutorService loggingExecutor; - private final RuntimeValidatorExecutor runtimeValidatorExecutor; - private volatile Set> loggingFutures = new HashSet<>(2); - - public RunNiFi(final File bootstrapConfigFile) throws IOException { - this.bootstrapConfigFile = bootstrapConfigFile; - - loggingExecutor = Executors.newFixedThreadPool(2, runnable -> { - final Thread t = Executors.defaultThreadFactory().newThread(runnable); - t.setDaemon(true); - t.setName("NiFi logging handler"); - return t; - }); - - runtimeValidatorExecutor = new RuntimeValidatorExecutor(); - } - - private static void printUsage() { - System.out.println("Usage:"); - System.out.println(); - System.out.println("java org.apache.nifi.bootstrap.RunNiFi [options]"); - System.out.println(); - System.out.println("Valid commands include:"); - System.out.println(); - System.out.println("Start : Start a new instance of Apache NiFi"); - System.out.println("Stop : Stop a running instance of Apache NiFi"); - System.out.println("Restart : Stop Apache NiFi, if it is running, and then start a new instance"); - System.out.println("Decommission : Disconnects Apache NiFi from its cluster, offloads its data to other nodes in the cluster, removes itself from the cluster, and shuts down the instance"); - System.out.println("Status : Determine if there is a running instance of Apache NiFi"); - System.out.println("Dump : Write a Thread Dump to the file specified by [options], or to the log if no file is given"); - System.out.println("Diagnostics : Write diagnostic information to the file specified by [options], or to the log if no file is given. The --verbose flag may be provided as an option before " + - "the filename, which may result in additional diagnostic information being written."); - System.out.println("Status-history : Save the status history to the file specified by [options]. The expected command parameters are: " + - "status-history . The parameter is optional and defaults to 1 day."); - System.out.println("Run : Start a new instance of Apache NiFi and monitor the Process, restarting if the instance dies"); - System.out.println(); - } - - public static void main(String[] args) throws IOException { - if (args.length < 1 || args.length > 3) { - printUsage(); - return; - } - - File dumpFile = null; - boolean verbose = false; - String statusHistoryDays = null; - - final String cmd = args[0]; - if (cmd.equalsIgnoreCase("dump")) { - if (args.length > 1) { - dumpFile = new File(args[1]); - } - } else if (cmd.equalsIgnoreCase("diagnostics")) { - if (args.length > 2) { - verbose = args[1].equalsIgnoreCase("--verbose"); - dumpFile = new File(args[2]); - } else if (args.length > 1) { - if (args[1].equalsIgnoreCase("--verbose")) { - verbose = true; - } else { - dumpFile = new File(args[1]); - } - } - } else if (cmd.equalsIgnoreCase("cluster-status")) { - if (args.length > 1) { - dumpFile = new File(args[1]); - } - } else if (cmd.equalsIgnoreCase("status-history")) { - if (args.length < 2) { - System.err.printf("Wrong number of arguments: %d instead of 1 or 2, the command parameters are: " + - "status-history %n", 0); - System.exit(INVALID_CMD_ARGUMENT); - } - if (args.length == 3) { - statusHistoryDays = args[1]; - try { - final int numberOfDays = Integer.parseInt(statusHistoryDays); - if (numberOfDays < 1) { - System.err.println("The parameter must be positive and greater than zero. The command parameters are:" + - " status-history "); - System.exit(INVALID_CMD_ARGUMENT); - } - } catch (NumberFormatException e) { - System.err.println("The parameter value is not a number. The command parameters are: status-history "); - System.exit(INVALID_CMD_ARGUMENT); - } - try { - Paths.get(args[2]); - } catch (InvalidPathException e) { - System.err.println("Invalid filename. The command parameters are: status-history "); - System.exit(INVALID_CMD_ARGUMENT); - } - dumpFile = new File(args[2]); - } else { - final boolean isValid = DumpFileValidator.validate(args[1]); - if (isValid) { - statusHistoryDays = DEFAULT_STATUS_HISTORY_DAYS; - dumpFile = new File(args[1]); - } else { - System.exit(INVALID_CMD_ARGUMENT); - } - } - } - - switch (cmd.toLowerCase()) { - case "start": - case "run": - case "stop": - case "decommission": - case "status": - case "is_loaded": - case "dump": - case "diagnostics": - case "status-history": - case "restart": - case "env": - case "cluster-status": - break; - default: - printUsage(); - return; - } - - final File configFile = getDefaultBootstrapConfFile(); - final RunNiFi runNiFi = new RunNiFi(configFile); - - Integer exitStatus = null; - switch (cmd.toLowerCase()) { - case "start": - runNiFi.start(true); - break; - case "run": - runNiFi.start(true); - break; - case "stop": - runNiFi.stop(); - break; - case "decommission": - final boolean shutdown = args.length < 2 || !"--shutdown=false".equals(args[1]); - exitStatus = runNiFi.decommission(shutdown); - break; - case "status": - exitStatus = runNiFi.status(); - break; - case "is_loaded": - try { - System.out.println(runNiFi.isNiFiFullyLoaded()); - } catch (NiFiNotRunningException e) { - System.out.println("not_running"); - } - break; - case "restart": - runNiFi.stop(); - runNiFi.start(true); - break; - case "dump": - runNiFi.dump(dumpFile); - break; - case "diagnostics": - runNiFi.diagnostics(dumpFile, verbose); - break; - case "cluster-status": - runNiFi.clusterStatus(dumpFile); - break; - case "status-history": - runNiFi.statusHistory(dumpFile, statusHistoryDays); - break; - case "env": - runNiFi.env(); - break; - } - if (exitStatus != null) { - System.exit(exitStatus); - } - } - - private static File getDefaultBootstrapConfFile() { - String configFilename = System.getProperty("org.apache.nifi.bootstrap.config.file"); - - if (configFilename == null) { - final String nifiHome = System.getenv("NIFI_HOME"); - if (nifiHome != null) { - final File nifiHomeFile = new File(nifiHome.trim()); - final File configFile = new File(nifiHomeFile, DEFAULT_CONFIG_FILE); - configFilename = configFile.getAbsolutePath(); - } - } - - if (configFilename == null) { - configFilename = DEFAULT_CONFIG_FILE; - } - - return new File(configFilename); - } - - protected File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException { - - final File confDir = bootstrapConfigFile.getParentFile(); - final File nifiHome = confDir.getParentFile(); - - String confFileDir = System.getProperty(directory); - - final File fileDir; - - if (confFileDir != null) { - fileDir = new File(confFileDir.trim()); - } else { - fileDir = new File(nifiHome, defaultDirectory); - } - - FileUtils.ensureDirectoryExistAndCanAccess(fileDir); - final File statusFile = new File(fileDir, fileName); - logger.debug("Status File: {}", statusFile); - return statusFile; - } - - protected File getPidFile(final Logger logger) throws IOException { - return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_PID_FILE_NAME); - } - - protected File getStatusFile(final Logger logger) throws IOException { - return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_STATUS_FILE_NAME); - } - - protected File getLockFile(final Logger logger) throws IOException { - return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_LOCK_FILE_NAME); - } - - protected File getStatusFile() throws IOException { - return getStatusFile(defaultLogger); - } - - private Properties loadProperties(final Logger logger) throws IOException { - final Properties props = new Properties(); - final File statusFile = getStatusFile(logger); - if (statusFile == null || !statusFile.exists()) { - logger.debug("No status file to load properties from"); - return props; - } - - try (final FileInputStream fis = new FileInputStream(getStatusFile(logger))) { - props.load(fis); - } - - final Map modified = new HashMap<>(props); - modified.remove("secret.key"); - logger.debug("Properties: {}", modified); - - return props; - } - - private synchronized void savePidProperties(final Properties pidProperties, final Logger logger) throws IOException { - final String pid = pidProperties.getProperty(PID_KEY); - if (pid != null && !pid.isBlank()) { - writePidFile(pid, logger); - } - - final File statusFile = getStatusFile(logger); - if (statusFile.exists() && !statusFile.delete()) { - logger.warn("Failed to delete {}", statusFile); - } - - if (!statusFile.createNewFile()) { - throw new IOException("Failed to create file " + statusFile); - } - - try { - final Set perms = new HashSet<>(); - perms.add(PosixFilePermission.OWNER_READ); - perms.add(PosixFilePermission.OWNER_WRITE); - Files.setPosixFilePermissions(statusFile.toPath(), perms); - } catch (final Exception e) { - logger.warn("Failed to set permissions so that only the owner can read status file {}; " - + "this may allows others to have access to the key needed to communicate with NiFi. " - + "Permissions should be changed so that only the owner can read this file", statusFile); - } - - try (final FileOutputStream fos = new FileOutputStream(statusFile)) { - pidProperties.store(fos, null); - fos.getFD().sync(); - } - - logger.debug("Saved Properties {} to {}", pidProperties, statusFile); - } - - private synchronized void writePidFile(final String pid, final Logger logger) throws IOException { - final File pidFile = getPidFile(logger); - if (pidFile.exists() && !pidFile.delete()) { - logger.warn("Failed to delete {}", pidFile); - } - - if (!pidFile.createNewFile()) { - throw new IOException("Failed to create file " + pidFile); - } - - try { - final Set perms = new HashSet<>(); - perms.add(PosixFilePermission.OWNER_WRITE); - perms.add(PosixFilePermission.OWNER_READ); - perms.add(PosixFilePermission.GROUP_READ); - perms.add(PosixFilePermission.OTHERS_READ); - Files.setPosixFilePermissions(pidFile.toPath(), perms); - } catch (final Exception e) { - logger.warn("Failed to set permissions so that only the owner can read pid file {}; " - + "this may allows others to have access to the key needed to communicate with NiFi. " - + "Permissions should be changed so that only the owner can read this file", pidFile); - } - - try (final FileOutputStream fos = new FileOutputStream(pidFile)) { - fos.write(pid.getBytes(StandardCharsets.UTF_8)); - fos.getFD().sync(); - } - - logger.debug("Saved PID [{}] to [{}]", pid, pidFile); - } - - private boolean isPingSuccessful(final int port, final String secretKey, final Logger logger) { - logger.debug("Pinging {}", port); - - try (final Socket socket = new Socket("localhost", port)) { - final OutputStream out = socket.getOutputStream(); - out.write((PING_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - - logger.debug("Sent PING command"); - socket.setSoTimeout(5000); - final InputStream in = socket.getInputStream(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(in)); - final String response = reader.readLine(); - logger.debug("PING response: {}", response); - out.close(); - reader.close(); - - return PING_CMD.equals(response); - } catch (final IOException ioe) { - return false; - } - } - - private Integer getCurrentPort(final Logger logger) throws IOException { - final Properties props = loadProperties(logger); - final String portVal = props.getProperty("port"); - if (portVal == null) { - logger.debug("No Port found in status file"); - return null; - } else { - logger.debug("Port defined in status file: {}", portVal); - } - - final int port = Integer.parseInt(portVal); - final boolean success = isPingSuccessful(port, props.getProperty("secret.key"), logger); - if (success) { - logger.debug("Successful PING on port {}", port); - return port; - } - - final String pid = props.getProperty(PID_KEY); - logger.debug("PID in status file is {}", pid); - if (pid != null) { - final boolean procRunning = isProcessRunning(pid, logger); - if (procRunning) { - return port; - } else { - return null; - } - } - - return null; - } - - private boolean isProcessRunning(final String pid, final Logger logger) { - try { - // We use the "ps" command to check if the process is still running. - final ProcessBuilder builder = new ProcessBuilder(); - - builder.command("ps", "-p", pid); - final Process proc = builder.start(); - - // Look for the pid in the output of the 'ps' command. - boolean running = false; - String line; - try (final InputStream in = proc.getInputStream(); - final Reader streamReader = new InputStreamReader(in); - final BufferedReader reader = new BufferedReader(streamReader)) { - - while ((line = reader.readLine()) != null) { - if (line.trim().startsWith(pid)) { - running = true; - } - } - } - - // If output of the ps command had our PID, the process is running. - if (running) { - logger.debug("Process with PID {} is running", pid); - } else { - logger.debug("Process with PID {} is not running", pid); - } - - return running; - } catch (final IOException ioe) { - System.err.println("Failed to determine if Process " + pid + " is running; assuming that it is not"); - return false; - } - } - - private Status getStatus(final Logger logger) { - final Properties props; - try { - props = loadProperties(logger); - } catch (final IOException ioe) { - return new Status(null, null, false, false); - } - - if (props == null) { - return new Status(null, null, false, false); - } - - final String portValue = props.getProperty("port"); - final String pid = props.getProperty(PID_KEY); - final String secretKey = props.getProperty("secret.key"); - - if (portValue == null && pid == null) { - return new Status(null, null, false, false); - } - - Integer port = null; - boolean pingSuccess = false; - if (portValue != null) { - try { - port = Integer.parseInt(portValue); - pingSuccess = isPingSuccessful(port, secretKey, logger); - } catch (final NumberFormatException nfe) { - return new Status(null, null, false, false); - } - } - - if (pingSuccess) { - return new Status(port, pid, true, true); - } - - final boolean alive = pid != null && isProcessRunning(pid, logger); - return new Status(port, pid, pingSuccess, alive); - } - - public int status() throws IOException { - final Logger logger = cmdLogger; - final Status status = getStatus(logger); - if (status.isRespondingToPing()) { - logger.info("Apache NiFi PID [{}] running with Bootstrap Port [{}]", status.getPid(), status.getPort()); - return 0; - } - - if (status.isProcessRunning()) { - logger.info("Apache NiFi PID [{}] running but not responding with Bootstrap Port [{}]", status.getPid(), status.getPort()); - return 4; - } - - if (status.getPort() == null) { - logger.info("Apache NiFi is not running"); - return 3; - } - - if (status.getPid() == null) { - logger.info("Apache NiFi is not responding to Ping requests. The process may have died or may be hung"); - } else { - logger.info("Apache NiFi is not running"); - } - return 3; - } - - public void env() { - final Logger logger = cmdLogger; - final Status status = getStatus(logger); - if (status.getPid() == null) { - logger.info("Apache NiFi is not running"); - return; - } - final Class virtualMachineClass; - try { - virtualMachineClass = Class.forName("com.sun.tools.attach.VirtualMachine"); - } catch (final ClassNotFoundException cnfe) { - logger.error("Seems tools.jar (Linux / Windows JDK) or classes.jar (Mac OS) is not available in classpath"); - return; - } - final Method attachMethod; - final Method detachMethod; - - try { - attachMethod = virtualMachineClass.getMethod("attach", String.class); - detachMethod = virtualMachineClass.getDeclaredMethod("detach"); - } catch (final Exception e) { - logger.error("Methods required for getting environment not available", e); - return; - } - - final Object virtualMachine; - try { - virtualMachine = attachMethod.invoke(null, status.getPid()); - } catch (final Throwable t) { - logger.error("Problem attaching to NiFi", t); - return; - } - - try { - final Method getSystemPropertiesMethod = virtualMachine.getClass().getMethod("getSystemProperties"); - - final Properties sysProps = (Properties) getSystemPropertiesMethod.invoke(virtualMachine); - for (Entry syspropEntry : sysProps.entrySet()) { - logger.info("{} = {}", syspropEntry.getKey(), syspropEntry.getValue()); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } finally { - try { - detachMethod.invoke(virtualMachine); - } catch (final Exception e) { - logger.warn("Caught exception detaching from process", e); - } - } - } - - /** - * Writes NiFi diagnostic information to the given file; if the file is null, logs at INFO level instead. - */ - public void diagnostics(final File dumpFile, final boolean verbose) throws IOException { - final String args = verbose ? "--verbose=true" : null; - makeRequest(DIAGNOSTICS_CMD, args, dumpFile, null, "diagnostics information"); - } - - public void clusterStatus(final File dumpFile) throws IOException { - try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - makeRequest(CLUSTER_STATUS_CMD, null, dumpFile, baos, "cluster status"); - - final String response = baos.toString(StandardCharsets.UTF_8); - System.out.println("Cluster Status: " + response); - } - } - - /** - * Writes a NiFi thread dump to the given file; if file is null, logs at - * INFO level instead. - * - * @param dumpFile the file to write the dump content to - * @throws IOException if any issues occur while writing the dump file - */ - public void dump(final File dumpFile) throws IOException { - makeRequest(DUMP_CMD, null, dumpFile, null, "thread dump"); - } - - /** - * Writes NiFi status history information to the given file. - * - * @param dumpFile the file to write the dump content to - * @throws IOException if any issues occur while writing the dump file - */ - public void statusHistory(final File dumpFile, final String days) throws IOException { - // Due to input validation, the dumpFile cannot currently be null in this scenario. - makeRequest(STATUS_HISTORY_CMD, days, dumpFile, null, "status history information"); - } - - private boolean isNiFiFullyLoaded() throws IOException, NiFiNotRunningException { - final Logger logger = defaultLogger; - final Integer port = getCurrentPort(logger); - if (port == null) { - logger.info("Apache NiFi is not currently running"); - throw new NiFiNotRunningException(); - } - - try (final Socket socket = new Socket()) { - sendRequest(socket, port, IS_LOADED_CMD, null, logger); - - final InputStream in = socket.getInputStream(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line = reader.readLine(); - return Boolean.parseBoolean(line); - } - } - } - - /** - * Makes a request to the Bootstrap Listener - * @param request the request to send - * @param arguments any arguments for the command, or null if the command takes no arguments - * @param dumpFile a file to write the results to, or null to skip writing the results to any file - * @param outputStream an OutputStream to write the results to, or null to skip writing the results to any OutputStream - * @param contentsDescription a description of the contents being written; used for logging purposes - * @throws IOException if unable to communicate with the NiFi instance or write out the results - */ - private void makeRequest(final String request, final String arguments, final File dumpFile, final OutputStream outputStream, final String contentsDescription) throws IOException { - final Logger logger = defaultLogger; // dump to bootstrap log file by default - final Integer port = getCurrentPort(logger); - if (port == null) { - cmdLogger.info("Apache NiFi is not currently running"); - logger.info("Apache NiFi is not currently running"); - return; - } - - final OutputStream fileOut = dumpFile == null ? null : new FileOutputStream(dumpFile); - try { - try (final Socket socket = new Socket()) { - sendRequest(socket, port, request, arguments, logger); - - final InputStream in = socket.getInputStream(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line; - while ((line = reader.readLine()) != null) { - boolean written = false; - if (fileOut != null) { - fileOut.write(line.getBytes(StandardCharsets.UTF_8)); - fileOut.write('\n'); - written = true; - } - - if (outputStream != null) { - outputStream.write(line.getBytes(StandardCharsets.UTF_8)); - outputStream.write('\n'); - written = true; - } - - if (!written) { - logger.info(line); - } - } - } - } - } finally { - if (fileOut != null) { - fileOut.close(); - cmdLogger.info("Successfully wrote {} to {}", contentsDescription, dumpFile.getAbsolutePath()); - } - } - } - - private void sendRequest(Socket socket, Integer port, String request, String arguments, Logger logger) throws IOException { - logger.debug("Connecting to NiFi instance"); - socket.setSoTimeout(60000); - socket.connect(new InetSocketAddress("localhost", port)); - logger.debug("Established connection to NiFi instance."); - socket.setSoTimeout(60000); - - logger.debug("Sending {} Command to port {}", request, port); - final OutputStream socketOut = socket.getOutputStream(); - - final Properties nifiProps = loadProperties(logger); - final String secretKey = nifiProps.getProperty("secret.key"); - - if (arguments == null) { - socketOut.write((request + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - } else { - socketOut.write((request + " " + secretKey + " " + arguments + "\n").getBytes(StandardCharsets.UTF_8)); - } - - socketOut.flush(); - } - - - public Integer decommission(final boolean shutdown) throws IOException { - final Logger logger = cmdLogger; - final Integer port = getCurrentPort(logger); - if (port == null) { - logger.info("Apache NiFi is not currently running"); - return 15; - } - - // indicate that a stop command is in progress - final File lockFile = getLockFile(logger); - if (!lockFile.exists()) { - lockFile.createNewFile(); - } - - final Properties nifiProps = loadProperties(logger); - final String secretKey = nifiProps.getProperty("secret.key"); - final String pid = nifiProps.getProperty(PID_KEY); - final File statusFile = getStatusFile(logger); - final File pidFile = getPidFile(logger); - - try (final Socket socket = new Socket()) { - logger.debug("Connecting to NiFi instance"); - socket.setSoTimeout(10000); - socket.connect(new InetSocketAddress("localhost", port)); - logger.debug("Established connection to NiFi instance."); - - // We don't know how long it will take for the offloading to complete. It could be a while. So don't timeout. - // User can press Ctrl+C to terminate if they don't want to wait - socket.setSoTimeout(0); - - logger.debug("Sending DECOMMISSION Command to port {}", port); - final OutputStream out = socket.getOutputStream(); - final String command = DECOMMISSION_CMD + " " + secretKey + " --shutdown=" + shutdown + "\n"; - out.write(command.getBytes(StandardCharsets.UTF_8)); - out.flush(); - socket.shutdownOutput(); - - final String response = readResponse(socket.getInputStream()); - - if (DECOMMISSION_CMD.equals(response)) { - logger.debug("Received response to DECOMMISSION command: {}", response); - - if (pid != null) { - waitForShutdown(pid, logger, statusFile, pidFile); - } - - return null; - } else { - logger.error("When sending DECOMMISSION command to NiFi, got unexpected response {}", response); - return 18; - } - } finally { - if (lockFile.exists() && !lockFile.delete()) { - logger.error("Failed to delete lock file {}; this file should be cleaned up manually", lockFile); - } - } - } - - public void stop() throws IOException { - final Logger logger = cmdLogger; - final Integer port = getCurrentPort(logger); - if (port == null) { - logger.info("Apache NiFi is not currently running"); - return; - } - - // indicate that a stop command is in progress - final File lockFile = getLockFile(logger); - if (!lockFile.exists()) { - lockFile.createNewFile(); - } - - final Properties nifiProps = loadProperties(logger); - final String secretKey = nifiProps.getProperty("secret.key"); - final String pid = nifiProps.getProperty(PID_KEY); - final File statusFile = getStatusFile(logger); - final File pidFile = getPidFile(logger); - - try (final Socket socket = new Socket()) { - logger.debug("Connecting to NiFi instance"); - socket.setSoTimeout(10000); - socket.connect(new InetSocketAddress("localhost", port)); - logger.debug("Established connection to NiFi instance."); - socket.setSoTimeout(10000); - - logger.debug("Sending SHUTDOWN Command to port {}", port); - final OutputStream out = socket.getOutputStream(); - out.write((SHUTDOWN_CMD + " " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - socket.shutdownOutput(); - - final String response = readResponse(socket.getInputStream()); - logger.debug("Received response to SHUTDOWN command: {}", response); - - if (SHUTDOWN_CMD.equals(response)) { - logger.info("Apache NiFi has accepted the Shutdown Command and is shutting down now"); - - if (pid != null) { - waitForShutdown(pid, logger, statusFile, pidFile); - } - } else { - logger.error("When sending SHUTDOWN command to NiFi, got unexpected response: {}", response); - } - } catch (final IOException ioe) { - if (pid == null) { - logger.error("Failed to send shutdown command to port {} due to {}. No PID found for the NiFi process, so unable to kill process; " - + "the process should be killed manually.", port, ioe.toString()); - } else { - logger.error("Failed to send shutdown command to port {} due to {}. Will kill the NiFi Process with PID {}.", port, ioe, pid); - killProcessTree(pid, logger); - if (statusFile.exists() && !statusFile.delete()) { - logger.error("Failed to delete status file {}; this file should be cleaned up manually", statusFile); - } - } - } finally { - if (lockFile.exists() && !lockFile.delete()) { - logger.error("Failed to delete lock file {}; this file should be cleaned up manually", lockFile); - } - } - } - - private String readResponse(final InputStream in) throws IOException { - int lastChar; - final StringBuilder sb = new StringBuilder(); - while ((lastChar = in.read()) > -1) { - sb.append((char) lastChar); - } - - return sb.toString().trim(); - } - - private void waitForShutdown(final String pid, final Logger logger, final File statusFile, final File pidFile) throws IOException { - final Properties bootstrapProperties = new Properties(); - try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) { - bootstrapProperties.load(fis); - } - - String gracefulShutdown = bootstrapProperties.getProperty(GRACEFUL_SHUTDOWN_PROP, DEFAULT_GRACEFUL_SHUTDOWN_VALUE); - int gracefulShutdownSeconds; - try { - gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown); - } catch (final NumberFormatException nfe) { - gracefulShutdownSeconds = Integer.parseInt(DEFAULT_GRACEFUL_SHUTDOWN_VALUE); - } - - final long startWait = System.nanoTime(); - while (isProcessRunning(pid, logger)) { - logger.info("NiFi PID [{}] shutdown in progress...", pid); - final long waitNanos = System.nanoTime() - startWait; - final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); - if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) { - if (isProcessRunning(pid, logger)) { - logger.warn("NiFi PID [{}] shutdown not completed after {} seconds: Killing process", pid, gracefulShutdownSeconds); - try { - killProcessTree(pid, logger); - } catch (final IOException ioe) { - logger.error("Failed to kill Process with PID {}", pid); - } - } - break; - } else { - try { - Thread.sleep(GRACEFUL_SHUTDOWN_RETRY_MILLIS); - } catch (final InterruptedException ie) { - } - } - } - - if (statusFile.exists() && !statusFile.delete()) { - logger.error("Failed to delete status file {}; this file should be cleaned up manually", statusFile); - } - - if (pidFile.exists() && !pidFile.delete()) { - logger.error("Failed to delete pid file {}; this file should be cleaned up manually", pidFile); - } - - logger.info("NiFi PID [{}] shutdown completed", pid); - } - - private static List getChildProcesses(final String ppid) throws IOException { - final Process proc = Runtime.getRuntime().exec(new String[]{"ps", "-o", "pid", "--no-headers", "--ppid", ppid}); - final List childPids = new ArrayList<>(); - try (final InputStream in = proc.getInputStream(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - - String line; - while ((line = reader.readLine()) != null) { - childPids.add(line.trim()); - } - } - - return childPids; - } - - private void killProcessTree(final String pid, final Logger logger) throws IOException { - logger.debug("Killing Process Tree for PID {}", pid); - - final List children = getChildProcesses(pid); - logger.debug("Children of PID {}: {}", pid, children); - - for (final String childPid : children) { - killProcessTree(childPid, logger); - } - - Runtime.getRuntime().exec(new String[]{"kill", "-9", pid}); - } - - public static boolean isAlive(final Process process) { - try { - process.exitValue(); - return false; - } catch (final IllegalStateException | IllegalThreadStateException itse) { - return true; - } - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - public void start(final boolean monitor) throws IOException { - final Integer port = getCurrentPort(cmdLogger); - if (port != null) { - cmdLogger.info("Apache NiFi is already running, listening to Bootstrap on port {}", port); - return; - } - - final File prevLockFile = getLockFile(cmdLogger); - if (prevLockFile.exists() && !prevLockFile.delete()) { - cmdLogger.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile); - } - - runtimeValidatorExecutor.execute(); - - final ProcessBuilder builder = new ProcessBuilder(); - - if (!bootstrapConfigFile.exists()) { - throw new FileNotFoundException(bootstrapConfigFile.getAbsolutePath()); - } - - final Properties properties = new Properties(); - try (final FileInputStream fis = new FileInputStream(bootstrapConfigFile)) { - properties.load(fis); - } - - final Map props = new HashMap<>(); - props.putAll((Map) properties); - - final String specifiedWorkingDir = props.get("working.dir"); - if (specifiedWorkingDir != null) { - builder.directory(new File(specifiedWorkingDir)); - } - - final File bootstrapConfigAbsoluteFile = bootstrapConfigFile.getAbsoluteFile(); - final File binDir = bootstrapConfigAbsoluteFile.getParentFile(); - final File workingDir = binDir.getParentFile(); - - if (specifiedWorkingDir == null) { - builder.directory(workingDir); - } - - final String nifiLogDir = replaceNull(System.getProperty("org.apache.nifi.bootstrap.config.log.dir"), DEFAULT_LOG_DIR).trim(); - - final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim(); - File libDir = getFile(libFilename, workingDir); - - final String confFilename = replaceNull(props.get("conf.dir"), "./conf").trim(); - File confDir = getFile(confFilename, workingDir); - - String nifiPropsFilename = props.get("props.file"); - if (nifiPropsFilename == null) { - if (confDir.exists()) { - nifiPropsFilename = new File(confDir, "nifi.properties").getAbsolutePath(); - } else { - nifiPropsFilename = DEFAULT_CONFIG_FILE; - } - } - - nifiPropsFilename = nifiPropsFilename.trim(); - - String maximumHeapSize = null; - String minimumHeapSize = null; - - final List javaAdditionalArgs = new ArrayList<>(); - for (final Map.Entry entry : props.entrySet()) { - final String key = entry.getKey(); - final String value = entry.getValue(); - - if (key.startsWith("java.arg")) { - javaAdditionalArgs.add(value); - if (value.startsWith("-Xms")) { - minimumHeapSize = value.replace("-Xms", ""); - } - if (value.startsWith("-Xmx")) { - maximumHeapSize = value.replace("-Xmx", ""); - } - } - } - - final File[] libFiles = libDir.listFiles(new FilenameFilter() { - @Override - public boolean accept(final File dir, final String filename) { - return filename.toLowerCase().endsWith(".jar"); - } - }); - - if (libFiles == null || libFiles.length == 0) { - throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath()); - } - - final File[] confFiles = confDir.listFiles(); - if (confFiles == null || confFiles.length == 0) { - throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath()); - } - - final List cpFiles = new ArrayList<>(confFiles.length + libFiles.length); - cpFiles.add(confDir.getAbsolutePath()); - for (final File file : libFiles) { - cpFiles.add(file.getAbsolutePath()); - } - - defaultLogger.info(getPlatformDetails(minimumHeapSize, maximumHeapSize)); - - final StringBuilder classPathBuilder = new StringBuilder(); - for (int i = 0; i < cpFiles.size(); i++) { - final String filename = cpFiles.get(i); - classPathBuilder.append(filename); - if (i < cpFiles.size() - 1) { - classPathBuilder.append(File.pathSeparatorChar); - } - } - - final String classPath = classPathBuilder.toString(); - String javaCmd = props.get("java"); - if (javaCmd == null) { - javaCmd = DEFAULT_JAVA_CMD; - } - if (javaCmd.equals(DEFAULT_JAVA_CMD)) { - String javaHome = System.getenv("JAVA_HOME"); - if (javaHome != null) { - String fileExtension = isWindows() ? ".exe" : ""; - File javaFile = new File(javaHome + File.separatorChar + "bin" - + File.separatorChar + "java" + fileExtension); - if (javaFile.exists() && javaFile.canExecute()) { - javaCmd = javaFile.getAbsolutePath(); - } - } - } - - try { - final ApplicationPropertyHandler propertyHandler = new SecurityApplicationPropertyHandler(cmdLogger); - final Path applicationPropertiesLocation = Paths.get(nifiPropsFilename); - propertyHandler.handleProperties(applicationPropertiesLocation); - } catch (final RuntimeException e) { - cmdLogger.error("Self-Signed Certificate Generation Failed", e); - } - - final String listenPortPropString = props.get(NIFI_BOOTSTRAP_LISTEN_PORT_PROP); - int listenPortPropInt = 0; // default to zero (random ephemeral port) - if (listenPortPropString != null) { - try { - listenPortPropInt = Integer.parseInt(listenPortPropString.trim()); - } catch (final Exception e) { - // no-op, use the default - } - } - - final NiFiListener listener = new NiFiListener(); - final int listenPort = listener.start(this, listenPortPropInt); - - final List cmd = new ArrayList<>(); - - cmd.add(javaCmd); - cmd.add("-classpath"); - cmd.add(classPath); - cmd.addAll(javaAdditionalArgs); - - cmd.add("-Dnifi.properties.file.path=" + nifiPropsFilename); - cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort); - cmd.add("-Dapp=NiFi"); - cmd.add("-Dorg.apache.nifi.bootstrap.config.log.dir=" + nifiLogDir); - cmd.add("org.apache.nifi.NiFi"); - - builder.command(cmd); - - final StringBuilder cmdBuilder = new StringBuilder(); - for (final String s : cmd) { - cmdBuilder.append(s).append(" "); - } - - cmdLogger.info("Starting Apache NiFi..."); - cmdLogger.info("Working Directory: {}", workingDir.getAbsolutePath()); - cmdLogger.info("Command: {}", cmdBuilder); - - String gracefulShutdown = props.get(GRACEFUL_SHUTDOWN_PROP); - if (gracefulShutdown == null) { - gracefulShutdown = DEFAULT_GRACEFUL_SHUTDOWN_VALUE; - } - - final int gracefulShutdownSeconds; - try { - gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown); - } catch (final NumberFormatException nfe) { - throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File " - + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); - } - - if (gracefulShutdownSeconds < 0) { - throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File " - + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); - } - - Process process = builder.start(); - handleLogging(process); - nifiPid = process.pid(); - final Properties pidProperties = new Properties(); - pidProperties.setProperty(PID_KEY, String.valueOf(nifiPid)); - savePidProperties(pidProperties, cmdLogger); - cmdLogger.info("Application Process [{}] launched", nifiPid); - - shutdownHook = new ShutdownHook(process, nifiPid, this, secretKey, gracefulShutdownSeconds, loggingExecutor); - - if (monitor) { - final Runtime runtime = Runtime.getRuntime(); - runtime.addShutdownHook(shutdownHook); - - while (true) { - final boolean alive = isAlive(process); - - if (alive) { - try { - Thread.sleep(1000L); - } catch (final InterruptedException ie) { - } - } else { - try { - runtime.removeShutdownHook(shutdownHook); - } catch (final IllegalStateException ise) { - // happens when already shutting down - } - - if (autoRestartNiFi) { - final File statusFile = getStatusFile(defaultLogger); - if (!statusFile.exists()) { - defaultLogger.info("Status File no longer exists. Will not restart NiFi"); - return; - } - - final File lockFile = getLockFile(defaultLogger); - if (lockFile.exists()) { - defaultLogger.info("A shutdown was initiated. Will not restart NiFi"); - return; - } - - final boolean previouslyStarted = getNifiStarted(); - if (!previouslyStarted) { - defaultLogger.info("NiFi never started. Will not restart NiFi"); - return; - } else { - setNiFiStarted(false); - } - - defaultLogger.warn("Apache NiFi appears to have died. Restarting..."); - secretKey = null; - process = builder.start(); - handleLogging(process); - - nifiPid = process.pid(); - pidProperties.setProperty(PID_KEY, String.valueOf(nifiPid)); - savePidProperties(pidProperties, defaultLogger); - cmdLogger.info("Application Process [{}] launched", nifiPid); - - shutdownHook = new ShutdownHook(process, nifiPid, this, secretKey, gracefulShutdownSeconds, loggingExecutor); - runtime.addShutdownHook(shutdownHook); - - final boolean started = waitForStart(); - - if (started) { - cmdLogger.info("Application Process [{}] started", nifiPid); - } else { - defaultLogger.error("Application Process [{}] not started", nifiPid); - } - } else { - return; - } - } - } - } - } - - private String getPlatformDetails(final String minimumHeapSize, final String maximumHeapSize) { - final Map details = new LinkedHashMap(6); - - details.put("javaVersion", System.getProperty("java.version")); - details.put("availableProcessors", Integer.toString(Runtime.getRuntime().availableProcessors())); - - try { - final ObjectName osObjectName = ManagementFactory.getOperatingSystemMXBean().getObjectName(); - final MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); - final Object maxOpenFileCount = mBeanServer.getAttribute(osObjectName, "MaxFileDescriptorCount"); - if (maxOpenFileCount != null) { - details.put("maxOpenFileDescriptors", String.valueOf(maxOpenFileCount)); - } - final Object totalPhysicalMemory = mBeanServer.getAttribute(osObjectName, "TotalPhysicalMemorySize"); - if (totalPhysicalMemory != null) { - details.put("totalPhysicalMemoryMB", String.valueOf(((Long) totalPhysicalMemory) / (1024 * 1024))); - } - } catch (final Throwable t) { - // Ignore. This will throw either ClassNotFound or NoClassDefFoundError if unavailable in this JVM. - } - - if (minimumHeapSize != null) { - details.put("minimumHeapSize", minimumHeapSize); - } - if (maximumHeapSize != null) { - details.put("maximumHeapSize", maximumHeapSize); - } - - return details.toString(); - } - - private void handleLogging(final Process process) { - final Set> existingFutures = loggingFutures; - if (existingFutures != null) { - for (final Future future : existingFutures) { - future.cancel(false); - } - } - - final Future stdOutFuture = loggingExecutor.submit(new Runnable() { - @Override - public void run() { - final Logger stdOutLogger = LoggerFactory.getLogger("org.apache.nifi.StdOut"); - final InputStream in = process.getInputStream(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line; - while ((line = reader.readLine()) != null) { - stdOutLogger.info(line); - } - } catch (IOException e) { - defaultLogger.error("Failed to read from NiFi's Standard Out stream", e); - } - } - }); - - final Future stdErrFuture = loggingExecutor.submit(new Runnable() { - @Override - public void run() { - final Logger stdErrLogger = LoggerFactory.getLogger("org.apache.nifi.StdErr"); - final InputStream in = process.getErrorStream(); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { - String line; - while ((line = reader.readLine()) != null) { - stdErrLogger.error(line); - } - } catch (IOException e) { - defaultLogger.error("Failed to read from NiFi's Standard Error stream", e); - } - } - }); - - final Set> futures = new HashSet<>(); - futures.add(stdOutFuture); - futures.add(stdErrFuture); - this.loggingFutures = futures; - } - - - private boolean isWindows() { - final String osName = System.getProperty("os.name"); - return osName != null && osName.toLowerCase().contains("win"); - } - - private boolean waitForStart() { - lock.lock(); - try { - final long startTime = System.nanoTime(); - - while (ccPort < 1) { - try { - startupCondition.await(1, TimeUnit.SECONDS); - } catch (final InterruptedException ie) { - return false; - } - - final long waitNanos = System.nanoTime() - startTime; - final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); - if (waitSeconds > STARTUP_WAIT_SECONDS) { - return false; - } - } - } finally { - lock.unlock(); - } - return true; - } - - private File getFile(final String filename, final File workingDir) { - File file = new File(filename); - if (!file.isAbsolute()) { - file = new File(workingDir, filename); - } - - return file; - } - - private String replaceNull(final String value, final String replacement) { - return (value == null) ? replacement : value; - } - - void setAutoRestartNiFi(final boolean restart) { - this.autoRestartNiFi = restart; - } - - void setNiFiCommandControlPort(final int port, final String secretKey) throws IOException { - - if (this.secretKey != null && this.ccPort != UNINITIALIZED_CC_PORT) { - defaultLogger.warn("Blocking attempt to change NiFi command port and secret after they have already been initialized. requestedPort={}", port); - return; - } - - this.ccPort = port; - this.secretKey = secretKey; - - if (shutdownHook != null) { - shutdownHook.setSecretKey(secretKey); - } - - final File statusFile = getStatusFile(defaultLogger); - - final Properties nifiProps = new Properties(); - if (nifiPid != -1) { - nifiProps.setProperty(PID_KEY, String.valueOf(nifiPid)); - } - nifiProps.setProperty("port", String.valueOf(ccPort)); - nifiProps.setProperty("secret.key", secretKey); - - try { - savePidProperties(nifiProps, defaultLogger); - } catch (final IOException ioe) { - defaultLogger.warn("Apache NiFi has started but failed to persist NiFi Port information to {} due to {}", statusFile.getAbsolutePath(), ioe); - } - - defaultLogger.info("Apache NiFi now running and listening for Bootstrap requests on port {}", port); - } - - int getNiFiCommandControlPort() { - return this.ccPort; - } - - void setNiFiStarted(final boolean nifiStarted) { - startedLock.lock(); - try { - this.nifiStarted = nifiStarted; - } finally { - startedLock.unlock(); - } - } - - boolean getNifiStarted() { - startedLock.lock(); - try { - return nifiStarted; - } finally { - startedLock.unlock(); - } - } - - private static class Status { - - private final Integer port; - private final String pid; - - private final Boolean respondingToPing; - private final Boolean processRunning; - - public Status(final Integer port, final String pid, final Boolean respondingToPing, final Boolean processRunning) { - this.port = port; - this.pid = pid; - this.respondingToPing = respondingToPing; - this.processRunning = processRunning; - } - - public String getPid() { - return pid; - } - - public Integer getPort() { - return port; - } - - public boolean isRespondingToPing() { - return Boolean.TRUE.equals(respondingToPing); - } - - public boolean isProcessRunning() { - return Boolean.TRUE.equals(processRunning); - } - } - - private static class NiFiNotRunningException extends Exception { - @Override - public synchronized Throwable fillInStackTrace() { - return this; - } - } -} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java deleted file mode 100644 index 502834c3c0c9..000000000000 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.bootstrap; - -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; - -public class ShutdownHook extends Thread { - - private final Process nifiProcess; - private final Long pid; - private final RunNiFi runner; - private final int gracefulShutdownSeconds; - private final ExecutorService executor; - - private volatile String secretKey; - - public ShutdownHook(final Process nifiProcess, final Long pid, final RunNiFi runner, final String secretKey, final int gracefulShutdownSeconds, final ExecutorService executor) { - this.nifiProcess = nifiProcess; - this.pid = pid; - this.runner = runner; - this.secretKey = secretKey; - this.gracefulShutdownSeconds = gracefulShutdownSeconds; - this.executor = executor; - } - - void setSecretKey(final String secretKey) { - this.secretKey = secretKey; - } - - @Override - public void run() { - executor.shutdown(); - runner.setAutoRestartNiFi(false); - final int ccPort = runner.getNiFiCommandControlPort(); - if (ccPort > 0) { - System.out.printf("NiFi PID [%d] shutdown started%n", pid); - - try { - final Socket socket = new Socket("localhost", ccPort); - final OutputStream out = socket.getOutputStream(); - out.write(("SHUTDOWN " + secretKey + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - - socket.close(); - } catch (final IOException ioe) { - System.out.println("Failed to Shutdown NiFi due to " + ioe); - } - } - - System.out.printf("NiFi PID [%d] shutdown in progress...%n", pid); - final long startWait = System.nanoTime(); - while (RunNiFi.isAlive(nifiProcess)) { - final long waitNanos = System.nanoTime() - startWait; - final long waitSeconds = TimeUnit.NANOSECONDS.toSeconds(waitNanos); - if (waitSeconds >= gracefulShutdownSeconds && gracefulShutdownSeconds > 0) { - if (RunNiFi.isAlive(nifiProcess)) { - System.out.println("NiFi has not finished shutting down after " + gracefulShutdownSeconds + " seconds. Killing process."); - nifiProcess.destroy(); - } - break; - } else { - try { - Thread.sleep(1000L); - } catch (final InterruptedException ie) { - } - } - } - - try { - final File statusFile = runner.getStatusFile(); - if (!statusFile.delete()) { - System.err.println("Failed to delete status file " + statusFile.getAbsolutePath() + "; this file should be cleaned up manually"); - } - } catch (IOException ex) { - System.err.println("Failed to retrieve status file " + ex); - } - } -} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommand.java new file mode 100644 index 000000000000..6abbd97af167 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommand.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; +import java.util.Optional; + +/** + * Bootstrap Command to return the status of the application process as a child of the bootstrap process + */ +class ApplicationProcessStatusBootstrapCommand implements BootstrapCommand { + + private static final Logger logger = LoggerFactory.getLogger(ApplicationProcessStatusBootstrapCommand.class); + + private final ProcessHandle processHandle; + + private CommandStatus commandStatus = CommandStatus.ERROR; + + ApplicationProcessStatusBootstrapCommand(final ProcessHandle processHandle) { + this.processHandle = Objects.requireNonNull(processHandle); + } + + @Override + public CommandStatus getCommandStatus() { + return commandStatus; + } + + @Override + public void run() { + final Optional childProcessHandleFound = processHandle.children().findFirst(); + + if (childProcessHandleFound.isEmpty()) { + logger.info("Application Process not found"); + commandStatus = CommandStatus.STOPPED; + } else { + final ProcessHandle childProcessHandle = childProcessHandleFound.get(); + if (childProcessHandle.isAlive()) { + logger.info("Application Process [{}] running", childProcessHandle.pid()); + commandStatus = CommandStatus.SUCCESS; + } else { + logger.info("Application Process [{}] stopped", childProcessHandle.pid()); + commandStatus = CommandStatus.COMMUNICATION_FAILED; + } + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommand.java new file mode 100644 index 000000000000..25833a3ac968 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommand.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +/** + * Bootstrap Command extension of Runnable with command status + */ +public interface BootstrapCommand extends Runnable { + /** + * Get Command Status on completion + * + * @return Command Status + */ + CommandStatus getCommandStatus(); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommandProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommandProvider.java new file mode 100644 index 000000000000..18a0f2191aae --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/BootstrapCommandProvider.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +/** + * Abstraction for parsing arguments and returning runnable Bootstrap Command + */ +public interface BootstrapCommandProvider { + /** + * Get Bootstrap Command + * + * @param arguments Application arguments + * @return Bootstrap Command to run + */ + BootstrapCommand getBootstrapCommand(String[] arguments); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/CommandStatus.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/CommandStatus.java new file mode 100644 index 000000000000..b2d050f0f974 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/CommandStatus.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +/** + * Enumeration of Bootstrap Command Statuses with status codes + */ +public enum CommandStatus { + RUNNING(-1), + + SUCCESS(0), + + ERROR(1), + + STOPPED(3), + + COMMUNICATION_FAILED(4), + + FAILED(5); + + private final int status; + + CommandStatus(final int status) { + this.status = status; + } + + /** + * Get Status Code for use with System.exit() + * + * @return Status Code + */ + public int getStatus() { + return status; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommand.java new file mode 100644 index 000000000000..b854d6ded554 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommand.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.apache.nifi.bootstrap.command.process.ManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.ProcessBuilderProvider; +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.command.process.StandardManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.StandardProcessBuilderProvider; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.apache.nifi.bootstrap.property.ApplicationPropertyHandler; +import org.apache.nifi.bootstrap.property.SecurityApplicationPropertyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.PrintStream; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Bootstrap Command to get the command arguments for running the application + */ +class GetRunCommandBootstrapCommand implements BootstrapCommand { + + private static final String SPACE_SEPARATOR = " "; + + private static final Logger logger = LoggerFactory.getLogger(GetRunCommandBootstrapCommand.class); + + private final ConfigurationProvider configurationProvider; + + private final ProcessHandleProvider processHandleProvider; + + private final ManagementServerAddressProvider managementServerAddressProvider; + + private final PrintStream outputStream; + + private CommandStatus commandStatus = CommandStatus.ERROR; + + public GetRunCommandBootstrapCommand(final ConfigurationProvider configurationProvider, final ProcessHandleProvider processHandleProvider, final PrintStream outputStream) { + this.configurationProvider = Objects.requireNonNull(configurationProvider); + this.processHandleProvider = Objects.requireNonNull(processHandleProvider); + this.outputStream = Objects.requireNonNull(outputStream); + this.managementServerAddressProvider = new StandardManagementServerAddressProvider(configurationProvider); + } + + @Override + public CommandStatus getCommandStatus() { + return commandStatus; + } + + @Override + public void run() { + try { + final Optional applicationProcessHandle = processHandleProvider.findApplicationProcessHandle(); + + if (applicationProcessHandle.isEmpty()) { + final ApplicationPropertyHandler securityApplicationPropertyHandler = new SecurityApplicationPropertyHandler(logger); + securityApplicationPropertyHandler.handleProperties(configurationProvider.getApplicationProperties()); + + final ProcessBuilderProvider processBuilderProvider = new StandardProcessBuilderProvider(configurationProvider, managementServerAddressProvider); + final ProcessBuilder processBuilder = processBuilderProvider.getApplicationProcessBuilder(); + final List command = processBuilder.command(); + final String processCommand = String.join(SPACE_SEPARATOR, command); + outputStream.println(processCommand); + + commandStatus = CommandStatus.SUCCESS; + } else { + logger.info("Application Process [{}] running", applicationProcessHandle.get().pid()); + commandStatus = CommandStatus.ERROR; + } + } catch (final Throwable e) { + logger.warn("Application Process command building failed", e); + commandStatus = CommandStatus.FAILED; + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ManagementServerBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ManagementServerBootstrapCommand.java new file mode 100644 index 000000000000..e462249ac2eb --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/ManagementServerBootstrapCommand.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.apache.nifi.bootstrap.command.io.HttpRequestMethod; +import org.apache.nifi.bootstrap.command.io.ResponseStreamHandler; +import org.apache.nifi.bootstrap.command.process.ManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.ProcessHandleManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.apache.nifi.bootstrap.configuration.ManagementServerPath; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; + +/** + * Sequence of Bootstrap Commands + */ +class ManagementServerBootstrapCommand implements BootstrapCommand { + + private static final Logger commandLogger = LoggerFactory.getLogger(ApplicationClassName.BOOTSTRAP_COMMAND.getName()); + + private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(5); + + private static final Duration READ_TIMEOUT = Duration.ofSeconds(15); + + private static final String SERVER_URI = "http://%s%s"; + + private static final char QUERY_SEPARATOR = '?'; + + private final ProcessHandleProvider processHandleProvider; + + private final HttpRequestMethod httpRequestMethod; + + private final ManagementServerPath managementServerPath; + + private final String managementServerQuery; + + private final int successStatusCode; + + private final ResponseStreamHandler responseStreamHandler; + + private CommandStatus commandStatus = CommandStatus.ERROR; + + ManagementServerBootstrapCommand( + final ProcessHandleProvider processHandleProvider, + final ManagementServerPath managementServerPath, + final ResponseStreamHandler responseStreamHandler + ) { + this(processHandleProvider, HttpRequestMethod.GET, managementServerPath, null, HttpURLConnection.HTTP_OK, responseStreamHandler); + } + + ManagementServerBootstrapCommand( + final ProcessHandleProvider processHandleProvider, + final HttpRequestMethod httpRequestMethod, + final ManagementServerPath managementServerPath, + final String managementServerQuery, + final int successStatusCode, + final ResponseStreamHandler responseStreamHandler + ) { + this.processHandleProvider = Objects.requireNonNull(processHandleProvider); + this.httpRequestMethod = Objects.requireNonNull(httpRequestMethod); + this.managementServerPath = Objects.requireNonNull(managementServerPath); + this.managementServerQuery = managementServerQuery; + this.successStatusCode = successStatusCode; + this.responseStreamHandler = Objects.requireNonNull(responseStreamHandler); + } + + @Override + public CommandStatus getCommandStatus() { + return commandStatus; + } + + @Override + public void run() { + final Optional applicationProcessHandle = processHandleProvider.findApplicationProcessHandle(); + + if (applicationProcessHandle.isEmpty()) { + commandStatus = CommandStatus.STOPPED; + getCommandLogger().info("Application Process STOPPED"); + } else { + run(applicationProcessHandle.get()); + } + } + + protected void run(final ProcessHandle applicationProcessHandle) { + final ManagementServerAddressProvider managementServerAddressProvider = new ProcessHandleManagementServerAddressProvider(applicationProcessHandle); + final Optional managementServerAddress = managementServerAddressProvider.getAddress(); + + final long pid = applicationProcessHandle.pid(); + if (managementServerAddress.isEmpty()) { + getCommandLogger().info("Application Process [{}] Management Server address not found", pid); + commandStatus = CommandStatus.ERROR; + } else { + final URI managementServerUri = getManagementServerUri(managementServerAddress.get()); + try (HttpClient httpClient = getHttpClient()) { + final HttpRequest httpRequest = getHttpRequest(managementServerUri); + + final HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofInputStream()); + final int statusCode = response.statusCode(); + try (InputStream responseStream = response.body()) { + onResponseStatus(applicationProcessHandle, statusCode, responseStream); + } + } catch (final Exception e) { + commandStatus = CommandStatus.COMMUNICATION_FAILED; + getCommandLogger().info("Application Process [{}] Management Server [{}] communication failed", pid, managementServerUri); + } + } + } + + protected void onResponseStatus(final ProcessHandle applicationProcessHandle, final int statusCode, final InputStream responseStream) { + final long pid = applicationProcessHandle.pid(); + + if (successStatusCode == statusCode) { + commandStatus = CommandStatus.SUCCESS; + getCommandLogger().info("Application Process [{}] Command Status [{}] HTTP {}", pid, commandStatus, statusCode); + responseStreamHandler.onResponseStream(responseStream); + } else { + commandStatus = CommandStatus.COMMUNICATION_FAILED; + getCommandLogger().warn("Application Process [{}] Command Status [{}] HTTP {}", pid, commandStatus, statusCode); + } + } + + protected Logger getCommandLogger() { + return commandLogger; + } + + protected HttpRequest getHttpRequest(final URI managementServerUri) { + return HttpRequest.newBuilder() + .method(httpRequestMethod.name(), HttpRequest.BodyPublishers.noBody()) + .uri(managementServerUri) + .timeout(READ_TIMEOUT) + .build(); + } + + protected URI getManagementServerUri(final String managementServerAddress) { + final StringBuilder builder = new StringBuilder(); + + final String serverUri = SERVER_URI.formatted(managementServerAddress, managementServerPath.getPath()); + builder.append(serverUri); + + if (managementServerQuery != null) { + builder.append(QUERY_SEPARATOR); + builder.append(managementServerQuery); + } + + return URI.create(builder.toString()); + } + + protected HttpClient getHttpClient() { + final HttpClient.Builder builder = HttpClient.newBuilder(); + builder.connectTimeout(CONNECT_TIMEOUT); + return builder.build(); + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/RunBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/RunBootstrapCommand.java new file mode 100644 index 000000000000..a2be6549faca --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/RunBootstrapCommand.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import com.sun.management.UnixOperatingSystemMXBean; +import org.apache.nifi.bootstrap.command.process.ManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.ProcessBuilderProvider; +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.command.process.StandardManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.StandardProcessBuilderProvider; +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.apache.nifi.bootstrap.process.RuntimeValidatorExecutor; +import org.apache.nifi.bootstrap.property.ApplicationPropertyHandler; +import org.apache.nifi.bootstrap.property.SecurityApplicationPropertyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; +import java.util.Objects; +import java.util.Optional; + +/** + * Bootstrap Command to run the application + */ +class RunBootstrapCommand implements BootstrapCommand { + + private static final String SPACE_SEPARATOR = " "; + + private static final Logger commandLogger = LoggerFactory.getLogger(ApplicationClassName.BOOTSTRAP_COMMAND.getName()); + + private static final Logger logger = LoggerFactory.getLogger(RunBootstrapCommand.class); + + private static final RuntimeValidatorExecutor runtimeValidatorExecutor = new RuntimeValidatorExecutor(); + + private final ConfigurationProvider configurationProvider; + + private final ProcessHandleProvider processHandleProvider; + + private final ManagementServerAddressProvider managementServerAddressProvider; + + private CommandStatus commandStatus = CommandStatus.ERROR; + + public RunBootstrapCommand(final ConfigurationProvider configurationProvider, final ProcessHandleProvider processHandleProvider) { + this.configurationProvider = Objects.requireNonNull(configurationProvider); + this.processHandleProvider = Objects.requireNonNull(processHandleProvider); + this.managementServerAddressProvider = new StandardManagementServerAddressProvider(configurationProvider); + } + + @Override + public CommandStatus getCommandStatus() { + return commandStatus; + } + + @Override + public void run() { + try { + final Optional applicationProcessHandle = processHandleProvider.findApplicationProcessHandle(); + + if (applicationProcessHandle.isEmpty()) { + writePlatformProperties(); + + runtimeValidatorExecutor.execute(); + + final ApplicationPropertyHandler securityApplicationPropertyHandler = new SecurityApplicationPropertyHandler(logger); + securityApplicationPropertyHandler.handleProperties(configurationProvider.getApplicationProperties()); + + final ProcessBuilderProvider processBuilderProvider = new StandardProcessBuilderProvider(configurationProvider, managementServerAddressProvider); + + final ProcessBuilder processBuilder = processBuilderProvider.getApplicationProcessBuilder(); + processBuilder.inheritIO(); + + final String command = String.join(SPACE_SEPARATOR, processBuilder.command()); + logger.info(command); + + final Process process = processBuilder.start(); + if (process.isAlive()) { + commandStatus = CommandStatus.SUCCESS; + commandLogger.info("Application Process [{}] started", process.pid()); + } else { + commandStatus = CommandStatus.STOPPED; + commandLogger.error("Application Process [{}] start failed", process.pid()); + } + } else { + commandLogger.info("Application Process [{}] running", applicationProcessHandle.get().pid()); + commandStatus = CommandStatus.ERROR; + } + } catch (final Throwable e) { + commandLogger.warn("Application Process run failed", e); + commandStatus = CommandStatus.FAILED; + } + } + + private void writePlatformProperties() { + final Runtime.Version version = Runtime.version(); + logger.info("Java Version: {}", version); + + final Runtime runtime = Runtime.getRuntime(); + logger.info("Available Processors: {}", runtime.availableProcessors()); + + final OperatingSystemMXBean operatingSystem = ManagementFactory.getOperatingSystemMXBean(); + if (operatingSystem instanceof UnixOperatingSystemMXBean unixOperatingSystem) { + logger.info("Total Memory: {}", unixOperatingSystem.getTotalMemorySize()); + logger.info("Maximum File Descriptors: {}", unixOperatingSystem.getMaxFileDescriptorCount()); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/SequenceBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/SequenceBootstrapCommand.java new file mode 100644 index 000000000000..f3d11c1cc2f1 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/SequenceBootstrapCommand.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import java.util.List; +import java.util.Objects; + +/** + * Sequence of Bootstrap Commands + */ +class SequenceBootstrapCommand implements BootstrapCommand { + + private final List bootstrapCommands; + + private CommandStatus commandStatus = CommandStatus.ERROR; + + SequenceBootstrapCommand(final List bootstrapCommands) { + this.bootstrapCommands = Objects.requireNonNull(bootstrapCommands); + } + + @Override + public CommandStatus getCommandStatus() { + return commandStatus; + } + + @Override + public void run() { + for (final BootstrapCommand bootstrapCommand : bootstrapCommands) { + bootstrapCommand.run(); + commandStatus = bootstrapCommand.getCommandStatus(); + + if (CommandStatus.SUCCESS != commandStatus) { + break; + } + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProvider.java new file mode 100644 index 000000000000..bfaada97b454 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProvider.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.apache.nifi.bootstrap.command.io.BootstrapArgument; +import org.apache.nifi.bootstrap.command.io.BootstrapArgumentParser; +import org.apache.nifi.bootstrap.command.io.FileResponseStreamHandler; +import org.apache.nifi.bootstrap.command.io.LoggerResponseStreamHandler; +import org.apache.nifi.bootstrap.command.io.ResponseStreamHandler; +import org.apache.nifi.bootstrap.command.io.StandardBootstrapArgumentParser; +import org.apache.nifi.bootstrap.command.process.StandardProcessHandleProvider; +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.apache.nifi.bootstrap.configuration.StandardConfigurationProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static java.net.HttpURLConnection.HTTP_ACCEPTED; +import static java.net.HttpURLConnection.HTTP_OK; +import static org.apache.nifi.bootstrap.command.io.HttpRequestMethod.DELETE; +import static org.apache.nifi.bootstrap.command.io.HttpRequestMethod.GET; +import static org.apache.nifi.bootstrap.configuration.ManagementServerPath.HEALTH; +import static org.apache.nifi.bootstrap.configuration.ManagementServerPath.HEALTH_CLUSTER; +import static org.apache.nifi.bootstrap.configuration.ManagementServerPath.HEALTH_DIAGNOSTICS; +import static org.apache.nifi.bootstrap.configuration.ManagementServerPath.HEALTH_STATUS_HISTORY; + +/** + * Standard implementation of Bootstrap Command Provider with parsing of supported commands + */ +public class StandardBootstrapCommandProvider implements BootstrapCommandProvider { + private static final String SHUTDOWN_REQUESTED = "--shutdown=true"; + + private static final String VERBOSE_REQUESTED = "--verbose"; + + private static final String VERBOSE_QUERY = "verbose=true"; + + private static final String DAYS_QUERY = "days=%d"; + + private static final String EMPTY_QUERY = null; + + private static final int FIRST_ARGUMENT = 1; + + private static final int SECOND_ARGUMENT = 2; + + private static final int PATH_ARGUMENTS = 2; + + private static final int DAYS_PATH_ARGUMENTS = 3; + + private static final int DAYS_REQUESTED_DEFAULT = 1; + + private static final BootstrapArgumentParser bootstrapArgumentParser = new StandardBootstrapArgumentParser(); + + private static final Logger commandLogger = LoggerFactory.getLogger(ApplicationClassName.BOOTSTRAP_COMMAND.getName()); + + /** + * Get Bootstrap Command + * + * @param arguments Application arguments + * @return Bootstrap Command to run + */ + @Override + public BootstrapCommand getBootstrapCommand(final String[] arguments) { + final BootstrapCommand bootstrapCommand; + + final Optional bootstrapArgumentFound = bootstrapArgumentParser.getBootstrapArgument(arguments); + if (bootstrapArgumentFound.isPresent()) { + final BootstrapArgument bootstrapArgument = bootstrapArgumentFound.get(); + bootstrapCommand = getBootstrapCommand(bootstrapArgument, arguments); + } else { + bootstrapCommand = new UnknownBootstrapCommand(); + } + + return bootstrapCommand; + } + + private BootstrapCommand getBootstrapCommand(final BootstrapArgument bootstrapArgument, final String[] arguments) { + final ConfigurationProvider configurationProvider = new StandardConfigurationProvider(System.getenv(), System.getProperties()); + final ProcessHandleProvider processHandleProvider = new StandardProcessHandleProvider(configurationProvider); + final ResponseStreamHandler commandLoggerStreamHandler = new LoggerResponseStreamHandler(commandLogger); + final BootstrapCommand stopBootstrapCommand = new StopBootstrapCommand(processHandleProvider, configurationProvider); + + final BootstrapCommand bootstrapCommand; + + if (BootstrapArgument.CLUSTER_STATUS == bootstrapArgument) { + bootstrapCommand = new ManagementServerBootstrapCommand(processHandleProvider, HEALTH_CLUSTER, commandLoggerStreamHandler); + } else if (BootstrapArgument.DECOMMISSION == bootstrapArgument) { + bootstrapCommand = getDecommissionCommand(processHandleProvider, stopBootstrapCommand, arguments); + } else if (BootstrapArgument.DIAGNOSTICS == bootstrapArgument) { + bootstrapCommand = getDiagnosticsCommand(processHandleProvider, arguments); + } else if (BootstrapArgument.GET_RUN_COMMAND == bootstrapArgument) { + bootstrapCommand = new GetRunCommandBootstrapCommand(configurationProvider, processHandleProvider, System.out); + } else if (BootstrapArgument.START == bootstrapArgument) { + final BootstrapCommand runBootstrapCommand = new RunBootstrapCommand(configurationProvider, processHandleProvider); + final ProcessHandle currentProcessHandle = ProcessHandle.current(); + final BootstrapCommand statusBootstrapCommand = new ApplicationProcessStatusBootstrapCommand(currentProcessHandle); + bootstrapCommand = new StartBootstrapCommand(runBootstrapCommand, statusBootstrapCommand); + } else if (BootstrapArgument.STATUS == bootstrapArgument) { + bootstrapCommand = new ManagementServerBootstrapCommand(processHandleProvider, HEALTH, commandLoggerStreamHandler); + } else if (BootstrapArgument.STATUS_HISTORY == bootstrapArgument) { + bootstrapCommand = getStatusHistoryCommand(processHandleProvider, arguments); + } else if (BootstrapArgument.STOP == bootstrapArgument) { + bootstrapCommand = stopBootstrapCommand; + } else { + bootstrapCommand = new UnknownBootstrapCommand(); + } + + return bootstrapCommand; + } + + private BootstrapCommand getDecommissionCommand(final ProcessHandleProvider processHandleProvider, final BootstrapCommand stopBootstrapCommand, final String[] arguments) { + final ResponseStreamHandler responseStreamHandler = new LoggerResponseStreamHandler(commandLogger); + final List bootstrapCommands = new ArrayList<>(); + final BootstrapCommand decommissionCommand = new ManagementServerBootstrapCommand(processHandleProvider, DELETE, HEALTH_CLUSTER, EMPTY_QUERY, HTTP_ACCEPTED, responseStreamHandler); + bootstrapCommands.add(decommissionCommand); + if (isShutdownRequested(arguments)) { + bootstrapCommands.add(stopBootstrapCommand); + } + return new SequenceBootstrapCommand(bootstrapCommands); + } + + private BootstrapCommand getDiagnosticsCommand(final ProcessHandleProvider processHandleProvider, final String[] arguments) { + final String verboseQuery = getVerboseQuery(arguments); + final ResponseStreamHandler responseStreamHandler = getDiagnosticsResponseStreamHandler(arguments); + return new ManagementServerBootstrapCommand(processHandleProvider, GET, HEALTH_DIAGNOSTICS, verboseQuery, HTTP_OK, responseStreamHandler); + } + + private ResponseStreamHandler getDiagnosticsResponseStreamHandler(final String[] arguments) { + final ResponseStreamHandler responseStreamHandler; + + if (arguments.length == PATH_ARGUMENTS) { + final String outputPathArgument = arguments[FIRST_ARGUMENT]; + final Path outputPath = Paths.get(outputPathArgument); + responseStreamHandler = new FileResponseStreamHandler(outputPath); + } else { + final Logger logger = LoggerFactory.getLogger(StandardBootstrapCommandProvider.class); + responseStreamHandler = new LoggerResponseStreamHandler(logger); + } + + return responseStreamHandler; + } + + private BootstrapCommand getStatusHistoryCommand(final ProcessHandleProvider processHandleProvider, final String[] arguments) { + final String daysQuery = getStatusHistoryDaysQuery(arguments); + final ResponseStreamHandler responseStreamHandler = getStatusHistoryResponseStreamHandler(arguments); + return new ManagementServerBootstrapCommand(processHandleProvider, GET, HEALTH_STATUS_HISTORY, daysQuery, HTTP_OK, responseStreamHandler); + } + + private boolean isShutdownRequested(final String[] arguments) { + boolean shutdownRequested = false; + + for (final String argument : arguments) { + if (SHUTDOWN_REQUESTED.contentEquals(argument)) { + shutdownRequested = true; + break; + } + } + + return shutdownRequested; + } + + private String getVerboseQuery(final String[] arguments) { + String query = null; + + for (final String argument : arguments) { + if (VERBOSE_REQUESTED.contentEquals(argument)) { + query = VERBOSE_QUERY; + break; + } + } + + return query; + } + + private String getStatusHistoryDaysQuery(final String[] arguments) { + final int daysRequested; + + if (arguments.length == DAYS_PATH_ARGUMENTS) { + final String daysRequestArgument = arguments[FIRST_ARGUMENT]; + daysRequested = getStatusHistoryDaysRequested(daysRequestArgument); + } else { + daysRequested = DAYS_REQUESTED_DEFAULT; + } + + return DAYS_QUERY.formatted(daysRequested); + } + + private int getStatusHistoryDaysRequested(final String daysRequestArgument) { + int daysRequested; + + try { + daysRequested = Integer.parseInt(daysRequestArgument); + } catch (final NumberFormatException e) { + throw new IllegalArgumentException("Status History Days requested not valid"); + } + + return daysRequested; + } + + private ResponseStreamHandler getStatusHistoryResponseStreamHandler(final String[] arguments) { + final ResponseStreamHandler responseStreamHandler; + + if (arguments.length == PATH_ARGUMENTS) { + final String outputPathArgument = arguments[FIRST_ARGUMENT]; + final Path outputPath = Paths.get(outputPathArgument); + responseStreamHandler = new FileResponseStreamHandler(outputPath); + } else if (arguments.length == DAYS_PATH_ARGUMENTS) { + final String outputPathArgument = arguments[SECOND_ARGUMENT]; + final Path outputPath = Paths.get(outputPathArgument); + responseStreamHandler = new FileResponseStreamHandler(outputPath); + } else { + final Logger logger = LoggerFactory.getLogger(StandardBootstrapCommandProvider.class); + responseStreamHandler = new LoggerResponseStreamHandler(logger); + } + + return responseStreamHandler; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StartBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StartBootstrapCommand.java new file mode 100644 index 000000000000..0c49e0800e3b --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StartBootstrapCommand.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Start Bootstrap Command executes the Run Command and monitors status + */ +class StartBootstrapCommand implements BootstrapCommand { + + private static final long MONITOR_INTERVAL = 5; + + private static final Logger logger = LoggerFactory.getLogger(StartBootstrapCommand.class); + + private final BootstrapCommand runCommand; + + private final BootstrapCommand statusCommand; + + private final ScheduledExecutorService scheduledExecutorService; + + private CommandStatus commandStatus = CommandStatus.ERROR; + + StartBootstrapCommand(final BootstrapCommand runCommand, final BootstrapCommand statusCommand) { + this.runCommand = Objects.requireNonNull(runCommand); + this.statusCommand = Objects.requireNonNull(statusCommand); + + this.scheduledExecutorService = Executors.newScheduledThreadPool(1, command -> { + final Thread thread = new Thread(command); + thread.setName(StartBootstrapCommand.class.getSimpleName()); + return thread; + }); + } + + @Override + public CommandStatus getCommandStatus() { + return commandStatus; + } + + @Override + public void run() { + runCommand.run(); + commandStatus = runCommand.getCommandStatus(); + + if (CommandStatus.SUCCESS == commandStatus) { + logger.info("Application watch started"); + final WatchCommand watchCommand = new WatchCommand(); + scheduledExecutorService.scheduleAtFixedRate(watchCommand, MONITOR_INTERVAL, MONITOR_INTERVAL, TimeUnit.SECONDS); + commandStatus = CommandStatus.RUNNING; + } else { + scheduledExecutorService.shutdown(); + } + } + + private class WatchCommand implements Runnable { + + @Override + public void run() { + statusCommand.run(); + final CommandStatus status = statusCommand.getCommandStatus(); + if (CommandStatus.SUCCESS == status) { + logger.debug("Application running"); + } else if (CommandStatus.FAILED == status) { + logger.error("Application watch failed"); + scheduledExecutorService.shutdown(); + logger.info("Application watch stopped"); + commandStatus = CommandStatus.FAILED; + } else { + logger.warn("Application not running [{}]", status); + runCommand.run(); + } + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StopBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StopBootstrapCommand.java new file mode 100644 index 000000000000..3e4f47f6c913 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/StopBootstrapCommand.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** + * Bootstrap Command to run for stopping application + */ +class StopBootstrapCommand implements BootstrapCommand { + + private static final Duration FORCE_TERMINATION_TIMEOUT = Duration.ofSeconds(5); + + private static final Logger logger = LoggerFactory.getLogger(ApplicationClassName.BOOTSTRAP_COMMAND.getName()); + + private final ProcessHandleProvider processHandleProvider; + + private final ConfigurationProvider configurationProvider; + + private CommandStatus commandStatus = CommandStatus.ERROR; + + StopBootstrapCommand(final ProcessHandleProvider processHandleProvider, final ConfigurationProvider configurationProvider) { + this.processHandleProvider = Objects.requireNonNull(processHandleProvider); + this.configurationProvider = Objects.requireNonNull(configurationProvider); + } + + @Override + public CommandStatus getCommandStatus() { + return commandStatus; + } + + @Override + public void run() { + final Optional processHandle = processHandleProvider.findApplicationProcessHandle(); + + if (processHandle.isEmpty()) { + commandStatus = CommandStatus.SUCCESS; + logger.info("Application Process not running"); + } else { + stopBootstrapProcess(); + destroy(processHandle.get()); + } + } + + private void stopBootstrapProcess() { + final Optional bootstrapProcessHandleFound = processHandleProvider.findBootstrapProcessHandle(); + if (bootstrapProcessHandleFound.isPresent()) { + final ProcessHandle bootstrapProcessHandle = bootstrapProcessHandleFound.get(); + + final boolean destroyRequested = bootstrapProcessHandle.destroy(); + final long pid = bootstrapProcessHandle.pid(); + if (destroyRequested) { + logger.info("Bootstrap Process [{}] termination requested", pid); + onBootstrapDestroyCompleted(bootstrapProcessHandle); + } else { + logger.warn("Bootstrap Process [{}] termination request failed", pid); + } + } + } + + private void onBootstrapDestroyCompleted(final ProcessHandle bootstrapProcessHandle) { + final long pid = bootstrapProcessHandle.pid(); + final CompletableFuture onExitHandle = bootstrapProcessHandle.onExit(); + try { + final ProcessHandle completedProcessHandle = onExitHandle.get(FORCE_TERMINATION_TIMEOUT.toSeconds(), TimeUnit.SECONDS); + logger.info("Bootstrap Process [{}] termination completed", completedProcessHandle.pid()); + } catch (final Exception e) { + logger.warn("Bootstrap Process [{}] termination failed", pid); + } + } + + private void destroy(final ProcessHandle applicationProcessHandle) { + final boolean destroyRequested = applicationProcessHandle.destroy(); + logger.info("Application Process [{}] termination requested", applicationProcessHandle.pid()); + if (destroyRequested) { + onDestroyCompleted(applicationProcessHandle); + } else { + logger.warn("Application Process [{}] termination request failed", applicationProcessHandle.pid()); + destroyForcibly(applicationProcessHandle); + } + } + + private void destroyForcibly(final ProcessHandle applicationProcessHandle) { + final boolean destroyForciblyRequested = applicationProcessHandle.destroyForcibly(); + if (destroyForciblyRequested) { + logger.warn("Application Process [{}] force termination failed", applicationProcessHandle.pid()); + } else { + onDestroyForciblyCompleted(applicationProcessHandle); + } + } + + private void onDestroyCompleted(final ProcessHandle applicationProcessHandle) { + final long pid = applicationProcessHandle.pid(); + final CompletableFuture onExitHandle = applicationProcessHandle.onExit(); + final Duration gracefulShutdownTimeout = configurationProvider.getGracefulShutdownTimeout(); + try { + final ProcessHandle completedProcessHandle = onExitHandle.get(gracefulShutdownTimeout.toSeconds(), TimeUnit.SECONDS); + logger.info("Application Process [{}] termination completed", completedProcessHandle.pid()); + commandStatus = CommandStatus.SUCCESS; + } catch (final Exception e) { + logger.warn("Application Process [{}] termination failed", pid); + destroyForcibly(applicationProcessHandle); + } + } + + private void onDestroyForciblyCompleted(final ProcessHandle applicationProcessHandle) { + final long pid = applicationProcessHandle.pid(); + final CompletableFuture onExitHandle = applicationProcessHandle.onExit(); + try { + final ProcessHandle completedProcessHandle = onExitHandle.get(FORCE_TERMINATION_TIMEOUT.toSeconds(), TimeUnit.SECONDS); + logger.warn("Application Process [{}] force termination completed", completedProcessHandle.pid()); + commandStatus = CommandStatus.SUCCESS; + } catch (final Exception e) { + logger.warn("Application Process [{}] force termination request failed", pid); + commandStatus = CommandStatus.ERROR; + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/UnknownBootstrapCommand.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/UnknownBootstrapCommand.java new file mode 100644 index 000000000000..a46b03018dbd --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/UnknownBootstrapCommand.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +/** + * Bootstrap Command to run for an unknown command requested + */ +class UnknownBootstrapCommand implements BootstrapCommand { + + @Override + public CommandStatus getCommandStatus() { + return CommandStatus.ERROR; + } + + @Override + public void run() { + + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgument.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgument.java new file mode 100644 index 000000000000..e1f208661757 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgument.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +/** + * Enumeration of supported arguments for the bootstrap application + */ +public enum BootstrapArgument { + CLUSTER_STATUS, + + DECOMMISSION, + + DIAGNOSTICS, + + GET_RUN_COMMAND, + + STATUS, + + STATUS_HISTORY, + + START, + + STOP +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgumentParser.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgumentParser.java new file mode 100644 index 000000000000..66500f15da4c --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/BootstrapArgumentParser.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +import java.util.Optional; + +/** + * Abstraction for parsing application arguments to Bootstrap Arguments + */ +public interface BootstrapArgumentParser { + /** + * Get Bootstrap Argument from application arguments + * + * @param arguments Application array of arguments + * @return Bootstrap Argument or empty when not found + */ + Optional getBootstrapArgument(String[] arguments); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandler.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandler.java new file mode 100644 index 000000000000..d4127641d72c --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandler.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +/** + * File implementation responsible for reading and transferring responses + */ +public class FileResponseStreamHandler implements ResponseStreamHandler { + private static final Logger logger = LoggerFactory.getLogger(FileResponseStreamHandler.class); + + private final Path outputPath; + + public FileResponseStreamHandler(final Path outputPath) { + this.outputPath = Objects.requireNonNull(outputPath); + } + + @Override + public void onResponseStream(final InputStream responseStream) { + try (OutputStream outputStream = Files.newOutputStream(outputPath)) { + responseStream.transferTo(outputStream); + } catch (final IOException e) { + logger.warn("Write response stream failed for [%s]".formatted(outputPath), e); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFiEntryPoint.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/HttpRequestMethod.java similarity index 81% rename from nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFiEntryPoint.java rename to nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/HttpRequestMethod.java index 386733325e23..e09a5436e67f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFiEntryPoint.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/HttpRequestMethod.java @@ -14,11 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi; +package org.apache.nifi.bootstrap.command.io; -public interface NiFiEntryPoint { - - NiFiServer getServer(); +/** + * Enumeration of supported HTTP Request Methods for Management Server + */ +public enum HttpRequestMethod { + DELETE, - void shutdownHook(boolean isReload); + GET } diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/LoggerResponseStreamHandler.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/LoggerResponseStreamHandler.java new file mode 100644 index 000000000000..b1926cbe5e8d --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/LoggerResponseStreamHandler.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +import org.slf4j.Logger; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Objects; + +/** + * Logger implementation responsible for reading and logging stream of lines + */ +public class LoggerResponseStreamHandler implements ResponseStreamHandler { + private final Logger logger; + + public LoggerResponseStreamHandler(final Logger logger) { + this.logger = Objects.requireNonNull(logger); + } + + @Override + public void onResponseStream(final InputStream responseStream) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(responseStream))) { + reader.lines().forEach(logger::info); + } catch (final IOException e) { + logger.warn("Read response stream failed", e); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/ResponseStreamHandler.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/ResponseStreamHandler.java new file mode 100644 index 000000000000..3cbf08dc09e2 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/ResponseStreamHandler.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +import java.io.InputStream; + +/** + * Response Stream Handler abstraction for reading Management Server responses + */ +public interface ResponseStreamHandler { + /** + * Handle response stream from Management Server + * + * @param responseStream Response Stream + */ + void onResponseStream(InputStream responseStream); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParser.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParser.java new file mode 100644 index 000000000000..10c59b301dab --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParser.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +import java.util.Arrays; +import java.util.Optional; + +/** + * Standard implementation of Bootstrap Argument Parser supporting enumerated arguments as the first element in an array + */ +public class StandardBootstrapArgumentParser implements BootstrapArgumentParser { + private static final char HYPHEN = '-'; + + private static final char UNDERSCORE = '_'; + + /** + * Get Bootstrap Argument from first argument provided + * + * @param arguments Application array of arguments + * @return Bootstrap Argument or empty when not found + */ + @Override + public Optional getBootstrapArgument(final String[] arguments) { + final Optional bootstrapArgumentFound; + + if (arguments == null || arguments.length == 0) { + bootstrapArgumentFound = Optional.empty(); + } else { + final String firstArgument = arguments[0]; + final String formattedArgument = getFormattedArgument(firstArgument); + bootstrapArgumentFound = Arrays.stream(BootstrapArgument.values()) + .filter(bootstrapArgument -> bootstrapArgument.name().equals(formattedArgument)) + .findFirst(); + } + + return bootstrapArgumentFound; + } + + private String getFormattedArgument(final String argument) { + final String upperCased = argument.toUpperCase(); + return upperCased.replace(HYPHEN, UNDERSCORE); + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ManagementServerAddressProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ManagementServerAddressProvider.java new file mode 100644 index 000000000000..9585b15a33bc --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ManagementServerAddressProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import java.util.Optional; + +/** + * Abstraction for producing or locating the Management Server socket address + */ +public interface ManagementServerAddressProvider { + /** + * Get Management Server Address with port number + * + * @return Management Server Address or empty when not found + */ + Optional getAddress(); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessBuilderProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessBuilderProvider.java new file mode 100644 index 000000000000..477b7101272a --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessBuilderProvider.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +/** + * Abstraction for creating a Process Builder + */ +public interface ProcessBuilderProvider { + /** + * Get Application Process Builder + * + * @return Process Builder for Application with command arguments configured + */ + ProcessBuilder getApplicationProcessBuilder(); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProvider.java new file mode 100644 index 000000000000..9a40233369fe --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProvider.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.SystemProperty; + +import java.util.Objects; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Provider implementation resolves the Management Server Address from command arguments of the application Process Handle + */ +public class ProcessHandleManagementServerAddressProvider implements ManagementServerAddressProvider { + private static final Pattern ADDRESS_ARGUMENT_PATTERN = Pattern.compile("^-D%s=(.+?)$".formatted(SystemProperty.MANAGEMENT_SERVER_ADDRESS.getProperty())); + + private static final int ADDRESS_GROUP = 1; + + private final ProcessHandle processHandle; + + public ProcessHandleManagementServerAddressProvider(final ProcessHandle processHandle) { + this.processHandle = Objects.requireNonNull(processHandle); + } + + /** + * Get Management Server Address with port number from command argument in Process Handle + * + * @return Management Server Address or null when not found + */ + @Override + public Optional getAddress() { + final ProcessHandle.Info info = processHandle.info(); + + final String managementServerAddress; + + final Optional argumentsFound = info.arguments(); + if (argumentsFound.isPresent()) { + final String[] arguments = argumentsFound.get(); + managementServerAddress = findManagementServerAddress(arguments); + } else { + managementServerAddress = null; + } + + return Optional.ofNullable(managementServerAddress); + } + + private String findManagementServerAddress(final String[] arguments) { + String managementServerAddress = null; + + for (final String argument : arguments) { + final Matcher matcher = ADDRESS_ARGUMENT_PATTERN.matcher(argument); + if (matcher.matches()) { + managementServerAddress = matcher.group(ADDRESS_GROUP); + break; + } + } + + return managementServerAddress; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleProvider.java new file mode 100644 index 000000000000..e2ba230e7751 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/ProcessHandleProvider.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import java.util.Optional; + +/** + * Abstraction for finding a Process Handle + */ +public interface ProcessHandleProvider { + /** + * Find Application Process Handle + * + * @return Application Process Handle or empty when not found + */ + Optional findApplicationProcessHandle(); + + /** + * Find Bootstrap Process Handle + * + * @return Bootstrap Process Handle or empty when not found + */ + Optional findBootstrapProcessHandle(); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProvider.java new file mode 100644 index 000000000000..7a13b40e6156 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProvider.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; + +import java.io.IOException; +import java.net.ServerSocket; +import java.net.URI; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.stream.IntStream; + +/** + * Standard Provider reads optional Server Address from Configuration Provider or selects from available ports + */ +public class StandardManagementServerAddressProvider implements ManagementServerAddressProvider { + private static final int STANDARD_PORT = 52020; + + private static final int MAXIMUM_PORT = 52050; + + private static final String NOT_AVAILABLE_MESSAGE = "Management Server Port not available in range [%d-%d]".formatted(STANDARD_PORT, MAXIMUM_PORT); + + private static final String LOCALHOST_ADDRESS = "127.0.0.1:%d"; + + private static final String HOST_ADDRESS = "%s:%d"; + + private final ConfigurationProvider configurationProvider; + + public StandardManagementServerAddressProvider(final ConfigurationProvider configurationProvider) { + this.configurationProvider = Objects.requireNonNull(configurationProvider); + } + + /** + * Get Management Server Address with port number + * + * @return Management Server Address + */ + @Override + public Optional getAddress() { + final Optional address; + + final Optional managementServerAddress = configurationProvider.getManagementServerAddress(); + if (managementServerAddress.isPresent()) { + final URI serverAddress = managementServerAddress.get(); + final String hostAddress = HOST_ADDRESS.formatted(serverAddress.getHost(), serverAddress.getPort()); + address = Optional.of(hostAddress); + } else { + final int serverPort = getServerPort(); + address = Optional.of(LOCALHOST_ADDRESS.formatted(serverPort)); + } + + return address; + } + + private int getServerPort() { + final OptionalInt portFound = IntStream.range(STANDARD_PORT, MAXIMUM_PORT) + .filter(StandardManagementServerAddressProvider::isPortFree) + .findFirst(); + + return portFound.orElseThrow(() -> new IllegalStateException(NOT_AVAILABLE_MESSAGE)); + } + + private static boolean isPortFree(final int port) { + try (ServerSocket ignored = new ServerSocket(port)) { + return true; + } catch (final IOException e) { + return false; + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProvider.java new file mode 100644 index 000000000000..f7803bf904cd --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProvider.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.apache.nifi.bootstrap.configuration.SystemProperty; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.BiPredicate; +import java.util.stream.Stream; + +/** + * Standard implementation of Process Builder Provider for constructing application command arguments + */ +public class StandardProcessBuilderProvider implements ProcessBuilderProvider { + private static final String JAR_FILE_EXTENSION = ".jar"; + + private static final BiPredicate JAR_FILE_MATCHER = (path, attributes) -> path.getFileName().toString().endsWith(JAR_FILE_EXTENSION); + + private static final int LIBRARY_JAR_DEPTH = 1; + + private static final String SYSTEM_PROPERTY = "-D%s=%s"; + + private static final String CLASS_PATH_ARGUMENT = "--class-path"; + + private final ConfigurationProvider configurationProvider; + + private final ManagementServerAddressProvider managementServerAddressProvider; + + public StandardProcessBuilderProvider(final ConfigurationProvider configurationProvider, final ManagementServerAddressProvider managementServerAddressProvider) { + this.configurationProvider = Objects.requireNonNull(configurationProvider); + this.managementServerAddressProvider = Objects.requireNonNull(managementServerAddressProvider); + } + + @Override + public ProcessBuilder getApplicationProcessBuilder() { + final ProcessBuilder processBuilder = new ProcessBuilder(); + + final List command = getCommand(); + processBuilder.command(command); + + return processBuilder; + } + + private List getCommand() { + final List command = new ArrayList<>(); + + final ProcessHandle.Info currentProcessHandleInfo = ProcessHandle.current().info(); + final String currentProcessCommand = getCurrentProcessCommand(currentProcessHandleInfo); + command.add(currentProcessCommand); + + final String classPath = getClassPath(); + command.add(CLASS_PATH_ARGUMENT); + command.add(classPath); + + final Path logDirectory = configurationProvider.getLogDirectory(); + final String logDirectoryProperty = SYSTEM_PROPERTY.formatted(SystemProperty.LOG_DIRECTORY.getProperty(), logDirectory); + command.add(logDirectoryProperty); + + final Path applicationProperties = configurationProvider.getApplicationProperties(); + final String applicationPropertiesProperty = SYSTEM_PROPERTY.formatted(SystemProperty.APPLICATION_PROPERTIES.getProperty(), applicationProperties); + command.add(applicationPropertiesProperty); + + final String managementServerAddress = managementServerAddressProvider.getAddress().orElseThrow(() -> new IllegalStateException("Management Server Address not configured")); + final String managementServerAddressProperty = SYSTEM_PROPERTY.formatted(SystemProperty.MANAGEMENT_SERVER_ADDRESS.getProperty(), managementServerAddress); + command.add(managementServerAddressProperty); + + final List additionalArguments = configurationProvider.getAdditionalArguments(); + command.addAll(additionalArguments); + + command.add(ApplicationClassName.APPLICATION.getName()); + return command; + } + + private String getCurrentProcessCommand(final ProcessHandle.Info currentProcessHandleInfo) { + final Optional currentProcessHandleCommand = currentProcessHandleInfo.command(); + return currentProcessHandleCommand.orElseThrow(IllegalStateException::new); + } + + private String getClassPath() { + final Path libraryDirectory = configurationProvider.getLibraryDirectory(); + try ( + Stream libraryFiles = Files.find(libraryDirectory, LIBRARY_JAR_DEPTH, JAR_FILE_MATCHER) + ) { + final List libraryPaths = new ArrayList<>(libraryFiles.map(Path::toString).toList()); + + final Path configurationDirectory = configurationProvider.getConfigurationDirectory(); + libraryPaths.add(configurationDirectory.toString()); + + return String.join(File.pathSeparator, libraryPaths); + } catch (final IOException e) { + throw new IllegalStateException("Read Library Directory [%s] failed".formatted(libraryDirectory), e); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProvider.java new file mode 100644 index 000000000000..f0f7cd14539e --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProvider.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.apache.nifi.bootstrap.configuration.SystemProperty; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; + +/** + * Process Handle Provider searches running Processes and locates the first handles match based on command arguments + */ +public class StandardProcessHandleProvider implements ProcessHandleProvider { + + private static final String PROPERTIES_ARGUMENT = "-D%s=%s"; + + private final ConfigurationProvider configurationProvider; + + public StandardProcessHandleProvider(final ConfigurationProvider configurationProvider) { + this.configurationProvider = Objects.requireNonNull(configurationProvider); + } + + /** + * Find Process Handle for Application based on matching argument for path to application properties + * + * @return Application Process Handle or empty when not found + */ + @Override + public Optional findApplicationProcessHandle() { + final Path applicationProperties = configurationProvider.getApplicationProperties(); + return findProcessHandle(SystemProperty.APPLICATION_PROPERTIES, applicationProperties); + } + + /** + * Find Process Handle for Bootstrap based on matching argument for path to bootstrap configuration + * + * @return Bootstrap Process Handle or empty when not found + */ + @Override + public Optional findBootstrapProcessHandle() { + final Path bootstrapConfiguration = configurationProvider.getBootstrapConfiguration(); + return findProcessHandle(SystemProperty.BOOTSTRAP_CONFIGURATION, bootstrapConfiguration); + } + + private Optional findProcessHandle(final SystemProperty systemProperty, final Path configuration) { + final String propertiesArgument = PROPERTIES_ARGUMENT.formatted(systemProperty.getProperty(), configuration); + final ProcessHandle currentProcessHandle = ProcessHandle.current(); + + return ProcessHandle.allProcesses() + .filter(Predicate.not(currentProcessHandle::equals)) + .filter(processHandle -> { + final ProcessHandle.Info processHandleInfo = processHandle.info(); + final Optional processArguments = processHandleInfo.arguments(); + final boolean matched; + if (processArguments.isPresent()) { + final String[] arguments = processArguments.get(); + matched = Arrays.stream(arguments).anyMatch(propertiesArgument::contentEquals); + } else { + matched = false; + } + return matched; + }) + .findFirst(); + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ApplicationClassName.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ApplicationClassName.java new file mode 100644 index 000000000000..154e6ce9a056 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ApplicationClassName.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.configuration; + +/** + * Enumeration of application class names for processes and logging + */ +public enum ApplicationClassName { + APPLICATION("org.apache.nifi.NiFi"), + + BOOTSTRAP_COMMAND("org.apache.nifi.bootstrap.Command"); + + private final String name; + + ApplicationClassName(final String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/BootstrapProperty.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/BootstrapProperty.java new file mode 100644 index 000000000000..571cc04cc298 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/BootstrapProperty.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.configuration; + +/** + * Enumeration of supported bootstrap properties for the application + */ +public enum BootstrapProperty { + CONFIGURATION_DIRECTORY("conf.dir"), + + GRACEFUL_SHUTDOWN_SECONDS("graceful.shutdown.seconds"), + + JAVA_ARGUMENT("java.arg"), + + LIBRARY_DIRECTORY("lib.dir"), + + MANAGEMENT_SERVER_ADDRESS("management.server.address"), + + WORKING_DIRECTORY("working.dir"); + + private final String property; + + BootstrapProperty(final String property) { + this.property = property; + } + + public String getProperty() { + return property; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ConfigurationProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ConfigurationProvider.java new file mode 100644 index 000000000000..2b6d95c38f87 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ConfigurationProvider.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.configuration; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +/** + * Abstraction for access to application configuration properties + */ +public interface ConfigurationProvider { + /** + * Get additional arguments for application command + * + * @return Additional arguments + */ + List getAdditionalArguments(); + + /** + * Get file containing application properties + * + * @return Application properties + */ + Path getApplicationProperties(); + + /** + * Get file containing bootstrap configuration + * + * @return Bootstrap configuration + */ + Path getBootstrapConfiguration(); + + /** + * Get directory containing application configuration + * + * @return Configuration directory + */ + Path getConfigurationDirectory(); + + /** + * Get directory containing application libraries + * + * @return Library directory + */ + Path getLibraryDirectory(); + + /** + * Get directory containing logs + * + * @return Log directory + */ + Path getLogDirectory(); + + /** + * Get timeout configured for graceful shutdown of application process + * + * @return Graceful Shutdown Timeout duration + */ + Duration getGracefulShutdownTimeout(); + + /** + * Get Management Server Address from the bootstrap configuration + * + * @return Management Server Address or empty when not configured + */ + Optional getManagementServerAddress(); + + /** + * Get directory for current operations and resolving relative paths + * + * @return Working directory + */ + Path getWorkingDirectory(); +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/EnvironmentVariable.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/EnvironmentVariable.java new file mode 100644 index 000000000000..940d53571000 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/EnvironmentVariable.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.configuration; + +/** + * Enumeration of supported bootstrap and application environment variables + */ +public enum EnvironmentVariable { + NIFI_HOME +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/InvalidCommandException.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ManagementServerPath.java similarity index 63% rename from nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/InvalidCommandException.java rename to nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ManagementServerPath.java index 36873e7a8a4e..3a3ff050ad35 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/InvalidCommandException.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/ManagementServerPath.java @@ -14,25 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.bootstrap; +package org.apache.nifi.bootstrap.configuration; -public class InvalidCommandException extends Exception { +/** + * Enumeration of Management Server HTTP resource paths + */ +public enum ManagementServerPath { + HEALTH("/health"), - private static final long serialVersionUID = 1L; + HEALTH_CLUSTER("/health/cluster"), - public InvalidCommandException() { - super(); - } + HEALTH_DIAGNOSTICS("/health/diagnostics"), - public InvalidCommandException(final String message) { - super(message); - } + HEALTH_STATUS_HISTORY("/health/status-history"); + + private final String path; - public InvalidCommandException(final Throwable t) { - super(t); + ManagementServerPath(final String path) { + this.path = path; } - public InvalidCommandException(final String message, final Throwable t) { - super(message, t); + public String getPath() { + return path; } } diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProvider.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProvider.java new file mode 100644 index 000000000000..1901bfde935e --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProvider.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.configuration; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; + +/** + * Standard implementation of Configuration Provider based on NIFI_HOME environment variable base directory + */ +public class StandardConfigurationProvider implements ConfigurationProvider { + + private static final String CONFIGURATION_DIRECTORY = "conf"; + + private static final String LIBRARY_DIRECTORY = "lib"; + + private static final String LOG_DIRECTORY = "logs"; + + private static final String APPLICATION_PROPERTIES = "nifi.properties"; + + private static final String BOOTSTRAP_CONFIGURATION = "bootstrap.conf"; + + private static final String CURRENT_DIRECTORY = ""; + + private static final Duration GRACEFUL_SHUTDOWN_TIMEOUT = Duration.ofSeconds(20); + + private final Map environmentVariables; + + private final Properties systemProperties; + + private final Properties bootstrapProperties = new Properties(); + + public StandardConfigurationProvider(final Map environmentVariables, final Properties systemProperties) { + this.environmentVariables = Objects.requireNonNull(environmentVariables); + this.systemProperties = Objects.requireNonNull(systemProperties); + setBootstrapProperties(); + } + + /** + * Get additional arguments for application command from Bootstrap Properties starting with java.arg + * + * @return Additional arguments + */ + @Override + public List getAdditionalArguments() { + final List additionalArguments = new ArrayList<>(); + + for (final String propertyName : bootstrapProperties.stringPropertyNames()) { + if (propertyName.startsWith(BootstrapProperty.JAVA_ARGUMENT.getProperty())) { + final String additionalArgument = bootstrapProperties.getProperty(propertyName); + if (!additionalArgument.isBlank()) { + additionalArguments.add(additionalArgument); + } + } + } + + return additionalArguments; + } + + /** + * Get Application Properties relative to configuration directory + * + * @return Application Properties + */ + @Override + public Path getApplicationProperties() { + final Path configurationDirectory = getConfigurationDirectory(); + final Path applicationProperties = configurationDirectory.resolve(APPLICATION_PROPERTIES); + + if (Files.notExists(applicationProperties)) { + throw new IllegalStateException("Application Properties [%s] not found".formatted(applicationProperties)); + } + + return applicationProperties; + } + + /** + * Get Bootstrap Configuration from either System Property or relative to configuration directory + * + * @return Bootstrap Configuration + */ + @Override + public Path getBootstrapConfiguration() { + final Path bootstrapConfiguration; + + final String bootstrapConfigurationProperty = System.getProperty(SystemProperty.BOOTSTRAP_CONFIGURATION.getProperty()); + if (isEmpty(bootstrapConfigurationProperty)) { + final Path configurationDirectory = getConfigurationDirectory(); + bootstrapConfiguration = configurationDirectory.resolve(BOOTSTRAP_CONFIGURATION); + } else { + bootstrapConfiguration = Paths.get(bootstrapConfigurationProperty).toAbsolutePath(); + } + + if (Files.notExists(bootstrapConfiguration)) { + throw new IllegalStateException("Bootstrap Configuration [%s] not found".formatted(bootstrapConfiguration)); + } + + return bootstrapConfiguration; + } + + /** + * Get Library Directory from Bootstrap Configuration or relative to configuration directory + * + * @return Library Directory + */ + @Override + public Path getLibraryDirectory() { + final Path libraryDirectory = getResolvedDirectory(BootstrapProperty.LIBRARY_DIRECTORY, LIBRARY_DIRECTORY); + + if (Files.notExists(libraryDirectory)) { + throw new IllegalStateException("Library Directory [%s] not found".formatted(libraryDirectory)); + } + + return libraryDirectory; + } + + /** + * Get Log Directory from System Property or relative to application home directory + * + * @return Log Directory + */ + @Override + public Path getLogDirectory() { + final Path logDirectory; + + final String logDirectoryProperty = systemProperties.getProperty(SystemProperty.LOG_DIRECTORY.getProperty()); + + if (isEmpty(logDirectoryProperty)) { + final Path applicationHome = getApplicationHome(); + logDirectory = applicationHome.resolve(LOG_DIRECTORY); + } else { + logDirectory = Paths.get(logDirectoryProperty); + } + + return logDirectory; + } + + /** + * Get timeout configured for graceful shutdown of application process + * + * @return Graceful Shutdown Timeout duration + */ + @Override + public Duration getGracefulShutdownTimeout() { + final Duration gracefulShutdownTimeout; + + final String gracefulShutdownSecondsProperty = bootstrapProperties.getProperty(BootstrapProperty.GRACEFUL_SHUTDOWN_SECONDS.getProperty()); + if (gracefulShutdownSecondsProperty == null || gracefulShutdownSecondsProperty.isEmpty()) { + gracefulShutdownTimeout = GRACEFUL_SHUTDOWN_TIMEOUT; + } else { + final int gracefulShutdownSeconds = Integer.parseInt(gracefulShutdownSecondsProperty); + gracefulShutdownTimeout = Duration.ofSeconds(gracefulShutdownSeconds); + } + + return gracefulShutdownTimeout; + } + + /** + * Get Management Server Address from the bootstrap configuration + * + * @return Management Server Address or empty when not configured + */ + @Override + public Optional getManagementServerAddress() { + final Optional managementServerAddress; + + final String managementServerAddressProperty = bootstrapProperties.getProperty(BootstrapProperty.MANAGEMENT_SERVER_ADDRESS.getProperty()); + if (managementServerAddressProperty == null || managementServerAddressProperty.isEmpty()) { + managementServerAddress = Optional.empty(); + } else { + final URI serverAddress = URI.create(managementServerAddressProperty); + managementServerAddress = Optional.of(serverAddress); + } + + return managementServerAddress; + } + + /** + * Get Configuration Directory from Bootstrap Configuration or relative to application home directory + * + * @return Configuration Directory + */ + @Override + public Path getConfigurationDirectory() { + final Path configurationDirectory = getResolvedDirectory(BootstrapProperty.CONFIGURATION_DIRECTORY, CONFIGURATION_DIRECTORY); + + if (Files.notExists(configurationDirectory)) { + throw new IllegalStateException("Configuration Directory [%s] not found".formatted(configurationDirectory)); + } + + return configurationDirectory; + } + + /** + * Get Working Directory from Bootstrap Configuration or current working directory + * + * @return Working Directory + */ + @Override + public Path getWorkingDirectory() { + final Path workingDirectory; + + final String workingDirectoryProperty = bootstrapProperties.getProperty(BootstrapProperty.WORKING_DIRECTORY.getProperty()); + if (isEmpty(workingDirectoryProperty)) { + workingDirectory = Paths.get(CURRENT_DIRECTORY).toAbsolutePath(); + } else { + workingDirectory = Paths.get(workingDirectoryProperty).toAbsolutePath(); + } + + return workingDirectory; + } + + private Path getResolvedDirectory(final BootstrapProperty bootstrapProperty, final String relativeDirectory) { + final Path resolvedDirectory; + + final String directoryProperty = bootstrapProperties.getProperty(bootstrapProperty.getProperty()); + if (isEmpty(directoryProperty)) { + final Path applicationHome = getApplicationHome(); + resolvedDirectory = applicationHome.resolve(relativeDirectory); + } else { + final Path directoryPropertyResolved = Paths.get(directoryProperty); + if (directoryPropertyResolved.isAbsolute()) { + resolvedDirectory = directoryPropertyResolved; + } else { + final Path workingDirectory = getWorkingDirectory(); + resolvedDirectory = workingDirectory.resolve(directoryPropertyResolved); + } + } + + // Normalize Path removing relative directory elements + return resolvedDirectory.normalize(); + } + + private Path getApplicationHome() { + final Path applicationHome; + + final String applicationHomeVariable = environmentVariables.get(EnvironmentVariable.NIFI_HOME.name()); + if (isEmpty(applicationHomeVariable)) { + throw new IllegalStateException("Application Home Environment Variable [NIFI_HOME] not configured"); + } else { + applicationHome = Paths.get(applicationHomeVariable).toAbsolutePath(); + } + + if (Files.notExists(applicationHome)) { + throw new IllegalStateException("Application Home [%s] not found".formatted(applicationHome)); + } + + return applicationHome; + } + + private boolean isEmpty(final String property) { + return property == null || property.isEmpty(); + } + + private void setBootstrapProperties() { + final Path bootstrapConfiguration = getBootstrapConfiguration(); + + try (InputStream inputStream = Files.newInputStream(bootstrapConfiguration)) { + bootstrapProperties.load(inputStream); + } catch (final IOException e) { + throw new UncheckedIOException("Bootstrap Properties [%s] loading failed".formatted(bootstrapConfiguration), e); + } + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/SystemProperty.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/SystemProperty.java new file mode 100644 index 000000000000..54daa12b664e --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/configuration/SystemProperty.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.configuration; + +/** + * Enumeration of supported system properties for the application + */ +public enum SystemProperty { + /** Path to application properties file */ + APPLICATION_PROPERTIES("nifi.properties.file.path"), + + /** Path to bootstrap configuration file */ + BOOTSTRAP_CONFIGURATION("org.apache.nifi.bootstrap.config.file"), + + /** Path to log directory */ + LOG_DIRECTORY("org.apache.nifi.bootstrap.config.log.dir"), + + /** Socket address and port number for management server */ + MANAGEMENT_SERVER_ADDRESS("org.apache.nifi.management.server.address"); + + private final String property; + + SystemProperty(final String property) { + this.property = property; + } + + public String getProperty() { + return property; + } +} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DumpFileValidator.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DumpFileValidator.java deleted file mode 100644 index 8eaef041e715..000000000000 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/DumpFileValidator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.bootstrap.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; - -public final class DumpFileValidator { - - private static final Logger logger = LoggerFactory.getLogger(DumpFileValidator.class); - - private DumpFileValidator() { - } - - public static boolean validate(final String filePath) { - try { - final Path path = Paths.get(filePath); - return checkFileCanBeCreated(path); - } catch (InvalidPathException e) { - System.err.println("Invalid filename. The command parameters are: status-history "); - return false; - } - } - - private static boolean checkFileCanBeCreated(final Path path) { - try (final FileOutputStream outputStream = new FileOutputStream(path.toString()); - final Closeable onClose = () -> Files.delete(path)) { - } catch (FileNotFoundException e) { - System.err.println("Invalid filename or there's no write permission to the currently selected file path."); - return false; - } catch (IOException e) { - logger.error("Could not delete file while validating file path."); - } - return true; - } -} diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java deleted file mode 100644 index 214934222cce..000000000000 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/util/LimitingInputStream.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.bootstrap.util; - -import java.io.IOException; -import java.io.InputStream; - -public class LimitingInputStream extends InputStream { - - private final InputStream in; - private final long limit; - private long bytesRead = 0; - - public LimitingInputStream(final InputStream in, final long limit) { - this.in = in; - this.limit = limit; - } - - @Override - public int read() throws IOException { - if (bytesRead >= limit) { - return -1; - } - - final int val = in.read(); - if (val > -1) { - bytesRead++; - } - return val; - } - - @Override - public int read(final byte[] b) throws IOException { - if (bytesRead >= limit) { - return -1; - } - - final int maxToRead = (int) Math.min(b.length, limit - bytesRead); - - final int val = in.read(b, 0, maxToRead); - if (val > 0) { - bytesRead += val; - } - return val; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (bytesRead >= limit) { - return -1; - } - - final int maxToRead = (int) Math.min(len, limit - bytesRead); - - final int val = in.read(b, off, maxToRead); - if (val > 0) { - bytesRead += val; - } - return val; - } - - @Override - public long skip(final long n) throws IOException { - final long skipped = in.skip(Math.min(n, limit - bytesRead)); - bytesRead += skipped; - return skipped; - } - - @Override - public int available() throws IOException { - return in.available(); - } - - @Override - public void close() throws IOException { - in.close(); - } - - @Override - public void mark(int readlimit) { - in.mark(readlimit); - } - - @Override - public boolean markSupported() { - return in.markSupported(); - } - - @Override - public void reset() throws IOException { - in.reset(); - } -} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommandTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommandTest.java new file mode 100644 index 000000000000..f83cf51007b7 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/ApplicationProcessStatusBootstrapCommandTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ApplicationProcessStatusBootstrapCommandTest { + @Mock + private ProcessHandle processHandle; + + @Mock + private ProcessHandle applicationProcessHandle; + + private ApplicationProcessStatusBootstrapCommand command; + + @BeforeEach + void setCommand() { + command = new ApplicationProcessStatusBootstrapCommand(processHandle); + } + + @Test + void testRunStopped() { + when(processHandle.children()).thenReturn(Stream.empty()); + + command.run(); + final CommandStatus commandStatus = command.getCommandStatus(); + + assertEquals(CommandStatus.STOPPED, commandStatus); + } + + @Test + void testRunSuccess() { + when(processHandle.children()).thenReturn(Stream.of(applicationProcessHandle)); + when(applicationProcessHandle.isAlive()).thenReturn(true); + + command.run(); + final CommandStatus commandStatus = command.getCommandStatus(); + + assertEquals(CommandStatus.SUCCESS, commandStatus); + } + + @Test + void testRunCommunicationFailed() { + when(processHandle.children()).thenReturn(Stream.of(applicationProcessHandle)); + when(applicationProcessHandle.isAlive()).thenReturn(false); + + command.run(); + final CommandStatus commandStatus = command.getCommandStatus(); + + assertEquals(CommandStatus.COMMUNICATION_FAILED, commandStatus); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommandTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommandTest.java new file mode 100644 index 000000000000..4e9cc2960e21 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/GetRunCommandBootstrapCommandTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class GetRunCommandBootstrapCommandTest { + private static final String CONFIGURATION_DIRECTORY = "conf"; + + private static final String SPACE_SEPARATOR = " "; + + @Mock + private ConfigurationProvider configurationProvider; + + @Mock + private ProcessHandleProvider processHandleProvider; + + @Test + void testRun(@TempDir final Path workingDirectory) throws IOException { + final Path configurationDirectory = workingDirectory.resolve(CONFIGURATION_DIRECTORY); + assertTrue(configurationDirectory.toFile().mkdir()); + + final Path applicationProperties = configurationDirectory.resolve(Properties.class.getSimpleName()); + Files.writeString(applicationProperties, SPACE_SEPARATOR); + + when(configurationProvider.getApplicationProperties()).thenReturn(applicationProperties); + when(configurationProvider.getLibraryDirectory()).thenReturn(workingDirectory); + when(configurationProvider.getConfigurationDirectory()).thenReturn(configurationDirectory); + + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + final PrintStream printStream = new PrintStream(outputStream); + + final GetRunCommandBootstrapCommand command = new GetRunCommandBootstrapCommand(configurationProvider, processHandleProvider, printStream); + command.run(); + + final CommandStatus commandStatus = command.getCommandStatus(); + + assertNotNull(commandStatus); + assertEquals(CommandStatus.SUCCESS, commandStatus); + + final String runCommand = outputStream.toString().trim(); + final List runCommands = List.of(runCommand.split(SPACE_SEPARATOR)); + + final String lastCommand = runCommands.getLast(); + assertEquals(ApplicationClassName.APPLICATION.getName(), lastCommand); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/RunBootstrapCommandTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/RunBootstrapCommandTest.java new file mode 100644 index 000000000000..a8870c23616c --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/RunBootstrapCommandTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +@ExtendWith(MockitoExtension.class) +class RunBootstrapCommandTest { + @Mock + private ConfigurationProvider configurationProvider; + + @Mock + private ProcessHandleProvider processHandleProvider; + + private RunBootstrapCommand command; + + @BeforeEach + void setCommand() { + command = new RunBootstrapCommand(configurationProvider, processHandleProvider); + } + + @Test + void testRun() { + command.run(); + + final CommandStatus commandStatus = command.getCommandStatus(); + + assertNotNull(commandStatus); + assertEquals(CommandStatus.FAILED, commandStatus); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProviderTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProviderTest.java new file mode 100644 index 000000000000..53a3377e888b --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StandardBootstrapCommandProviderTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class StandardBootstrapCommandProviderTest { + private StandardBootstrapCommandProvider provider; + + @BeforeEach + void setProvider() { + provider = new StandardBootstrapCommandProvider(); + } + + @Test + void testGetBootstrapCommandNull() { + final BootstrapCommand bootstrapCommand = provider.getBootstrapCommand(null); + + assertNotNull(bootstrapCommand); + assertInstanceOf(UnknownBootstrapCommand.class, bootstrapCommand); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StartBootstrapCommandTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StartBootstrapCommandTest.java new file mode 100644 index 000000000000..2d9b0a642fd1 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StartBootstrapCommandTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class StartBootstrapCommandTest { + @Mock + private BootstrapCommand runBootstrapCommand; + + @Mock + private BootstrapCommand statusBootstrapCommand; + + private StartBootstrapCommand command; + + @BeforeEach + void setCommand() { + command = new StartBootstrapCommand(runBootstrapCommand, statusBootstrapCommand); + } + + @Test + void testRunError() { + final CommandStatus runCommandStatus = CommandStatus.ERROR; + when(runBootstrapCommand.getCommandStatus()).thenReturn(runCommandStatus); + + command.run(); + + final CommandStatus commandStatus = command.getCommandStatus(); + assertEquals(runCommandStatus, commandStatus); + } + + @Test + void testRunSuccessFailed() { + final CommandStatus runCommandStatus = CommandStatus.SUCCESS; + when(runBootstrapCommand.getCommandStatus()).thenReturn(runCommandStatus); + + final CommandStatus statusCommandStatus = CommandStatus.FAILED; + lenient().when(statusBootstrapCommand.getCommandStatus()).thenReturn(statusCommandStatus); + + command.run(); + + final CommandStatus commandStatus = command.getCommandStatus(); + assertEquals(CommandStatus.RUNNING, commandStatus); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StopBootstrapCommandTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StopBootstrapCommandTest.java new file mode 100644 index 000000000000..37f293d933cd --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/StopBootstrapCommandTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command; + +import org.apache.nifi.bootstrap.command.process.ProcessHandleProvider; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class StopBootstrapCommandTest { + @Mock + private ConfigurationProvider configurationProvider; + + @Mock + private ProcessHandleProvider processHandleProvider; + + @Mock + private ProcessHandle applicationProcessHandle; + + @Mock + private ProcessHandle.Info applicationProcessHandleInfo; + + private StopBootstrapCommand command; + + @BeforeEach + void setCommand() { + command = new StopBootstrapCommand(processHandleProvider, configurationProvider); + } + + @Test + void testRunProcessHandleNotFound() { + when(processHandleProvider.findApplicationProcessHandle()).thenReturn(Optional.empty()); + + command.run(); + final CommandStatus commandStatus = command.getCommandStatus(); + + assertEquals(CommandStatus.SUCCESS, commandStatus); + } + + @Test + void testRunDestroyCompleted() { + when(processHandleProvider.findApplicationProcessHandle()).thenReturn(Optional.of(applicationProcessHandle)); + when(applicationProcessHandle.destroy()).thenReturn(true); + when(applicationProcessHandle.onExit()).thenReturn(CompletableFuture.completedFuture(applicationProcessHandle)); + + command.run(); + final CommandStatus commandStatus = command.getCommandStatus(); + + assertEquals(CommandStatus.SUCCESS, commandStatus); + } + + @Test + void testRunDestroyFailed() { + when(processHandleProvider.findApplicationProcessHandle()).thenReturn(Optional.of(applicationProcessHandle)); + when(applicationProcessHandle.destroy()).thenReturn(true); + when(applicationProcessHandle.onExit()).thenReturn(CompletableFuture.failedFuture(new RuntimeException())); + + command.run(); + final CommandStatus commandStatus = command.getCommandStatus(); + + assertEquals(CommandStatus.ERROR, commandStatus); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandlerTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandlerTest.java new file mode 100644 index 000000000000..c8fbef5878dd --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/FileResponseStreamHandlerTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +class FileResponseStreamHandlerTest { + + @Test + void testOnResponseStream(@TempDir final Path outputDirectory) throws IOException { + final Path outputPath = outputDirectory.resolve(FileResponseStreamHandlerTest.class.getSimpleName()); + + final FileResponseStreamHandler handler = new FileResponseStreamHandler(outputPath); + + final byte[] bytes = String.class.getName().getBytes(StandardCharsets.UTF_8); + final InputStream inputStream = new ByteArrayInputStream(bytes); + + handler.onResponseStream(inputStream); + + final byte[] read = Files.readAllBytes(outputPath); + assertArrayEquals(bytes, read); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParserTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParserTest.java new file mode 100644 index 000000000000..7201309589c3 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/io/StandardBootstrapArgumentParserTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.io; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StandardBootstrapArgumentParserTest { + private static final String CLUSTER_STATUS_ARGUMENT = "cluster-status"; + + private StandardBootstrapArgumentParser parser; + + @BeforeEach + void setParser() { + parser = new StandardBootstrapArgumentParser(); + } + + @Test + void testGetBootstrapArgumentNull() { + final Optional bootstrapArgumentFound = parser.getBootstrapArgument(null); + + assertTrue(bootstrapArgumentFound.isEmpty()); + } + + @Test + void testGetBootstrapArgumentClusterStatus() { + final Optional bootstrapArgumentFound = parser.getBootstrapArgument(new String[]{CLUSTER_STATUS_ARGUMENT}); + + assertTrue(bootstrapArgumentFound.isPresent()); + assertEquals(BootstrapArgument.CLUSTER_STATUS, bootstrapArgumentFound.get()); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProviderTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProviderTest.java new file mode 100644 index 000000000000..6103349d83b2 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/ProcessHandleManagementServerAddressProviderTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.SystemProperty; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ProcessHandleManagementServerAddressProviderTest { + private static final String SYSTEM_PROPERTY_ARGUMENT = "-D%s=%s"; + + private static final String ADDRESS = "127.0.0.1:52020"; + + @Mock + private ProcessHandle processHandle; + + @Mock + private ProcessHandle.Info processHandleInfo; + + private ProcessHandleManagementServerAddressProvider provider; + + @BeforeEach + void setProvider() { + provider = new ProcessHandleManagementServerAddressProvider(processHandle); + } + + @Test + void testGetAddressNotFound() { + when(processHandle.info()).thenReturn(processHandleInfo); + + final Optional managementServerAddress = provider.getAddress(); + + assertTrue(managementServerAddress.isEmpty()); + } + + @Test + void testGetAddressArgumentNotFound() { + when(processHandle.info()).thenReturn(processHandleInfo); + when(processHandleInfo.arguments()).thenReturn(Optional.empty()); + + final Optional managementServerAddress = provider.getAddress(); + + assertTrue(managementServerAddress.isEmpty()); + } + + @Test + void testGetAddress() { + when(processHandle.info()).thenReturn(processHandleInfo); + + final String systemPropertyArgument = SYSTEM_PROPERTY_ARGUMENT.formatted(SystemProperty.MANAGEMENT_SERVER_ADDRESS.getProperty(), ADDRESS); + final String[] arguments = new String[]{systemPropertyArgument}; + + when(processHandleInfo.arguments()).thenReturn(Optional.of(arguments)); + + final Optional managementServerAddress = provider.getAddress(); + + assertTrue(managementServerAddress.isPresent()); + + final String address = managementServerAddress.get(); + assertEquals(ADDRESS, address); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProviderTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProviderTest.java new file mode 100644 index 000000000000..3581423b2c7e --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardManagementServerAddressProviderTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(MockitoExtension.class) +class StandardManagementServerAddressProviderTest { + @Mock + private ConfigurationProvider configurationProvider; + + private StandardManagementServerAddressProvider provider; + + @BeforeEach + void setProvider() { + provider = new StandardManagementServerAddressProvider(configurationProvider); + } + + @Test + void testGetAddressFound() { + final Optional managementServerAddress = provider.getAddress(); + + assertTrue(managementServerAddress.isPresent()); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProviderTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProviderTest.java new file mode 100644 index 000000000000..b1cc3c705df4 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessBuilderProviderTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.ApplicationClassName; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.apache.nifi.bootstrap.configuration.SystemProperty; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class StandardProcessBuilderProviderTest { + private static final String SERVER_ADDRESS = "127.0.0.1:52020"; + + private static final String SERVER_ADDRESS_PROPERTY = "-D%s=%s".formatted(SystemProperty.MANAGEMENT_SERVER_ADDRESS.getProperty(), SERVER_ADDRESS); + + @Mock + private ConfigurationProvider configurationProvider; + + @Mock + private ManagementServerAddressProvider managementServerAddressProvider; + + private StandardProcessBuilderProvider provider; + + @BeforeEach + void setProvider() { + provider = new StandardProcessBuilderProvider(configurationProvider, managementServerAddressProvider); + } + + @Test + void testGetApplicationProcessBuilder(@TempDir final Path workingDirectory) { + when(configurationProvider.getLibraryDirectory()).thenReturn(workingDirectory); + when(configurationProvider.getConfigurationDirectory()).thenReturn(workingDirectory); + when(managementServerAddressProvider.getAddress()).thenReturn(Optional.of(SERVER_ADDRESS)); + + final ProcessBuilder processBuilder = provider.getApplicationProcessBuilder(); + + assertNotNull(processBuilder); + + final List command = processBuilder.command(); + + final String currentCommand = ProcessHandle.current().info().command().orElse(null); + final String firstCommand = command.getFirst(); + assertEquals(currentCommand, firstCommand); + + final String lastCommand = command.getLast(); + assertEquals(ApplicationClassName.APPLICATION.getName(), lastCommand); + + assertTrue(command.contains(SERVER_ADDRESS_PROPERTY)); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProviderTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProviderTest.java new file mode 100644 index 000000000000..8e73b66c2e65 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/command/process/StandardProcessHandleProviderTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.command.process; + +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(MockitoExtension.class) +class StandardProcessHandleProviderTest { + @Mock + private ConfigurationProvider configurationProvider; + + private StandardProcessHandleProvider provider; + + @BeforeEach + void setProvider() { + provider = new StandardProcessHandleProvider(configurationProvider); + } + + @Test + void testFindApplicationProcessHandleEmpty() { + final Optional applicationProcessHandle = provider.findApplicationProcessHandle(); + + assertTrue(applicationProcessHandle.isEmpty()); + } + + @Test + void testFindBootstrapProcessHandleEmpty() { + final Optional bootstrapProcessHandle = provider.findBootstrapProcessHandle(); + + assertTrue(bootstrapProcessHandle.isEmpty()); + } +} diff --git a/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProviderTest.java b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProviderTest.java new file mode 100644 index 000000000000..74af0687b016 --- /dev/null +++ b/nifi-bootstrap/src/test/java/org/apache/nifi/bootstrap/configuration/StandardConfigurationProviderTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap.configuration; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StandardConfigurationProviderTest { + + private static final String CONFIGURATION_DIRECTORY = "conf"; + + private static final String BOOTSTRAP_CONFIGURATION = "bootstrap.conf"; + + private static final String CURRENT_DIRECTORY = ""; + + private static final String MANAGEMENT_SERVER_ADDRESS = "http://127.0.0.1:52020"; + + private final Map environmentVariables = new LinkedHashMap<>(); + + private final Properties systemProperties = new Properties(); + + @BeforeEach + void setProvider() { + environmentVariables.clear(); + systemProperties.clear(); + } + + @Test + void testApplicationHomeNotConfigured() { + assertThrows(IllegalStateException.class, () -> new StandardConfigurationProvider(environmentVariables, systemProperties)); + } + + @Test + void testGetBootstrapConfiguration(@TempDir final Path applicationHomeDirectory) throws IOException { + environmentVariables.put(EnvironmentVariable.NIFI_HOME.name(), applicationHomeDirectory.toString()); + final Path configurationDirectory = createConfigurationDirectory(applicationHomeDirectory); + final Path bootstrapConfiguration = setRequiredConfiguration(applicationHomeDirectory); + + final StandardConfigurationProvider provider = new StandardConfigurationProvider(environmentVariables, systemProperties); + + final Path configurationDirectoryProvided = provider.getConfigurationDirectory(); + assertEquals(configurationDirectory, configurationDirectoryProvided); + + final Path bootstrapConfigurationProvided = provider.getBootstrapConfiguration(); + assertEquals(bootstrapConfiguration, bootstrapConfigurationProvided); + } + + @Test + void testGetWorkingDirectory(@TempDir final Path applicationHomeDirectory) throws IOException { + setRequiredConfiguration(applicationHomeDirectory); + + final StandardConfigurationProvider provider = new StandardConfigurationProvider(environmentVariables, systemProperties); + + final Path workingDirectory = provider.getWorkingDirectory(); + final Path workingDirectoryExpected = Paths.get(CURRENT_DIRECTORY).toAbsolutePath(); + + assertEquals(workingDirectoryExpected, workingDirectory); + } + + @Test + void testGetManagementServerAddressNotConfigured(@TempDir final Path applicationHomeDirectory) throws IOException { + setRequiredConfiguration(applicationHomeDirectory); + + final StandardConfigurationProvider provider = new StandardConfigurationProvider(environmentVariables, systemProperties); + + final Optional managementServerAddress = provider.getManagementServerAddress(); + + assertTrue(managementServerAddress.isEmpty()); + } + + @Test + void testGetManagementServerAddress(@TempDir final Path applicationHomeDirectory) throws IOException { + final Path bootstrapConfiguration = setRequiredConfiguration(applicationHomeDirectory); + + final Properties bootstrapProperties = new Properties(); + bootstrapProperties.put(BootstrapProperty.MANAGEMENT_SERVER_ADDRESS.getProperty(), MANAGEMENT_SERVER_ADDRESS); + try (OutputStream outputStream = Files.newOutputStream(bootstrapConfiguration)) { + bootstrapProperties.store(outputStream, Properties.class.getSimpleName()); + } + + final StandardConfigurationProvider provider = new StandardConfigurationProvider(environmentVariables, systemProperties); + + final Optional managementServerAddress = provider.getManagementServerAddress(); + + assertTrue(managementServerAddress.isPresent()); + final URI address = managementServerAddress.get(); + assertEquals(MANAGEMENT_SERVER_ADDRESS, address.toString()); + } + + private Path setRequiredConfiguration(final Path applicationHomeDirectory) throws IOException { + environmentVariables.put(EnvironmentVariable.NIFI_HOME.name(), applicationHomeDirectory.toString()); + final Path configurationDirectory = createConfigurationDirectory(applicationHomeDirectory); + return createBootstrapConfiguration(configurationDirectory); + } + + private Path createConfigurationDirectory(final Path applicationHomeDirectory) { + final Path configurationDirectory = applicationHomeDirectory.resolve(CONFIGURATION_DIRECTORY); + if (configurationDirectory.toFile().mkdir()) { + assertTrue(Files.isReadable(configurationDirectory)); + } + return configurationDirectory; + } + + private Path createBootstrapConfiguration(final Path configurationDirectory) throws IOException { + final Path bootstrapConfiguration = configurationDirectory.resolve(BOOTSTRAP_CONFIGURATION); + assertTrue(bootstrapConfiguration.toFile().createNewFile()); + return bootstrapConfiguration; + } +} diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index 6ab2fea53f3c..922ab26c686d 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -2842,7 +2842,6 @@ properties for minimum and maximum Java Heap size, the garbage collector to use, |`nifi.diagnostics.on.shutdown.directory`|This property specifies the location of the NiFi diagnostics directory. The default value is `./diagnostics`. |`nifi.diagnostics.on.shutdown.max.filecount`|This property specifies the maximum permitted number of diagnostic files. If the limit is exceeded, the oldest files are deleted. The default value is `10`. |`nifi.diagnostics.on.shutdown.max.directory.size`|This property specifies the maximum permitted size of the diagnostics directory. If the limit is exceeded, the oldest files are deleted. The default value is `10 MB`. -|`nifi.bootstrap.listen.port`|This property defines the port used to listen for communications from NiFi. If this property is missing, empty, or `0`, a random ephemeral port is used. |==== [[proxy_configuration]] diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.cmd b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.cmd index 157cad3295db..6a5748760677 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.cmd +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.cmd @@ -29,17 +29,16 @@ set BOOTSTRAP_LIB_DIR=%NIFI_HOME%\lib\bootstrap set CONF_DIR=%NIFI_HOME%\conf set LOG_DIR_PROPERTY=-Dorg.apache.nifi.bootstrap.config.log.dir=%NIFI_LOG_DIR% -set PID_DIR_PROPERTY=-Dorg.apache.nifi.bootstrap.config.pid.dir=%NIFI_PID_DIR% set CONFIG_FILE_PROPERTY=-Dorg.apache.nifi.bootstrap.config.file=%CONF_DIR%\bootstrap.conf set PROPERTIES_FILE_PROPERTY=-Dnifi.properties.file.path=%CONF_DIR%\nifi.properties set BOOTSTRAP_HEAP_SIZE=48m -set JAVA_ARGS=%LOG_DIR_PROPERTY% %PID_DIR_PROPERTY% %CONFIG_FILE_PROPERTY% %PROPERTIES_FILE_PROPERTY% +set JAVA_ARGS=%LOG_DIR_PROPERTY% %CONFIG_FILE_PROPERTY% %PROPERTIES_FILE_PROPERTY% set JAVA_PARAMS=-cp %BOOTSTRAP_LIB_DIR%\*;%CONF_DIR% %JAVA_ARGS% set JAVA_MEMORY=-Xms%BOOTSTRAP_HEAP_SIZE% -Xmx%BOOTSTRAP_HEAP_SIZE% -echo JAVA_HOME: %JAVA_HOME% -echo NIFI_HOME: %NIFI_HOME% +echo JAVA_HOME=%JAVA_HOME% +echo NIFI_HOME=%NIFI_HOME% echo. pushd %NIFI_HOME% @@ -54,7 +53,7 @@ if %RUN_COMMAND% == "set-single-user-credentials" ( ) else if %RUN_COMMAND% == "set-sensitive-properties-algorithm" ( call "%JAVA_EXE%" %JAVA_PARAMS% org.apache.nifi.flow.encryptor.command.SetSensitivePropertiesAlgorithm %~2 ) else ( - call "%JAVA_EXE%" %JAVA_MEMORY% %JAVA_PARAMS% org.apache.nifi.bootstrap.RunNiFi %RUN_COMMAND% + call "%JAVA_EXE%" %JAVA_MEMORY% %JAVA_PARAMS% org.apache.nifi.bootstrap.BootstrapProcess %RUN_COMMAND% ) popd diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh index d8d406fab44e..30474bdd076e 100755 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh @@ -143,15 +143,6 @@ locateJava() { fi fi fi - # if command is env, attempt to add more to the classpath - if [ "$1" = "env" ]; then - [ "x${TOOLS_JAR}" = "x" ] && [ -n "${JAVA_HOME}" ] && TOOLS_JAR=$(find -H "${JAVA_HOME}" -name "tools.jar") - [ "x${TOOLS_JAR}" = "x" ] && [ -n "${JAVA_HOME}" ] && TOOLS_JAR=$(find -H "${JAVA_HOME}" -name "classes.jar") - if [ "x${TOOLS_JAR}" = "x" ]; then - warn "Could not locate tools.jar or classes.jar. Please set manually to avail all command features." - fi - fi - } init() { @@ -162,7 +153,7 @@ init() { unlimitFD # Locate the Java VM to execute - locateJava "$1" + locateJava } is_nonzero_integer() { @@ -197,7 +188,6 @@ run() { NIFI_HOME=$(cygpath --path --windows "${NIFI_HOME}") NIFI_LOG_DIR=$(cygpath --path --windows "${NIFI_LOG_DIR}") - NIFI_PID_DIR=$(cygpath --path --windows "${NIFI_PID_DIR}") BOOTSTRAP_CONF=$(cygpath --path --windows "${BOOTSTRAP_CONF}") BOOTSTRAP_CONF_DIR=$(cygpath --path --windows "${BOOTSTRAP_CONF_DIR}") BOOTSTRAP_LIBS=$(cygpath --path --windows "${BOOTSTRAP_LIBS}") @@ -220,26 +210,21 @@ run() { fi echo - echo "Java home: ${JAVA_HOME}" - echo "NiFi home: ${NIFI_HOME}" - echo - echo "Bootstrap Config File: ${BOOTSTRAP_CONF}" + echo "JAVA_HOME=${JAVA_HOME}" + echo "NIFI_HOME=${NIFI_HOME}" echo - # run 'start' in the background because the process will continue to run, monitoring NiFi. - # all other commands will terminate quickly so want to just wait for them - #setup directory parameters BOOTSTRAP_LOG_PARAMS="-Dorg.apache.nifi.bootstrap.config.log.dir='${NIFI_LOG_DIR}'" - BOOTSTRAP_PID_PARAMS="-Dorg.apache.nifi.bootstrap.config.pid.dir='${NIFI_PID_DIR}'" BOOTSTRAP_CONF_PARAMS="-Dorg.apache.nifi.bootstrap.config.file='${BOOTSTRAP_CONF}'" # uncomment to allow debugging of the bootstrap process #BOOTSTRAP_DEBUG_PARAMS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000" - BOOTSTRAP_DIR_PARAMS="${BOOTSTRAP_LOG_PARAMS} ${BOOTSTRAP_PID_PARAMS} ${BOOTSTRAP_CONF_PARAMS}" + BOOTSTRAP_DIR_PARAMS="${BOOTSTRAP_LOG_PARAMS} ${BOOTSTRAP_CONF_PARAMS}" - run_bootstrap_cmd="'${JAVA}' -cp '${BOOTSTRAP_CLASSPATH}' -Xms48m -Xmx48m ${BOOTSTRAP_DIR_PARAMS} ${BOOTSTRAP_DEBUG_PARAMS} ${BOOTSTRAP_JAVA_OPTS} org.apache.nifi.bootstrap.RunNiFi" + MAXIMUM_HEAP_SIZE="-Xmx48m" + run_bootstrap_cmd="'${JAVA}' -cp '${BOOTSTRAP_CLASSPATH}' ${MAXIMUM_HEAP_SIZE} ${BOOTSTRAP_DIR_PARAMS} ${BOOTSTRAP_DEBUG_PARAMS} ${BOOTSTRAP_JAVA_OPTS} org.apache.nifi.bootstrap.BootstrapProcess" run_nifi_cmd="${run_bootstrap_cmd} $@" if [ -n "${run_as_user}" ]; then @@ -252,11 +237,6 @@ run() { run_nifi_cmd="${SUDO} -u ${run_as_user} sh -c \"SCRIPT_DIR='${SCRIPT_DIR}' && . '${SCRIPT_DIR}/nifi-env.sh' && ${run_nifi_cmd}\"" fi - if [ "$1" = "run" ]; then - # Use exec to handover PID to RunNiFi java process, instead of foking it as a child process - run_nifi_cmd="exec ${run_nifi_cmd}" - fi - if [ "$1" = "set-sensitive-properties-algorithm" ]; then run_command="'${JAVA}' -cp '${BOOTSTRAP_CLASSPATH}' '-Dnifi.properties.file.path=${NIFI_HOME}/conf/nifi.properties' 'org.apache.nifi.flow.encryptor.command.SetSensitivePropertiesAlgorithm'" eval "cd ${NIFI_HOME}" @@ -287,8 +267,20 @@ run() { return; fi - if [ "$1" = "start" ]; then - ( eval "cd ${NIFI_HOME} && ${run_nifi_cmd}" & )> /dev/null 1>&- + eval "cd ${NIFI_HOME}" + + if [ "$1" = "run" ]; then + RUN_COMMAND=$(eval "${run_bootstrap_cmd} get-run-command") + RUN_COMMAND_STATUS=$? + if [ $RUN_COMMAND_STATUS = 0 ]; then + exec $RUN_COMMAND + else + echo "Failed to get run command" + echo "${RUN_COMMAND}" + exit 1 + fi + elif [ "$1" = "start" ]; then + eval "${run_nifi_cmd}" > /dev/null 1>&- & if [ "$2" = "--wait-for-init" ]; then @@ -304,63 +296,57 @@ run() { time_since_feedback=0 not_running_counter=0 - is_nifi_loaded="false" # 3 possible values: "true", "false", "not_running". "not_running" means NiFi has not been started. - while [ "$is_nifi_loaded" != "true" ]; do + PROCESS_STATUS=1 + while [ $PROCESS_STATUS != 0 ]; do time_at_previous_loop=$current_time current_time=$(date +%s) if [ "$current_time" -ge "$endtime" ]; then - echo "Exited the script due to --wait-for-init timeout" + echo "Initialization failed after $wait_timeout seconds" break; fi time_since_feedback=$(($time_since_feedback+($current_time-$time_at_previous_loop))) if [ "$time_since_feedback" -ge "$WAIT_FOR_INIT_FEEDBACK_INTERVAL" ]; then time_since_feedback=0 - echo "NiFi has not fully initialized yet..." fi - is_nifi_loaded=$( eval "cd ${NIFI_HOME} && ${run_bootstrap_cmd} is_loaded" ) + eval "cd ${NIFI_HOME} && ${run_bootstrap_cmd} status" + PROCESS_STATUS=$? - if [ "$is_nifi_loaded" = "not_running" ]; then + if [ $PROCESS_STATUS = 3 ]; then not_running_counter=$(($not_running_counter+1)) if [ "$not_running_counter" -ge 3 ]; then - echo "NiFi is not running. Stopped waiting for it to initialize." + echo "Initialization failed" break; fi fi sleep $WAIT_FOR_INIT_SLEEP_TIME done - if [ "$is_nifi_loaded" = "true" ]; then - echo "NiFi initialized." - echo "Exiting startup script..." + if [ $PROCESS_STATUS = 0 ]; then + echo "Initialization completed" fi fi + + # Wait for logging initialization before returning to shell after starting + sleep 1 else - eval "cd ${NIFI_HOME} && ${run_nifi_cmd}" + eval "${run_nifi_cmd}" fi EXIT_STATUS=$? - # Wait just a bit (3 secs) to wait for the logging to finish and then echo a new-line. - # We do this to avoid having logs spewed on the console after running the command and then not giving - # control back to the user - sleep 1 echo } main() { - init "$1" + init run "$@" } case "$1" in - install) - install "$@" - ;; - - start|stop|decommission|run|status|is_loaded|dump|diagnostics|status-history|env|set-sensitive-properties-algorithm|set-sensitive-properties-key|set-single-user-credentials|cluster-status) + start|stop|decommission|run|status|cluster-status|diagnostics|status-history|set-sensitive-properties-algorithm|set-sensitive-properties-key|set-single-user-credentials) main "$@" ;; @@ -370,6 +356,6 @@ case "$1" in run "start" ;; *) - echo "Usage nifi {start|stop|decommission|run|restart|status|dump|diagnostics|status-history|set-sensitive-properties-algorithm|set-sensitive-properties-key|set-single-user-credentials|cluster-status}" + echo "Usage nifi.sh {start|stop|decommission|run|restart|status|cluster-status|diagnostics|status-history|set-sensitive-properties-algorithm|set-sensitive-properties-key|set-single-user-credentials}" ;; esac diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf index 53094181b79b..09394b7afd9e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf @@ -15,9 +15,6 @@ # limitations under the License. # -# Java command to use when running NiFi -java=java - # Username to use when running NiFi. This value will be ignored on Windows. run.as=${nifi.run.as} @@ -61,5 +58,3 @@ java.arg.securityAuthUseSubjectCredsOnly=-Djavax.security.auth.useSubjectCredsOn # org.apache.jasper.servlet.JasperLoader,org.jvnet.hk2.internal.DelegatingClassLoader,org.apache.nifi.nar.NarClassLoader # End of Java Agent config for native library loading. -# Port used to listen for communications from NiFi. If this property is missing, empty, or 0, a random ephemeral port is used. -nifi.bootstrap.listen.port=0 diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml index cbeff47ef6bb..baef9d3881c6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml @@ -15,8 +15,6 @@ --> - - true diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/BootstrapListener.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/BootstrapListener.java deleted file mode 100644 index d6537603d7fa..000000000000 --- a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/BootstrapListener.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import org.apache.nifi.cluster.ClusterDetailsFactory; -import org.apache.nifi.cluster.ConnectionState; -import org.apache.nifi.controller.DecommissionTask; -import org.apache.nifi.controller.status.history.StatusHistoryDump; -import org.apache.nifi.diagnostics.DiagnosticsDump; -import org.apache.nifi.util.LimitingInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class BootstrapListener { - - private static final Logger logger = LoggerFactory.getLogger(BootstrapListener.class); - - private final NiFiEntryPoint nifi; - private final int bootstrapPort; - private final String secretKey; - - private volatile Listener listener; - private volatile ServerSocket serverSocket; - private volatile boolean nifiLoaded = false; - - public BootstrapListener(final NiFiEntryPoint nifi, final int bootstrapPort) { - this.nifi = nifi; - this.bootstrapPort = bootstrapPort; - secretKey = UUID.randomUUID().toString(); - } - - public void start(final int listenPort) throws IOException { - logger.debug("Starting Bootstrap Listener to communicate with Bootstrap Port {}", bootstrapPort); - - serverSocket = new ServerSocket(); - serverSocket.bind(new InetSocketAddress("localhost", listenPort)); - serverSocket.setSoTimeout(2000); - - final int localPort = serverSocket.getLocalPort(); - logger.info("Started Bootstrap Listener, Listening for incoming requests on port {}", localPort); - - listener = new Listener(serverSocket); - final Thread listenThread = new Thread(listener); - listenThread.setDaemon(true); - listenThread.setName("Listen to Bootstrap"); - listenThread.start(); - - logger.debug("Notifying Bootstrap that local port is {}", localPort); - sendCommand("PORT", new String[]{String.valueOf(localPort), secretKey}); - } - - public void reload() throws IOException { - if (listener != null) { - listener.stop(); - } - sendCommand("RELOAD", new String[]{}); - } - - public void stop() { - if (listener != null) { - listener.stop(); - } - } - - public void setNiFiLoaded(boolean nifiLoaded) { - this.nifiLoaded = nifiLoaded; - } - - public void sendStartedStatus(boolean status) throws IOException { - logger.debug("Notifying Bootstrap that the status of starting NiFi is {}", status); - sendCommand("STARTED", new String[]{String.valueOf(status)}); - } - - private void sendCommand(final String command, final String[] args) throws IOException { - try (final Socket socket = new Socket()) { - socket.setSoTimeout(60000); - socket.connect(new InetSocketAddress("localhost", bootstrapPort)); - socket.setSoTimeout(60000); - - final StringBuilder commandBuilder = new StringBuilder(command); - for (final String arg : args) { - commandBuilder.append(" ").append(arg); - } - commandBuilder.append("\n"); - - final String commandWithArgs = commandBuilder.toString(); - logger.debug("Sending command to Bootstrap: {}", commandWithArgs); - - final OutputStream out = socket.getOutputStream(); - out.write((commandWithArgs).getBytes(StandardCharsets.UTF_8)); - out.flush(); - - logger.debug("Awaiting response from Bootstrap..."); - final BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); - final String response = reader.readLine(); - if ("OK".equals(response)) { - logger.info("Successfully initiated communication with Bootstrap"); - } else { - logger.error("Failed to communicate with Bootstrap. Bootstrap may be unable to issue or receive commands from NiFi"); - } - } - } - - private class Listener implements Runnable { - - private final ServerSocket serverSocket; - private final ExecutorService executor; - private volatile boolean stopped = false; - - public Listener(final ServerSocket serverSocket) { - this.serverSocket = serverSocket; - this.executor = Executors.newFixedThreadPool(2); - } - - public void stop() { - stopped = true; - - executor.shutdownNow(); - - try { - serverSocket.close(); - } catch (final IOException ioe) { - // nothing to really do here. we could log this, but it would just become - // confusing in the logs, as we're shutting down and there's no real benefit - } - } - - @Override - public void run() { - while (!stopped) { - try { - final Socket socket; - try { - logger.debug("Listening for Bootstrap Requests"); - socket = serverSocket.accept(); - } catch (final SocketTimeoutException ste) { - if (stopped) { - return; - } - - continue; - } catch (final IOException ioe) { - if (stopped) { - return; - } - - throw ioe; - } - - logger.debug("Received connection from Bootstrap"); - socket.setSoTimeout(5000); - - executor.submit(new Runnable() { - @Override - public void run() { - try { - final BootstrapRequest request = readRequest(socket.getInputStream()); - final BootstrapRequest.RequestType requestType = request.getRequestType(); - - switch (requestType) { - case PING: - logger.debug("Received PING request from Bootstrap; responding"); - sendAnswer(socket.getOutputStream(), "PING"); - logger.debug("Responded to PING request from Bootstrap"); - break; - case RELOAD: - logger.info("Received RELOAD request from Bootstrap"); - sendAnswer(socket.getOutputStream(), "RELOAD"); - nifi.shutdownHook(true); - return; - case SHUTDOWN: - logger.info("Received SHUTDOWN request from Bootstrap"); - sendAnswer(socket.getOutputStream(), "SHUTDOWN"); - socket.close(); - nifi.shutdownHook(false); - return; - case DUMP: - logger.info("Received DUMP request from Bootstrap"); - writeDump(socket.getOutputStream()); - break; - case CLUSTER_STATUS: - logger.info("Received CLUSTER_STATUS request from Bootstrap"); - final String clusterStatus = getClusterStatus(); - logger.debug("Responding to CLUSTER_STATUS request from Bootstrap with {}", clusterStatus); - sendAnswer(socket.getOutputStream(), clusterStatus); - break; - case DECOMMISSION: - logger.info("Received DECOMMISSION request from Bootstrap"); - - boolean shutdown = true; - final String[] decommissionArgs = request.getArgs(); - if (decommissionArgs != null) { - for (final String arg : decommissionArgs) { - if ("--shutdown=false".equalsIgnoreCase(arg)) { - shutdown = false; - break; - } - } - } - - logger.info("Command indicates that after decommission, shutdown={}", shutdown); - - try { - decommission(); - sendAnswer(socket.getOutputStream(), "DECOMMISSION"); - - if (shutdown) { - nifi.shutdownHook(false); - } - } catch (final Exception e) { - final OutputStream out = socket.getOutputStream(); - - out.write(("Failed to decommission node: " + e + "; see app-log for additional details").getBytes(StandardCharsets.UTF_8)); - out.flush(); - } finally { - if (shutdown) { - socket.close(); - } - } - - break; - case DIAGNOSTICS: - logger.info("Received DIAGNOSTICS request from Bootstrap"); - final String[] args = request.getArgs(); - boolean verbose = false; - if (args == null) { - verbose = false; - } else { - for (final String arg : args) { - if ("--verbose=true".equalsIgnoreCase(arg)) { - verbose = true; - break; - } - } - } - - writeDiagnostics(socket.getOutputStream(), verbose); - break; - case STATUS_HISTORY: - logger.info("Received STATUS_HISTORY request from Bootstrap"); - final String[] statusHistoryArgs = request.getArgs(); - final int days = Integer.parseInt(statusHistoryArgs[0]); - writeNodeStatusHistory(socket.getOutputStream(), days); - break; - case IS_LOADED: - logger.debug("Received IS_LOADED request from Bootstrap"); - String answer = String.valueOf(nifiLoaded); - sendAnswer(socket.getOutputStream(), answer); - logger.debug("Responded to IS_LOADED request from Bootstrap with value: {}", answer); - break; - } - } catch (final Throwable t) { - logger.error("Failed to process request from Bootstrap", t); - } finally { - try { - socket.close(); - } catch (final IOException ioe) { - logger.warn("Failed to close socket to Bootstrap", ioe); - } - } - } - }); - } catch (final Throwable t) { - logger.error("Failed to process request from Bootstrap", t); - } - } - } - } - - private void writeDump(final OutputStream out) throws IOException { - final DiagnosticsDump diagnosticsDump = nifi.getServer().getThreadDumpFactory().create(true); - diagnosticsDump.writeTo(out); - } - - private String getClusterStatus() { - final ClusterDetailsFactory clusterDetailsFactory = nifi.getServer().getClusterDetailsFactory(); - if (clusterDetailsFactory == null) { - return ConnectionState.UNKNOWN.name(); - } - - final ConnectionState connectionState = clusterDetailsFactory.getConnectionState(); - return connectionState == null ? ConnectionState.UNKNOWN.name() : connectionState.name(); - } - - private void decommission() throws InterruptedException { - final DecommissionTask decommissionTask = nifi.getServer().getDecommissionTask(); - if (decommissionTask == null) { - throw new IllegalArgumentException("This NiFi instance does not support decommissioning"); - } - - decommissionTask.decommission(); - } - - private void writeDiagnostics(final OutputStream out, final boolean verbose) throws IOException { - final DiagnosticsDump diagnosticsDump = nifi.getServer().getDiagnosticsFactory().create(verbose); - diagnosticsDump.writeTo(out); - } - - private void writeNodeStatusHistory(final OutputStream out, final int days) throws IOException { - final StatusHistoryDump statusHistoryDump = nifi.getServer().getStatusHistoryDumpFactory().create(days); - statusHistoryDump.writeTo(out); - } - - private void sendAnswer(final OutputStream out, final String answer) throws IOException { - out.write((answer + "\n").getBytes(StandardCharsets.UTF_8)); - out.flush(); - } - - @SuppressWarnings("resource") // we don't want to close the stream, as the caller will do that - private BootstrapRequest readRequest(final InputStream in) throws IOException { - // We want to ensure that we don't try to read data from an InputStream directly - // by a BufferedReader because any user on the system could open a socket and send - // a multi-gigabyte file without any new lines in order to crash the NiFi instance - // (or at least cause OutOfMemoryErrors, which can wreak havoc on the running instance). - // So we will limit the Input Stream to only 4 KB, which should be plenty for any request. - final LimitingInputStream limitingIn = new LimitingInputStream(in, 4096); - final BufferedReader reader = new BufferedReader(new InputStreamReader(limitingIn)); - - final String line = reader.readLine(); - final String[] splits = line.split(" "); - if (splits.length < 1) { - throw new IOException("Received invalid request from Bootstrap: " + line); - } - - final String requestType = splits[0]; - final String[] args; - if (splits.length == 1) { - throw new IOException("Received invalid request from Bootstrap; request did not have a secret key; request type = " + requestType); - } else if (splits.length == 2) { - args = new String[0]; - } else { - args = Arrays.copyOfRange(splits, 2, splits.length); - } - - final String requestKey = splits[1]; - if (!secretKey.equals(requestKey)) { - throw new IOException("Received invalid Secret Key for request type " + requestType); - } - - try { - return new BootstrapRequest(requestType, args); - } catch (final Exception e) { - throw new IOException("Received invalid request from Bootstrap; request type = " + requestType); - } - } - - private static class BootstrapRequest { - public enum RequestType { - RELOAD, - SHUTDOWN, - DUMP, - DIAGNOSTICS, - DECOMMISSION, - PING, - IS_LOADED, - STATUS_HISTORY, - CLUSTER_STATUS - } - - private final RequestType requestType; - private final String[] args; - - public BootstrapRequest(final String request, final String[] args) { - this.requestType = RequestType.valueOf(request); - this.args = args; - } - - public RequestType getRequestType() { - return requestType; - } - - @SuppressWarnings("unused") - public String[] getArgs() { - return args; - } - } -} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java index deb7529967b4..c21ec70bd0cd 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java @@ -25,6 +25,8 @@ import org.apache.nifi.nar.NarUnpacker; import org.apache.nifi.nar.SystemBundle; import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.runtime.ManagementServer; +import org.apache.nifi.runtime.StandardManagementServer; import org.apache.nifi.util.DiagnosticUtils; import org.apache.nifi.util.FileUtils; import org.apache.nifi.util.NiFiProperties; @@ -38,6 +40,7 @@ import java.io.OutputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; @@ -50,19 +53,32 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; -public class NiFi implements NiFiEntryPoint { +public class NiFi { - public static final String BOOTSTRAP_PORT_PROPERTY = "nifi.bootstrap.listen.port"; - public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss"); + private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss"); + + private static final String MANAGEMENT_SERVER_ADDRESS = "org.apache.nifi.management.server.address"; + + private static final Pattern MANAGEMENT_SERVER_ADDRESS_PATTERN = Pattern.compile("^(.+?):([1-9][0-9]{3,4})$"); + + private static final String MANAGEMENT_SERVER_DEFAULT_ADDRESS = "127.0.0.1:52020"; + + private static final int ADDRESS_GROUP = 1; + + private static final int PORT_GROUP = 2; private static final Logger LOGGER = LoggerFactory.getLogger(NiFi.class); private final NiFiServer nifiServer; - private final BootstrapListener bootstrapListener; + private final NiFiProperties properties; + private final ManagementServer managementServer; + private volatile boolean shutdown = false; public NiFi(final NiFiProperties properties) @@ -89,25 +105,6 @@ public NiFi(final NiFiProperties properties, ClassLoader rootClassLoader) // register the shutdown hook addShutdownHook(); - final String bootstrapPort = System.getProperty(BOOTSTRAP_PORT_PROPERTY); - if (bootstrapPort != null) { - try { - final int port = Integer.parseInt(bootstrapPort); - - if (port < 1 || port > 65535) { - throw new RuntimeException("Failed to start NiFi because system property '" + BOOTSTRAP_PORT_PROPERTY + "' is not a valid integer in the range 1 - 65535"); - } - - bootstrapListener = new BootstrapListener(this, port); - bootstrapListener.start(properties.getDefaultListenerBootstrapPort()); - } catch (final NumberFormatException nfe) { - throw new RuntimeException("Failed to start NiFi because system property '" + BOOTSTRAP_PORT_PROPERTY + "' is not a valid integer in the range 1 - 65535"); - } - } else { - LOGGER.info("NiFi started without Bootstrap Port information provided; will not listen for requests from Bootstrap"); - bootstrapListener = null; - } - // delete the web working dir - if the application does not start successfully // the web app directories might be in an invalid state. when this happens // jetty will not attempt to re-extract the war into the directory. by removing @@ -151,15 +148,12 @@ public NiFi(final NiFiProperties properties, ClassLoader rootClassLoader) narBundles, extensionMapping); + managementServer = getManagementServer(); if (shutdown) { LOGGER.info("NiFi has been shutdown via NiFi Bootstrap. Will not start Controller"); } else { nifiServer.start(); - - if (bootstrapListener != null) { - bootstrapListener.setNiFiLoaded(true); - bootstrapListener.sendStartedStatus(true); - } + managementServer.start(); final long duration = System.nanoTime() - startTime; final double durationSeconds = TimeUnit.NANOSECONDS.toMillis(duration) / 1000.0; @@ -172,14 +166,16 @@ public NiFiServer getServer() { } protected void setDefaultUncaughtExceptionHandler() { - Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> LOGGER.error("An Unknown Error Occurred in Thread {}", thread, exception)); + Thread.setDefaultUncaughtExceptionHandler(new ExceptionHandler()); } protected void addShutdownHook() { - Runtime.getRuntime().addShutdownHook(new Thread(() -> - // shutdown the jetty server - shutdownHook(false) - )); + final Thread shutdownHook = Thread.ofPlatform() + .name(NiFi.class.getSimpleName()) + .uncaughtExceptionHandler(new ExceptionHandler()) + .unstarted(this::stop); + + Runtime.getRuntime().addShutdownHook(shutdownHook); } protected void initLogging() { @@ -205,12 +201,15 @@ private static ClassLoader createBootstrapClassLoader() { return new URLClassLoader(urls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); } - public void shutdownHook(final boolean isReload) { + /** + * Stop Application and shutdown server + */ + public void stop() { try { runDiagnosticsOnShutdown(); shutdown(); } catch (final Throwable t) { - LOGGER.warn("Problem occurred ensuring Jetty web server was properly terminated", t); + LOGGER.warn("Application Controller shutdown failed", t); } } @@ -238,18 +237,39 @@ private void diagnose(final File file, final boolean verbose) throws IOException } } - protected void shutdown() { this.shutdown = true; - LOGGER.info("Application Server shutdown started"); - if (nifiServer != null) { + LOGGER.info("Application Controller shutdown started"); + + managementServer.stop(); + + if (nifiServer == null) { + LOGGER.info("Application Server not running"); + } else { nifiServer.stop(); } - if (bootstrapListener != null) { - bootstrapListener.stop(); + + LOGGER.info("Application Controller shutdown completed"); + } + + private ManagementServer getManagementServer() { + final String managementServerAddressProperty = System.getProperty(MANAGEMENT_SERVER_ADDRESS, MANAGEMENT_SERVER_DEFAULT_ADDRESS); + if (managementServerAddressProperty.isBlank()) { + throw new IllegalStateException("Management Server Address System Property [%s] not configured".formatted(MANAGEMENT_SERVER_ADDRESS)); + } + + final Matcher matcher = MANAGEMENT_SERVER_ADDRESS_PATTERN.matcher(managementServerAddressProperty); + if (matcher.matches()) { + final String addressGroup = matcher.group(ADDRESS_GROUP); + final String portGroup = matcher.group(PORT_GROUP); + final int port = Integer.parseInt(portGroup); + + final InetSocketAddress bindAddress = new InetSocketAddress(addressGroup, port); + return new StandardManagementServer(bindAddress, nifiServer); + } else { + throw new IllegalStateException("Management Server Address System Property [%s] not valid [%s]".formatted(MANAGEMENT_SERVER_ADDRESS, managementServerAddressProperty)); } - LOGGER.info("Application Server shutdown completed"); } /** @@ -301,4 +321,12 @@ private static NiFiProperties initializeProperties(final ClassLoader boostrapLoa Thread.currentThread().setContextClassLoader(contextClassLoader); } } + + private static class ExceptionHandler implements Thread.UncaughtExceptionHandler { + + @Override + public void uncaughtException(final Thread thread, Throwable exception) { + LOGGER.error("An Unknown Error Occurred in Thread {}", thread, exception); + } + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthClusterHttpHandler.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthClusterHttpHandler.java new file mode 100644 index 000000000000..6a11031a1b10 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthClusterHttpHandler.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.runtime; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import org.apache.nifi.NiFiServer; +import org.apache.nifi.cluster.ClusterDetailsFactory; +import org.apache.nifi.cluster.ConnectionState; +import org.apache.nifi.controller.DecommissionTask; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +import static java.net.HttpURLConnection.HTTP_ACCEPTED; +import static java.net.HttpURLConnection.HTTP_BAD_METHOD; +import static java.net.HttpURLConnection.HTTP_OK; + +/** + * HTTP Handler for Cluster Health status operations + */ +class HealthClusterHttpHandler implements HttpHandler { + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + + private static final String TEXT_PLAIN = "text/plain"; + + private static final int NO_RESPONSE_BODY = -1; + + private static final String GET_METHOD = "GET"; + + private static final String DELETE_METHOD = "DELETE"; + + private static final String STATUS = "Cluster Status: %s\n"; + + private final NiFiServer server; + + HealthClusterHttpHandler(final NiFiServer server) { + this.server = Objects.requireNonNull(server); + } + + @Override + public void handle(final HttpExchange exchange) throws IOException { + final String requestMethod = exchange.getRequestMethod(); + + final OutputStream responseBody = exchange.getResponseBody(); + + if (GET_METHOD.contentEquals(requestMethod)) { + exchange.getResponseHeaders().set(CONTENT_TYPE_HEADER, TEXT_PLAIN); + final ConnectionState connectionState = getConnectionState(); + final String status = STATUS.formatted(connectionState); + final byte[] response = status.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(HTTP_OK, response.length); + responseBody.write(response); + } else if (DELETE_METHOD.contentEquals(requestMethod)) { + startDecommission(); + + exchange.getResponseHeaders().set(CONTENT_TYPE_HEADER, TEXT_PLAIN); + final String status = STATUS.formatted(ConnectionState.OFFLOADING); + final byte[] response = status.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(HTTP_ACCEPTED, response.length); + responseBody.write(response); + } else { + exchange.sendResponseHeaders(HTTP_BAD_METHOD, NO_RESPONSE_BODY); + } + } + + private void startDecommission() { + final DecommissionTask decommissionTask = server.getDecommissionTask(); + Thread.ofVirtual().name(DecommissionTask.class.getSimpleName()).start(() -> { + try { + decommissionTask.decommission(); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + } + + private ConnectionState getConnectionState() { + final ConnectionState connectionState; + + final ClusterDetailsFactory clusterDetailsFactory = server.getClusterDetailsFactory(); + if (clusterDetailsFactory == null) { + connectionState = ConnectionState.UNKNOWN; + } else { + connectionState = clusterDetailsFactory.getConnectionState(); + } + + return connectionState; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthDiagnosticsHttpHandler.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthDiagnosticsHttpHandler.java new file mode 100644 index 000000000000..3f4e37b81b45 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthDiagnosticsHttpHandler.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.runtime; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import org.apache.nifi.NiFiServer; +import org.apache.nifi.diagnostics.DiagnosticsDump; +import org.apache.nifi.diagnostics.DiagnosticsFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.Objects; + +import static java.net.HttpURLConnection.HTTP_BAD_METHOD; +import static java.net.HttpURLConnection.HTTP_OK; + +/** + * HTTP Handler for Health Diagnostics operations + */ +class HealthDiagnosticsHttpHandler implements HttpHandler { + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + + private static final String TEXT_PLAIN = "text/plain"; + + private static final int STREAM_RESPONSE_BODY = 0; + + private static final int NO_RESPONSE_BODY = -1; + + private static final String GET_METHOD = "GET"; + + private static final String VERBOSE_QUERY_ENABLED = "verbose=true"; + + private final NiFiServer server; + + HealthDiagnosticsHttpHandler(final NiFiServer server) { + this.server = Objects.requireNonNull(server); + } + + @Override + public void handle(final HttpExchange exchange) throws IOException { + final String requestMethod = exchange.getRequestMethod(); + + if (GET_METHOD.contentEquals(requestMethod)) { + exchange.getResponseHeaders().set(CONTENT_TYPE_HEADER, TEXT_PLAIN); + exchange.sendResponseHeaders(HTTP_OK, STREAM_RESPONSE_BODY); + + final URI requestUri = exchange.getRequestURI(); + final boolean verboseRequested = getVerboseRequested(requestUri); + + final DiagnosticsFactory diagnosticsFactory = server.getDiagnosticsFactory(); + final DiagnosticsDump diagnosticsDump = diagnosticsFactory.create(verboseRequested); + try (OutputStream responseBody = exchange.getResponseBody()) { + diagnosticsDump.writeTo(responseBody); + } + } else { + exchange.sendResponseHeaders(HTTP_BAD_METHOD, NO_RESPONSE_BODY); + } + } + + private boolean getVerboseRequested(final URI requestUri) { + final boolean verboseRequested; + + final String query = requestUri.getQuery(); + if (query == null) { + verboseRequested = false; + } else { + verboseRequested = query.contains(VERBOSE_QUERY_ENABLED); + } + + return verboseRequested; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthHttpHandler.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthHttpHandler.java new file mode 100644 index 000000000000..f37e980fa928 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthHttpHandler.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.runtime; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +import static java.net.HttpURLConnection.HTTP_BAD_METHOD; +import static java.net.HttpURLConnection.HTTP_OK; + +/** + * HTTP Handler for Health status operations + */ +class HealthHttpHandler implements HttpHandler { + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + + private static final String TEXT_PLAIN = "text/plain"; + + private static final int NO_RESPONSE_BODY = -1; + + private static final String GET_METHOD = "GET"; + + private static final String STATUS_UP = "Status: UP\n"; + + @Override + public void handle(final HttpExchange exchange) throws IOException { + final String requestMethod = exchange.getRequestMethod(); + + final OutputStream responseBody = exchange.getResponseBody(); + + if (GET_METHOD.contentEquals(requestMethod)) { + exchange.getResponseHeaders().set(CONTENT_TYPE_HEADER, TEXT_PLAIN); + final byte[] response = STATUS_UP.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(HTTP_OK, response.length); + responseBody.write(response); + } else { + exchange.sendResponseHeaders(HTTP_BAD_METHOD, NO_RESPONSE_BODY); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthStatusHistoryHttpHandler.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthStatusHistoryHttpHandler.java new file mode 100644 index 000000000000..6483e0d92bc7 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/HealthStatusHistoryHttpHandler.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.runtime; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import org.apache.nifi.NiFiServer; +import org.apache.nifi.controller.status.history.StatusHistoryDump; +import org.apache.nifi.controller.status.history.StatusHistoryDumpFactory; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.net.HttpURLConnection.HTTP_BAD_METHOD; +import static java.net.HttpURLConnection.HTTP_OK; + +/** + * HTTP Handler for Health Status History operations + */ +class HealthStatusHistoryHttpHandler implements HttpHandler { + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + + private static final String APPLICATION_JSON = "application/json"; + + private static final int STREAM_RESPONSE_BODY = 0; + + private static final int NO_RESPONSE_BODY = -1; + + private static final String GET_METHOD = "GET"; + + private static final Pattern DAYS_QUERY_PATTERN = Pattern.compile("^days=(\\d+)$"); + + private static final int DAYS_GROUP = 1; + + private static final int DAYS_DEFAULT = 1; + + private final NiFiServer server; + + HealthStatusHistoryHttpHandler(final NiFiServer server) { + this.server = Objects.requireNonNull(server); + } + + @Override + public void handle(final HttpExchange exchange) throws IOException { + final String requestMethod = exchange.getRequestMethod(); + + if (GET_METHOD.contentEquals(requestMethod)) { + exchange.getResponseHeaders().set(CONTENT_TYPE_HEADER, APPLICATION_JSON); + exchange.sendResponseHeaders(HTTP_OK, STREAM_RESPONSE_BODY); + + final URI requestUri = exchange.getRequestURI(); + final int daysRequested = getDaysRequested(requestUri); + + final StatusHistoryDumpFactory statusHistoryDumpFactory = server.getStatusHistoryDumpFactory(); + final StatusHistoryDump statusHistoryDump = statusHistoryDumpFactory.create(daysRequested); + + try (OutputStream responseBody = exchange.getResponseBody()) { + statusHistoryDump.writeTo(responseBody); + } + } else { + exchange.sendResponseHeaders(HTTP_BAD_METHOD, NO_RESPONSE_BODY); + } + } + + private int getDaysRequested(final URI requestUri) { + final int daysRequested; + + final String query = requestUri.getQuery(); + if (query == null) { + daysRequested = DAYS_DEFAULT; + } else { + final Matcher matcher = DAYS_QUERY_PATTERN.matcher(query); + if (matcher.matches()) { + final String daysGroup = matcher.group(DAYS_GROUP); + daysRequested = Integer.parseInt(daysGroup); + } else { + daysRequested = DAYS_DEFAULT; + } + } + + return daysRequested; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/ManagementServer.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/ManagementServer.java new file mode 100644 index 000000000000..6ecf1e5d9671 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/ManagementServer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.runtime; + +/** + * Abstraction for controlling Management Server operation + */ +public interface ManagementServer { + /** + * Start Server and bind to configured address + */ + void start(); + + /** + * Stop Server + */ + void stop(); +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/StandardManagementServer.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/StandardManagementServer.java new file mode 100644 index 000000000000..01966da5427c --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/runtime/StandardManagementServer.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.runtime; + +import com.sun.net.httpserver.HttpServer; +import org.apache.nifi.NiFiServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.InetSocketAddress; +import java.util.Objects; + +/** + * Standard Management Server based on Java HttpServer + */ +public class StandardManagementServer implements ManagementServer { + + private static final Logger logger = LoggerFactory.getLogger(StandardManagementServer.class); + + private static final String HEALTH_PATH = "/health"; + + private static final String HEALTH_CLUSTER_PATH = "/health/cluster"; + + private static final String HEALTH_DIAGNOSTICS_PATH = "/health/diagnostics"; + + private static final String HEALTH_STATUS_HISTORY_PATH = "/health/status-history"; + + private static final int STOP_DELAY = 0; + + private static final int CONNECTION_BACKLOG = 10; + + private final InetSocketAddress bindAddress; + + private final NiFiServer server; + + private HttpServer httpServer; + + public StandardManagementServer(final InetSocketAddress bindAddress, final NiFiServer server) { + this.bindAddress = Objects.requireNonNull(bindAddress, "Bind Address required"); + this.server = Objects.requireNonNull(server, "Server required"); + } + + @Override + public void start() { + if (httpServer == null) { + try { + httpServer = HttpServer.create(); + + httpServer.createContext(HEALTH_PATH, new HealthHttpHandler()); + httpServer.createContext(HEALTH_CLUSTER_PATH, new HealthClusterHttpHandler(server)); + httpServer.createContext(HEALTH_DIAGNOSTICS_PATH, new HealthDiagnosticsHttpHandler(server)); + httpServer.createContext(HEALTH_STATUS_HISTORY_PATH, new HealthStatusHistoryHttpHandler(server)); + + httpServer.bind(bindAddress, CONNECTION_BACKLOG); + httpServer.start(); + + final InetSocketAddress serverAddress = getServerAddress(); + + logger.info("Started Management Server on http://{}:{}", serverAddress.getHostString(), serverAddress.getPort()); + } catch (final IOException e) { + throw new UncheckedIOException("Management Server start failed", e); + } + } else { + throw new IllegalStateException("Management Server running"); + } + } + + @Override + public void stop() { + if (httpServer == null) { + logger.info("Management Server not running"); + } else { + httpServer.stop(STOP_DELAY); + logger.info("Management Server stopped"); + } + } + + protected InetSocketAddress getServerAddress() { + return httpServer == null ? bindAddress : httpServer.getAddress(); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/java/org/apache/nifi/runtime/StandardManagementServerTest.java b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/java/org/apache/nifi/runtime/StandardManagementServerTest.java new file mode 100644 index 000000000000..1a6711338da3 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/java/org/apache/nifi/runtime/StandardManagementServerTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.runtime; + +import org.apache.nifi.NiFiServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; + +@ExtendWith(MockitoExtension.class) +class StandardManagementServerTest { + private static final String LOCALHOST = "127.0.0.1"; + + private static final String HEALTH_URI = "http://%s:%d/health"; + + private static final String GET_METHOD = "GET"; + + private static final String DELETE_METHOD = "DELETE"; + + private static final Duration TIMEOUT = Duration.ofSeconds(15); + + @Mock + private NiFiServer server; + + @Test + void testStartStop() { + final InetSocketAddress initialBindAddress = new InetSocketAddress(LOCALHOST, 0); + final StandardManagementServer standardManagementServer = new StandardManagementServer(initialBindAddress, server); + + try { + standardManagementServer.start(); + + final InetSocketAddress bindAddress = standardManagementServer.getServerAddress(); + assertNotSame(initialBindAddress.getPort(), bindAddress.getPort()); + } finally { + standardManagementServer.stop(); + } + } + + @Test + void testGetHealth() throws Exception { + final InetSocketAddress initialBindAddress = new InetSocketAddress(LOCALHOST, 0); + final StandardManagementServer standardManagementServer = new StandardManagementServer(initialBindAddress, server); + + try { + standardManagementServer.start(); + + final InetSocketAddress serverAddress = standardManagementServer.getServerAddress(); + assertNotSame(initialBindAddress.getPort(), serverAddress.getPort()); + + assertResponseStatusCode(serverAddress, GET_METHOD, HttpURLConnection.HTTP_OK); + } finally { + standardManagementServer.stop(); + } + } + + @Test + void testDeleteHealth() throws Exception { + final InetSocketAddress initialBindAddress = new InetSocketAddress(LOCALHOST, 0); + + final StandardManagementServer standardManagementServer = new StandardManagementServer(initialBindAddress, server); + + try { + standardManagementServer.start(); + + final InetSocketAddress serverAddress = standardManagementServer.getServerAddress(); + assertNotSame(initialBindAddress.getPort(), serverAddress.getPort()); + + assertResponseStatusCode(serverAddress, DELETE_METHOD, HttpURLConnection.HTTP_BAD_METHOD); + } finally { + standardManagementServer.stop(); + } + } + + private void assertResponseStatusCode(final InetSocketAddress serverAddress, final String method, final int responseStatusCode) throws IOException, InterruptedException { + final URI healthUri = URI.create(HEALTH_URI.formatted(serverAddress.getHostString(), serverAddress.getPort())); + + try (HttpClient httpClient = HttpClient.newBuilder().connectTimeout(TIMEOUT).build()) { + final HttpRequest request = HttpRequest.newBuilder(healthUri) + .method(method, HttpRequest.BodyPublishers.noBody()) + .timeout(TIMEOUT) + .build(); + + final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + final int statusCode = response.statusCode(); + + assertEquals(responseStatusCode, statusCode); + + final String responseBody = response.body(); + assertNotNull(responseBody); + } + } +} diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java index 0ab74e03db34..db1565ccd028 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java @@ -16,7 +16,12 @@ */ package org.apache.nifi.tests.system; -import org.apache.nifi.bootstrap.RunNiFi; +import org.apache.nifi.bootstrap.command.process.ManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.ProcessBuilderProvider; +import org.apache.nifi.bootstrap.command.process.StandardManagementServerAddressProvider; +import org.apache.nifi.bootstrap.command.process.StandardProcessBuilderProvider; +import org.apache.nifi.bootstrap.configuration.ConfigurationProvider; +import org.apache.nifi.bootstrap.configuration.StandardConfigurationProvider; import org.apache.nifi.registry.security.util.KeystoreType; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientConfig; @@ -51,7 +56,7 @@ public SpawnedStandaloneNiFiInstanceFactory(final InstanceConfiguration instance @Override public NiFiInstance createInstance() { - return new RunNiFiInstance(instanceConfig); + return new ProcessNiFiInstance(instanceConfig); } @Override @@ -78,14 +83,14 @@ public int hashCode() { return Objects.hash(instanceConfig); } - private static class RunNiFiInstance implements NiFiInstance { + private static class ProcessNiFiInstance implements NiFiInstance { private final File instanceDirectory; private final File configDir; private final InstanceConfiguration instanceConfiguration; private File bootstrapConfigFile; - private RunNiFi runNiFi; + private Process process; - public RunNiFiInstance(final InstanceConfiguration instanceConfiguration) { + public ProcessNiFiInstance(final InstanceConfiguration instanceConfiguration) { this.instanceDirectory = instanceConfiguration.getInstanceDirectory(); this.bootstrapConfigFile = instanceConfiguration.getBootstrapConfigFile(); this.instanceConfiguration = instanceConfiguration; @@ -108,30 +113,33 @@ public RunNiFiInstance(final InstanceConfiguration instanceConfiguration) { @Override public String toString() { - return "RunNiFiInstance[dir=" + instanceDirectory + "]"; + return "ProcessNiFiInstance[home=%s,process=%s]".formatted(instanceDirectory, process); } @Override public void start(final boolean waitForCompletion) { - if (runNiFi != null) { + if (process != null) { throw new IllegalStateException("NiFi has already been started"); } logger.info("Starting NiFi [{}]", instanceDirectory.getName()); - try { - this.runNiFi = new RunNiFi(bootstrapConfigFile); - } catch (IOException e) { - throw new RuntimeException("Failed to start NiFi", e); - } + final Map environmentVariables = Map.of("NIFI_HOME", instanceDirectory.getAbsolutePath()); + final ConfigurationProvider configurationProvider = new StandardConfigurationProvider(environmentVariables, new Properties()); + final ManagementServerAddressProvider managementServerAddressProvider = new StandardManagementServerAddressProvider(configurationProvider); + final ProcessBuilderProvider processBuilderProvider = new StandardProcessBuilderProvider(configurationProvider, managementServerAddressProvider); try { - runNiFi.start(false); + final ProcessBuilder processBuilder = processBuilderProvider.getApplicationProcessBuilder(); + processBuilder.directory(instanceDirectory); + process = processBuilder.start(); + + logger.info("Started NiFi [{}] PID [{}]", instanceDirectory.getName(), process.pid()); if (waitForCompletion) { waitForStartup(); } - } catch (IOException e) { + } catch (final IOException e) { throw new RuntimeException("Failed to start NiFi", e); } } @@ -225,7 +233,7 @@ private void copyContents(final File dir, final File destinationDir) throws IOEx @Override public boolean isAccessible() { - if (runNiFi == null) { + if (process == null) { return false; } @@ -263,20 +271,27 @@ private void waitForStartup() throws IOException { @Override public void stop() { - if (runNiFi == null) { + if (process == null) { logger.info("NiFi Shutdown Ignored (runNiFi==null) [{}]", instanceDirectory.getName()); return; } - logger.info("NiFi Shutdown Started [{}]", instanceDirectory.getName()); + logger.info("NiFi Process [{}] Shutdown Started [{}]", process.pid(), instanceDirectory.getName()); try { - runNiFi.stop(); - logger.info("NiFi Shutdown Completed [{}]", instanceDirectory.getName()); - } catch (IOException e) { + process.destroy(); + logger.info("NiFi Process [{}] Shutdown Requested [{}]", process.pid(), instanceDirectory.getName()); + process.waitFor(15, TimeUnit.SECONDS); + logger.info("NiFi Process [{}] Shutdown Completed [{}]", process.pid(), instanceDirectory.getName()); + } catch (final Exception e) { throw new RuntimeException("Failed to stop NiFi", e); } finally { - runNiFi = null; + try { + process.destroyForcibly(); + } catch (final Exception e) { + logger.warn("NiFi Process [{}] force termination failed", process.pid(), e); + } + process = null; } } @@ -352,11 +367,8 @@ public void quarantineTroubleshootingInfo(final File destinationDir, final Throw copyContents(new File(getInstanceDirectory(), dirToCopy), new File(destinationDir, dirToCopy)); } - if (runNiFi == null) { + if (process == null) { logger.warn("NiFi instance is not running so will not capture diagnostics for {}", getInstanceDirectory()); - } else { - final File diagnosticsFile = new File(destinationDir, "diagnostics.txt"); - runNiFi.diagnostics(diagnosticsFile, false); } final File causeFile = new File(destinationDir, "test-failure-stack-trace.txt"); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml index e29ca67b3a9d..e2d5b11a1dac 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml @@ -15,8 +15,6 @@ --> - - true diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml index e29ca67b3a9d..e2d5b11a1dac 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml @@ -15,8 +15,6 @@ --> - - true diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml index aa8e72d4522e..28e7c63f7ddc 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml @@ -15,8 +15,6 @@ --> - - true diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml index a3047818dd57..d387a430b82f 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/logback.xml @@ -15,8 +15,6 @@ --> - - true diff --git a/pom.xml b/pom.xml index f248106962b5..7e1e33b0ccea 100644 --- a/pom.xml +++ b/pom.xml @@ -129,7 +129,7 @@ 4.4.16 1.78.1 1.20.1 - 2.0.15 + 2.0.16 2.9.0 10.17.1.0 12.0.12