diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 9e083e7c545c..db2c337628d1 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -26,7 +26,7 @@ on: - 'nifi-system-tests/**' - 'nifi-api/**' - 'nifi-framework-api/**' - - 'nifi-nifi-framework-bundle/**' + - 'nifi-framework-bundle/**' - 'nifi-extension-bundles/nifi-py4j-bundle/**' - 'nifi-stateless/**' pull_request: @@ -36,7 +36,7 @@ on: - 'nifi-system-tests/**' - 'nifi-api/**' - 'nifi-framework-api/**' - - 'nifi-nifi-framework-bundle/**' + - 'nifi-framework-bundle/**' - 'nifi-extension-bundles/nifi-py4j-bundle/**' - 'nifi-stateless/**' diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java index 8e62196f6e0b..f9aa09a39692 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java @@ -89,6 +89,7 @@ import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; +import org.apache.nifi.web.api.entity.ParameterContextsEntity; import org.apache.nifi.web.api.entity.ParameterEntity; import org.apache.nifi.web.api.entity.ParameterGroupConfigurationEntity; import org.apache.nifi.web.api.entity.ParameterProviderApplyParametersRequestEntity; @@ -96,6 +97,7 @@ import org.apache.nifi.web.api.entity.ParameterProviderEntity; import org.apache.nifi.web.api.entity.ParameterProviderParameterApplicationEntity; import org.apache.nifi.web.api.entity.ParameterProviderParameterFetchEntity; +import org.apache.nifi.web.api.entity.ParameterProvidersEntity; import org.apache.nifi.web.api.entity.PortEntity; import org.apache.nifi.web.api.entity.ProcessGroupEntity; import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity; @@ -126,11 +128,13 @@ import java.util.Collections; import java.util.Date; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Collectors; public class NiFiClientUtil { @@ -472,6 +476,38 @@ public void deleteFlowAnalysisRules() throws NiFiClientException, IOException { } } + public void deleteParameterContexts() throws NiFiClientException, IOException { + final ParameterContextsEntity parameterContextsEntity = nifiClient.getParamContextClient().getParamContexts(); + final Map parameterContextMap = parameterContextsEntity.getParameterContexts().stream() + .collect(Collectors.toMap(ParameterContextEntity::getId, Function.identity())); + + // If parameter context have inherited contexts then they needed to be deleted from the bottom up, so we just keep iterating + // over the set of ids and retrying deletes until all have been deleted, knowing that some delete calls will fail the first time + final Set parameterContextIds = new HashSet<>(parameterContextMap.keySet()); + while (!parameterContextIds.isEmpty()) { + final Iterator parameterContextIdIterator = parameterContextIds.iterator(); + while (parameterContextIdIterator.hasNext()) { + final String parameterContextId = parameterContextIdIterator.next(); + final ParameterContextEntity parameterContextEntity = parameterContextMap.get(parameterContextId); + try { + final String version = String.valueOf(parameterContextEntity.getRevision().getVersion()); + nifiClient.getParamContextClient().deleteParamContext(parameterContextId, version, true); + parameterContextIdIterator.remove(); + } catch (final Exception e) { + logger.warn("Failed to delete parameter context [{}] due to: {}", parameterContextId, e.getMessage()); + } + } + } + } + + public void deleteParameterProviders() throws NiFiClientException, IOException { + final ParameterProvidersEntity parameterProvidersEntity = nifiClient.getFlowClient().getParamProviders(); + for (final ParameterProviderEntity parameterProviderEntity : parameterProvidersEntity.getParameterProviders()) { + final String version = String.valueOf(parameterProviderEntity.getRevision().getVersion()); + nifiClient.getParamProviderClient().deleteParamProvider(parameterProviderEntity.getId(), version, true); + } + } + public void waitForFlowAnalysisRuleState(final String desiredState, final Collection ruleIdsOfInterest) throws NiFiClientException, IOException { final long maxTimestamp = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(2L); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java index 57e2dbe0c607..cb6e955d6d96 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java @@ -249,6 +249,8 @@ protected void destroyFlow() throws NiFiClientException, IOException, Interrupte getClientUtil().deleteControllerLevelServices(); getClientUtil().deleteReportingTasks(); getClientUtil().deleteFlowAnalysisRules(); + getClientUtil().deleteParameterContexts(); + getClientUtil().deleteParameterProviders(); logger.info("Finished destroyFlow"); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java index 265271dc0aa6..607a264bca13 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java @@ -32,6 +32,8 @@ public interface ParamContextClient { ParameterContextEntity deleteParamContext(String id, String version) throws NiFiClientException, IOException; + ParameterContextEntity deleteParamContext(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException; + ParameterContextUpdateRequestEntity updateParamContext(ParameterContextEntity paramContext) throws NiFiClientException, IOException; ParameterContextUpdateRequestEntity getParamContextUpdateRequest(String contextId, String updateRequestId) throws NiFiClientException, IOException; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java index ea17aaa57aff..9a7be680dedc 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java @@ -32,6 +32,8 @@ public interface ParamProviderClient { ParameterProviderEntity deleteParamProvider(String id, String version) throws NiFiClientException, IOException; + ParameterProviderEntity deleteParamProvider(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException; + ParameterProviderEntity fetchParameters(ParameterProviderParameterFetchEntity parameterFetchEntity) throws NiFiClientException, IOException; ParameterProviderApplyParametersRequestEntity applyParameters(ParameterProviderParameterApplicationEntity parameterApplicationEntity) throws NiFiClientException, IOException; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java index 27dd8f53fb54..0b25c6540796 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java @@ -83,6 +83,11 @@ public ParameterContextEntity createParamContext(final ParameterContextEntity pa @Override public ParameterContextEntity deleteParamContext(final String id, final String version) throws NiFiClientException, IOException { + return deleteParamContext(id, version, false); + } + + @Override + public ParameterContextEntity deleteParamContext(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { if (StringUtils.isBlank(id)) { throw new IllegalArgumentException("Parameter context id cannot be null or blank"); } @@ -92,9 +97,14 @@ public ParameterContextEntity deleteParamContext(final String id, final String v } return executeAction("Error deleting parameter context", () -> { - final WebTarget target = paramContextTarget.path("{id}") + WebTarget target = paramContextTarget.path("{id}") .resolveTemplate("id", id) .queryParam("version", version); + + if (disconnectedNodeAcknowledged) { + target = target.queryParam("disconnectedNodeAcknowledged", "true"); + } + return getRequestBuilder(target).delete(ParameterContextEntity.class); }); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java index 6026c9c5b34a..698e5279e1ef 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java @@ -75,6 +75,11 @@ public ParameterProviderEntity updateParamProvider(final ParameterProviderEntity @Override public ParameterProviderEntity deleteParamProvider(final String id, final String version) throws NiFiClientException, IOException { + return deleteParamProvider(id, version, false); + } + + @Override + public ParameterProviderEntity deleteParamProvider(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException { if (StringUtils.isBlank(id)) { throw new IllegalArgumentException("Parameter provider id cannot be null or blank"); } @@ -84,9 +89,14 @@ public ParameterProviderEntity deleteParamProvider(final String id, final String } return executeAction("Error deleting parameter provider", () -> { - final WebTarget target = paramProviderTarget.path("{id}") + WebTarget target = paramProviderTarget.path("{id}") .resolveTemplate("id", id) .queryParam("version", version); + + if (disconnectedNodeAcknowledged) { + target = target.queryParam("disconnectedNodeAcknowledged", "true"); + } + return getRequestBuilder(target).delete(ParameterProviderEntity.class); }); }