NIFI-13581 Remove Parameter Contexts and Providers in destroyFlow of …

…NiFiSystemIT (#9110) Signed-off-by: David Handermann <exceptionfactory@apache.org>
apache · Jul 24, 2024 · 94f8688 · 94f8688
1 parent 2dd7cf7
commit 94f8688
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 4 deletions.
diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml
@@ -26,7 +26,7 @@ on:
       - 'nifi-system-tests/**'
       - 'nifi-api/**'
       - 'nifi-framework-api/**'
-      - 'nifi-nifi-framework-bundle/**'
+      - 'nifi-framework-bundle/**'
       - 'nifi-extension-bundles/nifi-py4j-bundle/**'
       - 'nifi-stateless/**'
   pull_request:
@@ -36,7 +36,7 @@ on:
       - 'nifi-system-tests/**'
       - 'nifi-api/**'
       - 'nifi-framework-api/**'
-      - 'nifi-nifi-framework-bundle/**'
+      - 'nifi-framework-bundle/**'
       - 'nifi-extension-bundles/nifi-py4j-bundle/**'
       - 'nifi-stateless/**'
 

diff --git a/...sts/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/...sts/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java
@@ -89,13 +89,15 @@
 import org.apache.nifi.web.api.entity.ParameterContextEntity;
 import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity;
 import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity;
+import org.apache.nifi.web.api.entity.ParameterContextsEntity;
 import org.apache.nifi.web.api.entity.ParameterEntity;
 import org.apache.nifi.web.api.entity.ParameterGroupConfigurationEntity;
 import org.apache.nifi.web.api.entity.ParameterProviderApplyParametersRequestEntity;
 import org.apache.nifi.web.api.entity.ParameterProviderConfigurationEntity;
 import org.apache.nifi.web.api.entity.ParameterProviderEntity;
 import org.apache.nifi.web.api.entity.ParameterProviderParameterApplicationEntity;
 import org.apache.nifi.web.api.entity.ParameterProviderParameterFetchEntity;
+import org.apache.nifi.web.api.entity.ParameterProvidersEntity;
 import org.apache.nifi.web.api.entity.PortEntity;
 import org.apache.nifi.web.api.entity.ProcessGroupEntity;
 import org.apache.nifi.web.api.entity.ProcessGroupFlowEntity;
@@ -126,11 +128,13 @@
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class NiFiClientUtil {
@@ -472,6 +476,38 @@ public void deleteFlowAnalysisRules() throws NiFiClientException, IOException {
         }
     }
 
+    public void deleteParameterContexts() throws NiFiClientException, IOException {
+        final ParameterContextsEntity parameterContextsEntity = nifiClient.getParamContextClient().getParamContexts();
+        final Map<String, ParameterContextEntity> parameterContextMap = parameterContextsEntity.getParameterContexts().stream()
+                .collect(Collectors.toMap(ParameterContextEntity::getId, Function.identity()));
+
+        // If parameter context have inherited contexts then they needed to be deleted from the bottom up, so we just keep iterating
+        // over the set of ids and retrying deletes until all have been deleted, knowing that some delete calls will fail the first time
+        final Set<String> parameterContextIds = new HashSet<>(parameterContextMap.keySet());
+        while (!parameterContextIds.isEmpty()) {
+            final Iterator<String> parameterContextIdIterator = parameterContextIds.iterator();
+            while (parameterContextIdIterator.hasNext()) {
+                final String parameterContextId = parameterContextIdIterator.next();
+                final ParameterContextEntity parameterContextEntity = parameterContextMap.get(parameterContextId);
+                try {
+                    final String version = String.valueOf(parameterContextEntity.getRevision().getVersion());
+                    nifiClient.getParamContextClient().deleteParamContext(parameterContextId, version, true);
+                    parameterContextIdIterator.remove();
+                } catch (final Exception e) {
+                    logger.warn("Failed to delete parameter context [{}] due to: {}", parameterContextId, e.getMessage());
+                }
+            }
+        }
+    }
+
+    public void deleteParameterProviders() throws NiFiClientException, IOException {
+        final ParameterProvidersEntity parameterProvidersEntity = nifiClient.getFlowClient().getParamProviders();
+        for (final ParameterProviderEntity parameterProviderEntity : parameterProvidersEntity.getParameterProviders()) {
+            final String version = String.valueOf(parameterProviderEntity.getRevision().getVersion());
+            nifiClient.getParamProviderClient().deleteParamProvider(parameterProviderEntity.getId(), version, true);
+        }
+    }
+
     public void waitForFlowAnalysisRuleState(final String desiredState, final Collection<String> ruleIdsOfInterest) throws NiFiClientException, IOException {
         final long maxTimestamp = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(2L);
 

diff --git a/...tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java b/...tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java
@@ -249,6 +249,8 @@ protected void destroyFlow() throws NiFiClientException, IOException, Interrupte
         getClientUtil().deleteControllerLevelServices();
         getClientUtil().deleteReportingTasks();
         getClientUtil().deleteFlowAnalysisRules();
+        getClientUtil().deleteParameterContexts();
+        getClientUtil().deleteParameterProviders();
 
         logger.info("Finished destroyFlow");
     }

diff --git a/...it-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java b/...it-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java
@@ -32,6 +32,8 @@ public interface ParamContextClient {
 
     ParameterContextEntity deleteParamContext(String id, String version) throws NiFiClientException, IOException;
 
+    ParameterContextEntity deleteParamContext(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException;
+
     ParameterContextUpdateRequestEntity updateParamContext(ParameterContextEntity paramContext) throws NiFiClientException, IOException;
 
     ParameterContextUpdateRequestEntity getParamContextUpdateRequest(String contextId, String updateRequestId) throws NiFiClientException, IOException;

diff --git a/...t-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java b/...t-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamProviderClient.java
@@ -32,6 +32,8 @@ public interface ParamProviderClient {
 
     ParameterProviderEntity deleteParamProvider(String id, String version) throws NiFiClientException, IOException;
 
+    ParameterProviderEntity deleteParamProvider(String id, String version, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException;
+
     ParameterProviderEntity fetchParameters(ParameterProviderParameterFetchEntity parameterFetchEntity) throws NiFiClientException, IOException;
 
     ParameterProviderApplyParametersRequestEntity applyParameters(ParameterProviderParameterApplicationEntity parameterApplicationEntity) throws NiFiClientException, IOException;

diff --git a/...main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java b/...main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java
@@ -83,6 +83,11 @@ public ParameterContextEntity createParamContext(final ParameterContextEntity pa
 
     @Override
     public ParameterContextEntity deleteParamContext(final String id, final String version) throws NiFiClientException, IOException {
+        return deleteParamContext(id, version, false);
+    }
+
+    @Override
+    public ParameterContextEntity deleteParamContext(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException {
         if (StringUtils.isBlank(id)) {
             throw new IllegalArgumentException("Parameter context id cannot be null or blank");
         }
@@ -92,9 +97,14 @@ public ParameterContextEntity deleteParamContext(final String id, final String v
         }
 
         return executeAction("Error deleting parameter context", () -> {
-            final WebTarget target = paramContextTarget.path("{id}")
+            WebTarget target = paramContextTarget.path("{id}")
                     .resolveTemplate("id", id)
                     .queryParam("version", version);
+
+            if (disconnectedNodeAcknowledged) {
+                target = target.queryParam("disconnectedNodeAcknowledged", "true");
+            }
+
             return getRequestBuilder(target).delete(ParameterContextEntity.class);
         });
     }

diff --git a/...ain/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java b/...ain/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamProviderClient.java
@@ -75,6 +75,11 @@ public ParameterProviderEntity updateParamProvider(final ParameterProviderEntity
 
     @Override
     public ParameterProviderEntity deleteParamProvider(final String id, final String version) throws NiFiClientException, IOException {
+        return deleteParamProvider(id, version, false);
+    }
+
+    @Override
+    public ParameterProviderEntity deleteParamProvider(final String id, final String version, final boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException {
         if (StringUtils.isBlank(id)) {
             throw new IllegalArgumentException("Parameter provider id cannot be null or blank");
         }
@@ -84,9 +89,14 @@ public ParameterProviderEntity deleteParamProvider(final String id, final String
         }
 
         return executeAction("Error deleting parameter provider", () -> {
-            final WebTarget target = paramProviderTarget.path("{id}")
+            WebTarget target = paramProviderTarget.path("{id}")
                     .resolveTemplate("id", id)
                     .queryParam("version", version);
+
+            if (disconnectedNodeAcknowledged) {
+                target = target.queryParam("disconnectedNodeAcknowledged", "true");
+            }
+
             return getRequestBuilder(target).delete(ParameterProviderEntity.class);
         });
     }