NIFI-2933 Remote input/output ports at any PG

nifi-docs/src/main/asciidoc/user-guide.adoc

@@ -1238,12 +1238,14 @@ value entered is not valid, the Remote Process Group will not be valid and will
 [[Site-to-Site_Input_Port]]
 *Input Port*: In order to allow another NiFi instance to push data to your local instance, you can simply drag an <<input_port,Input Port>> onto the Root Process Group of your canvas. After entering a name for the port, it will be added to your flow. You can now right-click on the Input Port and choose Configure in order to adjust the name and the number of concurrent tasks that are used for the port.
 
+Also, you can create an Input Port for Site-to-Site in child Process Groups by selecting "Receive Data From" as "Site-to-Site connections".
+
 If Site-to-Site is configured to run securely, you will need to manage the port's "receive data via site-to-site" component access policy. Only those users who have been added to the policy will be able to communicate with the port.
 
 [[Site-to-Site_Output_Port]]
-*Output Port*: Similar to an Input Port, a DataFlow Manager may choose to add an <<output_port,Output Port>> to the Root Process Group. The Output Port allows an authorized NiFi instance to remotely connect to your instance and pull data from the Output Port. Configuring the Output Port and managing the port's access policies will again allow the DFM to control how many concurrent tasks are allowed, as well as which users are authorized to pull data from the instance being configured.
+*Output Port*: Similar to an Input Port, a DataFlow Manager may choose to add an <<output_port,Output Port>> to the Root Process Group. Or an Output Port in child Process Groups for Site-to-Site connections. The Output Port allows an authorized NiFi instance to remotely connect to your instance and pull data from the Output Port. Configuring the Output Port and managing the port's access policies will again allow the DFM to control how many concurrent tasks are allowed, as well as which users are authorized to pull data from the instance being configured.
 
-In addition to other instances of NiFi, some other applications may use a Site-to-Site client in order to push data to or receive data from a NiFi instance. For example, NiFi provides an Apache Storm spout and an Apache Spark Receiver that are able to pull data from NiFi's Root Group Output Ports.
+In addition to other instances of NiFi, some other applications may use a Site-to-Site client in order to push data to or receive data from a NiFi instance. For example, NiFi provides an Apache Storm spout and an Apache Spark Receiver that are able to pull data from NiFi's Root Group Output Ports, and Output Ports in child Process Groups for Site-to-Site connections.
 
 For information on how to enable and configure Site-to-Site on a NiFi instance, see the
 link:administration-guide.html#site_to_site_properties[Site-to-Site Properties] section of the

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/PortDTO.java

@@ -36,6 +36,7 @@ public class PortDTO extends ComponentDTO {
     private Integer concurrentlySchedulableTaskCount;
     private Set<String> userAccessControl;
     private Set<String> groupAccessControl;
+    private Boolean allowRemoteAccess;
 
     private Collection<String> validationErrors;
 
@@ -114,10 +115,10 @@ public void setComments(String comments) {
     }
 
     /**
-     * @return whether this port has incoming or outgoing connections to a remote NiFi. This is only applicable when the port is running on the root group
+     * @return whether this port has incoming or outgoing connections to a remote NiFi. This is only applicable when the port is allowed to be accessed remotely.
      */
     @ApiModelProperty(
-            value = "Whether the port has incoming or output connections to a remote NiFi. This is only applicable when the port is running in the root group."
+            value = "Whether the port has incoming or output connections to a remote NiFi. This is only applicable when the port is allowed to be accessed remotely."
     )
     public Boolean isTransmitting() {
         return transmitting;
@@ -171,4 +172,18 @@ public void setValidationErrors(Collection<String> validationErrors) {
         this.validationErrors = validationErrors;
     }
 
+
+    /**
+     * @return whether this port can be accessed remotely via Site-to-Site protocol.
+     */
+    @ApiModelProperty(
+            value = "Whether this port can be accessed remotely via Site-to-Site protocol."
+    )
+    public Boolean getAllowRemoteAccess() {
+        return allowRemoteAccess;
+    }
+
+    public void setAllowRemoteAccess(Boolean allowRemoteAccess) {
+        this.allowRemoteAccess = allowRemoteAccess;
+    }
 }

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ProcessGroupDTO.java

@@ -17,6 +17,7 @@
 package org.apache.nifi.web.api.dto;
 
 import io.swagger.annotations.ApiModelProperty;
+import org.apache.nifi.web.api.dto.util.NumberUtil;
 
 import javax.xml.bind.annotation.XmlType;
 import java.util.Map;
@@ -45,8 +46,11 @@ public class ProcessGroupDTO extends ComponentDTO {
     private Integer locallyModifiedAndStaleCount;
     private Integer syncFailureCount;
 
-    private Integer inputPortCount;
-    private Integer outputPortCount;
+    private Integer localInputPortCount;
+    private Integer localOutputPortCount;
+
+    private Integer publicInputPortCount;
+    private Integer publicOutputPortCount;
 
     private FlowSnippetDTO contents;
 
@@ -102,14 +106,46 @@ public void setContents(FlowSnippetDTO contents) {
      * @return number of input ports contained in this process group
      */
     @ApiModelProperty(
-            value = "The number of input ports in the process group."
+            value = "The number of input ports in the process group.",
+            readOnly = true
     )
     public Integer getInputPortCount() {
-        return inputPortCount;
+        return NumberUtil.sumNullableIntegers(localInputPortCount, publicInputPortCount);
     }
 
     public void setInputPortCount(Integer inputPortCount) {
-        this.inputPortCount = inputPortCount;
+        // Without having setter for 'inputPortCount', deserialization fails.
+        // If we use Jackson annotation @JsonIgnoreProperties, this empty setter is not needed.
+        // Ex. @JsonIgnoreProperties(value={"inputPortCount", "outputPortCount"}, allowGetters=true)
+        // But in order to minimize dependencies, we don't use Jackson annotations in this module.
+    }
+
+    /**
+     * @return number of local input ports contained in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of local input ports in the process group."
+    )
+    public Integer getLocalInputPortCount() {
+        return localInputPortCount;
+    }
+
+    public void setLocalInputPortCount(Integer localInputPortCount) {
+        this.localInputPortCount = localInputPortCount;
+    }
+
+    /**
+     * @return number of public input ports contained in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of public input ports in the process group."
+    )
+    public Integer getPublicInputPortCount() {
+        return publicInputPortCount;
+    }
+
+    public void setPublicInputPortCount(Integer publicInputPortCount) {
+        this.publicInputPortCount = publicInputPortCount;
     }
 
     /**
@@ -130,14 +166,43 @@ public void setInvalidCount(Integer invalidCount) {
      * @return number of output ports in this process group
      */
     @ApiModelProperty(
-            value = "The number of output ports in the process group."
+            value = "The number of output ports in the process group.",
+            readOnly = true
     )
     public Integer getOutputPortCount() {
-        return outputPortCount;
+        return NumberUtil.sumNullableIntegers(localOutputPortCount, publicOutputPortCount);
     }
 
     public void setOutputPortCount(Integer outputPortCount) {
-        this.outputPortCount = outputPortCount;
+        // See setInputPortCount for the reason why this is needed.
+    }
+
+    /**
+     * @return number of local output ports in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of local output ports in the process group."
+    )
+    public Integer getLocalOutputPortCount() {
+        return localOutputPortCount;
+    }
+
+    public void setLocalOutputPortCount(Integer localOutputPortCount) {
+        this.localOutputPortCount = localOutputPortCount;
+    }
+
+    /**
+     * @return number of public output ports in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of public output ports in the process group."
+    )
+    public Integer getPublicOutputPortCount() {
+        return publicOutputPortCount;
+    }
+
+    public void setPublicOutputPortCount(Integer publicOutputPortCount) {
+        this.publicOutputPortCount = publicOutputPortCount;
     }
 
     /**

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/util/NumberUtil.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto.util;
+
+/**
+ * Utility class for numbers.
+ */
+public class NumberUtil {
+
+    /**
+     * Calculate sum of Integers those can be null.
+     * This method can be used to avoid getting NullPointerException when a null Integer being auto-boxed into an int.
+     * @param values Integers to add
+     * @return the sum of given values or null if all values are null
+     */
+    public static Integer sumNullableIntegers(Integer ... values) {
+        int sum = 0;
+        int count = 0;
+        for (Integer value : values) {
+            if (value == null) {
+                continue;
+            }
+            sum += value;
+            count++;
+        }
+        return count == 0 ? null : sum;
+    }
+
+}

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/PortEntity.java

@@ -33,6 +33,7 @@ public class PortEntity extends ComponentEntity implements Permissible<PortDTO>,
     private PortStatusDTO status;
     private String portType;
     private PermissionsDTO operatePermissions;
+    private Boolean allowRemoteAccess;
 
     /**
      * @return input PortDTO that are being serialized
@@ -84,4 +85,18 @@ public PermissionsDTO getOperatePermissions() {
     public void setOperatePermissions(PermissionsDTO permissions) {
         this.operatePermissions = permissions;
     }
+
+    /**
+     * @return whether this port can be accessed remotely via Site-to-Site protocol.
+     */
+    @ApiModelProperty(
+        value = "Whether this port can be accessed remotely via Site-to-Site protocol."
+    )
+    public Boolean isAllowRemoteAccess() {
+        return allowRemoteAccess;
+    }
+
+    public void setAllowRemoteAccess(Boolean allowRemoteAccess) {
+        this.allowRemoteAccess = allowRemoteAccess;
+    }
 }

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/ProcessGroupEntity.java

@@ -20,6 +20,7 @@
 import org.apache.nifi.registry.flow.VersionedFlowSnapshot;
 import org.apache.nifi.web.api.dto.ProcessGroupDTO;
 import org.apache.nifi.web.api.dto.status.ProcessGroupStatusDTO;
+import org.apache.nifi.web.api.dto.util.NumberUtil;
 
 import javax.xml.bind.annotation.XmlRootElement;
 
@@ -48,8 +49,10 @@ public class ProcessGroupEntity extends ComponentEntity implements Permissible<P
     private Integer locallyModifiedAndStaleCount;
     private Integer syncFailureCount;
 
-    private Integer inputPortCount;
-    private Integer outputPortCount;
+    private Integer localInputPortCount;
+    private Integer localOutputPortCount;
+    private Integer publicInputPortCount;
+    private Integer publicOutputPortCount;
 
     /**
      * The ProcessGroupDTO that is being serialized.
@@ -84,14 +87,43 @@ public void setStatus(ProcessGroupStatusDTO status) {
      * @return number of input ports contained in this process group
      */
     @ApiModelProperty(
-        value = "The number of input ports in the process group."
+        value = "The number of input ports in the process group.",
+        readOnly = true
     )
     public Integer getInputPortCount() {
-        return inputPortCount;
+        return NumberUtil.sumNullableIntegers(localInputPortCount, publicInputPortCount);
     }
 
     public void setInputPortCount(Integer inputPortCount) {
-        this.inputPortCount = inputPortCount;
+        // See ProcessGroupDTO.setInputPortCount for the reason why this is needed.
+    }
+
+    /**
+     * @return number of local input ports contained in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of local input ports in the process group."
+    )
+    public Integer getLocalInputPortCount() {
+        return localInputPortCount;
+    }
+
+    public void setLocalInputPortCount(Integer localInputPortCount) {
+        this.localInputPortCount = localInputPortCount;
+    }
+
+    /**
+     * @return number of public input ports contained in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of public input ports in the process group."
+    )
+    public Integer getPublicInputPortCount() {
+        return publicInputPortCount;
+    }
+
+    public void setPublicInputPortCount(Integer publicInputPortCount) {
+        this.publicInputPortCount = publicInputPortCount;
     }
 
     /**
@@ -112,14 +144,43 @@ public void setInvalidCount(Integer invalidCount) {
      * @return number of output ports in this process group
      */
     @ApiModelProperty(
-        value = "The number of output ports in the process group."
+        value = "The number of output ports in the process group.",
+        readOnly = true
     )
     public Integer getOutputPortCount() {
-        return outputPortCount;
+        return NumberUtil.sumNullableIntegers(localOutputPortCount, publicOutputPortCount);
     }
 
     public void setOutputPortCount(Integer outputPortCount) {
-        this.outputPortCount = outputPortCount;
+        // See ProcessGroupDTO.setInputPortCount for the reason why this is needed.
+    }
+
+    /**
+     * @return number of local output ports in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of local output ports in the process group."
+    )
+    public Integer getLocalOutputPortCount() {
+        return localOutputPortCount;
+    }
+
+    public void setLocalOutputPortCount(Integer localOutputPortCount) {
+        this.localOutputPortCount = localOutputPortCount;
+    }
+
+    /**
+     * @return number of public output ports in this process group
+     */
+    @ApiModelProperty(
+        value = "The number of public output ports in the process group."
+    )
+    public Integer getPublicOutputPortCount() {
+        return publicOutputPortCount;
+    }
+
+    public void setPublicOutputPortCount(Integer publicOutputPortCount) {
+        this.publicOutputPortCount = publicOutputPortCount;
     }
 
     /**