Conformance: Merging SupportedFeatures and ExemptFeatures, adding TLSRoute feature

Author: robscott

conformance/conformance_test.go

@@ -47,36 +47,28 @@ func TestConformance(t *testing.T) {
 	t.Logf("Running conformance tests with %s GatewayClass", *flags.GatewayClassName)
 
 	supportedFeatures := parseSupportedFeatures(*flags.SupportedFeatures)
-	exemptFeatures := parseExemptFeatures(*flags.ExemptFeatures)
+	exemptFeatures := parseSupportedFeatures(*flags.ExemptFeatures)
+	for feature := range exemptFeatures {
+		supportedFeatures[feature] = false
+	}
 
 	cSuite := suite.New(suite.Options{
 		Client:               client,
 		GatewayClassName:     *flags.GatewayClassName,
 		Debug:                *flags.ShowDebug,
 		CleanupBaseResources: *flags.CleanupBaseResources,
 		SupportedFeatures:    supportedFeatures,
-		ExemptFeatures:       exemptFeatures,
 	})
 	cSuite.Setup(t)
 	cSuite.Run(t, tests.ConformanceTests)
 }
 
-// parseSupportedFeatures parses the arguments for supported-features flag,
-// then converts the string to []suite.SupportedFeature
-func parseSupportedFeatures(f string) []suite.SupportedFeature {
-	var res []suite.SupportedFeature
-	for _, value := range strings.Split(f, ",") {
-		res = append(res, suite.SupportedFeature(value))
-	}
-	return res
-}
-
-// parseExemptFeatures parses the arguments for exempt-features flag,
-// then converts the string to []suite.ExemptFeature
-func parseExemptFeatures(f string) []suite.ExemptFeature {
-	var res []suite.ExemptFeature
+// parseSupportedFeatures parses flag arguments and converts the string to
+// map[suite.SupportedFeature]bool
+func parseSupportedFeatures(f string) map[suite.SupportedFeature]bool {
+	res := map[suite.SupportedFeature]bool{}
 	for _, value := range strings.Split(f, ",") {
-		res = append(res, suite.ExemptFeature(value))
+		res[suite.SupportedFeature(value)] = true
 	}
 	return res
 }

conformance/tests/gateway-unsupported-route-kind.go conformance/tests/gateway-invalid-route-kind.go

@@ -28,16 +28,16 @@ import (
 )
 
 func init() {
-	ConformanceTests = append(ConformanceTests, GatewayUnsupportedRouteKind)
+	ConformanceTests = append(ConformanceTests, GatewayInvalidRouteKind)
 }
 
-var GatewayUnsupportedRouteKind = suite.ConformanceTest{
-	ShortName:   "GatewayUnsupportedRouteKind",
-	Description: "A Gateway in the gateway-conformance-infra namespace should fail to become ready if explicitly supports any route type incompatible with the protocol type, even if there are types which are valid as well.",
-	Manifests:   []string{"tests/gateway-unsupported-route-kind.yaml"},
+var GatewayInvalidRouteKind = suite.ConformanceTest{
+	ShortName:   "GatewayInvalidRouteKind",
+	Description: "A Gateway in the gateway-conformance-infra namespace should fail to become ready an invalid Route kind is specified.",
+	Manifests:   []string{"tests/gateway-invalid-route-kind.yaml"},
 	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
 		t.Run("Gateway listener should have a false ResolvedRefs condition with reason InvalidRouteKinds and no supportedKinds", func(t *testing.T) {
-			gwNN := types.NamespacedName{Name: "gateway-only-unsupported-route-kind", Namespace: "gateway-conformance-infra"}
+			gwNN := types.NamespacedName{Name: "gateway-only-invalid-route-kind", Namespace: "gateway-conformance-infra"}
 			listeners := []v1beta1.ListenerStatus{{
 				Name:           v1beta1.SectionName("http"),
 				SupportedKinds: []v1beta1.RouteGroupKind{},
@@ -52,7 +52,7 @@ var GatewayUnsupportedRouteKind = suite.ConformanceTest{
 		})
 
 		t.Run("Gateway listener should have a false ResolvedRefs condition with reason InvalidRouteKinds and HTTPRoute must be put in the supportedKinds", func(t *testing.T) {
-			gwNN := types.NamespacedName{Name: "gateway-supported-and-unsupported-route-kind", Namespace: "gateway-conformance-infra"}
+			gwNN := types.NamespacedName{Name: "gateway-supported-and-invalid-route-kind", Namespace: "gateway-conformance-infra"}
 			listeners := []v1beta1.ListenerStatus{{
 				Name: v1beta1.SectionName("http"),
 				SupportedKinds: []v1beta1.RouteGroupKind{{

conformance/tests/gateway-unsupported-route-kind.yaml conformance/tests/gateway-invalid-route-kind.yaml

@@ -1,7 +1,7 @@
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: Gateway
 metadata:
-  name: gateway-only-unsupported-route-kind
+  name: gateway-only-invalid-route-kind
   namespace: gateway-conformance-infra
 spec:
   gatewayClassName: "{GATEWAY_CLASS_NAME}"
@@ -13,12 +13,12 @@ spec:
         namespaces:
           from: All
         kinds:
-        - kind: UDPRoute
+        - kind: InvalidRoute
 ---
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: Gateway
 metadata:
-  name: gateway-supported-and-unsupported-route-kind
+  name: gateway-supported-and-invalid-route-kind
   namespace: gateway-conformance-infra
 spec:
   gatewayClassName: "{GATEWAY_CLASS_NAME}"
@@ -30,5 +30,5 @@ spec:
         namespaces:
           from: All
         kinds:
-        - kind: UDPRoute
+        - kind: InvalidRoute
         - kind: HTTPRoute

conformance/tests/gateway-secret-invalid-reference-grant.go

@@ -34,7 +34,7 @@ func init() {
 var GatewaySecretInvalidReferenceGrant = suite.ConformanceTest{
 	ShortName:   "GatewaySecretInvalidReferenceGrant",
 	Description: "A Gateway in the gateway-conformance-infra namespace should fail to become ready if the Gateway has a certificateRef for a Secret in the gateway-conformance-web-backend namespace and a ReferenceGrant exists but does not grant permission to that specific Secret",
-	Exemptions:  []suite.ExemptFeature{suite.ExemptReferenceGrant},
+	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
 	Manifests:   []string{"tests/gateway-secret-invalid-reference-grant.yaml"},
 	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
 		gwNN := types.NamespacedName{Name: "gateway-secret-invalid-reference-grant", Namespace: "gateway-conformance-infra"}

conformance/tests/gateway-secret-missing-reference-grant.go

@@ -34,7 +34,7 @@ func init() {
 var GatewaySecretMissingReferenceGrant = suite.ConformanceTest{
 	ShortName:   "GatewaySecretMissingReferenceGrant",
 	Description: "A Gateway in the gateway-conformance-infra namespace should fail to become ready if the Gateway has a certificateRef for a Secret in the gateway-conformance-web-backend namespace and a ReferenceGrant granting permission to the Secret does not exist",
-	Exemptions:  []suite.ExemptFeature{suite.ExemptReferenceGrant},
+	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
 	Manifests:   []string{"tests/gateway-secret-missing-reference-grant.yaml"},
 	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
 		gwNN := types.NamespacedName{Name: "gateway-secret-missing-reference-grant", Namespace: "gateway-conformance-infra"}

conformance/tests/gateway-secret-reference-grant-all-in-namespace.go

@@ -34,7 +34,7 @@ func init() {
 var GatewaySecretReferenceGrantAllInNamespace = suite.ConformanceTest{
 	ShortName:   "GatewaySecretReferenceGrantAllInNamespace",
 	Description: "A Gateway in the gateway-conformance-infra namespace should become ready if the Gateway has a certificateRef for a Secret in the gateway-conformance-web-backend namespace and a ReferenceGrant granting permission to all Secrets in the namespace exists",
-	Exemptions:  []suite.ExemptFeature{suite.ExemptReferenceGrant},
+	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
 	Manifests:   []string{"tests/gateway-secret-reference-grant-all-in-namespace.yaml"},
 	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
 		gwNN := types.NamespacedName{Name: "gateway-secret-reference-grant", Namespace: "gateway-conformance-infra"}

conformance/tests/gateway-secret-reference-grant-specific.go

@@ -34,7 +34,7 @@ func init() {
 var GatewaySecretReferenceGrantSpecific = suite.ConformanceTest{
 	ShortName:   "GatewaySecretReferenceGrantSpecific",
 	Description: "A Gateway in the gateway-conformance-infra namespace should become ready if the Gateway has a certificateRef for a Secret in the gateway-conformance-web-backend namespace and a ReferenceGrant granting permission to the specific Secret exists",
-	Exemptions:  []suite.ExemptFeature{suite.ExemptReferenceGrant},
+	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
 	Manifests:   []string{"tests/gateway-secret-reference-grant-specific.yaml"},
 	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
 		gwNN := types.NamespacedName{Name: "gateway-secret-reference-grant", Namespace: "gateway-conformance-infra"}

conformance/tests/httproute-disallowed-kind.go

@@ -32,6 +32,7 @@ func init() {
 var HTTPRouteDisallowedKind = suite.ConformanceTest{
 	ShortName:   "HTTPRouteDisallowedKind",
 	Description: "A single HTTPRoute in the gateway-conformance-infra namespace should fail to attach to a Listener that does not allow the HTTPRoute kind",
+	Features:    []suite.SupportedFeature{suite.SupportTLSRoute},
 	Manifests:   []string{"tests/httproute-disallowed-kind.yaml"},
 	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
 		// This test creates an additional Gateway in the gateway-conformance-infra

conformance/tests/httproute-invalid-cross-namespace-backend-ref.go

@@ -35,7 +35,7 @@ func init() {
 var HTTPRouteInvalidCrossNamespaceBackendRef = suite.ConformanceTest{
 	ShortName:   "HTTPRouteInvalidCrossNamespaceBackendRef",
 	Description: "A single HTTPRoute in the gateway-conformance-infra namespace should set a ResolvedRefs status False with reason RefNotPermitted when attempting to bind to a Gateway in the same namespace if the route has a BackendRef Service in the gateway-conformance-web-backend namespace and a ReferenceGrant granting permission to route to that Service does not exist",
-	Exemptions:  []suite.ExemptFeature{suite.ExemptReferenceGrant},
+	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
 	Manifests:   []string{"tests/httproute-invalid-cross-namespace-backend-ref.yaml"},
 	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
 		routeNN := types.NamespacedName{Name: "invalid-cross-namespace-backend-ref", Namespace: "gateway-conformance-infra"}

conformance/tests/httproute-invalid-reference-grant.go

@@ -35,7 +35,7 @@ func init() {
 var HTTPRouteInvalidReferenceGrant = suite.ConformanceTest{
 	ShortName:   "HTTPRouteInvalidReferenceGrant",
 	Description: "A single HTTPRoute in the gateway-conformance-infra namespace should fail to attach to a Gateway in the same namespace if the route has a backendRef Service in the gateway-conformance-app-backend namespace and a ReferenceGrant exists but does not grant permission to route to that specific Service",
-	Exemptions:  []suite.ExemptFeature{suite.ExemptReferenceGrant},
+	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
 	Manifests:   []string{"tests/httproute-invalid-reference-grant.yaml"},
 	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
 		routeNN := types.NamespacedName{Name: "invalid-reference-grant", Namespace: "gateway-conformance-infra"}

conformance/tests/httproute-reference-grant.go

@@ -33,7 +33,7 @@ func init() {
 var HTTPRouteReferenceGrant = suite.ConformanceTest{
 	ShortName:   "HTTPRouteReferenceGrant",
 	Description: "A single HTTPRoute in the gateway-conformance-infra namespace, with a backendRef in the gateway-conformance-web-backend namespace, should attach to Gateway in the gateway-conformance-infra namespace",
-	Exemptions:  []suite.ExemptFeature{suite.ExemptReferenceGrant},
+	Features:    []suite.SupportedFeature{suite.SupportReferenceGrant},
 	Manifests:   []string{"tests/httproute-reference-grant.yaml"},
 	Test: func(t *testing.T, s *suite.ConformanceTestSuite) {
 		routeNN := types.NamespacedName{Name: "reference-grant", Namespace: "gateway-conformance-infra"}

conformance/utils/suite/suite.go

@@ -19,7 +19,6 @@ package suite
 import (
 	"testing"
 
-	"golang.org/x/exp/slices"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"sigs.k8s.io/gateway-api/apis/v1beta1"
@@ -28,22 +27,19 @@ import (
 	"sigs.k8s.io/gateway-api/conformance/utils/roundtripper"
 )
 
-// ExemptFeature allows opting out of core conformance tests at an
-// individual feature granularity.
-type ExemptFeature string
-
-const (
-	// This option indicates the implementation is exempting itself from the
-	// requirement of a ReferenceGrant to allow cross-namespace references,
-	// and has instead implemented alternative safeguards.
-	ExemptReferenceGrant ExemptFeature = "ReferenceGrant"
-)
-
 // SupportedFeature allows opting in to additional conformance tests at an
 // individual feature granularity.
 type SupportedFeature string
 
 const (
+	// This option indicates support for ReferenceGrant (core conformance).
+	// Opting out of this requires an implementation to have clearly implemented
+	// and documented equivalent safeguards.
+	SupportReferenceGrant SupportedFeature = "ReferenceGrant"
+
+	// This option indicates support for TLSRoute (extended conformance).
+	SupportTLSRoute SupportedFeature = "TLSRoute"
+
 	// This option indicates support for HTTPRoute query param matching (extended conformance).
 	SupportHTTPRouteQueryParamMatching SupportedFeature = "HTTPRouteQueryParamMatching"
 
@@ -54,6 +50,12 @@ const (
 	SupportHTTPResponseHeaderModification SupportedFeature = "HTTPResponseHeaderModification"
 )
 
+// StandardCoreFeatures are the features that are required to be conformant with
+// the Core API features that are part of the Standard release channel.
+var StandardCoreFeatures = map[SupportedFeature]bool{
+	SupportReferenceGrant: true,
+}
+
 // ConformanceTestSuite defines the test suite used to run Gateway API
 // conformance tests.
 type ConformanceTestSuite struct {
@@ -65,8 +67,7 @@ type ConformanceTestSuite struct {
 	Cleanup           bool
 	BaseManifests     string
 	Applier           kubernetes.Applier
-	ExemptFeatures    []ExemptFeature
-	SupportedFeatures []SupportedFeature
+	SupportedFeatures map[SupportedFeature]bool
 	TimeoutConfig     config.TimeoutConfig
 }
 
@@ -89,8 +90,7 @@ type Options struct {
 	// CleanupBaseResources indicates whether or not the base test
 	// resources such as Gateways should be cleaned up after the run.
 	CleanupBaseResources bool
-	ExemptFeatures       []ExemptFeature
-	SupportedFeatures    []SupportedFeature
+	SupportedFeatures    map[SupportedFeature]bool
 	TimeoutConfig        config.TimeoutConfig
 }
 
@@ -103,6 +103,16 @@ func New(s Options) *ConformanceTestSuite {
 		roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig}
 	}
 
+	if s.SupportedFeatures == nil {
+		s.SupportedFeatures = StandardCoreFeatures
+	} else {
+		for feature, val := range StandardCoreFeatures {
+			if _, ok := s.SupportedFeatures[feature]; !ok {
+				s.SupportedFeatures[feature] = val
+			}
+		}
+	}
+
 	suite := &ConformanceTestSuite{
 		Client:           s.Client,
 		RoundTripper:     roundTripper,
@@ -114,7 +124,6 @@ func New(s Options) *ConformanceTestSuite {
 			NamespaceLabels:          s.NamespaceLabels,
 			ValidUniqueListenerPorts: s.ValidUniqueListenerPorts,
 		},
-		ExemptFeatures:    s.ExemptFeatures,
 		SupportedFeatures: s.SupportedFeatures,
 		TimeoutConfig:     s.TimeoutConfig,
 	}
@@ -164,7 +173,6 @@ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
 type ConformanceTest struct {
 	ShortName   string
 	Description string
-	Exemptions  []ExemptFeature
 	Features    []SupportedFeature
 	Manifests   []string
 	Slow        bool
@@ -182,19 +190,11 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
 	// Check that all features exercised by the test have been opted into by
 	// the suite.
 	for _, feature := range test.Features {
-		if !slices.Contains(suite.SupportedFeatures, feature) {
+		if supported, ok := suite.SupportedFeatures[feature]; !ok || !supported {
 			t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature)
 		}
 	}
 
-	// Check that no features exercised by the test have been opted out of by
-	// the suite.
-	for _, feature := range test.Exemptions {
-		if slices.Contains(suite.ExemptFeatures, feature) {
-			t.Skipf("Skipping %s: suite exempts %s", test.ShortName, feature)
-		}
-	}
-
 	for _, manifestLocation := range test.Manifests {
 		t.Logf("Applying %s", manifestLocation)
 		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, suite.GatewayClassName, true)