diff --git a/pkg/gateway/model/model_build_target_group.go b/pkg/gateway/model/model_build_target_group.go index da430cb2bf..c99f26544b 100644 --- a/pkg/gateway/model/model_build_target_group.go +++ b/pkg/gateway/model/model_build_target_group.go @@ -100,7 +100,7 @@ func (t *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack, gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend, backendSGIDToken core.StringToken) (*elbv2model.TargetGroup, error) { targetGroupProps := backend.ELBV2TargetGroupProps - tgResID := t.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backend.Service), routeDescriptor.GetRouteNamespacedName(), backend.ServicePort.TargetPort) + tgResID := t.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backend.Service), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backend.ServicePort.TargetPort) if tg, exists := t.tgByResID[tgResID]; exists { return tg, nil } @@ -129,7 +129,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate if targetType == elbv2api.TargetTypeInstance { targetPort = intstr.FromInt32(backend.ServicePort.NodePort) } - tgbNetworking := builder.buildTargetGroupBindingNetworking(targetPort, *tgSpec.HealthCheckConfig.Port, *backend.ServicePort, backendSGIDToken) + tgbNetworking := builder.buildTargetGroupBindingNetworking(targetPort, *tgSpec.HealthCheckConfig.Port, tgSpec.Protocol, backendSGIDToken) multiClusterEnabled := builder.buildTargetGroupBindingMultiClusterFlag(tgProps) @@ -175,14 +175,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate } } -func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetPort intstr.IntOrString, healthCheckPort intstr.IntOrString, svcPort corev1.ServicePort, backendSGIDToken core.StringToken) *elbv2model.TargetGroupBindingNetworking { +func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetPort intstr.IntOrString, healthCheckPort intstr.IntOrString, tgProtocol elbv2model.Protocol, backendSGIDToken core.StringToken) *elbv2model.TargetGroupBindingNetworking { if backendSGIDToken == nil { return nil } protocolTCP := elbv2api.NetworkingProtocolTCP protocolUDP := elbv2api.NetworkingProtocolUDP - udpSupported := svcPort.Protocol == corev1.ProtocolUDP + udpSupported := tgProtocol == elbv2model.ProtocolUDP || tgProtocol == elbv2model.ProtocolTCP_UDP if builder.disableRestrictedSGRules { ports := []elbv2api.NetworkingPort{ @@ -282,7 +282,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro return elbv2model.TargetGroupSpec{}, err } tgPort := builder.buildTargetGroupPort(targetType, *backend.ServicePort) - name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), k8s.NamespacedName(backend.Service), tgPort, targetType, tgProtocol, tgProtocolVersion) + name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), route.GetRouteKind(), k8s.NamespacedName(backend.Service), tgPort, targetType, tgProtocol, tgProtocolVersion) if tgPort == 0 { if targetType == elbv2model.TargetTypeIP { @@ -290,7 +290,6 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro } return elbv2model.TargetGroupSpec{}, errors.Errorf("TargetGroup port is empty. When using Instance targets, your service be must of type 'NodePort' or 'LoadBalancer'") } - return elbv2model.TargetGroupSpec{ Name: name, TargetType: targetType, @@ -308,7 +307,7 @@ var invalidTargetGroupNamePattern = regexp.MustCompile("[[:^alnum:]]") // buildTargetGroupName will calculate the targetGroup's name. func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *elbv2gw.TargetGroupProps, - gwKey types.NamespacedName, routeKey types.NamespacedName, svcKey types.NamespacedName, tgPort int32, + gwKey types.NamespacedName, routeKey types.NamespacedName, routeKind routeutils.RouteKind, svcKey types.NamespacedName, tgPort int32, targetType elbv2model.TargetType, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion) string { if targetGroupProps != nil && targetGroupProps.TargetGroupName != nil { @@ -321,6 +320,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *el _, _ = uuidHash.Write([]byte(gwKey.Name)) _, _ = uuidHash.Write([]byte(routeKey.Namespace)) _, _ = uuidHash.Write([]byte(routeKey.Name)) + _, _ = uuidHash.Write([]byte(routeKind)) _, _ = uuidHash.Write([]byte(svcKey.Namespace)) _, _ = uuidHash.Write([]byte(svcKey.Name)) _, _ = uuidHash.Write([]byte(strconv.Itoa(int(tgPort)))) @@ -664,8 +664,8 @@ func (builder *targetGroupBuilderImpl) convertMapToAttributes(attributeMap map[s return convertedAttributes } -func (builder *targetGroupBuilderImpl) buildTargetGroupResourceID(gwKey types.NamespacedName, svcKey types.NamespacedName, routeKey types.NamespacedName, port intstr.IntOrString) string { - return fmt.Sprintf("%s/%s:%s-%s:%s-%s:%s", gwKey.Namespace, gwKey.Name, routeKey.Namespace, routeKey.Name, svcKey.Namespace, svcKey.Name, port.String()) +func (builder *targetGroupBuilderImpl) buildTargetGroupResourceID(gwKey types.NamespacedName, svcKey types.NamespacedName, routeKey types.NamespacedName, routeKind routeutils.RouteKind, port intstr.IntOrString) string { + return fmt.Sprintf("%s/%s:%s-%s:%s-%s-%s:%s", gwKey.Namespace, gwKey.Name, routeKey.Namespace, routeKey.Name, routeKind, svcKey.Namespace, svcKey.Name, port.String()) } func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNodeSelector(tgProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType) *metav1.LabelSelector { diff --git a/pkg/gateway/model/model_build_target_group_test.go b/pkg/gateway/model/model_build_target_group_test.go index 11a79d35f5..92578d0625 100644 --- a/pkg/gateway/model/model_build_target_group_test.go +++ b/pkg/gateway/model/model_build_target_group_test.go @@ -68,7 +68,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-d02da2803b", + Name: "k8s-myrouten-myroute-8d8111f6ac", TargetType: elbv2model.TargetTypeInstance, Port: awssdk.Int32(8080), Protocol: elbv2model.ProtocolTCP, @@ -123,7 +123,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-d146029dfb", + Name: "k8s-myrouten-myroute-224f4b6ea6", TargetType: elbv2model.TargetTypeInstance, Port: awssdk.Int32(8080), Protocol: elbv2model.ProtocolHTTP, @@ -183,7 +183,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-d9d6c4e6eb", + Name: "k8s-myrouten-myroute-3bce8b0f70", TargetType: elbv2model.TargetTypeIP, Port: awssdk.Int32(80), Protocol: elbv2model.ProtocolTCP, @@ -238,7 +238,7 @@ func Test_buildTargetGroupSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-400113e816", + Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, Port: awssdk.Int32(80), Protocol: elbv2model.ProtocolHTTP, @@ -448,7 +448,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-d146029dfb", + Name: "k8s-myrouten-myroute-224f4b6ea6", TargetType: elbv2model.TargetTypeInstance, Port: awssdk.Int32(8080), Protocol: elbv2model.ProtocolHTTP, @@ -476,7 +476,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-d146029dfb", + Name: "k8s-myrouten-myroute-224f4b6ea6", Annotations: make(map[string]string), Labels: make(map[string]string), }, @@ -527,7 +527,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-d9d6c4e6eb", + Name: "k8s-myrouten-myroute-3bce8b0f70", TargetType: elbv2model.TargetTypeIP, Port: awssdk.Int32(80), Protocol: elbv2model.ProtocolTCP, @@ -550,7 +550,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-d9d6c4e6eb", + Name: "k8s-myrouten-myroute-3bce8b0f70", Annotations: make(map[string]string), Labels: make(map[string]string), }, @@ -601,7 +601,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-400113e816", + Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, Port: awssdk.Int32(80), Protocol: elbv2model.ProtocolHTTP, @@ -629,7 +629,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-400113e816", + Name: "k8s-myrouten-myroute-a44a20bcbf", Annotations: make(map[string]string), Labels: make(map[string]string), }, @@ -690,7 +690,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { }, }, expectedTgSpec: elbv2model.TargetGroupSpec{ - Name: "k8s-myrouten-myroute-400113e816", + Name: "k8s-myrouten-myroute-a44a20bcbf", TargetType: elbv2model.TargetTypeIP, Port: awssdk.Int32(80), Protocol: elbv2model.ProtocolHTTP, @@ -718,7 +718,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) { Template: elbv2model.TargetGroupBindingTemplate{ ObjectMeta: metav1.ObjectMeta{ Namespace: "my-svc-ns", - Name: "k8s-myrouten-myroute-400113e816", + Name: "k8s-myrouten-myroute-a44a20bcbf", Annotations: map[string]string{ "foo": "bar", }, @@ -771,7 +771,7 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { targetPort intstr.IntOrString healthCheckPort intstr.IntOrString - svcPort corev1.ServicePort + tgProtocol elbv2model.Protocol backendSGIDToken core.StringToken expected *elbv2model.TargetGroupBindingNetworking @@ -804,9 +804,7 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { name: "disable restricted sg rules - with udp", disableRestrictedSGRules: true, backendSGIDToken: core.LiteralStringToken("foo"), - svcPort: corev1.ServicePort{ - Protocol: corev1.ProtocolUDP, - }, + tgProtocol: elbv2model.ProtocolUDP, expected: &elbv2model.TargetGroupBindingNetworking{ Ingress: []elbv2model.NetworkingIngressRule{ { @@ -834,11 +832,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { { name: "use restricted sg rules - int hc port", backendSGIDToken: core.LiteralStringToken("foo"), - svcPort: corev1.ServicePort{ - Protocol: corev1.ProtocolTCP, - }, - targetPort: intstr80, - healthCheckPort: intstr80, + tgProtocol: elbv2model.ProtocolTCP, + targetPort: intstr80, + healthCheckPort: intstr80, expected: &elbv2model.TargetGroupBindingNetworking{ Ingress: []elbv2model.NetworkingIngressRule{ { @@ -862,11 +858,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { { name: "use restricted sg rules - int hc port - udp traffic", backendSGIDToken: core.LiteralStringToken("foo"), - svcPort: corev1.ServicePort{ - Protocol: corev1.ProtocolUDP, - }, - targetPort: intstr80, - healthCheckPort: intstr80, + tgProtocol: elbv2model.ProtocolUDP, + targetPort: intstr80, + healthCheckPort: intstr80, expected: &elbv2model.TargetGroupBindingNetworking{ Ingress: []elbv2model.NetworkingIngressRule{ { @@ -905,11 +899,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { { name: "use restricted sg rules - str hc port", backendSGIDToken: core.LiteralStringToken("foo"), - svcPort: corev1.ServicePort{ - Protocol: corev1.ProtocolTCP, - }, - targetPort: intstr80, - healthCheckPort: intstrTrafficPort, + tgProtocol: elbv2model.ProtocolHTTP, + targetPort: intstr80, + healthCheckPort: intstrTrafficPort, expected: &elbv2model.TargetGroupBindingNetworking{ Ingress: []elbv2model.NetworkingIngressRule{ { @@ -933,11 +925,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { { name: "use restricted sg rules - str hc port - udp", backendSGIDToken: core.LiteralStringToken("foo"), - svcPort: corev1.ServicePort{ - Protocol: corev1.ProtocolUDP, - }, - targetPort: intstr80, - healthCheckPort: intstrTrafficPort, + tgProtocol: elbv2model.ProtocolUDP, + targetPort: intstr80, + healthCheckPort: intstrTrafficPort, expected: &elbv2model.TargetGroupBindingNetworking{ Ingress: []elbv2model.NetworkingIngressRule{ { @@ -976,11 +966,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { { name: "use restricted sg rules - diff hc port", backendSGIDToken: core.LiteralStringToken("foo"), - svcPort: corev1.ServicePort{ - Protocol: corev1.ProtocolTCP, - }, - targetPort: intstr80, - healthCheckPort: intstr85, + tgProtocol: elbv2model.ProtocolHTTP, + targetPort: intstr80, + healthCheckPort: intstr85, expected: &elbv2model.TargetGroupBindingNetworking{ Ingress: []elbv2model.NetworkingIngressRule{ { @@ -1019,11 +1007,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { { name: "use restricted sg rules - str hc port - udp", backendSGIDToken: core.LiteralStringToken("foo"), - svcPort: corev1.ServicePort{ - Protocol: corev1.ProtocolUDP, - }, - targetPort: intstr80, - healthCheckPort: intstr85, + tgProtocol: elbv2model.ProtocolUDP, + targetPort: intstr80, + healthCheckPort: intstr85, expected: &elbv2model.TargetGroupBindingNetworking{ Ingress: []elbv2model.NetworkingIngressRule{ { @@ -1066,7 +1052,7 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) { disableRestrictedSGRules: tc.disableRestrictedSGRules, } - result := builder.buildTargetGroupBindingNetworking(tc.targetPort, tc.healthCheckPort, tc.svcPort, tc.backendSGIDToken) + result := builder.buildTargetGroupBindingNetworking(tc.targetPort, tc.healthCheckPort, tc.tgProtocol, tc.backendSGIDToken) assert.Equal(t, tc.expected, result) }) } @@ -1101,16 +1087,16 @@ func Test_buildTargetGroupName(t *testing.T) { { name: "no name in props", targetGroupProps: &elbv2gw.TargetGroupProps{}, - expected: "k8s-myns-myroute-719950e570", + expected: "k8s-myns-myroute-27d98b9190", }, { name: "no props", - expected: "k8s-myns-myroute-719950e570", + expected: "k8s-myns-myroute-27d98b9190", }, { name: "protocol specified props", protocolVersion: &http2, - expected: "k8s-myns-myroute-ce262fa9fe", + expected: "k8s-myns-myroute-d2bd5deaa7", }, } @@ -1120,7 +1106,7 @@ func Test_buildTargetGroupName(t *testing.T) { clusterName: clusterName, } - result := builder.buildTargetGroupName(tc.targetGroupProps, gwKey, routeKey, svcKey, 80, elbv2model.TargetTypeIP, elbv2model.ProtocolTCP, tc.protocolVersion) + result := builder.buildTargetGroupName(tc.targetGroupProps, gwKey, routeKey, routeutils.HTTPRouteKind, svcKey, 80, elbv2model.TargetTypeIP, elbv2model.ProtocolTCP, tc.protocolVersion) assert.Equal(t, tc.expected, result) }) } diff --git a/pkg/gateway/routeutils/backend.go b/pkg/gateway/routeutils/backend.go index 16734b514f..8e8036a709 100644 --- a/pkg/gateway/routeutils/backend.go +++ b/pkg/gateway/routeutils/backend.go @@ -98,6 +98,10 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci // Otherwise, general error. No need for status update. return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier)) } + + // TODO -- This should be updated, to handle UDP and TCP on the same service port. + // Currently, it will just arbitrarily take one. + var servicePort *corev1.ServicePort for _, svcPort := range svc.Spec.Ports { @@ -107,6 +111,12 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci } } + if servicePort == nil { + initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port) + wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) + return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil) + } + tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, k8s.NamespacedName(svc)) if err != nil { @@ -114,12 +124,6 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object")) } - if servicePort == nil { - initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port) - wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier) - return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil) - } - var tgProps *elbv2gw.TargetGroupProps if tgConfig != nil { diff --git a/pkg/gateway/routeutils/constants.go b/pkg/gateway/routeutils/constants.go index 4c1a70e9f1..a27c7b693f 100644 --- a/pkg/gateway/routeutils/constants.go +++ b/pkg/gateway/routeutils/constants.go @@ -27,10 +27,10 @@ var allRoutes = map[RouteKind]func(context context.Context, client client.Client } // Default protocol map used to infer accepted route kinds when a listener doesn't specify the `allowedRoutes` field. -var defaultProtocolToRouteKindMap = map[gwv1.ProtocolType]RouteKind{ - gwv1.TCPProtocolType: TCPRouteKind, - gwv1.UDPProtocolType: UDPRouteKind, - gwv1.TLSProtocolType: TLSRouteKind, - gwv1.HTTPProtocolType: HTTPRouteKind, - gwv1.HTTPSProtocolType: HTTPRouteKind, +var defaultProtocolToRouteKindMap = map[gwv1.ProtocolType][]RouteKind{ + gwv1.TCPProtocolType: {TCPRouteKind}, + gwv1.UDPProtocolType: {UDPRouteKind}, + gwv1.TLSProtocolType: {TLSRouteKind, TCPRouteKind}, + gwv1.HTTPProtocolType: {HTTPRouteKind}, + gwv1.HTTPSProtocolType: {HTTPRouteKind}, } diff --git a/pkg/gateway/routeutils/listener_attachment_helper.go b/pkg/gateway/routeutils/listener_attachment_helper.go index 883ea1c6b2..0564b9e612 100644 --- a/pkg/gateway/routeutils/listener_attachment_helper.go +++ b/pkg/gateway/routeutils/listener_attachment_helper.go @@ -125,7 +125,7 @@ func (attachmentHelper *listenerAttachmentHelperImpl) kindCheck(listener gwv1.Li ... */ if listener.AllowedRoutes == nil || listener.AllowedRoutes.Kinds == nil || len(listener.AllowedRoutes.Kinds) == 0 { - allowedRoutes = sets.New[RouteKind](defaultProtocolToRouteKindMap[listener.Protocol]) + allowedRoutes = sets.New[RouteKind](defaultProtocolToRouteKindMap[listener.Protocol]...) } else { // TODO - Not sure how to handle versioning (correctly) here. // So going to ignore the group checks for now :x @@ -134,7 +134,36 @@ func (attachmentHelper *listenerAttachmentHelperImpl) kindCheck(listener gwv1.Li allowedRoutes.Insert(RouteKind(v.Kind)) } } - return allowedRoutes.Has(route.GetRouteKind()) + + isAllowed := allowedRoutes.Has(route.GetRouteKind()) + + if !isAllowed { + return false + } + + if listener.Protocol == gwv1.TLSProtocolType { + + var tlsMode *gwv1.TLSModeType + + if listener.TLS != nil && listener.TLS.Mode != nil { + tlsMode = listener.TLS.Mode + } + switch route.GetRouteKind() { + case TCPRouteKind: + // Listener must allow termination at lb + return tlsMode == nil || *tlsMode == gwv1.TLSModeTerminate + case TLSRouteKind: + // This is kind of different. + // For AWS NLB, the original TLS will be terminated, however + // the LB will establish a new TLS connection to the backend. + // Users that want to persist the same TLS connection should use TCP + return tlsMode != nil && *tlsMode == gwv1.TLSModePassthrough + } + // Unsupported route type. + return false + } + + return true } func (attachmentHelper *listenerAttachmentHelperImpl) hostnameCheck(listener gwv1.Listener, route preLoadRouteDescriptor) (bool, error) { diff --git a/pkg/gateway/routeutils/listener_attachment_helper_test.go b/pkg/gateway/routeutils/listener_attachment_helper_test.go index ffb11a3bde..2e8dca5836 100644 --- a/pkg/gateway/routeutils/listener_attachment_helper_test.go +++ b/pkg/gateway/routeutils/listener_attachment_helper_test.go @@ -297,6 +297,8 @@ func Test_namespaceCheck(t *testing.T) { } func Test_kindCheck(t *testing.T) { + term := gwv1.TLSModeTerminate + pt := gwv1.TLSModePassthrough testCases := []struct { name string route preLoadRouteDescriptor @@ -347,6 +349,55 @@ func Test_kindCheck(t *testing.T) { }, expectedResult: true, }, + { + name: "tls listener, tcp route, terminate by default", + route: &tcpRouteDescription{}, + listener: gwv1.Listener{ + Protocol: gwv1.TCPProtocolType, + }, + expectedResult: true, + }, + { + name: "tls listener, tls route, terminate by default", + route: &tlsRouteDescription{}, + listener: gwv1.Listener{ + Protocol: gwv1.TCPProtocolType, + }, + expectedResult: false, + }, + { + name: "tls listener, tcp route, terminate specified", + route: &tcpRouteDescription{}, + listener: gwv1.Listener{ + Protocol: gwv1.TCPProtocolType, + TLS: &gwv1.GatewayTLSConfig{ + Mode: &term, + }, + }, + expectedResult: true, + }, + { + name: "tls listener, tcp route, passthrough specified", + route: &tcpRouteDescription{}, + listener: gwv1.Listener{ + Protocol: gwv1.TLSProtocolType, + TLS: &gwv1.GatewayTLSConfig{ + Mode: &pt, + }, + }, + expectedResult: false, + }, + { + name: "tls listener, tls route, passthrough specified", + route: &tlsRouteDescription{}, + listener: gwv1.Listener{ + Protocol: gwv1.TLSProtocolType, + TLS: &gwv1.GatewayTLSConfig{ + Mode: &pt, + }, + }, + expectedResult: true, + }, } for _, tc := range testCases { diff --git a/test/e2e/gateway/alb_instance_target_test.go b/test/e2e/gateway/alb_instance_target_test.go index dbf2223f51..6f02420462 100644 --- a/test/e2e/gateway/alb_instance_target_test.go +++ b/test/e2e/gateway/alb_instance_target_test.go @@ -9,6 +9,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" "sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier" + "strconv" ) var _ = Describe("test k8s alb gateway using instance targets reconciled by the aws load balancer controller", func() { @@ -54,6 +55,10 @@ var _ = Describe("test k8s alb gateway using instance targets reconciled by the Expect(lbARN).ToNot(BeEmpty()) }) + tgMap := map[string][]string{ + strconv.Itoa(int(stack.albResourceStack.commonStack.svcs[0].Spec.Ports[0].NodePort)): {"HTTP"}, + } + By("verifying AWS loadbalancer resources", func() { nodeList, err := stack.GetWorkerNodes(ctx, tf) Expect(err).ToNot(HaveOccurred()) @@ -62,7 +67,7 @@ var _ = Describe("test k8s alb gateway using instance targets reconciled by the Scheme: "internet-facing", TargetType: "instance", Listeners: stack.albResourceStack.getListenersPortMap(), - TargetGroups: stack.albResourceStack.getTargetGroupNodePortMap(), + TargetGroups: tgMap, NumTargets: len(nodeList), TargetGroupHC: &verifier.TargetGroupHC{ Protocol: "HTTP", diff --git a/test/e2e/gateway/alb_ip_target_test.go b/test/e2e/gateway/alb_ip_target_test.go index 4a991b3916..f486a700ff 100644 --- a/test/e2e/gateway/alb_ip_target_test.go +++ b/test/e2e/gateway/alb_ip_target_test.go @@ -59,11 +59,11 @@ var _ = Describe("test k8s alb gateway using ip targets reconciled by the aws lo Expect(lbARN).ToNot(BeEmpty()) }) - tgMap := map[string]string{ - "80": "HTTP", + tgMap := map[string][]string{ + "80": {"HTTP"}, } - targetNumber := int(*stack.albResourceStack.commonStack.dp.Spec.Replicas) + targetNumber := int(*stack.albResourceStack.commonStack.dps[0].Spec.Replicas) By("verifying AWS loadbalancer resources", func() { err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, lbARN, verifier.LoadBalancerExpectation{ diff --git a/test/e2e/gateway/alb_resource_stack.go b/test/e2e/gateway/alb_resource_stack.go index d0f27df6a0..7f9bcbcd5b 100644 --- a/test/e2e/gateway/alb_resource_stack.go +++ b/test/e2e/gateway/alb_resource_stack.go @@ -12,7 +12,7 @@ import ( func newALBResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, tgc *elbv2gw.TargetGroupConfiguration, httpr *gwv1.HTTPRoute, baseName string, enablePodReadinessGate bool) *albResourceStack { - commonStack := newCommonResourceStack(dp, svc, gwc, gw, lbc, tgc, baseName, enablePodReadinessGate) + commonStack := newCommonResourceStack([]*appsv1.Deployment{dp}, []*corev1.Service{svc}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{tgc}, baseName, enablePodReadinessGate) return &albResourceStack{ httpr: httpr, commonStack: commonStack, @@ -32,10 +32,6 @@ func (s *albResourceStack) Deploy(ctx context.Context, f *framework.Framework) e }) } -func (s *albResourceStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error { - return s.commonStack.ScaleDeployment(ctx, f, numReplicas) -} - func (s *albResourceStack) Cleanup(ctx context.Context, f *framework.Framework) { s.commonStack.Cleanup(ctx, f) } @@ -44,31 +40,10 @@ func (s *albResourceStack) GetLoadBalancerIngressHostname() string { return s.commonStack.GetLoadBalancerIngressHostname() } -func (s *albResourceStack) GetStackName() string { - return s.commonStack.GetStackName() -} - func (s *albResourceStack) getListenersPortMap() map[string]string { return s.commonStack.getListenersPortMap() } -func (s *albResourceStack) getTargetGroupNodePortMap() map[string]string { - res := s.commonStack.getTargetGroupNodePortMap() - - for p := range res { - // TODO - kinda a hack to get HTTP to work. - if res[p] == string(corev1.ProtocolTCP) { - res[p] = "HTTP" - } - } - - return res -} - -func (s *albResourceStack) getHealthCheckNodePort() string { - return s.commonStack.getHealthCheckNodePort() -} - func (s *albResourceStack) waitUntilDeploymentReady(ctx context.Context, f *framework.Framework) error { return s.commonStack.waitUntilDeploymentReady(ctx, f) } diff --git a/test/e2e/gateway/alb_test_helper.go b/test/e2e/gateway/alb_test_helper.go index 187523deb0..e3a214724b 100644 --- a/test/e2e/gateway/alb_test_helper.go +++ b/test/e2e/gateway/alb_test_helper.go @@ -16,19 +16,21 @@ func (s *ALBTestStack) Deploy(ctx context.Context, f *framework.Framework, lbCon dp := buildDeploymentSpec(f.Options.TestImageRegistry) svc := buildServiceSpec() gwc := buildGatewayClassSpec("gateway.k8s.aws/alb") - gw := buildBasicGatewaySpec(gwc, gwv1.HTTPProtocolType) + gw := buildBasicGatewaySpec(gwc, []gwv1.Listener{ + { + Name: "test-listener", + Port: 80, + Protocol: gwv1.HTTPProtocolType, + }, + }) lbc := buildLoadBalancerConfig(lbConfSpec) - tgc := buildTargetGroupConfig(tgConfSpec, svc) + tgc := buildTargetGroupConfig(defaultTgConfigName, tgConfSpec, svc) httpr := buildHTTPRoute() s.albResourceStack = newALBResourceStack(dp, svc, gwc, gw, lbc, tgc, httpr, "alb-gateway-e2e", false) return s.albResourceStack.Deploy(ctx, f) } -func (s *ALBTestStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error { - return s.albResourceStack.ScaleDeployment(ctx, f, numReplicas) -} - func (s *ALBTestStack) Cleanup(ctx context.Context, f *framework.Framework) { s.albResourceStack.Cleanup(ctx, f) } diff --git a/test/e2e/gateway/common_resource_stack.go b/test/e2e/gateway/common_resource_stack.go index cfc7d57062..477f2b3ee2 100644 --- a/test/e2e/gateway/common_resource_stack.go +++ b/test/e2e/gateway/common_resource_stack.go @@ -2,10 +2,8 @@ package gateway import ( "context" - "fmt" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" @@ -18,14 +16,14 @@ import ( "strconv" ) -func newCommonResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, tgc *elbv2gw.TargetGroupConfiguration, baseName string, enablePodReadinessGate bool) *commonResourceStack { +func newCommonResourceStack(dps []*appsv1.Deployment, svcs []*corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, tgcs []*elbv2gw.TargetGroupConfiguration, baseName string, enablePodReadinessGate bool) *commonResourceStack { return &commonResourceStack{ - dp: dp, - svc: svc, + dps: dps, + svcs: svcs, gwc: gwc, gw: gw, lbc: lbc, - tgc: tgc, + tgcs: tgcs, baseName: baseName, enablePodReadinessGate: enablePodReadinessGate, } @@ -34,31 +32,40 @@ func newCommonResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv // commonResourceStack contains resources that are common between nlb / alb gateways type commonResourceStack struct { // configurations - svc *corev1.Service - dp *appsv1.Deployment + svcs []*corev1.Service + dps []*appsv1.Deployment gwc *gwv1.GatewayClass gw *gwv1.Gateway lbc *elbv2gw.LoadBalancerConfiguration - tgc *elbv2gw.TargetGroupConfiguration + tgcs []*elbv2gw.TargetGroupConfiguration ns *corev1.Namespace baseName string enablePodReadinessGate bool // runtime variables - createdDP *appsv1.Deployment - createdSVC *corev1.Service - createdGW *gwv1.Gateway + createdDPs []*appsv1.Deployment + createdSVCs []*corev1.Service + createdGW *gwv1.Gateway } func (s *commonResourceStack) Deploy(ctx context.Context, f *framework.Framework, resourceSpecificCreation func(ctx context.Context, f *framework.Framework, namespace string) error) error { if err := s.allocateNamespace(ctx, f); err != nil { return err } - s.dp.Namespace = s.ns.Name - s.svc.Namespace = s.ns.Name + for _, v := range s.dps { + v.Namespace = s.ns.Name + } + + for _, v := range s.svcs { + v.Namespace = s.ns.Name + } + + for _, v := range s.tgcs { + v.Namespace = s.ns.Name + } + s.gw.Namespace = s.ns.Name s.lbc.Namespace = s.ns.Name - s.tgc.Namespace = s.ns.Name if err := s.createGatewayClass(ctx, f); err != nil { return err @@ -66,13 +73,13 @@ func (s *commonResourceStack) Deploy(ctx context.Context, f *framework.Framework if err := s.createLoadBalancerConfig(ctx, f); err != nil { return err } - if err := s.createTargetGroupConfig(ctx, f); err != nil { + if err := s.createTargetGroupConfigs(ctx, f); err != nil { return err } - if err := s.createDeployment(ctx, f); err != nil { + if err := s.createDeployments(ctx, f); err != nil { return err } - if err := s.createService(ctx, f); err != nil { + if err := s.createServices(ctx, f); err != nil { return err } if err := s.createGateway(ctx, f); err != nil { @@ -97,20 +104,6 @@ func (s *commonResourceStack) Deploy(ctx context.Context, f *framework.Framework return nil } -func (s *commonResourceStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error { - f.Logger.Info("scaling deployment", "dp", k8s.NamespacedName(s.dp), "currentReplicas", s.dp.Spec.Replicas, "desiredReplicas", numReplicas) - oldDP := s.dp.DeepCopy() - s.dp.Spec.Replicas = &numReplicas - if err := f.K8sClient.Patch(ctx, s.dp, client.MergeFrom(oldDP)); err != nil { - f.Logger.Info("failed to update deployment", "dp", k8s.NamespacedName(s.dp)) - return err - } - if err := s.waitUntilDeploymentReady(ctx, f); err != nil { - return err - } - return nil -} - func (s *commonResourceStack) Cleanup(ctx context.Context, f *framework.Framework) { _ = s.deleteNamespace(ctx, f) _ = s.deleteGatewayClass(ctx, f) @@ -120,10 +113,6 @@ func (s *commonResourceStack) GetLoadBalancerIngressHostname() string { return s.createdGW.Status.Addresses[0].Value } -func (s *commonResourceStack) GetStackName() string { - return fmt.Sprintf("%v/%v", s.ns.Name, s.svc.Name) -} - func (s *commonResourceStack) getListenersPortMap() map[string]string { listenersMap := map[string]string{} for _, l := range s.createdGW.Spec.Listeners { @@ -132,43 +121,41 @@ func (s *commonResourceStack) getListenersPortMap() map[string]string { return listenersMap } -func (s *commonResourceStack) getTargetGroupNodePortMap() map[string]string { - tgPortProtocolMap := map[string]string{} - for _, port := range s.createdSVC.Spec.Ports { - tgPortProtocolMap[strconv.Itoa(int(port.NodePort))] = string(port.Protocol) - } - return tgPortProtocolMap -} - -func (s *commonResourceStack) getHealthCheckNodePort() string { - return strconv.Itoa(int(s.svc.Spec.HealthCheckNodePort)) -} - -func (s *commonResourceStack) createDeployment(ctx context.Context, f *framework.Framework) error { - f.Logger.Info("creating deployment", "dp", k8s.NamespacedName(s.dp)) - if err := f.K8sClient.Create(ctx, s.dp); err != nil { - f.Logger.Info("failed to create deployment") - return err +func (s *commonResourceStack) createDeployments(ctx context.Context, f *framework.Framework) error { + for _, dp := range s.dps { + f.Logger.Info("creating deployment", "dp", k8s.NamespacedName(dp)) + if err := f.K8sClient.Create(ctx, dp); err != nil { + f.Logger.Info("failed to create deployment") + return err + } + f.Logger.Info("created deployment", "dp", k8s.NamespacedName(dp)) } - f.Logger.Info("created deployment", "dp", k8s.NamespacedName(s.dp)) return nil } func (s *commonResourceStack) waitUntilDeploymentReady(ctx context.Context, f *framework.Framework) error { - f.Logger.Info("waiting until deployment becomes ready", "dp", k8s.NamespacedName(s.dp)) - observedDP, err := f.DPManager.WaitUntilDeploymentReady(ctx, s.dp) - if err != nil { - f.Logger.Info("failed waiting for deployment") - return err + for _, dp := range s.dps { + f.Logger.Info("waiting until deployment becomes ready", "dp", k8s.NamespacedName(dp)) + _, err := f.DPManager.WaitUntilDeploymentReady(ctx, dp) + if err != nil { + f.Logger.Info("failed waiting for deployment") + return err + } + f.Logger.Info("deployment is ready", "dp", k8s.NamespacedName(dp)) } - f.Logger.Info("deployment is ready", "dp", k8s.NamespacedName(s.dp)) - s.createdDP = observedDP return nil } -func (s *commonResourceStack) createService(ctx context.Context, f *framework.Framework) error { - f.Logger.Info("creating service", "svc", k8s.NamespacedName(s.svc)) - return f.K8sClient.Create(ctx, s.svc) +func (s *commonResourceStack) createServices(ctx context.Context, f *framework.Framework) error { + for _, svc := range s.svcs { + f.Logger.Info("creating service", "svc", k8s.NamespacedName(svc)) + if err := f.K8sClient.Create(ctx, svc); err != nil { + f.Logger.Info("failed to create service") + return err + } + f.Logger.Info("created service", "svc", k8s.NamespacedName(svc)) + } + return nil } func (s *commonResourceStack) createGatewayClass(ctx context.Context, f *framework.Framework) error { @@ -181,9 +168,17 @@ func (s *commonResourceStack) createLoadBalancerConfig(ctx context.Context, f *f return f.K8sClient.Create(ctx, s.lbc) } -func (s *commonResourceStack) createTargetGroupConfig(ctx context.Context, f *framework.Framework) error { - f.Logger.Info("creating target group config", "tgc", k8s.NamespacedName(s.tgc)) - return f.K8sClient.Create(ctx, s.tgc) +func (s *commonResourceStack) createTargetGroupConfigs(ctx context.Context, f *framework.Framework) error { + for _, tgc := range s.tgcs { + f.Logger.Info("creating target group config", "tgc", k8s.NamespacedName(tgc)) + err := f.K8sClient.Create(ctx, tgc) + if err != nil { + f.Logger.Error(err, "failed to create target group config") + return err + } + f.Logger.Info("created target group config", "tgc", k8s.NamespacedName(tgc)) + } + return nil } func (s *commonResourceStack) createGateway(ctx context.Context, f *framework.Framework) error { @@ -192,12 +187,14 @@ func (s *commonResourceStack) createGateway(ctx context.Context, f *framework.Fr } func (s *commonResourceStack) waitUntilServiceReady(ctx context.Context, f *framework.Framework) error { - observedSvc := &corev1.Service{} - err := f.K8sClient.Get(ctx, k8s.NamespacedName(s.svc), observedSvc) - if err != nil { - return err + for _, svc := range s.svcs { + observedSvc := &corev1.Service{} + err := f.K8sClient.Get(ctx, k8s.NamespacedName(svc), observedSvc) + if err != nil { + f.Logger.Error(err, "unable to observe service go ready") + return err + } } - s.createdSVC = observedSvc return nil } @@ -225,63 +222,10 @@ func (s *commonResourceStack) waitUntilGatewayReady(ctx context.Context, f *fram return nil } -func (s *commonResourceStack) deleteDeployment(ctx context.Context, f *framework.Framework) error { - f.Logger.Info("deleting deployment", "dp", k8s.NamespacedName(s.dp)) - if err := f.K8sClient.Delete(ctx, s.dp); err != nil { - f.Logger.Info("failed to delete deployment", "dp", k8s.NamespacedName(s.dp)) - return err - } - if err := f.DPManager.WaitUntilDeploymentDeleted(ctx, s.dp); err != nil { - f.Logger.Info("failed to wait for deployment deletion", "dp", k8s.NamespacedName(s.dp)) - return err - } - f.Logger.Info("deleted deployment", "dp", k8s.NamespacedName(s.dp)) - return nil -} - -func (s *commonResourceStack) deleteService(ctx context.Context, f *framework.Framework) error { - f.Logger.Info("deleting service", "svc", k8s.NamespacedName(s.svc)) - if err := f.K8sClient.Delete(ctx, s.svc); err != nil { - f.Logger.Info("failed to delete service", "svc", k8s.NamespacedName(s.svc)) - return err - } - if err := f.SVCManager.WaitUntilServiceDeleted(ctx, s.svc); err != nil { - f.Logger.Info("failed to wait for service deletion", "svc", k8s.NamespacedName(s.svc)) - return err - } - f.Logger.Info("deleted service", "svc", k8s.NamespacedName(s.svc)) - return nil -} - -func (s *commonResourceStack) deleteGateway(ctx context.Context, f *framework.Framework) error { - err := f.K8sClient.Delete(ctx, s.gw) - if err != nil { - return err - } - observedGW := &gwv1.Gateway{} - return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) { - if err := f.K8sClient.Get(ctx, k8s.NamespacedName(s.gw), observedGW); err != nil { - if apierrs.IsNotFound(err) { - return true, nil - } - return false, err - } - return false, nil - }, ctx.Done()) -} - func (s *commonResourceStack) deleteGatewayClass(ctx context.Context, f *framework.Framework) error { return f.K8sClient.Delete(ctx, s.gwc) } -func (s *commonResourceStack) deleteLoadbalancerConfig(ctx context.Context, f *framework.Framework) error { - return f.K8sClient.Delete(ctx, s.lbc) -} - -func (s *commonResourceStack) deleteTargetGroupConfig(ctx context.Context, f *framework.Framework) error { - return f.K8sClient.Delete(ctx, s.tgc) -} - func (s *commonResourceStack) allocateNamespace(ctx context.Context, f *framework.Framework) error { f.Logger.Info("allocating namespace") ns, err := f.NSManager.AllocateNamespace(ctx, s.baseName) diff --git a/test/e2e/gateway/consts.go b/test/e2e/gateway/consts.go index 7cb70a39b0..2560fee2ab 100644 --- a/test/e2e/gateway/consts.go +++ b/test/e2e/gateway/consts.go @@ -2,9 +2,12 @@ package gateway const ( appContainerPort = 80 + udpContainerPort = 8080 defaultNumReplicas = 3 defaultName = "gateway-e2e" + udpDefaultName = defaultName + "-udp" defaultGatewayClassName = "gwclass-e2e" defaultLbConfigName = "lbconfig-e2e" defaultTgConfigName = "tgconfig-e2e" + udpDefaultTgConfigName = defaultTgConfigName + "-udp" ) diff --git a/test/e2e/gateway/nlb_instance_target_test.go b/test/e2e/gateway/nlb_instance_target_test.go index 34502210f2..b5d920e797 100644 --- a/test/e2e/gateway/nlb_instance_target_test.go +++ b/test/e2e/gateway/nlb_instance_target_test.go @@ -9,6 +9,8 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" "sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier" + "strconv" + "strings" ) var _ = Describe("test nlb gateway using instance targets reconciled by the aws load balancer controller", func() { @@ -35,6 +37,20 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws lbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ Scheme: &interf, } + + var hasTLS bool + if len(tf.Options.CertificateARNs) > 0 { + cert := strings.Split(tf.Options.CertificateARNs, ",")[0] + + lbcSpec.ListenerConfigurations = &[]elbv2gw.ListenerConfiguration{ + { + DefaultCertificate: &cert, + ProtocolPort: "TLS:443", + }, + } + hasTLS = true + } + tgSpec := elbv2gw.TargetGroupConfigurationSpec{} By("deploying stack", func() { err := stack.Deploy(ctx, tf, lbcSpec, tgSpec) @@ -53,6 +69,12 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws Expect(lbARN).ToNot(BeEmpty()) }) + // TODO -- This might be hacky. Currently, the TCP svc always is 0, while UDP is 1. + tgMap := map[string][]string{ + strconv.Itoa(int(stack.nlbResourceStack.commonStack.svcs[0].Spec.Ports[0].NodePort)): {"TCP"}, + strconv.Itoa(int(stack.nlbResourceStack.commonStack.svcs[1].Spec.Ports[1].NodePort)): {"UDP"}, + } + By("verifying AWS loadbalancer resources", func() { nodeList, err := stack.GetWorkerNodes(ctx, tf) Expect(err).ToNot(HaveOccurred()) @@ -61,7 +83,7 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws Scheme: "internet-facing", TargetType: "instance", Listeners: stack.nlbResourceStack.getListenersPortMap(), - TargetGroups: stack.nlbResourceStack.getTargetGroupNodePortMap(), + TargetGroups: tgMap, NumTargets: len(nodeList), TargetGroupHC: &verifier.TargetGroupHC{ Protocol: "TCP", @@ -89,6 +111,18 @@ var _ = Describe("test nlb gateway using instance targets reconciled by the aws err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) Expect(err).NotTo(HaveOccurred()) }) + By("sending https request to the lb", func() { + if hasTLS { + url := fmt.Sprintf("https://%v/any-path", dnsName) + err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + } + }) + By("sending udp request to the lb", func() { + endpoint := fmt.Sprintf("%v:8080", dnsName) + err := tf.UDPVerifier.VerifyUDP(endpoint) + Expect(err).NotTo(HaveOccurred()) + }) }) }) }) diff --git a/test/e2e/gateway/nlb_ip_target_test.go b/test/e2e/gateway/nlb_ip_target_test.go index 10f43f65c4..db27d17ba6 100644 --- a/test/e2e/gateway/nlb_ip_target_test.go +++ b/test/e2e/gateway/nlb_ip_target_test.go @@ -9,6 +9,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" "sigs.k8s.io/aws-load-balancer-controller/test/framework/verifier" + "strings" ) var _ = Describe("test nlb gateway using ip targets reconciled by the aws load balancer controller", func() { @@ -35,6 +36,20 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b lbcSpec := elbv2gw.LoadBalancerConfigurationSpec{ Scheme: &interf, } + + var hasTLS bool + if len(tf.Options.CertificateARNs) > 0 { + cert := strings.Split(tf.Options.CertificateARNs, ",")[0] + + lbcSpec.ListenerConfigurations = &[]elbv2gw.ListenerConfiguration{ + { + DefaultCertificate: &cert, + ProtocolPort: "TLS:443", + }, + } + hasTLS = true + } + ipTargetType := elbv2gw.TargetTypeIP tgSpec := elbv2gw.TargetGroupConfigurationSpec{ DefaultConfiguration: elbv2gw.TargetGroupProps{ @@ -44,6 +59,7 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b By("deploying stack", func() { err := stack.Deploy(ctx, tf, lbcSpec, tgSpec) Expect(err).NotTo(HaveOccurred()) + //time.Sleep(10 * time.Minute) }) By("checking gateway status for lb dns name", func() { @@ -58,11 +74,12 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b Expect(lbARN).ToNot(BeEmpty()) }) - tgMap := map[string]string{ - "80": "TCP", + tgMap := map[string][]string{ + "80": {"TCP"}, + "8080": {"UDP"}, } - targetNumber := int(*stack.nlbResourceStack.commonStack.dp.Spec.Replicas) + targetNumber := int(*stack.nlbResourceStack.commonStack.dps[0].Spec.Replicas) By("verifying AWS loadbalancer resources", func() { err := verifier.VerifyAWSLoadBalancerResources(ctx, tf, lbARN, verifier.LoadBalancerExpectation{ Type: "network", @@ -95,6 +112,18 @@ var _ = Describe("test nlb gateway using ip targets reconciled by the aws load b err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) Expect(err).NotTo(HaveOccurred()) }) + By("sending https request to the lb", func() { + if hasTLS { + url := fmt.Sprintf("https://%v/any-path", dnsName) + err := tf.HTTPVerifier.VerifyURL(url, http.ResponseCodeMatches(200)) + Expect(err).NotTo(HaveOccurred()) + } + }) + By("sending udp request to the lb", func() { + endpoint := fmt.Sprintf("%v:8080", dnsName) + err := tf.UDPVerifier.VerifyUDP(endpoint) + Expect(err).NotTo(HaveOccurred()) + }) }) }) }) diff --git a/test/e2e/gateway/nlb_resource_stack.go b/test/e2e/gateway/nlb_resource_stack.go index e7973ea826..7585b53d56 100644 --- a/test/e2e/gateway/nlb_resource_stack.go +++ b/test/e2e/gateway/nlb_resource_stack.go @@ -11,11 +11,13 @@ import ( gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) -func newNLBResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, tgc *elbv2gw.TargetGroupConfiguration, tcpr *gwalpha2.TCPRoute, baseName string, enablePodReadinessGate bool) *nlbResourceStack { +func newNLBResourceStack(dps []*appsv1.Deployment, svcs []*corev1.Service, gwc *gwv1.GatewayClass, gw *gwv1.Gateway, lbc *elbv2gw.LoadBalancerConfiguration, tgcs []*elbv2gw.TargetGroupConfiguration, tcpr *gwalpha2.TCPRoute, udpr *gwalpha2.UDPRoute, tlsr *gwalpha2.TLSRoute, baseName string, enablePodReadinessGate bool) *nlbResourceStack { - commonStack := newCommonResourceStack(dp, svc, gwc, gw, lbc, tgc, baseName, enablePodReadinessGate) + commonStack := newCommonResourceStack(dps, svcs, gwc, gw, lbc, tgcs, baseName, enablePodReadinessGate) return &nlbResourceStack{ tcpr: tcpr, + udpr: udpr, + tlsr: tlsr, commonStack: commonStack, } } @@ -24,19 +26,29 @@ func newNLBResourceStack(dp *appsv1.Deployment, svc *corev1.Service, gwc *gwv1.G type nlbResourceStack struct { commonStack *commonResourceStack tcpr *gwalpha2.TCPRoute + udpr *gwalpha2.UDPRoute + tlsr *gwalpha2.TLSRoute } func (s *nlbResourceStack) Deploy(ctx context.Context, f *framework.Framework) error { return s.commonStack.Deploy(ctx, f, func(ctx context.Context, f *framework.Framework, namespace string) error { s.tcpr.Namespace = namespace - return s.createTCPRoute(ctx, f) + s.udpr.Namespace = namespace + if s.tlsr != nil { + s.tlsr.Namespace = namespace + } + err := s.createTCPRoute(ctx, f) + if err != nil { + return err + } + err = s.createUDPRoute(ctx, f) + if err != nil { + return err + } + return s.createTLSRoute(ctx, f) }) } -func (s *nlbResourceStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error { - return s.commonStack.ScaleDeployment(ctx, f, numReplicas) -} - func (s *nlbResourceStack) Cleanup(ctx context.Context, f *framework.Framework) { s.commonStack.Cleanup(ctx, f) } @@ -45,22 +57,10 @@ func (s *nlbResourceStack) GetLoadBalancerIngressHostname() string { return s.commonStack.GetLoadBalancerIngressHostname() } -func (s *nlbResourceStack) GetStackName() string { - return s.commonStack.GetStackName() -} - func (s *nlbResourceStack) getListenersPortMap() map[string]string { return s.commonStack.getListenersPortMap() } -func (s *nlbResourceStack) getTargetGroupNodePortMap() map[string]string { - return s.commonStack.getTargetGroupNodePortMap() -} - -func (s *nlbResourceStack) getHealthCheckNodePort() string { - return s.commonStack.getHealthCheckNodePort() -} - func (s *nlbResourceStack) waitUntilDeploymentReady(ctx context.Context, f *framework.Framework) error { return s.commonStack.waitUntilDeploymentReady(ctx, f) } @@ -70,6 +70,15 @@ func (s *nlbResourceStack) createTCPRoute(ctx context.Context, f *framework.Fram return f.K8sClient.Create(ctx, s.tcpr) } -func (s *nlbResourceStack) deleteTCPRoute(ctx context.Context, f *framework.Framework) error { - return f.K8sClient.Delete(ctx, s.tcpr) +func (s *nlbResourceStack) createUDPRoute(ctx context.Context, f *framework.Framework) error { + f.Logger.Info("creating udp route", "udpr", k8s.NamespacedName(s.udpr)) + return f.K8sClient.Create(ctx, s.udpr) +} + +func (s *nlbResourceStack) createTLSRoute(ctx context.Context, f *framework.Framework) error { + if s.tlsr == nil { + return nil + } + f.Logger.Info("creating tls route", "tlsr", k8s.NamespacedName(s.tlsr)) + return f.K8sClient.Create(ctx, s.tlsr) } diff --git a/test/e2e/gateway/nlb_test_helper.go b/test/e2e/gateway/nlb_test_helper.go index ca9bf9af06..b8133dd6a6 100644 --- a/test/e2e/gateway/nlb_test_helper.go +++ b/test/e2e/gateway/nlb_test_helper.go @@ -2,10 +2,12 @@ package gateway import ( "context" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" elbv2gw "sigs.k8s.io/aws-load-balancer-controller/apis/gateway/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/test/framework" gwv1 "sigs.k8s.io/gateway-api/apis/v1" + gwalpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) type NLBTestStack struct { @@ -13,22 +15,52 @@ type NLBTestStack struct { } func (s *NLBTestStack) Deploy(ctx context.Context, f *framework.Framework, lbConfSpec elbv2gw.LoadBalancerConfigurationSpec, tgConfSpec elbv2gw.TargetGroupConfigurationSpec) error { - dp := buildDeploymentSpec(f.Options.TestImageRegistry) - svc := buildServiceSpec() + dpTCP := buildDeploymentSpec(f.Options.TestImageRegistry) + svcTCP := buildServiceSpec() + + dpUDP := buildUDPDeploymentSpec() + svcUDP := buildUDPServiceSpec() gwc := buildGatewayClassSpec("gateway.k8s.aws/nlb") - gw := buildBasicGatewaySpec(gwc, gwv1.TCPProtocolType) + + listeners := []gwv1.Listener{ + { + Name: "port80", + Port: 80, + Protocol: gwv1.TCPProtocolType, + }, + { + Name: "port8080", + Port: 8080, + Protocol: gwv1.UDPProtocolType, + }, + } + + var tlsr *gwalpha2.TLSRoute + if lbConfSpec.ListenerConfigurations != nil { + for _, lsr := range *lbConfSpec.ListenerConfigurations { + if lsr.ProtocolPort == "TLS:443" { + listeners = append(listeners, gwv1.Listener{ + Name: "port443", + Port: 443, + Protocol: gwv1.TLSProtocolType, + }) + //tlsr = buildTLSRoute() + break + } + } + } + gw := buildBasicGatewaySpec(gwc, listeners) + lbc := buildLoadBalancerConfig(lbConfSpec) - tgc := buildTargetGroupConfig(tgConfSpec, svc) + tgcTCP := buildTargetGroupConfig(defaultTgConfigName, tgConfSpec, svcTCP) + tgcUDP := buildTargetGroupConfig(udpDefaultTgConfigName, tgConfSpec, svcUDP) tcpr := buildTCPRoute() - s.nlbResourceStack = newNLBResourceStack(dp, svc, gwc, gw, lbc, tgc, tcpr, "nlb-gateway-e2e", false) + udpr := buildUDPRoute() + s.nlbResourceStack = newNLBResourceStack([]*appsv1.Deployment{dpTCP, dpUDP}, []*corev1.Service{svcTCP, svcUDP}, gwc, gw, lbc, []*elbv2gw.TargetGroupConfiguration{tgcTCP, tgcUDP}, tcpr, udpr, tlsr, "nlb-gateway-e2e", false) return s.nlbResourceStack.Deploy(ctx, f) } -func (s *NLBTestStack) ScaleDeployment(ctx context.Context, f *framework.Framework, numReplicas int32) error { - return s.nlbResourceStack.ScaleDeployment(ctx, f, numReplicas) -} - func (s *NLBTestStack) Cleanup(ctx context.Context, f *framework.Framework) { s.nlbResourceStack.Cleanup(ctx, f) } diff --git a/test/e2e/gateway/shared_resource_definitions.go b/test/e2e/gateway/shared_resource_definitions.go index 9b160aa6a6..f8dacbc776 100644 --- a/test/e2e/gateway/shared_resource_definitions.go +++ b/test/e2e/gateway/shared_resource_definitions.go @@ -1,6 +1,7 @@ package gateway import ( + awssdk "github.com/aws/aws-sdk-go-v2/aws" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,6 +52,50 @@ func buildDeploymentSpec(testImageRegistry string) *appsv1.Deployment { } } +func buildUDPDeploymentSpec() *appsv1.Deployment { + numReplicas := int32(defaultNumReplicas) + labels := map[string]string{ + "app.kubernetes.io/instance": udpDefaultName, + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: udpDefaultName, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &numReplicas, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "app", + ImagePullPolicy: corev1.PullAlways, + Image: utils.UDPImage, + Ports: []corev1.ContainerPort{ + { + ContainerPort: udpContainerPort, + Protocol: corev1.ProtocolUDP, + Name: "udp8080", + }, + { + ContainerPort: udpContainerPort, + Protocol: corev1.ProtocolTCP, + Name: "tcp8080", + }, + }, + }, + }, + }, + }, + }, + } +} + func buildServiceSpec() *corev1.Service { labels := map[string]string{ "app.kubernetes.io/name": "multi-port", @@ -75,6 +120,36 @@ func buildServiceSpec() *corev1.Service { return svc } +func buildUDPServiceSpec() *corev1.Service { + labels := map[string]string{ + "app.kubernetes.io/instance": udpDefaultName, + } + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: udpDefaultName, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Selector: labels, + Ports: []corev1.ServicePort{ + { + Name: "tcp8080", + Port: 8080, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "udp8080", + Port: 8080, + TargetPort: intstr.FromInt(8080), + Protocol: corev1.ProtocolUDP, + }, + }, + }, + } + return svc +} + func buildGatewayClassSpec(controllerName string) *gwv1.GatewayClass { lbType := strings.Split(controllerName, "/")[1] gwc := &gwv1.GatewayClass{ @@ -98,31 +173,25 @@ func buildLoadBalancerConfig(spec elbv2gw.LoadBalancerConfigurationSpec) *elbv2g return lbc } -func buildTargetGroupConfig(spec elbv2gw.TargetGroupConfigurationSpec, svc *corev1.Service) *elbv2gw.TargetGroupConfiguration { +func buildTargetGroupConfig(name string, spec elbv2gw.TargetGroupConfigurationSpec, svc *corev1.Service) *elbv2gw.TargetGroupConfiguration { spec.TargetReference.Name = svc.Name tgc := &elbv2gw.TargetGroupConfiguration{ ObjectMeta: metav1.ObjectMeta{ - Name: defaultTgConfigName, + Name: name, }, - Spec: spec, + Spec: *(spec.DeepCopy()), } return tgc } -func buildBasicGatewaySpec(gwc *gwv1.GatewayClass, protocol gwv1.ProtocolType) *gwv1.Gateway { +func buildBasicGatewaySpec(gwc *gwv1.GatewayClass, listeners []gwv1.Listener) *gwv1.Gateway { gw := &gwv1.Gateway{ ObjectMeta: metav1.ObjectMeta{ Name: defaultName, }, Spec: gwv1.GatewaySpec{ GatewayClassName: gwv1.ObjectName(gwc.Name), - Listeners: []gwv1.Listener{ - { - Name: "test-listener", - Port: 80, - Protocol: protocol, - }, - }, + Listeners: listeners, Infrastructure: &gwv1.GatewayInfrastructure{ ParametersRef: &gwv1.LocalParametersReference{ Group: "gateway.k8s.aws", @@ -145,7 +214,12 @@ func buildTCPRoute() *gwalpha2.TCPRoute { CommonRouteSpec: gwalpha2.CommonRouteSpec{ ParentRefs: []gwv1.ParentReference{ { - Name: defaultName, + Name: defaultName, + SectionName: (*gwv1.SectionName)(awssdk.String("port80")), + }, + { + Name: defaultName, + SectionName: (*gwv1.SectionName)(awssdk.String("port443")), }, }, }, @@ -166,6 +240,73 @@ func buildTCPRoute() *gwalpha2.TCPRoute { return tcpr } +func buildUDPRoute() *gwalpha2.UDPRoute { + port := gwalpha2.PortNumber(8080) + udpr := &gwalpha2.UDPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + Spec: gwalpha2.UDPRouteSpec{ + CommonRouteSpec: gwalpha2.CommonRouteSpec{ + ParentRefs: []gwv1.ParentReference{ + { + Name: defaultName, + SectionName: (*gwv1.SectionName)(awssdk.String("port8080")), + }, + }, + }, + Rules: []gwalpha2.UDPRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: udpDefaultName, + Port: &port, + }, + }, + }, + }, + }, + }, + } + return udpr +} + +/* +func buildTLSRoute() *gwalpha2.TLSRoute { + port := gwalpha2.PortNumber(80) + tlrs := &gwalpha2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultName, + }, + Spec: gwalpha2.TLSRouteSpec{ + CommonRouteSpec: gwalpha2.CommonRouteSpec{ + ParentRefs: []gwv1.ParentReference{ + { + Name: defaultName, + SectionName: (*gwv1.SectionName)(awssdk.String("port443")), + }, + }, + }, + Rules: []gwalpha2.TLSRouteRule{ + { + BackendRefs: []gwalpha2.BackendRef{ + { + BackendObjectReference: gwalpha2.BackendObjectReference{ + Name: defaultName, + Port: &port, + }, + }, + }, + }, + }, + }, + } + return tlrs +} + +*/ + func buildHTTPRoute() *gwv1.HTTPRoute { port := gwalpha2.PortNumber(80) httpr := &gwv1.HTTPRoute{ diff --git a/test/e2e/service/nlb_ip_target_test.go b/test/e2e/service/nlb_ip_target_test.go index 24c923c10a..12c00d0956 100644 --- a/test/e2e/service/nlb_ip_target_test.go +++ b/test/e2e/service/nlb_ip_target_test.go @@ -128,8 +128,8 @@ var _ = Describe("k8s service reconciled by the aws load balancer", func() { Listeners: map[string]string{ "80": "TCP", }, - TargetGroups: map[string]string{ - "80": "TCP", + TargetGroups: map[string][]string{ + "80": {"TCP"}, }, NumTargets: int(numReplicas), TargetGroupHC: &verifier.TargetGroupHC{ @@ -174,8 +174,8 @@ var _ = Describe("k8s service reconciled by the aws load balancer", func() { Listeners: map[string]string{ "80": "TCP", }, - TargetGroups: map[string]string{ - "80": "TCP", + TargetGroups: map[string][]string{ + "80": {"TCP"}, }, NumTargets: int(numReplicas), TargetGroupHC: &verifier.TargetGroupHC{ @@ -239,8 +239,8 @@ var _ = Describe("k8s service reconciled by the aws load balancer", func() { Listeners: map[string]string{ "80": "TCP", }, - TargetGroups: map[string]string{ - "80": "TCP", + TargetGroups: map[string][]string{ + "80": {"TCP"}, }, NumTargets: int(numReplicas) + 1, TargetGroupHC: &verifier.TargetGroupHC{ @@ -272,8 +272,8 @@ var _ = Describe("k8s service reconciled by the aws load balancer", func() { Listeners: map[string]string{ "80": "TCP", }, - TargetGroups: map[string]string{ - "80": "TCP", + TargetGroups: map[string][]string{ + "80": {"TCP"}, }, NumTargets: int(numReplicas), TargetGroupHC: &verifier.TargetGroupHC{ @@ -363,10 +363,10 @@ var _ = Describe("k8s service reconciled by the aws load balancer", func() { "443": "TLS", "333": "TLS", }, - TargetGroups: map[string]string{ - "80": "TCP", - "443": "TCP", - "333": "TCP", + TargetGroups: map[string][]string{ + "80": {"TCP"}, + "443": {"TCP"}, + "333": {"TCP"}, }, NumTargets: int(numReplicas), }) @@ -394,10 +394,10 @@ var _ = Describe("k8s service reconciled by the aws load balancer", func() { "443": "TLS", "333": "TLS", }, - TargetGroups: map[string]string{ - "80": "TCP", - "443": "TCP", - "333": "TCP", + TargetGroups: map[string][]string{ + "80": {"TCP"}, + "443": {"TCP"}, + "333": {"TCP"}, }, NumTargets: int(numReplicas), }) @@ -492,8 +492,8 @@ var _ = Describe("k8s service reconciled by the aws load balancer", func() { Listeners: map[string]string{ "80": "TCP", }, - TargetGroups: map[string]string{ - "80": "TCP", + TargetGroups: map[string][]string{ + "80": {"TCP"}, }, NumTargets: int(numReplicas), TargetGroupHC: &verifier.TargetGroupHC{ diff --git a/test/e2e/service/resource_stack.go b/test/e2e/service/resource_stack.go index 7b55caa757..39cbefa340 100644 --- a/test/e2e/service/resource_stack.go +++ b/test/e2e/service/resource_stack.go @@ -129,10 +129,10 @@ func (s *resourceStack) getListenersPortMap() map[string]string { return listenersMap } -func (s *resourceStack) getTargetGroupNodePortMap() map[string]string { - tgPortProtocolMap := map[string]string{} +func (s *resourceStack) getTargetGroupNodePortMap() map[string][]string { + tgPortProtocolMap := map[string][]string{} for _, port := range s.createdSVC.Spec.Ports { - tgPortProtocolMap[strconv.Itoa(int(port.NodePort))] = string(port.Protocol) + tgPortProtocolMap[strconv.Itoa(int(port.NodePort))] = []string{string(port.Protocol)} } return tgPortProtocolMap } diff --git a/test/framework/framework.go b/test/framework/framework.go index 1f09e51d46..877ab18501 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/aws-load-balancer-controller/test/framework/http" awsresources "sigs.k8s.io/aws-load-balancer-controller/test/framework/resources/aws" k8sresources "sigs.k8s.io/aws-load-balancer-controller/test/framework/resources/k8s" + "sigs.k8s.io/aws-load-balancer-controller/test/framework/udp" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,6 +40,7 @@ type Framework struct { TGManager awsresources.TargetGroupManager HTTPVerifier http.Verifier + UDPVerifier udp.Verifier Logger logr.Logger LoggerReporter httpexpect.LoggerReporter @@ -91,6 +93,7 @@ func InitFramework() (*Framework, error) { TGManager: awsresources.NewDefaultTargetGroupManager(cloud.ELBV2(), logger), HTTPVerifier: http.NewDefaultVerifier(), + UDPVerifier: udp.NewDefaultVerifier(), Logger: logger, LoggerReporter: loggerReporter, diff --git a/test/framework/http/verifier.go b/test/framework/http/verifier.go index 769daf4bee..3d55394ee2 100644 --- a/test/framework/http/verifier.go +++ b/test/framework/http/verifier.go @@ -1,6 +1,7 @@ package http import ( + "crypto/tls" gohttp "net/http" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" ) @@ -11,8 +12,14 @@ type Verifier interface { } func NewDefaultVerifier() *defaultVerifier { + httpClient := &gohttp.Client{} + httpClient.Transport = &gohttp.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } return &defaultVerifier{ - httpClient: &gohttp.Client{}, + httpClient: httpClient, } } diff --git a/test/framework/udp/verifier.go b/test/framework/udp/verifier.go new file mode 100644 index 0000000000..e40e752d06 --- /dev/null +++ b/test/framework/udp/verifier.go @@ -0,0 +1,80 @@ +package udp + +import ( + "fmt" + "github.com/pkg/errors" + "net" + "time" +) + +const ( + maxTries = 100 + consecutiveSuccessesThreshold = 5 +) + +// Verifier is responsible for verify the behavior of an HTTP endpoint. +type Verifier interface { + VerifyUDP(endpoint string) error +} + +func NewDefaultVerifier() *defaultVerifier { + return &defaultVerifier{} +} + +var _ Verifier = &defaultVerifier{} + +// default implementation for Verifier. +type defaultVerifier struct { +} + +func (v *defaultVerifier) VerifyUDP(endpoint string) error { + tries := 0 + consecutiveSuccesses := 0 + for tries < maxTries { + err := v.doCall(endpoint) + if err == nil { + consecutiveSuccesses++ + } else { + fmt.Printf("Got an error during UDP call (%+v\n)", err) + } + if consecutiveSuccesses >= consecutiveSuccessesThreshold { + return nil + } + tries++ + time.Sleep(1 * time.Second) + } + return errors.New("Unable to observe stable UDP connection") +} + +func (v *defaultVerifier) doCall(endpoint string) error { + serverAddr, err := net.ResolveUDPAddr("udp", endpoint) + if err != nil { + return err + } + + conn, err := net.DialUDP("udp", nil, serverAddr) + if err != nil { + return err + } + defer conn.Close() + + message := []byte("Hello, UDP Server!") + _, err = conn.Write(message) + if err != nil { + return err + } + + // Set a read deadline to avoid blocking indefinitely + err = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if err != nil { + return err + } + + buffer := make([]byte, 1024) + _, _, err = conn.ReadFromUDP(buffer) + if err != nil { + return err + } + + return nil +} diff --git a/test/framework/utils/constants.go b/test/framework/utils/constants.go index 2ae6e75c98..fe0413e79a 100644 --- a/test/framework/utils/constants.go +++ b/test/framework/utils/constants.go @@ -3,4 +3,6 @@ package utils const ( HelloImage = "networking-e2e-test-images/hello-multi:latest" ColortellerImage = "networking-e2e-test-images/colorteller:latest" + // UDPImage TODO -- Maybe something more official? + UDPImage = "public.ecr.aws/u6k2n8q7/nixozach/udp-echoserver:latest" ) diff --git a/test/framework/verifier/aws_resource_verifier.go b/test/framework/verifier/aws_resource_verifier.go index ead49c94a9..296a62a9ed 100644 --- a/test/framework/verifier/aws_resource_verifier.go +++ b/test/framework/verifier/aws_resource_verifier.go @@ -2,6 +2,7 @@ package verifier import ( "context" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/aws-load-balancer-controller/test/framework/utils" "sort" "strconv" @@ -28,8 +29,8 @@ type LoadBalancerExpectation struct { Type string Scheme string TargetType string - Listeners map[string]string // listener port, protocol - TargetGroups map[string]string // target group port, protocol + Listeners map[string]string // listener port, protocol + TargetGroups map[string][]string // target group port, list of protocols NumTargets int TargetGroupHC *TargetGroupHC } @@ -176,15 +177,41 @@ func VerifyLoadBalancerListenerCertificates(ctx context.Context, f *framework.Fr func VerifyLoadBalancerTargetGroups(ctx context.Context, f *framework.Framework, lbARN string, expected LoadBalancerExpectation) error { targetGroups, err := f.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN) Expect(err).ToNot(HaveOccurred()) - Expect(len(targetGroups)).To(Equal(len(expected.TargetGroups))) + + expectedTgCount := 0 + expectedProtocolsPerPort := make(map[string]sets.Set[string]) + + for port, protocols := range expected.TargetGroups { + expectedTgCount += len(protocols) + for _, protocol := range protocols { + _, ok := expectedProtocolsPerPort[port] + if !ok { + expectedProtocolsPerPort[port] = make(sets.Set[string]) + } + + expectedProtocolsPerPort[port].Insert(protocol) + } + } + + Expect(len(targetGroups)).To(Equal(expectedTgCount)) for _, tg := range targetGroups { + port := strconv.Itoa(int(*tg.Port)) Expect(string(tg.TargetType)).To(Equal(expected.TargetType)) - Expect(string(tg.Protocol)).To(Equal(expected.TargetGroups[strconv.Itoa(int(awssdk.ToInt32(tg.Port)))])) + protocolSet := expectedProtocolsPerPort[port] + protocolFound := protocolSet.Has(string(tg.Protocol)) + Expect(protocolFound).To(BeTrue()) + expectedProtocolsPerPort[port] = protocolSet.Delete(string(tg.Protocol)) + err = VerifyTargetGroupHealthCheckConfig(tg, expected.TargetGroupHC) Expect(err).NotTo(HaveOccurred()) err = VerifyTargetGroupNumRegistered(ctx, f, awssdk.ToString(tg.TargetGroupArn), expected.NumTargets) Expect(err).NotTo(HaveOccurred()) } + + for _, protocols := range expectedProtocolsPerPort { + Expect(len(protocols)).To(Equal(0)) + } + return nil }