Skip to content

[gw api] add TLS and UDP listener tests. Fix bugs that were found from the tests #4250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 30, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions pkg/gateway/model/model_build_target_group.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func (t *targetGroupBuilderImpl) buildTargetGroup(stack core.Stack,
gw *gwv1.Gateway, lbConfig elbv2gw.LoadBalancerConfiguration, lbIPType elbv2model.IPAddressType, routeDescriptor routeutils.RouteDescriptor, backend routeutils.Backend, backendSGIDToken core.StringToken) (*elbv2model.TargetGroup, error) {

targetGroupProps := backend.ELBV2TargetGroupProps
tgResID := t.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backend.Service), routeDescriptor.GetRouteNamespacedName(), backend.ServicePort.TargetPort)
tgResID := t.buildTargetGroupResourceID(k8s.NamespacedName(gw), k8s.NamespacedName(backend.Service), routeDescriptor.GetRouteNamespacedName(), routeDescriptor.GetRouteKind(), backend.ServicePort.TargetPort)
if tg, exists := t.tgByResID[tgResID]; exists {
return tg, nil
}
Expand Down Expand Up @@ -129,7 +129,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate
if targetType == elbv2api.TargetTypeInstance {
targetPort = intstr.FromInt32(backend.ServicePort.NodePort)
}
tgbNetworking := builder.buildTargetGroupBindingNetworking(targetPort, *tgSpec.HealthCheckConfig.Port, *backend.ServicePort, backendSGIDToken)
tgbNetworking := builder.buildTargetGroupBindingNetworking(targetPort, *tgSpec.HealthCheckConfig.Port, tgSpec.Protocol, backendSGIDToken)

multiClusterEnabled := builder.buildTargetGroupBindingMultiClusterFlag(tgProps)

Expand Down Expand Up @@ -175,14 +175,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupBindingSpec(gw *gwv1.Gate
}
}

func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetPort intstr.IntOrString, healthCheckPort intstr.IntOrString, svcPort corev1.ServicePort, backendSGIDToken core.StringToken) *elbv2model.TargetGroupBindingNetworking {
func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNetworking(targetPort intstr.IntOrString, healthCheckPort intstr.IntOrString, tgProtocol elbv2model.Protocol, backendSGIDToken core.StringToken) *elbv2model.TargetGroupBindingNetworking {
if backendSGIDToken == nil {
return nil
}
protocolTCP := elbv2api.NetworkingProtocolTCP
protocolUDP := elbv2api.NetworkingProtocolUDP

udpSupported := svcPort.Protocol == corev1.ProtocolUDP
udpSupported := tgProtocol == elbv2model.ProtocolUDP || tgProtocol == elbv2model.ProtocolTCP_UDP

if builder.disableRestrictedSGRules {
ports := []elbv2api.NetworkingPort{
Expand Down Expand Up @@ -282,15 +282,14 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupSpec(gw *gwv1.Gateway, ro
return elbv2model.TargetGroupSpec{}, err
}
tgPort := builder.buildTargetGroupPort(targetType, *backend.ServicePort)
name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), k8s.NamespacedName(backend.Service), tgPort, targetType, tgProtocol, tgProtocolVersion)
name := builder.buildTargetGroupName(targetGroupProps, k8s.NamespacedName(gw), route.GetRouteNamespacedName(), route.GetRouteKind(), k8s.NamespacedName(backend.Service), tgPort, targetType, tgProtocol, tgProtocolVersion)

if tgPort == 0 {
if targetType == elbv2model.TargetTypeIP {
return elbv2model.TargetGroupSpec{}, errors.Errorf("TargetGroup port is empty. Are you using the correct service type?")
}
return elbv2model.TargetGroupSpec{}, errors.Errorf("TargetGroup port is empty. When using Instance targets, your service be must of type 'NodePort' or 'LoadBalancer'")
}

return elbv2model.TargetGroupSpec{
Name: name,
TargetType: targetType,
Expand All @@ -308,7 +307,7 @@ var invalidTargetGroupNamePattern = regexp.MustCompile("[[:^alnum:]]")

// buildTargetGroupName will calculate the targetGroup's name.
func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *elbv2gw.TargetGroupProps,
gwKey types.NamespacedName, routeKey types.NamespacedName, svcKey types.NamespacedName, tgPort int32,
gwKey types.NamespacedName, routeKey types.NamespacedName, routeKind routeutils.RouteKind, svcKey types.NamespacedName, tgPort int32,
targetType elbv2model.TargetType, tgProtocol elbv2model.Protocol, tgProtocolVersion *elbv2model.ProtocolVersion) string {

if targetGroupProps != nil && targetGroupProps.TargetGroupName != nil {
Expand All @@ -321,6 +320,7 @@ func (builder *targetGroupBuilderImpl) buildTargetGroupName(targetGroupProps *el
_, _ = uuidHash.Write([]byte(gwKey.Name))
_, _ = uuidHash.Write([]byte(routeKey.Namespace))
_, _ = uuidHash.Write([]byte(routeKey.Name))
_, _ = uuidHash.Write([]byte(routeKind))
_, _ = uuidHash.Write([]byte(svcKey.Namespace))
_, _ = uuidHash.Write([]byte(svcKey.Name))
_, _ = uuidHash.Write([]byte(strconv.Itoa(int(tgPort))))
Expand Down Expand Up @@ -664,8 +664,8 @@ func (builder *targetGroupBuilderImpl) convertMapToAttributes(attributeMap map[s
return convertedAttributes
}

func (builder *targetGroupBuilderImpl) buildTargetGroupResourceID(gwKey types.NamespacedName, svcKey types.NamespacedName, routeKey types.NamespacedName, port intstr.IntOrString) string {
return fmt.Sprintf("%s/%s:%s-%s:%s-%s:%s", gwKey.Namespace, gwKey.Name, routeKey.Namespace, routeKey.Name, svcKey.Namespace, svcKey.Name, port.String())
func (builder *targetGroupBuilderImpl) buildTargetGroupResourceID(gwKey types.NamespacedName, svcKey types.NamespacedName, routeKey types.NamespacedName, routeKind routeutils.RouteKind, port intstr.IntOrString) string {
return fmt.Sprintf("%s/%s:%s-%s:%s-%s-%s:%s", gwKey.Namespace, gwKey.Name, routeKey.Namespace, routeKey.Name, routeKind, svcKey.Namespace, svcKey.Name, port.String())
}

func (builder *targetGroupBuilderImpl) buildTargetGroupBindingNodeSelector(tgProps *elbv2gw.TargetGroupProps, targetType elbv2model.TargetType) *metav1.LabelSelector {
Expand Down
88 changes: 37 additions & 51 deletions pkg/gateway/model/model_build_target_group_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func Test_buildTargetGroupSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-d02da2803b",
Name: "k8s-myrouten-myroute-8d8111f6ac",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what are those values, when do they need to be changed? is it because we added routeKind this time?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, it is because the kind is added to the name,. the value is just the hash of a bunch of fields.

TargetType: elbv2model.TargetTypeInstance,
Port: awssdk.Int32(8080),
Protocol: elbv2model.ProtocolTCP,
Expand Down Expand Up @@ -123,7 +123,7 @@ func Test_buildTargetGroupSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-d146029dfb",
Name: "k8s-myrouten-myroute-224f4b6ea6",
TargetType: elbv2model.TargetTypeInstance,
Port: awssdk.Int32(8080),
Protocol: elbv2model.ProtocolHTTP,
Expand Down Expand Up @@ -183,7 +183,7 @@ func Test_buildTargetGroupSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-d9d6c4e6eb",
Name: "k8s-myrouten-myroute-3bce8b0f70",
TargetType: elbv2model.TargetTypeIP,
Port: awssdk.Int32(80),
Protocol: elbv2model.ProtocolTCP,
Expand Down Expand Up @@ -238,7 +238,7 @@ func Test_buildTargetGroupSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-400113e816",
Name: "k8s-myrouten-myroute-a44a20bcbf",
TargetType: elbv2model.TargetTypeIP,
Port: awssdk.Int32(80),
Protocol: elbv2model.ProtocolHTTP,
Expand Down Expand Up @@ -448,7 +448,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-d146029dfb",
Name: "k8s-myrouten-myroute-224f4b6ea6",
TargetType: elbv2model.TargetTypeInstance,
Port: awssdk.Int32(8080),
Protocol: elbv2model.ProtocolHTTP,
Expand Down Expand Up @@ -476,7 +476,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
Template: elbv2model.TargetGroupBindingTemplate{
ObjectMeta: metav1.ObjectMeta{
Namespace: "my-svc-ns",
Name: "k8s-myrouten-myroute-d146029dfb",
Name: "k8s-myrouten-myroute-224f4b6ea6",
Annotations: make(map[string]string),
Labels: make(map[string]string),
},
Expand Down Expand Up @@ -527,7 +527,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-d9d6c4e6eb",
Name: "k8s-myrouten-myroute-3bce8b0f70",
TargetType: elbv2model.TargetTypeIP,
Port: awssdk.Int32(80),
Protocol: elbv2model.ProtocolTCP,
Expand All @@ -550,7 +550,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
Template: elbv2model.TargetGroupBindingTemplate{
ObjectMeta: metav1.ObjectMeta{
Namespace: "my-svc-ns",
Name: "k8s-myrouten-myroute-d9d6c4e6eb",
Name: "k8s-myrouten-myroute-3bce8b0f70",
Annotations: make(map[string]string),
Labels: make(map[string]string),
},
Expand Down Expand Up @@ -601,7 +601,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-400113e816",
Name: "k8s-myrouten-myroute-a44a20bcbf",
TargetType: elbv2model.TargetTypeIP,
Port: awssdk.Int32(80),
Protocol: elbv2model.ProtocolHTTP,
Expand Down Expand Up @@ -629,7 +629,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
Template: elbv2model.TargetGroupBindingTemplate{
ObjectMeta: metav1.ObjectMeta{
Namespace: "my-svc-ns",
Name: "k8s-myrouten-myroute-400113e816",
Name: "k8s-myrouten-myroute-a44a20bcbf",
Annotations: make(map[string]string),
Labels: make(map[string]string),
},
Expand Down Expand Up @@ -690,7 +690,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
},
},
expectedTgSpec: elbv2model.TargetGroupSpec{
Name: "k8s-myrouten-myroute-400113e816",
Name: "k8s-myrouten-myroute-a44a20bcbf",
TargetType: elbv2model.TargetTypeIP,
Port: awssdk.Int32(80),
Protocol: elbv2model.ProtocolHTTP,
Expand Down Expand Up @@ -718,7 +718,7 @@ func Test_buildTargetGroupBindingSpec(t *testing.T) {
Template: elbv2model.TargetGroupBindingTemplate{
ObjectMeta: metav1.ObjectMeta{
Namespace: "my-svc-ns",
Name: "k8s-myrouten-myroute-400113e816",
Name: "k8s-myrouten-myroute-a44a20bcbf",
Annotations: map[string]string{
"foo": "bar",
},
Expand Down Expand Up @@ -771,7 +771,7 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {

targetPort intstr.IntOrString
healthCheckPort intstr.IntOrString
svcPort corev1.ServicePort
tgProtocol elbv2model.Protocol
backendSGIDToken core.StringToken

expected *elbv2model.TargetGroupBindingNetworking
Expand Down Expand Up @@ -804,9 +804,7 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
name: "disable restricted sg rules - with udp",
disableRestrictedSGRules: true,
backendSGIDToken: core.LiteralStringToken("foo"),
svcPort: corev1.ServicePort{
Protocol: corev1.ProtocolUDP,
},
tgProtocol: elbv2model.ProtocolUDP,
expected: &elbv2model.TargetGroupBindingNetworking{
Ingress: []elbv2model.NetworkingIngressRule{
{
Expand Down Expand Up @@ -834,11 +832,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
{
name: "use restricted sg rules - int hc port",
backendSGIDToken: core.LiteralStringToken("foo"),
svcPort: corev1.ServicePort{
Protocol: corev1.ProtocolTCP,
},
targetPort: intstr80,
healthCheckPort: intstr80,
tgProtocol: elbv2model.ProtocolTCP,
targetPort: intstr80,
healthCheckPort: intstr80,
expected: &elbv2model.TargetGroupBindingNetworking{
Ingress: []elbv2model.NetworkingIngressRule{
{
Expand All @@ -862,11 +858,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
{
name: "use restricted sg rules - int hc port - udp traffic",
backendSGIDToken: core.LiteralStringToken("foo"),
svcPort: corev1.ServicePort{
Protocol: corev1.ProtocolUDP,
},
targetPort: intstr80,
healthCheckPort: intstr80,
tgProtocol: elbv2model.ProtocolUDP,
targetPort: intstr80,
healthCheckPort: intstr80,
expected: &elbv2model.TargetGroupBindingNetworking{
Ingress: []elbv2model.NetworkingIngressRule{
{
Expand Down Expand Up @@ -905,11 +899,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
{
name: "use restricted sg rules - str hc port",
backendSGIDToken: core.LiteralStringToken("foo"),
svcPort: corev1.ServicePort{
Protocol: corev1.ProtocolTCP,
},
targetPort: intstr80,
healthCheckPort: intstrTrafficPort,
tgProtocol: elbv2model.ProtocolHTTP,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why we changed from TCP protocol to HTTP?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it's using the TG protocol, so testing out various protocols get translated correctly to TCP. Previously we were just using the service port.

targetPort: intstr80,
healthCheckPort: intstrTrafficPort,
expected: &elbv2model.TargetGroupBindingNetworking{
Ingress: []elbv2model.NetworkingIngressRule{
{
Expand All @@ -933,11 +925,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
{
name: "use restricted sg rules - str hc port - udp",
backendSGIDToken: core.LiteralStringToken("foo"),
svcPort: corev1.ServicePort{
Protocol: corev1.ProtocolUDP,
},
targetPort: intstr80,
healthCheckPort: intstrTrafficPort,
tgProtocol: elbv2model.ProtocolUDP,
targetPort: intstr80,
healthCheckPort: intstrTrafficPort,
expected: &elbv2model.TargetGroupBindingNetworking{
Ingress: []elbv2model.NetworkingIngressRule{
{
Expand Down Expand Up @@ -976,11 +966,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
{
name: "use restricted sg rules - diff hc port",
backendSGIDToken: core.LiteralStringToken("foo"),
svcPort: corev1.ServicePort{
Protocol: corev1.ProtocolTCP,
},
targetPort: intstr80,
healthCheckPort: intstr85,
tgProtocol: elbv2model.ProtocolHTTP,
targetPort: intstr80,
healthCheckPort: intstr85,
expected: &elbv2model.TargetGroupBindingNetworking{
Ingress: []elbv2model.NetworkingIngressRule{
{
Expand Down Expand Up @@ -1019,11 +1007,9 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
{
name: "use restricted sg rules - str hc port - udp",
backendSGIDToken: core.LiteralStringToken("foo"),
svcPort: corev1.ServicePort{
Protocol: corev1.ProtocolUDP,
},
targetPort: intstr80,
healthCheckPort: intstr85,
tgProtocol: elbv2model.ProtocolUDP,
targetPort: intstr80,
healthCheckPort: intstr85,
expected: &elbv2model.TargetGroupBindingNetworking{
Ingress: []elbv2model.NetworkingIngressRule{
{
Expand Down Expand Up @@ -1066,7 +1052,7 @@ func Test_buildTargetGroupBindingNetworking(t *testing.T) {
disableRestrictedSGRules: tc.disableRestrictedSGRules,
}

result := builder.buildTargetGroupBindingNetworking(tc.targetPort, tc.healthCheckPort, tc.svcPort, tc.backendSGIDToken)
result := builder.buildTargetGroupBindingNetworking(tc.targetPort, tc.healthCheckPort, tc.tgProtocol, tc.backendSGIDToken)
assert.Equal(t, tc.expected, result)
})
}
Expand Down Expand Up @@ -1101,16 +1087,16 @@ func Test_buildTargetGroupName(t *testing.T) {
{
name: "no name in props",
targetGroupProps: &elbv2gw.TargetGroupProps{},
expected: "k8s-myns-myroute-719950e570",
expected: "k8s-myns-myroute-27d98b9190",
},
{
name: "no props",
expected: "k8s-myns-myroute-719950e570",
expected: "k8s-myns-myroute-27d98b9190",
},
{
name: "protocol specified props",
protocolVersion: &http2,
expected: "k8s-myns-myroute-ce262fa9fe",
expected: "k8s-myns-myroute-d2bd5deaa7",
},
}

Expand All @@ -1120,7 +1106,7 @@ func Test_buildTargetGroupName(t *testing.T) {
clusterName: clusterName,
}

result := builder.buildTargetGroupName(tc.targetGroupProps, gwKey, routeKey, svcKey, 80, elbv2model.TargetTypeIP, elbv2model.ProtocolTCP, tc.protocolVersion)
result := builder.buildTargetGroupName(tc.targetGroupProps, gwKey, routeKey, routeutils.HTTPRouteKind, svcKey, 80, elbv2model.TargetTypeIP, elbv2model.ProtocolTCP, tc.protocolVersion)
assert.Equal(t, tc.expected, result)
})
}
Expand Down
16 changes: 10 additions & 6 deletions pkg/gateway/routeutils/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci
// Otherwise, general error. No need for status update.
return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch svc object %+v", svcIdentifier))
}

// TODO -- This should be updated, to handle UDP and TCP on the same service port.
// Currently, it will just arbitrarily take one.

var servicePort *corev1.ServicePort

for _, svcPort := range svc.Spec.Ports {
Expand All @@ -107,19 +111,19 @@ func commonBackendLoader(ctx context.Context, k8sClient client.Client, typeSpeci
}
}

if servicePort == nil {
initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port)
wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier)
return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil)
}

tgConfig, err := LookUpTargetGroupConfiguration(ctx, k8sClient, k8s.NamespacedName(svc))

if err != nil {
// As of right now, this error can only be thrown because of a k8s api error hence no status update.
return nil, errors.Wrap(err, fmt.Sprintf("Unable to fetch tg config object"))
}

if servicePort == nil {
initialErrorMessage := fmt.Sprintf("Unable to find service port for port %d", *backendRef.Port)
wrappedGatewayErrorMessage := generateInvalidMessageWithRouteDetails(initialErrorMessage, routeKind, routeIdentifier)
return nil, wrapError(errors.Errorf("%s", initialErrorMessage), gwv1.GatewayReasonListenersNotValid, gwv1.RouteReasonBackendNotFound, &wrappedGatewayErrorMessage, nil)
}

var tgProps *elbv2gw.TargetGroupProps

if tgConfig != nil {
Expand Down
12 changes: 6 additions & 6 deletions pkg/gateway/routeutils/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ var allRoutes = map[RouteKind]func(context context.Context, client client.Client
}

// Default protocol map used to infer accepted route kinds when a listener doesn't specify the `allowedRoutes` field.
var defaultProtocolToRouteKindMap = map[gwv1.ProtocolType]RouteKind{
gwv1.TCPProtocolType: TCPRouteKind,
gwv1.UDPProtocolType: UDPRouteKind,
gwv1.TLSProtocolType: TLSRouteKind,
gwv1.HTTPProtocolType: HTTPRouteKind,
gwv1.HTTPSProtocolType: HTTPRouteKind,
var defaultProtocolToRouteKindMap = map[gwv1.ProtocolType][]RouteKind{
gwv1.TCPProtocolType: {TCPRouteKind},
gwv1.UDPProtocolType: {UDPRouteKind},
gwv1.TLSProtocolType: {TLSRouteKind, TCPRouteKind},
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i might missed it, other than the kindCheck, did we also check somewhere that TLS protocol can only use TCPRoute when mode=terminate

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good point. fixed.

gwv1.HTTPProtocolType: {HTTPRouteKind},
gwv1.HTTPSProtocolType: {HTTPRouteKind},
}
Loading
Loading