From dc80aad0307c2d816d5375f1504a928454200431 Mon Sep 17 00:00:00 2001 From: Steve Cathcart Date: Thu, 13 Feb 2025 17:50:04 +0100 Subject: [PATCH 1/7] TestKit backend: add custom DNS resolution hook Signed-off-by: Rouven Bauer --- neo4j/driver_with_context_testkit.go | 4 +++ neo4j/internal/bolt/hydrator.go | 5 ++-- neo4j/internal/connector/connector.go | 34 ++++++++++++++++++------ testkit-backend/backend.go | 37 ++++++++++++++++++++++++--- 4 files changed, 65 insertions(+), 15 deletions(-) diff --git a/neo4j/driver_with_context_testkit.go b/neo4j/driver_with_context_testkit.go index 5e645bce..31ad3e3d 100644 --- a/neo4j/driver_with_context_testkit.go +++ b/neo4j/driver_with_context_testkit.go @@ -51,6 +51,10 @@ func ForceRoutingTableUpdate(d DriverWithContext, database string, bookmarks []s return errorutil.WrapError(err) } +func RegisterDnsResolver(d DriverWithContext, hook func(address string) []string) { + d.(*driverWithContext).connector.TestKitDnsResolver = hook +} + func GetRoutingTable(d DriverWithContext, database string) (*RoutingTable, error) { driver := d.(*driverWithContext) router, ok := driver.router.(*router.Router) diff --git a/neo4j/internal/bolt/hydrator.go b/neo4j/internal/bolt/hydrator.go index 7f57a8c2..45de57de 100644 --- a/neo4j/internal/bolt/hydrator.go +++ b/neo4j/internal/bolt/hydrator.go @@ -24,12 +24,11 @@ import ( "github.com/neo4j/neo4j-go-driver/v5/neo4j/internal/gql" "time" - idb "github.com/neo4j/neo4j-go-driver/v5/neo4j/internal/db" - "github.com/neo4j/neo4j-go-driver/v5/neo4j/log" - "github.com/neo4j/neo4j-go-driver/v5/neo4j/db" "github.com/neo4j/neo4j-go-driver/v5/neo4j/dbtype" + idb "github.com/neo4j/neo4j-go-driver/v5/neo4j/internal/db" "github.com/neo4j/neo4j-go-driver/v5/neo4j/internal/packstream" + "github.com/neo4j/neo4j-go-driver/v5/neo4j/log" ) const containsSystemUpdatesKey = "contains-system-updates" diff --git a/neo4j/internal/connector/connector.go b/neo4j/internal/connector/connector.go index 97775c13..49ef624a 100644 --- a/neo4j/internal/connector/connector.go +++ b/neo4j/internal/connector/connector.go @@ -34,13 +34,14 @@ import ( ) type Connector struct { - SkipEncryption bool - SkipVerify bool - Log log.Logger - RoutingContext map[string]string - Network string - Config *config.Config - SupplyConnection func(context.Context, string) (net.Conn, error) + SkipEncryption bool + SkipVerify bool + Log log.Logger + RoutingContext map[string]string + Network string + Config *config.Config + SupplyConnection func(context.Context, string) (net.Conn, error) + TestKitDnsResolver func(string) []string } func (c Connector) Connect( @@ -138,7 +139,24 @@ func (c Connector) createConnection(ctx context.Context, address string) (net.Co dialer.KeepAlive = -1 * time.Second // Turns keep-alive off } - return dialer.DialContext(ctx, c.Network, address) + if c.TestKitDnsResolver == nil { + return dialer.DialContext(ctx, c.Network, address) + } + + addresses := c.TestKitDnsResolver(address) + + if len(addresses) == 0 { + return nil, errors.New("TestKit DNS resolver returned no address") + } + + var err error = nil + for _, address := range addresses { + con, err := dialer.DialContext(ctx, c.Network, address) + if err == nil { + return con, nil + } + } + return nil, err } func (c Connector) tlsConfig(serverName string) *tls.Config { diff --git a/testkit-backend/backend.go b/testkit-backend/backend.go index 9106674d..a994f5ea 100644 --- a/testkit-backend/backend.go +++ b/testkit-backend/backend.go @@ -52,6 +52,7 @@ type backend struct { explicitTransactions map[string]neo4j.ExplicitTransaction recordedErrors map[string]error resolvedAddresses map[string][]any + dnsResolutions map[string][]any authTokenManagers map[string]auth.TokenManager resolvedGetAuthTokens map[string]neo4j.AuthToken resolvedHandleSecurityException map[string]bool @@ -148,6 +149,7 @@ func newBackend(rd *bufio.Reader, wr io.Writer) *backend { explicitTransactions: make(map[string]neo4j.ExplicitTransaction), recordedErrors: make(map[string]error), resolvedAddresses: make(map[string][]any), + dnsResolutions: make(map[string][]any), authTokenManagers: make(map[string]auth.TokenManager), resolvedGetAuthTokens: make(map[string]neo4j.AuthToken), resolvedHandleSecurityException: make(map[string]bool), @@ -496,6 +498,27 @@ func (b *backend) customAddressResolverFunction() config.ServerAddressResolver { } } +func (b *backend) dnsResolverFunction() func(address string) []string { + return func(address string) []string { + id := b.nextId() + b.writeResponse("DomainNameResolutionRequired", map[string]string{ + "id": id, + "name": address, + }) + for { + b.process() + if addresses, ok := b.dnsResolutions[id]; ok { + delete(b.dnsResolutions, id) + result := make([]string, len(addresses)) + for i, address := range addresses { + result[i] = address.(string) + } + return result + } + } + } +} + type serverAddress struct { hostname string port string @@ -533,6 +556,11 @@ func (b *backend) handleRequest(req map[string]any) { fmt.Printf("REQ: %s %s\n", name, dataJson) switch name { + case "DomainNameResolutionCompleted": + requestId := data["requestId"].(string) + addresses := data["addresses"].([]any) + b.dnsResolutions[requestId] = addresses + case "ResolverResolutionCompleted": requestId := data["requestId"].(string) addresses := data["addresses"].([]any) @@ -637,6 +665,11 @@ func (b *backend) handleRequest(req map[string]any) { b.writeError(err) return } + + if data["domainNameResolverRegistered"] != nil && data["domainNameResolverRegistered"].(bool) { + neo4j.RegisterDnsResolver(driver, b.dnsResolverFunction()) + } + idKey := b.nextId() b.drivers[idKey] = driver b.writeResponse("Driver", map[string]any{"id": idKey}) @@ -1691,10 +1724,6 @@ func testSkips() map[string]string { "stub.routing.test_routing_v3.RoutingV3.test_should_fail_when_writing_on_unexpectedly_interrupting_writer_on_run_using_tx_run": "Won't fix - only Bolt 3 affected (not officially supported by this driver): broken servers are not removed from routing table", "stub.routing.test_routing_v3.RoutingV3.test_should_fail_when_writing_on_unexpectedly_interrupting_writer_using_tx_run": "Won't fix - only Bolt 3 affected (not officially supported by this driver): broken servers are not removed from routing table", - // Missing message support in testkit backend - "stub.routing.*.*.test_should_request_rt_from_all_initial_routers_until_successful_on_unknown_failure": "Add DNS resolver TestKit message and connection timeout support", - "stub.routing.*.*.test_should_request_rt_from_all_initial_routers_until_successful_on_authorization_expired": "Add DNS resolver TestKit message and connection timeout support", - // To fix/to decide whether to fix "stub.routing.test_routing_v*.RoutingV*.test_should_revert_to_initial_router_if_known_router_throws_protocol_errors": "Driver always uses configured URL first and custom resolver only if that fails", "stub.routing.test_routing_v*.RoutingV*.test_should_read_successfully_from_reachable_db_after_trying_unreachable_db": "Driver retries to fetch a routing table up to 100 times if it's empty", From 96f4d10cae0f3ca9249f591089038289d86082f9 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Thu, 13 Feb 2025 18:10:14 +0100 Subject: [PATCH 2/7] Fix DNS hook protocol: shouldn't send or expect port --- neo4j/internal/connector/connector.go | 9 ++++++--- testkit-backend/backend.go | 12 ++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/neo4j/internal/connector/connector.go b/neo4j/internal/connector/connector.go index 49ef624a..791cc375 100644 --- a/neo4j/internal/connector/connector.go +++ b/neo4j/internal/connector/connector.go @@ -44,7 +44,7 @@ type Connector struct { TestKitDnsResolver func(string) []string } -func (c Connector) Connect( +func (c *Connector) Connect( ctx context.Context, address string, auth *db.ReAuthToken, @@ -149,9 +149,12 @@ func (c Connector) createConnection(ctx context.Context, address string) (net.Co return nil, errors.New("TestKit DNS resolver returned no address") } - var err error = nil + var ( + err error + con net.Conn + ) for _, address := range addresses { - con, err := dialer.DialContext(ctx, c.Network, address) + con, err = dialer.DialContext(ctx, c.Network, address) if err == nil { return con, nil } diff --git a/testkit-backend/backend.go b/testkit-backend/backend.go index a994f5ea..9dc36ac2 100644 --- a/testkit-backend/backend.go +++ b/testkit-backend/backend.go @@ -29,6 +29,7 @@ import ( "github.com/neo4j/neo4j-go-driver/v5/neo4j/notifications" "io" "math" + "net" "net/url" "regexp" "strings" @@ -501,9 +502,16 @@ func (b *backend) customAddressResolverFunction() config.ServerAddressResolver { func (b *backend) dnsResolverFunction() func(address string) []string { return func(address string) []string { id := b.nextId() + host, port, err := net.SplitHostPort(address) + if err != nil { + b.writeError(fmt.Errorf( + "couldn't parse address for custom DNS resulution (probably a bug in backend): %w", err, + )) + return nil + } b.writeResponse("DomainNameResolutionRequired", map[string]string{ "id": id, - "name": address, + "name": host, }) for { b.process() @@ -511,7 +519,7 @@ func (b *backend) dnsResolverFunction() func(address string) []string { delete(b.dnsResolutions, id) result := make([]string, len(addresses)) for i, address := range addresses { - result[i] = address.(string) + result[i] = fmt.Sprintf("%s:%s", address, port) } return result } From 94e74584e7db7505966bc804c72cb6c6f3ad8563 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Thu, 13 Feb 2025 18:19:31 +0100 Subject: [PATCH 3/7] Improve connection logging --- neo4j/driver_with_context.go | 1 + neo4j/internal/connector/connector.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/neo4j/driver_with_context.go b/neo4j/driver_with_context.go index 6cff2cb5..3aeaf4bd 100644 --- a/neo4j/driver_with_context.go +++ b/neo4j/driver_with_context.go @@ -216,6 +216,7 @@ func NewDriverWithContext(target string, auth auth.TokenManager, configurers ... } d.connector.Log = d.log + d.connector.LogId = d.logId d.connector.RoutingContext = routingContext d.connector.Config = d.config diff --git a/neo4j/internal/connector/connector.go b/neo4j/internal/connector/connector.go index 791cc375..110e6bc2 100644 --- a/neo4j/internal/connector/connector.go +++ b/neo4j/internal/connector/connector.go @@ -37,6 +37,7 @@ type Connector struct { SkipEncryption bool SkipVerify bool Log log.Logger + LogId string RoutingContext map[string]string Network string Config *config.Config @@ -140,6 +141,7 @@ func (c Connector) createConnection(ctx context.Context, address string) (net.Co } if c.TestKitDnsResolver == nil { + c.Log.Debugf(log.Driver, c.LogId, "dialing %s", address) return dialer.DialContext(ctx, c.Network, address) } @@ -156,6 +158,7 @@ func (c Connector) createConnection(ctx context.Context, address string) (net.Co for _, address := range addresses { con, err = dialer.DialContext(ctx, c.Network, address) if err == nil { + c.Log.Debugf(log.Driver, c.LogId, "dialing %s", address) return con, nil } } From 9da454bbcec577dc0b290e1d9eed6ce84479e604 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Fri, 14 Feb 2025 09:36:13 +0100 Subject: [PATCH 4/7] Skip test the driver isn't compliant with --- testkit-backend/backend.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/testkit-backend/backend.go b/testkit-backend/backend.go index 9dc36ac2..6cfd0e6f 100644 --- a/testkit-backend/backend.go +++ b/testkit-backend/backend.go @@ -1733,14 +1733,15 @@ func testSkips() map[string]string { "stub.routing.test_routing_v3.RoutingV3.test_should_fail_when_writing_on_unexpectedly_interrupting_writer_using_tx_run": "Won't fix - only Bolt 3 affected (not officially supported by this driver): broken servers are not removed from routing table", // To fix/to decide whether to fix - "stub.routing.test_routing_v*.RoutingV*.test_should_revert_to_initial_router_if_known_router_throws_protocol_errors": "Driver always uses configured URL first and custom resolver only if that fails", - "stub.routing.test_routing_v*.RoutingV*.test_should_read_successfully_from_reachable_db_after_trying_unreachable_db": "Driver retries to fetch a routing table up to 100 times if it's empty", - "stub.routing.test_routing_v*.RoutingV*.test_should_write_successfully_after_leader_switch_using_tx_run": "Driver retries to fetch a routing table up to 100 times if it's empty", - "stub.routing.test_routing_v*.RoutingV*.test_should_fail_when_writing_without_writers_using_session_run": "Driver retries to fetch a routing table up to 100 times if it's empty", - "stub.routing.test_routing_v*.RoutingV*.test_should_accept_routing_table_without_writers_and_then_rediscover": "Driver retries to fetch a routing table up to 100 times if it's empty", - "stub.routing.test_routing_v*.RoutingV*.test_should_fail_on_routing_table_with_no_reader": "Driver retries to fetch a routing table up to 100 times if it's empty", - "stub.routing.test_routing_v*.RoutingV*.test_should_fail_discovery_when_router_fails_with_unknown_code": "Unify: other drivers have a list of fast failing errors during discover: on anything else, the driver will try the next router", - "stub.routing.test_routing_v*.RoutingV*.test_should_drop_connections_failing_liveness_check": "Liveness check error handling is not (yet) unified: https://github.com/neo-technology/drivers-adr/pull/83", + "stub.routing.test_routing_v*.RoutingV*.test_should_revert_to_initial_router_if_known_router_throws_protocol_errors": "Driver always uses configured URL first and custom resolver only if that fails", + "stub.routing.test_routing_v*.RoutingV*.test_should_request_rt_from_all_initial_routers_until_successful_on_authorization_expired": "Driver always uses configured URL first and custom resolver only if that fails", + "stub.routing.test_routing_v*.RoutingV*.test_should_read_successfully_from_reachable_db_after_trying_unreachable_db": "Driver retries to fetch a routing table up to 100 times if it's empty", + "stub.routing.test_routing_v*.RoutingV*.test_should_write_successfully_after_leader_switch_using_tx_run": "Driver retries to fetch a routing table up to 100 times if it's empty", + "stub.routing.test_routing_v*.RoutingV*.test_should_fail_when_writing_without_writers_using_session_run": "Driver retries to fetch a routing table up to 100 times if it's empty", + "stub.routing.test_routing_v*.RoutingV*.test_should_accept_routing_table_without_writers_and_then_rediscover": "Driver retries to fetch a routing table up to 100 times if it's empty", + "stub.routing.test_routing_v*.RoutingV*.test_should_fail_on_routing_table_with_no_reader": "Driver retries to fetch a routing table up to 100 times if it's empty", + "stub.routing.test_routing_v*.RoutingV*.test_should_fail_discovery_when_router_fails_with_unknown_code": "Unify: other drivers have a list of fast failing errors during discover: on anything else, the driver will try the next router", + "stub.routing.test_routing_v*.RoutingV*.test_should_drop_connections_failing_liveness_check": "Liveness check error handling is not (yet) unified: https://github.com/neo-technology/drivers-adr/pull/83", "stub.*.test_0_timeout": "Fixme: driver omits 0 as tx timeout value", "stub.summary.test_summary.TestSummaryBasicInfo.test_server_info": "pending unification: should the server address be pre or post DNS resolution?", } From 85fa5ef33b5ae75d0aec7db645fe5bfa6986358e Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Fri, 14 Feb 2025 09:54:45 +0100 Subject: [PATCH 5/7] More another test skipped --- testkit-backend/backend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/testkit-backend/backend.go b/testkit-backend/backend.go index 6cfd0e6f..3d3b464a 100644 --- a/testkit-backend/backend.go +++ b/testkit-backend/backend.go @@ -1735,6 +1735,7 @@ func testSkips() map[string]string { // To fix/to decide whether to fix "stub.routing.test_routing_v*.RoutingV*.test_should_revert_to_initial_router_if_known_router_throws_protocol_errors": "Driver always uses configured URL first and custom resolver only if that fails", "stub.routing.test_routing_v*.RoutingV*.test_should_request_rt_from_all_initial_routers_until_successful_on_authorization_expired": "Driver always uses configured URL first and custom resolver only if that fails", + "stub.routing.test_routing_v*.RoutingV*test_should_request_rt_from_all_initial_routers_until_successful_on_unknown_failure": "Driver always uses configured URL first and custom resolver only if that fails", "stub.routing.test_routing_v*.RoutingV*.test_should_read_successfully_from_reachable_db_after_trying_unreachable_db": "Driver retries to fetch a routing table up to 100 times if it's empty", "stub.routing.test_routing_v*.RoutingV*.test_should_write_successfully_after_leader_switch_using_tx_run": "Driver retries to fetch a routing table up to 100 times if it's empty", "stub.routing.test_routing_v*.RoutingV*.test_should_fail_when_writing_without_writers_using_session_run": "Driver retries to fetch a routing table up to 100 times if it's empty", From 6c231eec000270bea8fcd3373716dab50cd1e4f3 Mon Sep 17 00:00:00 2001 From: Robsdedude Date: Fri, 14 Feb 2025 09:55:00 +0100 Subject: [PATCH 6/7] Avoid backend getting stuck if TestKit fails to respond to resolver request --- testkit-backend/backend.go | 41 ++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/testkit-backend/backend.go b/testkit-backend/backend.go index 3d3b464a..0b4b40b8 100644 --- a/testkit-backend/backend.go +++ b/testkit-backend/backend.go @@ -66,6 +66,7 @@ type backend struct { bookmarkManagers map[string]neo4j.BookmarkManager clientCertificateProviders map[string]auth.ClientCertificateProvider resolvedClientCertificates map[string]auth.ClientCertificate + closed bool } // To implement transactional functions a bit of extra state is needed on the @@ -162,6 +163,7 @@ func newBackend(rd *bufio.Reader, wr io.Writer) *backend { consumedBookmarks: make(map[string]struct{}), clientCertificateProviders: make(map[string]auth.ClientCertificateProvider), resolvedClientCertificates: make(map[string]auth.ClientCertificate), + closed: false, } } @@ -298,6 +300,7 @@ func (b *backend) process() bool { for { line, err := b.rd.ReadString('\n') if err != nil { + b.closed = true return false } @@ -332,6 +335,10 @@ func (b *backend) writeResponse(name string, data any) { if err != nil { panic(err.Error()) } + if b.closed { + fmt.Print("RES ignored because backend is closed\n") + return + } // Make sure that logging framework doesn't write anything inbetween here... b.wrLock.Lock() defer b.wrLock.Unlock() @@ -446,8 +453,7 @@ func (b *backend) handleTransactionFunc(isRead bool, data map[string]any) { b.managedTransactions[txId] = tx b.writeResponse("RetryableTry", map[string]any{"id": txId}) // Process all things that the client might do within the transaction - for { - b.process() + for b.process() { switch sessionState.retryableState { case retryablePositive: // Client succeeded and wants to commit @@ -463,6 +469,7 @@ func (b *backend) handleTransactionFunc(isRead bool, data map[string]any) { // Client did something not related to the retryable state } } + return nil, nil } var err error if isRead { @@ -485,8 +492,7 @@ func (b *backend) customAddressResolverFunction() config.ServerAddressResolver { "id": id, "address": fmt.Sprintf("%s:%s", address.Hostname(), address.Port()), }) - for { - b.process() + for b.process() { if addresses, ok := b.resolvedAddresses[id]; ok { delete(b.resolvedAddresses, id) result := make([]config.ServerAddress, len(addresses)) @@ -496,6 +502,7 @@ func (b *backend) customAddressResolverFunction() config.ServerAddressResolver { return result } } + return nil } } @@ -513,8 +520,7 @@ func (b *backend) dnsResolverFunction() func(address string) []string { "id": id, "name": host, }) - for { - b.process() + for b.process() { if addresses, ok := b.dnsResolutions[id]; ok { delete(b.dnsResolutions, id) result := make([]string, len(addresses)) @@ -524,6 +530,7 @@ func (b *backend) dnsResolverFunction() func(address string) []string { return result } } + return nil } } @@ -1188,13 +1195,13 @@ func (b *backend) handleRequest(req map[string]any) { "id": id, "authTokenManagerId": managerId, }) - for { - b.process() + for b.process() { if token, ok := b.resolvedGetAuthTokens[id]; ok { delete(b.resolvedGetAuthTokens, id) return token } } + return neo4j.AuthToken{} }, HandleSecurityExceptionFunc: func(token neo4j.AuthToken, error *db.Neo4jError) bool { id := b.nextId() @@ -1206,13 +1213,13 @@ func (b *backend) handleRequest(req map[string]any) { "auth": serializeAuth(token), "errorCode": error.Code, }) - for { - b.process() + for b.process() { if handled, ok := b.resolvedHandleSecurityException[id]; ok { delete(b.resolvedHandleSecurityException, id) return handled } } + return false }, } b.authTokenManagers[managerId] = manager @@ -1241,13 +1248,13 @@ func (b *backend) handleRequest(req map[string]any) { "id": id, "basicAuthTokenManagerId": managerId, }) - for { - b.process() + for b.process() { if basicToken, ok := b.resolvedBasicTokens[id]; ok { delete(b.resolvedBasicTokens, id) return basicToken.token, nil } } + return neo4j.AuthToken{}, nil }) b.authTokenManagers[managerId] = manager b.writeResponse("BasicAuthTokenManager", map[string]any{"id": managerId}) @@ -1267,13 +1274,13 @@ func (b *backend) handleRequest(req map[string]any) { "id": id, "bearerAuthTokenManagerId": managerId, }) - for { - b.process() + for b.process() { if bearerToken, ok := b.resolvedBearerTokens[id]; ok { delete(b.resolvedBearerTokens, id) return bearerToken.token, bearerToken.expiration, nil } } + return neo4j.AuthToken{}, nil, nil }) b.authTokenManagers[managerId] = manager b.writeResponse("BearerAuthTokenManager", map[string]any{"id": managerId}) @@ -1830,13 +1837,13 @@ func (b *backend) consumeBookmarks(bookmarkManagerId string) func(context.Contex "bookmarkManagerId": bookmarkManagerId, "bookmarks": bookmarks, }) - for { - b.process() + for b.process() { if _, found := b.consumedBookmarks[id]; found { delete(b.consumedBookmarks, id) - return nil + break } } + return nil } } From e67168040beee5ac183fecf8d4758c8af11b359c Mon Sep 17 00:00:00 2001 From: Stephen Cathcart Date: Mon, 17 Feb 2025 11:17:50 +0000 Subject: [PATCH 7/7] Add skipped test from incorect merge --- testkit-backend/backend.go | 1 + 1 file changed, 1 insertion(+) diff --git a/testkit-backend/backend.go b/testkit-backend/backend.go index fbcddb7f..60ec08f6 100644 --- a/testkit-backend/backend.go +++ b/testkit-backend/backend.go @@ -1758,6 +1758,7 @@ func testSkips() map[string]string { "stub.routing.test_routing_v3.RoutingV3.test_should_fail_when_writing_on_unexpectedly_interrupting_writer_using_tx_run": "Won't fix - only Bolt 3 affected (not officially supported by this driver): broken servers are not removed from routing table", // To fix/to decide whether to fix + "stub.routing.*.*.test_should_successfully_acquire_rt_when_router_ip_changes": "Backend lacks custom DNS resolution and Go driver RT discovery differs.", "stub.routing.test_routing_v*.RoutingV*.test_should_revert_to_initial_router_if_known_router_throws_protocol_errors": "Driver always uses configured URL first and custom resolver only if that fails", "stub.routing.test_routing_v*.RoutingV*.test_should_request_rt_from_all_initial_routers_until_successful_on_authorization_expired": "Driver always uses configured URL first and custom resolver only if that fails", "stub.routing.test_routing_v*.RoutingV*test_should_request_rt_from_all_initial_routers_until_successful_on_unknown_failure": "Driver always uses configured URL first and custom resolver only if that fails",